From 0f3159709739d5bc1044d2876a83c0cf087ada4c Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 19 Oct 2026 18:12:47 +0200 Subject: [PATCH 01/32] fix(ai-anthropic): send the computer use beta header computerUseTool() went out without its anthropic-beta header, so Anthropic rejected the request. Add computer-use-2025-01-24 for computer_20250124 and computer-use-2024-10-22 for computer_20241022. --- .changeset/anthropic-computer-use-beta.md | 5 ++ packages/ai-anthropic/src/adapters/text.ts | 16 +++++ .../tests/computer-use-beta.test.ts | 68 +++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 .changeset/anthropic-computer-use-beta.md create mode 100644 packages/ai-anthropic/tests/computer-use-beta.test.ts diff --git a/.changeset/anthropic-computer-use-beta.md b/.changeset/anthropic-computer-use-beta.md new file mode 100644 index 0000000000..780f25cd02 --- /dev/null +++ b/.changeset/anthropic-computer-use-beta.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-anthropic': patch +--- + +Send the `anthropic-beta` header that `computerUseTool()` needs: `computer-use-2025-01-24` for `computer_20250124`, and `computer-use-2024-10-22` for `computer_20241022`. Before, a request with the tool had no computer use beta, so Anthropic rejected it. diff --git a/packages/ai-anthropic/src/adapters/text.ts b/packages/ai-anthropic/src/adapters/text.ts index d02313231b..dd3dec4dbc 100644 --- a/packages/ai-anthropic/src/adapters/text.ts +++ b/packages/ai-anthropic/src/adapters/text.ts @@ -16,6 +16,7 @@ import { import { BaseTextAdapter } from '@tanstack/ai/adapters' import { convertToolsToProviderFormat } from '../tools/tool-converter' import { getAnthropicProviderToolKind } from '../tools/anthropic-provider-tool' +import { convertComputerUseToolToAdapterFormat } from '../tools/computer-use-tool' import { readCodeExecutionConfig, readCodeExecutionSkills, @@ -216,6 +217,8 @@ function toAnthropicToolChoice( * - `interleaved-thinking-2025-05-14` when interleaved thinking is enabled, * - `code-execution-2025-08-25` when a `code_execution` tool is present, * - `skills-2025-10-02` when that tool carries skills, + * - `computer-use-2024-10-22` or `computer-use-2025-01-24` for the version + * of a `computer` tool, * - `context-management-2025-06-27` when `context_management` is set, * - `files-api-2025-04-14` when a message references an uploaded file handle, * - `mid-conversation-tool-changes-2026-07-01` in mid-conversation tool mode. @@ -291,6 +294,19 @@ export function computeAnthropicBetas( ) if (hasSkills) betas.add('skills-2025-10-02') + // Each computer use version needs its own beta header. + const computerTool = tools?.find( + (tool) => getAnthropicProviderToolKind(tool) === 'computer_use', + ) + if (computerTool) { + betas.add( + convertComputerUseToolToAdapterFormat(computerTool).type === + 'computer_20241022' + ? 'computer-use-2024-10-22' + : 'computer-use-2025-01-24', + ) + } + return betas.size > 0 ? Array.from(betas) : undefined } diff --git a/packages/ai-anthropic/tests/computer-use-beta.test.ts b/packages/ai-anthropic/tests/computer-use-beta.test.ts new file mode 100644 index 0000000000..7957398814 --- /dev/null +++ b/packages/ai-anthropic/tests/computer-use-beta.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it, vi } from 'vitest' +import { chat } from '@tanstack/ai' +import { AnthropicTextAdapter } from '../src/adapters/text' +import { computerUseTool } from '../src/tools' + +const mocks = vi.hoisted(() => { + const betaMessagesCreate = vi.fn() + const client = { beta: { messages: { create: betaMessagesCreate } } } + return { betaMessagesCreate, client } +}) + +vi.mock('@anthropic-ai/sdk', () => { + const { client } = mocks + class MockAnthropic { + beta = client.beta + constructor(_: { apiKey: string }) {} + } + return { default: MockAnthropic } +}) + +function mockEmptyStream() { + mocks.betaMessagesCreate.mockResolvedValueOnce( + (async function* () { + yield { + type: 'message_delta', + delta: { stop_reason: 'end_turn' }, + usage: { output_tokens: 1 }, + } + yield { type: 'message_stop' } + })(), + ) +} + +async function sentBetas(tool: ReturnType) { + mockEmptyStream() + const adapter = new AnthropicTextAdapter({ apiKey: 'k' }, 'claude-haiku-4-5') + for await (const _ of chat({ + adapter, + messages: [{ role: 'user', content: 'Open the browser.' }], + tools: [tool], + })) { + // consume + } + const [payload] = mocks.betaMessagesCreate.mock.calls.at(-1)! + return payload.betas ?? [] +} + +describe('anthropic computer use beta header', () => { + it('sends computer_20250124 with the computer-use-2025-01-24 beta', async () => { + const tool = computerUseTool({ + type: 'computer_20250124', + name: 'computer', + display_width_px: 1024, + display_height_px: 768, + }) + expect(await sentBetas(tool)).toContain('computer-use-2025-01-24') + }) + + it('sends computer_20241022 with the computer-use-2024-10-22 beta', async () => { + const tool = computerUseTool({ + type: 'computer_20241022', + name: 'computer', + display_width_px: 1024, + display_height_px: 768, + }) + expect(await sentBetas(tool)).toContain('computer-use-2024-10-22') + }) +}) From 07228a73adc30b30e50b0f4ed87a3dc44a959fac Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 19 Oct 2026 18:22:13 +0200 Subject: [PATCH 02/32] fix(openai-base): run the computer use loop A computer_call from the model is now a tool call to computer_use_preview, with the action and the pending safety checks as its input. The tool result goes back as computer_call_output, with the first image of the result as the screenshot. When the call does not fail, the output acknowledges the safety checks of the call. --- .changeset/openai-computer-call-loop.md | 5 + docs/adapters/openai.md | 50 ++++++- docs/tools/provider-tools.md | 2 +- .../src/adapters/responses-text.ts | 27 +++- .../src/adapters/responses-user-tools.ts | 103 ++++++++++++- .../tests/computer-use-loop.test.ts | 137 ++++++++++++++++++ 6 files changed, 310 insertions(+), 14 deletions(-) create mode 100644 .changeset/openai-computer-call-loop.md create mode 100644 packages/openai-base/tests/computer-use-loop.test.ts diff --git a/.changeset/openai-computer-call-loop.md b/.changeset/openai-computer-call-loop.md new file mode 100644 index 0000000000..9232a504f1 --- /dev/null +++ b/.changeset/openai-computer-call-loop.md @@ -0,0 +1,5 @@ +--- +'@tanstack/openai-base': patch +--- + +The computer use loop of `computerUseTool()` runs. A `computer_call` from the model is now a tool call to `computer_use_preview`, with the `action` and the `pending_safety_checks` as its input. The tool result goes back as `computer_call_output`, with the first image of the result as the screenshot. When the call does not fail, the output acknowledges the safety checks of the call. diff --git a/docs/adapters/openai.md b/docs/adapters/openai.md index 2812e181cd..6646c49201 100644 --- a/docs/adapters/openai.md +++ b/docs/adapters/openai.md @@ -996,25 +996,63 @@ Lets the model observe a virtual desktop via screenshots and interact with it using keyboard and mouse events. Provide the display dimensions and the execution environment type. +The model asks for one action at a time, for example a click. Your app runs the action, then returns a screenshot of the screen. Add an `execute` to the tool that does this: + ```typescript import { chat } from "@tanstack/ai"; import { openaiText } from "@tanstack/ai-openai"; import { computerUseTool } from "@tanstack/ai-openai/tools"; +interface SafetyCheck { + id: string; + code?: string; + message?: string; +} + +// Your code: run one action on the desktop, then return a PNG of the screen as base64. +declare function runAction(action: unknown): Promise; +// Your code: show the checks to a person and return their answer. +declare function askPerson(checks: Array): Promise; + const stream = chat({ adapter: openaiText("computer-use-preview"), messages: [{ role: "user", content: "Open Chrome and navigate to example.com" }], tools: [ - computerUseTool({ - type: "computer_use_preview", - display_width: 1024, - display_height: 768, - environment: "browser", - }), + { + ...computerUseTool({ + type: "computer_use_preview", + display_width: 1024, + display_height: 768, + environment: "browser", + }), + execute: async (input: { + action: unknown; + pending_safety_checks: Array; + }) => { + const checks = input.pending_safety_checks; + if (checks.length > 0 && !(await askPerson(checks))) { + throw new Error("A person did not allow this action."); + } + const screenshot = await runAction(input.action); + return [ + { + type: "image" as const, + source: { type: "data" as const, value: screenshot, mimeType: "image/png" }, + }, + ]; + }, + }, ], }); ``` +The input of each call has two fields: + +- `action`: the action to run, as OpenAI sends it. For example `{ type: "click", button: "left", x: 120, y: 340 }`. +- `pending_safety_checks`: the safety checks of the action. If the list is not empty, ask a person before you run the action. + +The adapter sends the first image of the result back as the screenshot. When the call does not fail, the adapter also tells OpenAI that the safety checks are accepted. When the call fails, they stay unaccepted. + **Supported models:** `computer-use-preview`. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). ### `localShellTool` diff --git a/docs/tools/provider-tools.md b/docs/tools/provider-tools.md index 25da0275c4..addf9bf719 100644 --- a/docs/tools/provider-tools.md +++ b/docs/tools/provider-tools.md @@ -90,7 +90,7 @@ const followUp = chat({ The search/fetch call surfaces as a provider-executed `tool-call` part on the assistant message; the agent loop never tries to run it client-side. -OpenAI `applyPatchTool`, `localShellTool`, and `shellTool({ environment: { type: "local" } })` are different. The model returns the call, and your app runs it. See [OpenAI adapter](../adapters/openai.md#applypatchtool). +OpenAI `applyPatchTool`, `localShellTool`, `computerUseTool`, and `shellTool({ environment: { type: "local" } })` are different. The model returns the call, and your app runs it. See [OpenAI adapter](../adapters/openai.md#applypatchtool) and [`computerUseTool`](../adapters/openai.md#computerusetool). ### Read web search sources diff --git a/packages/openai-base/src/adapters/responses-text.ts b/packages/openai-base/src/adapters/responses-text.ts index 8803b2fb0a..1628d75202 100644 --- a/packages/openai-base/src/adapters/responses-text.ts +++ b/packages/openai-base/src/adapters/responses-text.ts @@ -51,6 +51,7 @@ import type { } from '@tanstack/ai/adapters' import type { Response, + ResponseComputerToolCallOutputScreenshot, ResponseCreateParams, ResponseFunctionCallOutputItem, ResponseFunctionWebSearch, @@ -400,7 +401,10 @@ export interface OpenAIResponsesToolCallMetadata extends ProviderExecutedToolMet * `additional_tools` has one, and the API needs it on the replayed call. */ namespace?: string - /** Set for shell, local_shell, and apply_patch calls the app must run. */ + /** + * Set for shell, local_shell, apply_patch, and computer calls the app must + * run. + */ openaiUserTool?: OpenAIUserToolName /** Shell `action.max_output_length`, echoed on `shell_call_output`. */ maxOutputLength?: number | null @@ -2913,7 +2917,10 @@ export abstract class OpenAIBaseResponsesTextAdapter< ? userToolCalls.get(message.toolCallId) : undefined const userOutput = owner - ? userToolResultItem(owner, message.content) + ? userToolResultItem(owner, message.content, { + screenshot: this.screenshotOf(message.content), + failed: message.error !== undefined, + }) : null if (userOutput) { result.push(userOutput) @@ -3149,6 +3156,22 @@ export abstract class OpenAIBaseResponsesTextAdapter< return result } + /** The first image of a tool result, as a computer screenshot. */ + private screenshotOf( + content: ModelMessage['content'], + ): ResponseComputerToolCallOutputScreenshot { + const image = Array.isArray(content) + ? content.find((part) => part.type === 'image') + : undefined + const input = image ? this.convertContentPartToInput(image) : undefined + if (input?.type !== 'input_image') return { type: 'computer_screenshot' } + return { + type: 'computer_screenshot', + ...(input.image_url ? { image_url: input.image_url } : {}), + ...(input.file_id ? { file_id: input.file_id } : {}), + } + } + /** * Converts a ContentPart to Responses API input content item. * Handles text, image, audio, and document (PDF) content parts. diff --git a/packages/openai-base/src/adapters/responses-user-tools.ts b/packages/openai-base/src/adapters/responses-user-tools.ts index b357ec146e..01a02b2a8e 100644 --- a/packages/openai-base/src/adapters/responses-user-tools.ts +++ b/packages/openai-base/src/adapters/responses-user-tools.ts @@ -1,11 +1,20 @@ -import type { ResponseInputItem } from 'openai/resources/responses/responses' +import type { + ResponseComputerToolCall, + ResponseComputerToolCallOutputScreenshot, + ResponseInputItem, +} from 'openai/resources/responses/responses' /** * OpenAI Responses tools the app must run. * A container `shell` runs on the provider. `apply_patch`, `local_shell`, - * and `shell` with `environment.type: "local"` (or no environment) do not. + * `computer_use_preview`, and `shell` with `environment.type: "local"` (or + * no environment) do not. */ -export type OpenAIUserToolName = 'shell' | 'apply_patch' | 'local_shell' +export type OpenAIUserToolName = + | 'shell' + | 'apply_patch' + | 'local_shell' + | 'computer_use_preview' export interface OpenAIUserExecutedCall { name: OpenAIUserToolName @@ -71,12 +80,47 @@ export function hostedShellCallIds( export function readUserToolName(metadata: unknown): OpenAIUserToolName | null { if (!isRecord(metadata)) return null const name = metadata.openaiUserTool - if (name === 'shell' || name === 'apply_patch' || name === 'local_shell') { + if ( + name === 'shell' || + name === 'apply_patch' || + name === 'local_shell' || + name === 'computer_use_preview' + ) { return name } return null } +/** + * A computer action. It goes back to the API as the model sent it, so only + * its `type` is checked. + */ +function isComputerAction( + value: unknown, +): value is NonNullable { + return isRecord(value) && typeof value.type === 'string' +} + +/** The safety checks of a computer call, with only their known fields. */ +function safetyChecks( + value: unknown, +): Array { + if (!Array.isArray(value)) return [] + return value.flatMap((check) => + isRecord(check) && typeof check.id === 'string' + ? [ + { + id: check.id, + ...(typeof check.code === 'string' ? { code: check.code } : {}), + ...(typeof check.message === 'string' + ? { message: check.message } + : {}), + }, + ] + : [], + ) +} + function readItemId(metadata: unknown): string | undefined { if (!isRecord(metadata)) return undefined return typeof metadata.itemId === 'string' && metadata.itemId.length > 0 @@ -135,6 +179,19 @@ export function readUserExecutedCall( } } + if (item.type === 'computer_call') { + if (!callId || !isComputerAction(item.action)) return null + return { + name: 'computer_use_preview', + callId, + ...(itemId ? { itemId } : {}), + input: { + action: item.action, + pending_safety_checks: safetyChecks(item.pending_safety_checks), + }, + } + } + if (item.type === 'local_shell_call') { if (!isRecord(item.action)) return null const command = stringList(item.action.command) @@ -234,6 +291,18 @@ export function userToolRequestItem( } } + if (name === 'computer_use_preview') { + if (!isComputerAction(args.action)) return null + return { + type: 'computer_call', + id: itemId ?? toolCall.id, + call_id: toolCall.id, + status: 'completed', + action: args.action, + pending_safety_checks: safetyChecks(args.pending_safety_checks), + } + } + const commands = stringList(args.commands) if (!commands) return null return { @@ -294,13 +363,37 @@ function shellOutputList(content: unknown): Array<{ return [shellEntry(JSON.stringify(parsed ?? ''))] } -/** Replay the app's tool result as the matching Responses output item. */ +/** + * Replay the app's tool result as the matching Responses output item. + * `computer` is what a `computer_use_preview` result needs: the screenshot, + * and whether the call failed. The output of a call that did not fail + * acknowledges the safety checks of the call. + */ export function userToolResultItem( toolCall: ToolCallLike, content: unknown, + computer?: { + screenshot: ResponseComputerToolCallOutputScreenshot + failed: boolean + }, ): ResponseInputItem | null { const name = readUserToolName(toolCall.metadata) if (!name) return null + + if (name === 'computer_use_preview') { + const checks = computer?.failed + ? [] + : safetyChecks( + parseArguments(toolCall.function.arguments).pending_safety_checks, + ) + return { + type: 'computer_call_output', + call_id: toolCall.id, + output: computer?.screenshot ?? { type: 'computer_screenshot' }, + ...(checks.length > 0 ? { acknowledged_safety_checks: checks } : {}), + } + } + const parsed = parseContent(content) if (name === 'apply_patch') { diff --git a/packages/openai-base/tests/computer-use-loop.test.ts b/packages/openai-base/tests/computer-use-loop.test.ts new file mode 100644 index 0000000000..f15e0674c6 --- /dev/null +++ b/packages/openai-base/tests/computer-use-loop.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it, vi } from 'vitest' +import { chat } from '@tanstack/ai' +import { OpenAIBaseResponsesTextAdapter } from '../src/adapters/responses-text' +import { computerUseTool } from '../src/tools/computer-use-tool' +import type OpenAI from 'openai' +import type { StreamChunk } from '@tanstack/ai' + +const safetyCheck = { + id: 'sc_1', + code: 'malicious_instructions', + message: 'The page asks the model to do something else.', +} + +async function* completed(output: Array>) { + yield* [ + { + type: 'response.created', + response: { + id: 'resp_1', + model: 'computer-use-preview', + status: 'in_progress', + }, + }, + { + type: 'response.completed', + response: { + id: 'resp_1', + model: 'computer-use-preview', + status: 'completed', + output, + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + }, + }, + ] +} + +const clickCall = { + type: 'computer_call', + id: 'cu_1', + call_id: 'call_cu', + status: 'completed', + action: { type: 'click', button: 'left', x: 10, y: 20 }, + pending_safety_checks: [safetyCheck], +} + +const answer = { + type: 'message', + id: 'msg_1', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: 'Done.', annotations: [] }], +} + +class TestAdapter extends OpenAIBaseResponsesTextAdapter {} + +async function runLoop(execute: (input: unknown) => Promise) { + const create = vi + .fn() + .mockResolvedValueOnce(completed([clickCall])) + .mockResolvedValueOnce(completed([answer])) + const client = { + responses: { create: (params: unknown) => create(params) }, + } as unknown as OpenAI + const adapter = new TestAdapter('computer-use-preview', 'openai', client) + const tool = { + ...computerUseTool({ + type: 'computer_use_preview', + display_width: 1024, + display_height: 768, + environment: 'browser', + }), + execute: vi.fn(execute), + } + const chunks: Array = [] + for await (const chunk of chat({ + adapter, + messages: [{ role: 'user', content: 'Click the button.' }], + tools: [tool], + })) { + chunks.push(chunk) + } + return { create, execute: tool.execute, chunks } +} + +describe('computer use loop', () => { + it('runs the computer call and sends the screenshot back', async () => { + const { create, execute } = await runLoop(async () => [ + { + type: 'image', + source: { type: 'data', value: 'iVBORw0KGgo=', mimeType: 'image/png' }, + }, + ]) + + expect(execute).toHaveBeenCalledOnce() + expect(execute.mock.calls[0]?.[0]).toEqual({ + action: { type: 'click', button: 'left', x: 10, y: 20 }, + pending_safety_checks: [safetyCheck], + }) + expect(create).toHaveBeenCalledTimes(2) + expect(create.mock.calls[1]?.[0].input).toEqual( + expect.arrayContaining([ + { + type: 'computer_call', + id: 'cu_1', + call_id: 'call_cu', + status: 'completed', + action: { type: 'click', button: 'left', x: 10, y: 20 }, + pending_safety_checks: [safetyCheck], + }, + { + type: 'computer_call_output', + call_id: 'call_cu', + output: { + type: 'computer_screenshot', + image_url: 'data:image/png;base64,iVBORw0KGgo=', + }, + acknowledged_safety_checks: [safetyCheck], + }, + ]), + ) + }) + + it('does not acknowledge the safety checks of a failed call', async () => { + const { create } = await runLoop(async () => { + throw new Error('A person did not allow the click.') + }) + + const input: Array> = + create.mock.calls[1]?.[0].input + const output = input.find((item) => item.type === 'computer_call_output') + expect(output).toEqual({ + type: 'computer_call_output', + call_id: 'call_cu', + output: { type: 'computer_screenshot' }, + }) + }) +}) From fa2729d166667fd398dca364222da06cc2bbe896 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 19 Oct 2026 18:41:28 +0200 Subject: [PATCH 03/32] feat(openai-base, ai-openai): add the computer tool computerUseTool({ type: 'computer' }) sends the computer tool of the newer OpenAI models. Its tool name is computer. A computer_call is a call to the computer tool of the request, with the batched actions and the pending safety checks as input, and the replay sends the actions back as they came. The computer_use_preview tool works as before. --- .changeset/openai-computer-tool.md | 6 ++ docs/adapters/openai.md | 48 ++++++---- docs/config.json | 2 +- .../ai-openai/src/tools/computer-use-tool.ts | 10 +- .../tests/tools-per-model-type-safety.test.ts | 6 ++ .../src/adapters/responses-text.ts | 10 +- .../src/adapters/responses-user-tools.ts | 49 +++++++--- .../src/tools/computer-use-tool.ts | 17 +++- packages/openai-base/src/tools/tool-choice.ts | 1 + .../tests/computer-use-loop.test.ts | 96 +++++++++++++++---- .../tests/provider-tool-dispatch.test.ts | 7 ++ 11 files changed, 191 insertions(+), 61 deletions(-) create mode 100644 .changeset/openai-computer-tool.md diff --git a/.changeset/openai-computer-tool.md b/.changeset/openai-computer-tool.md new file mode 100644 index 0000000000..a2c1d63e9c --- /dev/null +++ b/.changeset/openai-computer-tool.md @@ -0,0 +1,6 @@ +--- +'@tanstack/openai-base': minor +'@tanstack/ai-openai': minor +--- + +`computerUseTool({ type: 'computer' })` adds the computer tool of the newer OpenAI models, for example `gpt-6.1-sol`. The tool name is `computer`. Each call has the batched `actions` of the model and the `pending_safety_checks`, and the replay sends the actions back as they came. `computerUseTool({ type: 'computer_use_preview', ... })` for the `computer-use-preview` model works as before. The Responses tool choice type also accepts `computer`. diff --git a/docs/adapters/openai.md b/docs/adapters/openai.md index 6646c49201..1a89ed4b82 100644 --- a/docs/adapters/openai.md +++ b/docs/adapters/openai.md @@ -993,10 +993,9 @@ const stream = chat({ ### `computerUseTool` Lets the model observe a virtual desktop via screenshots and interact with -it using keyboard and mouse events. Provide the display dimensions and the -execution environment type. +it using keyboard and mouse events. -The model asks for one action at a time, for example a click. Your app runs the action, then returns a screenshot of the screen. Add an `execute` to the tool that does this: +The model asks for actions, for example a click and then some typing. Your app runs the actions, then returns a screenshot of the screen. Add an `execute` to the tool that does this: ```typescript import { chat } from "@tanstack/ai"; @@ -1009,31 +1008,31 @@ interface SafetyCheck { message?: string; } -// Your code: run one action on the desktop, then return a PNG of the screen as base64. -declare function runAction(action: unknown): Promise; +// Your code: run one action on the desktop. +declare function runAction(action: unknown): Promise; +// Your code: return a PNG of the screen as base64. +declare function takeScreenshot(): Promise; // Your code: show the checks to a person and return their answer. declare function askPerson(checks: Array): Promise; const stream = chat({ - adapter: openaiText("computer-use-preview"), - messages: [{ role: "user", content: "Open Chrome and navigate to example.com" }], + adapter: openaiText("gpt-6.1-sol"), + messages: [{ role: "user", content: "Open example.com and sign up for the newsletter" }], tools: [ { - ...computerUseTool({ - type: "computer_use_preview", - display_width: 1024, - display_height: 768, - environment: "browser", - }), + ...computerUseTool({ type: "computer" }), execute: async (input: { - action: unknown; + actions: Array; pending_safety_checks: Array; }) => { const checks = input.pending_safety_checks; if (checks.length > 0 && !(await askPerson(checks))) { throw new Error("A person did not allow this action."); } - const screenshot = await runAction(input.action); + for (const action of input.actions) { + await runAction(action); + } + const screenshot = await takeScreenshot(); return [ { type: "image" as const, @@ -1048,12 +1047,25 @@ const stream = chat({ The input of each call has two fields: -- `action`: the action to run, as OpenAI sends it. For example `{ type: "click", button: "left", x: 120, y: 340 }`. -- `pending_safety_checks`: the safety checks of the action. If the list is not empty, ask a person before you run the action. +- `actions`: the actions to run, in order, as OpenAI sends them. For example `[{ type: "click", button: "left", x: 120, y: 340 }, { type: "type", text: "ada@example.com" }]`. +- `pending_safety_checks`: the safety checks of the actions. If the list is not empty, ask a person before you run the actions. The adapter sends the first image of the result back as the screenshot. When the call does not fail, the adapter also tells OpenAI that the safety checks are accepted. When the call fails, they stay unaccepted. -**Supported models:** `computer-use-preview`. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). +The `computer-use-preview` model takes the older tool type. Give it the display size and the environment. Each call then has one `action` field in place of `actions`: + +```typescript +import { computerUseTool } from "@tanstack/ai-openai/tools"; + +const previewTool = computerUseTool({ + type: "computer_use_preview", + display_width: 1024, + display_height: 768, + environment: "browser", +}); +``` + +**Supported models:** `type: "computer"` works with the models that list computer use, for example `gpt-6.1-sol`. `type: "computer_use_preview"` works with `computer-use-preview`. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). ### `localShellTool` diff --git a/docs/config.json b/docs/config.json index 5fed936ceb..cddd848128 100644 --- a/docs/config.json +++ b/docs/config.json @@ -1209,7 +1209,7 @@ "label": "OpenAI", "to": "adapters/openai", "addedAt": "2026-04-15", - "updatedAt": "2026-10-12" + "updatedAt": "2026-10-19" }, { "label": "Anthropic", diff --git a/packages/ai-openai/src/tools/computer-use-tool.ts b/packages/ai-openai/src/tools/computer-use-tool.ts index 3f65dddb7c..0ad3ecc555 100644 --- a/packages/ai-openai/src/tools/computer-use-tool.ts +++ b/packages/ai-openai/src/tools/computer-use-tool.ts @@ -9,11 +9,11 @@ export { } from '@tanstack/openai-base' // The brand discriminator (`computer_use`) intentionally differs from the -// runtime tool name (`computer_use_preview`). The brand matches the model-meta -// tool-capability union (`tools: ['computer_use', ...]`) used to gate which -// models can construct this tool at compile time, while the runtime name -// matches the OpenAI SDK's literal `'computer_use_preview'`. The base factory -// records the canonical `computer_use` kind separately in adapter metadata. +// runtime tool name (`computer` or `computer_use_preview`). The brand matches +// the model-meta tool-capability union (`tools: ['computer_use', ...]`) used to +// gate which models can construct this tool at compile time, while the runtime +// name matches the tool type in the OpenAI SDK. The base factory records the +// canonical `computer_use` kind separately in adapter metadata. export type OpenAIComputerUseTool = ProviderTool<'openai', 'computer_use'> /** diff --git a/packages/ai-openai/tests/tools-per-model-type-safety.test.ts b/packages/ai-openai/tests/tools-per-model-type-safety.test.ts index ad9ae3205c..cc89d94901 100644 --- a/packages/ai-openai/tests/tools-per-model-type-safety.test.ts +++ b/packages/ai-openai/tests/tools-per-model-type-safety.test.ts @@ -107,6 +107,12 @@ describe('OpenAI per-model tool gating', () => { // OpenAI's GPT-6 model pages list web_search, file_search, image_generation, // code_interpreter, hosted_shell, apply_patch, computer_use and mcp for the // Responses API, and neither web_search_preview nor local_shell. + it('gpt-6.1-sol accepts the computer tool', () => { + typedTools(openaiText('gpt-6.1-sol'), [ + computerUseTool({ type: 'computer' }), + ]) + }) + it('gpt-6-astra accepts the Responses tools its model page lists but rejects the rest', () => { const adapter = openaiText('gpt-6-astra') typedTools(adapter, [ diff --git a/packages/openai-base/src/adapters/responses-text.ts b/packages/openai-base/src/adapters/responses-text.ts index 1628d75202..4e8d55794d 100644 --- a/packages/openai-base/src/adapters/responses-text.ts +++ b/packages/openai-base/src/adapters/responses-text.ts @@ -1391,6 +1391,14 @@ export abstract class OpenAIBaseResponsesTextAdapter< hasEmittedRunStarted: boolean }, ): AsyncIterable { + // A `computer_call` is a call to the computer tool of this request. + const computerTool = options.tools?.some( + (tool) => + tool.name === 'computer' && + getOpenAIProviderToolKind(tool) === 'computer_use', + ) + ? 'computer' + : 'computer_use_preview' const normalizeToolInput = createToolInputNormalizer( options.tools, (schema, required) => @@ -1619,7 +1627,7 @@ export abstract class OpenAIBaseResponsesTextAdapter< outputIndex: number, bareShell: boolean, ): Array => { - const call = readUserExecutedCall(item, { bareShell }) + const call = readUserExecutedCall(item, { bareShell, computerTool }) if (!call || toolCallMetadata.get(call.callId)?.ended) return [] const tracked = toolCallMetadata.get(call.callId) ?? { callId: call.callId, diff --git a/packages/openai-base/src/adapters/responses-user-tools.ts b/packages/openai-base/src/adapters/responses-user-tools.ts index 01a02b2a8e..903f907a05 100644 --- a/packages/openai-base/src/adapters/responses-user-tools.ts +++ b/packages/openai-base/src/adapters/responses-user-tools.ts @@ -1,4 +1,5 @@ import type { + ComputerActionList, ResponseComputerToolCall, ResponseComputerToolCallOutputScreenshot, ResponseInputItem, @@ -7,14 +8,17 @@ import type { /** * OpenAI Responses tools the app must run. * A container `shell` runs on the provider. `apply_patch`, `local_shell`, - * `computer_use_preview`, and `shell` with `environment.type: "local"` (or - * no environment) do not. + * `computer`, `computer_use_preview`, and `shell` with + * `environment.type: "local"` (or no environment) do not. */ export type OpenAIUserToolName = | 'shell' | 'apply_patch' | 'local_shell' - | 'computer_use_preview' + | OpenAIComputerToolName + +/** The names of the computer tool: its two tool types. */ +export type OpenAIComputerToolName = 'computer' | 'computer_use_preview' export interface OpenAIUserExecutedCall { name: OpenAIUserToolName @@ -84,6 +88,7 @@ export function readUserToolName(metadata: unknown): OpenAIUserToolName | null { name === 'shell' || name === 'apply_patch' || name === 'local_shell' || + name === 'computer' || name === 'computer_use_preview' ) { return name @@ -101,6 +106,14 @@ function isComputerAction( return isRecord(value) && typeof value.type === 'string' } +/** The batched actions of a computer call. */ +function isComputerActionList(value: unknown): value is ComputerActionList { + return ( + Array.isArray(value) && + value.every((action) => isRecord(action) && typeof action.type === 'string') + ) +} + /** The safety checks of a computer call, with only their known fields. */ function safetyChecks( value: unknown, @@ -159,10 +172,12 @@ function applyPatchOperation( * `bareShell` is true only once the full response is known. A shell call * with no environment waits for that pass, so a hosted call that later * carries `shell_call_output` is not asked of the app. + * `computerTool` is the name of the computer tool of the request. A + * `computer_call` is a call to it. */ export function readUserExecutedCall( item: unknown, - options: { bareShell: boolean }, + options: { bareShell: boolean; computerTool?: OpenAIComputerToolName }, ): OpenAIUserExecutedCall | null { if (!isRecord(item) || typeof item.type !== 'string') return null const callId = typeof item.call_id === 'string' ? item.call_id : '' @@ -180,13 +195,17 @@ export function readUserExecutedCall( } if (item.type === 'computer_call') { - if (!callId || !isComputerAction(item.action)) return null + const { action, actions } = item + const hasAction = isComputerAction(action) + const hasActions = isComputerActionList(actions) + if (!callId || (!hasAction && !hasActions)) return null return { - name: 'computer_use_preview', + name: options.computerTool ?? 'computer_use_preview', callId, ...(itemId ? { itemId } : {}), input: { - action: item.action, + ...(hasAction ? { action } : {}), + ...(hasActions ? { actions } : {}), pending_safety_checks: safetyChecks(item.pending_safety_checks), }, } @@ -291,14 +310,18 @@ export function userToolRequestItem( } } - if (name === 'computer_use_preview') { - if (!isComputerAction(args.action)) return null + if (name === 'computer' || name === 'computer_use_preview') { + const { action, actions } = args + const hasAction = isComputerAction(action) + const hasActions = isComputerActionList(actions) + if (!hasAction && !hasActions) return null return { type: 'computer_call', id: itemId ?? toolCall.id, call_id: toolCall.id, status: 'completed', - action: args.action, + ...(hasAction ? { action } : {}), + ...(hasActions ? { actions } : {}), pending_safety_checks: safetyChecks(args.pending_safety_checks), } } @@ -365,8 +388,8 @@ function shellOutputList(content: unknown): Array<{ /** * Replay the app's tool result as the matching Responses output item. - * `computer` is what a `computer_use_preview` result needs: the screenshot, - * and whether the call failed. The output of a call that did not fail + * `computer` is what a computer tool result needs: the screenshot, and + * whether the call failed. The output of a call that did not fail * acknowledges the safety checks of the call. */ export function userToolResultItem( @@ -380,7 +403,7 @@ export function userToolResultItem( const name = readUserToolName(toolCall.metadata) if (!name) return null - if (name === 'computer_use_preview') { + if (name === 'computer' || name === 'computer_use_preview') { const checks = computer?.failed ? [] : safetyChecks( diff --git a/packages/openai-base/src/tools/computer-use-tool.ts b/packages/openai-base/src/tools/computer-use-tool.ts index 26bec9b451..fb5b6a66b0 100644 --- a/packages/openai-base/src/tools/computer-use-tool.ts +++ b/packages/openai-base/src/tools/computer-use-tool.ts @@ -2,10 +2,17 @@ import { getOpenAIProviderToolMetadata, openAIProviderTool, } from './openai-provider-tool' -import type { ComputerUsePreviewTool as ComputerUseToolConfig } from 'openai/resources/responses/responses' +import type { + ComputerTool, + ComputerUsePreviewTool, +} from 'openai/resources/responses/responses' import type { Tool } from '@tanstack/ai' -export type { ComputerUseToolConfig } +/** + * The computer tool: `{ type: 'computer' }`, or the `computer_use_preview` + * tool of the `computer-use-preview` model. + */ +export type ComputerUseToolConfig = ComputerTool | ComputerUsePreviewTool /** @deprecated Renamed to `ComputerUseToolConfig`. Will be removed in a future release. */ export type ComputerUseTool = ComputerUseToolConfig @@ -17,6 +24,7 @@ export function convertComputerUseToolToAdapterFormat( tool: Tool, ): ComputerUseToolConfig { const metadata = getOpenAIProviderToolMetadata(tool) as ComputerUseToolConfig + if (metadata.type === 'computer') return { type: 'computer' } return { type: 'computer_use_preview', display_height: metadata.display_height, @@ -26,7 +34,8 @@ export function convertComputerUseToolToAdapterFormat( } /** - * Creates a standard Tool from ComputerUseTool parameters. + * Creates a standard Tool from ComputerUseTool parameters. The tool name is + * the tool type: `computer` or `computer_use_preview`. * * Base (non-branded) factory. Providers that need branded return types should * re-wrap this in their own package. @@ -34,7 +43,7 @@ export function convertComputerUseToolToAdapterFormat( export function computerUseTool(toolData: ComputerUseToolConfig): Tool { return openAIProviderTool( { - name: 'computer_use_preview', + name: toolData.type, description: 'Control a virtual computer', metadata: { ...toolData, diff --git a/packages/openai-base/src/tools/tool-choice.ts b/packages/openai-base/src/tools/tool-choice.ts index 644e49c72e..3c8e825d16 100644 --- a/packages/openai-base/src/tools/tool-choice.ts +++ b/packages/openai-base/src/tools/tool-choice.ts @@ -37,6 +37,7 @@ interface HostedToolChoice { type: | 'file_search' | 'web_search_preview' + | 'computer' | 'computer_use_preview' | 'code_interpreter' | 'image_generation' diff --git a/packages/openai-base/tests/computer-use-loop.test.ts b/packages/openai-base/tests/computer-use-loop.test.ts index f15e0674c6..e8950ea1ac 100644 --- a/packages/openai-base/tests/computer-use-loop.test.ts +++ b/packages/openai-base/tests/computer-use-loop.test.ts @@ -2,8 +2,9 @@ import { describe, expect, it, vi } from 'vitest' import { chat } from '@tanstack/ai' import { OpenAIBaseResponsesTextAdapter } from '../src/adapters/responses-text' import { computerUseTool } from '../src/tools/computer-use-tool' +import { convertToolsToProviderFormat } from '../src/tools/tool-converter' import type OpenAI from 'openai' -import type { StreamChunk } from '@tanstack/ai' +import type { StreamChunk, Tool } from '@tanstack/ai' const safetyCheck = { id: 'sc_1', @@ -51,26 +52,43 @@ const answer = { content: [{ type: 'output_text', text: 'Done.', annotations: [] }], } -class TestAdapter extends OpenAIBaseResponsesTextAdapter {} +// The OpenAI adapter sends provider tools in their native shape. +class TestAdapter extends OpenAIBaseResponsesTextAdapter { + protected override convertTools(tools: Array) { + return convertToolsToProviderFormat(tools) + } +} + +const previewTool = computerUseTool({ + type: 'computer_use_preview', + display_width: 1024, + display_height: 768, + environment: 'browser', +}) -async function runLoop(execute: (input: unknown) => Promise) { +const screenshot = [ + { + type: 'image', + source: { type: 'data', value: 'iVBORw0KGgo=', mimeType: 'image/png' }, + }, +] + +async function runLoop( + execute: (input: unknown) => Promise, + { + computer = previewTool, + call = clickCall, + }: { computer?: Tool; call?: Record } = {}, +) { const create = vi .fn() - .mockResolvedValueOnce(completed([clickCall])) + .mockResolvedValueOnce(completed([call])) .mockResolvedValueOnce(completed([answer])) const client = { responses: { create: (params: unknown) => create(params) }, } as unknown as OpenAI const adapter = new TestAdapter('computer-use-preview', 'openai', client) - const tool = { - ...computerUseTool({ - type: 'computer_use_preview', - display_width: 1024, - display_height: 768, - environment: 'browser', - }), - execute: vi.fn(execute), - } + const tool = { ...computer, execute: vi.fn(execute) } const chunks: Array = [] for await (const chunk of chat({ adapter, @@ -84,12 +102,7 @@ async function runLoop(execute: (input: unknown) => Promise) { describe('computer use loop', () => { it('runs the computer call and sends the screenshot back', async () => { - const { create, execute } = await runLoop(async () => [ - { - type: 'image', - source: { type: 'data', value: 'iVBORw0KGgo=', mimeType: 'image/png' }, - }, - ]) + const { create, execute } = await runLoop(async () => screenshot) expect(execute).toHaveBeenCalledOnce() expect(execute.mock.calls[0]?.[0]).toEqual({ @@ -120,6 +133,51 @@ describe('computer use loop', () => { ) }) + it('runs the batched actions of the computer tool', async () => { + const actions = [ + { type: 'click', button: 'left', x: 10, y: 20 }, + { type: 'type', text: 'hello' }, + ] + const { create, execute } = await runLoop(async () => screenshot, { + computer: computerUseTool({ type: 'computer' }), + call: { + type: 'computer_call', + id: 'cu_2', + call_id: 'call_batch', + status: 'completed', + actions, + pending_safety_checks: [], + }, + }) + + expect(execute.mock.calls[0]?.[0]).toEqual({ + actions, + pending_safety_checks: [], + }) + const [first, second] = create.mock.calls.map(([params]) => params) + expect(first.tools).toEqual([{ type: 'computer' }]) + expect(second.input).toEqual( + expect.arrayContaining([ + { + type: 'computer_call', + id: 'cu_2', + call_id: 'call_batch', + status: 'completed', + actions, + pending_safety_checks: [], + }, + { + type: 'computer_call_output', + call_id: 'call_batch', + output: { + type: 'computer_screenshot', + image_url: 'data:image/png;base64,iVBORw0KGgo=', + }, + }, + ]), + ) + }) + it('does not acknowledge the safety checks of a failed call', async () => { const { create } = await runLoop(async () => { throw new Error('A person did not allow the click.') diff --git a/packages/openai-base/tests/provider-tool-dispatch.test.ts b/packages/openai-base/tests/provider-tool-dispatch.test.ts index 1f09aaaffd..e7292ae5a5 100644 --- a/packages/openai-base/tests/provider-tool-dispatch.test.ts +++ b/packages/openai-base/tests/provider-tool-dispatch.test.ts @@ -30,6 +30,7 @@ function isRoundTrippedTool(value: unknown): value is Tool { const PROVIDER_TOOL_NAMES = [ 'apply_patch', 'code_interpreter', + 'computer', 'computer_use_preview', 'custom', 'file_search', @@ -57,6 +58,12 @@ const FACTORY_CASES = [ }), wire: { type: 'code_interpreter' }, }, + { + name: 'computer', + kind: 'openai.computer_use', + tool: computerUseTool({ type: 'computer' }), + wire: { type: 'computer' }, + }, { name: 'computer_use_preview', kind: 'openai.computer_use', From 08a7b2a75217617535db996cf59954a627095249 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 19 Oct 2026 18:47:32 +0200 Subject: [PATCH 04/32] test(e2e): check the computer use beta header on the wire --- testing/e2e/src/routeTree.gen.ts | 22 +++++ .../routes/api.anthropic-computer-use-wire.ts | 96 +++++++++++++++++++ .../tests/anthropic-computer-use-wire.spec.ts | 28 ++++++ 3 files changed, 146 insertions(+) create mode 100644 testing/e2e/src/routes/api.anthropic-computer-use-wire.ts create mode 100644 testing/e2e/tests/anthropic-computer-use-wire.spec.ts diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index 98d5716a87..70bc3a477a 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -54,6 +54,7 @@ import { Route as ApiActivityTestRouteImport } from './routes/api.activity-test' import { Route as ApiAdapterInputModalitiesRouteImport } from './routes/api.adapter-input-modalities' import { Route as ApiAnthropicAuthWireRouteImport } from './routes/api.anthropic-auth-wire' import { Route as ApiAnthropicBugTestRouteImport } from './routes/api.anthropic-bug-test' +import { Route as ApiAnthropicComputerUseWireRouteImport } from './routes/api.anthropic-computer-use-wire' import { Route as ApiAnthropicHaiku55WireRouteImport } from './routes/api.anthropic-haiku-5-5-wire' import { Route as ApiAnthropicMaxTokensUsageRouteImport } from './routes/api.anthropic-max-tokens-usage' import { Route as ApiAnthropicMultiTurnStructuredWireRouteImport } from './routes/api.anthropic-multi-turn-structured-wire' @@ -413,6 +414,12 @@ const ApiAnthropicBugTestRoute = ApiAnthropicBugTestRouteImport.update({ path: '/api/anthropic-bug-test', getParentRoute: () => rootRouteImport, } as any) +const ApiAnthropicComputerUseWireRoute = + ApiAnthropicComputerUseWireRouteImport.update({ + id: '/api/anthropic-computer-use-wire', + path: '/api/anthropic-computer-use-wire', + getParentRoute: () => rootRouteImport, + } as any) const ApiAnthropicHaiku55WireRoute = ApiAnthropicHaiku55WireRouteImport.update({ id: '/api/anthropic-haiku-5-5-wire', path: '/api/anthropic-haiku-5-5-wire', @@ -1143,6 +1150,7 @@ export interface FileRoutesByFullPath { '/api/adapter-input-modalities': typeof ApiAdapterInputModalitiesRoute '/api/anthropic-auth-wire': typeof ApiAnthropicAuthWireRoute '/api/anthropic-bug-test': typeof ApiAnthropicBugTestRoute + '/api/anthropic-computer-use-wire': typeof ApiAnthropicComputerUseWireRoute '/api/anthropic-haiku-5-5-wire': typeof ApiAnthropicHaiku55WireRoute '/api/anthropic-max-tokens-usage': typeof ApiAnthropicMaxTokensUsageRoute '/api/anthropic-multi-turn-structured-wire': typeof ApiAnthropicMultiTurnStructuredWireRoute @@ -1319,6 +1327,7 @@ export interface FileRoutesByTo { '/api/adapter-input-modalities': typeof ApiAdapterInputModalitiesRoute '/api/anthropic-auth-wire': typeof ApiAnthropicAuthWireRoute '/api/anthropic-bug-test': typeof ApiAnthropicBugTestRoute + '/api/anthropic-computer-use-wire': typeof ApiAnthropicComputerUseWireRoute '/api/anthropic-haiku-5-5-wire': typeof ApiAnthropicHaiku55WireRoute '/api/anthropic-max-tokens-usage': typeof ApiAnthropicMaxTokensUsageRoute '/api/anthropic-multi-turn-structured-wire': typeof ApiAnthropicMultiTurnStructuredWireRoute @@ -1496,6 +1505,7 @@ export interface FileRoutesById { '/api/adapter-input-modalities': typeof ApiAdapterInputModalitiesRoute '/api/anthropic-auth-wire': typeof ApiAnthropicAuthWireRoute '/api/anthropic-bug-test': typeof ApiAnthropicBugTestRoute + '/api/anthropic-computer-use-wire': typeof ApiAnthropicComputerUseWireRoute '/api/anthropic-haiku-5-5-wire': typeof ApiAnthropicHaiku55WireRoute '/api/anthropic-max-tokens-usage': typeof ApiAnthropicMaxTokensUsageRoute '/api/anthropic-multi-turn-structured-wire': typeof ApiAnthropicMultiTurnStructuredWireRoute @@ -1674,6 +1684,7 @@ export interface FileRouteTypes { | '/api/adapter-input-modalities' | '/api/anthropic-auth-wire' | '/api/anthropic-bug-test' + | '/api/anthropic-computer-use-wire' | '/api/anthropic-haiku-5-5-wire' | '/api/anthropic-max-tokens-usage' | '/api/anthropic-multi-turn-structured-wire' @@ -1850,6 +1861,7 @@ export interface FileRouteTypes { | '/api/adapter-input-modalities' | '/api/anthropic-auth-wire' | '/api/anthropic-bug-test' + | '/api/anthropic-computer-use-wire' | '/api/anthropic-haiku-5-5-wire' | '/api/anthropic-max-tokens-usage' | '/api/anthropic-multi-turn-structured-wire' @@ -2026,6 +2038,7 @@ export interface FileRouteTypes { | '/api/adapter-input-modalities' | '/api/anthropic-auth-wire' | '/api/anthropic-bug-test' + | '/api/anthropic-computer-use-wire' | '/api/anthropic-haiku-5-5-wire' | '/api/anthropic-max-tokens-usage' | '/api/anthropic-multi-turn-structured-wire' @@ -2203,6 +2216,7 @@ export interface RootRouteChildren { ApiAdapterInputModalitiesRoute: typeof ApiAdapterInputModalitiesRoute ApiAnthropicAuthWireRoute: typeof ApiAnthropicAuthWireRoute ApiAnthropicBugTestRoute: typeof ApiAnthropicBugTestRoute + ApiAnthropicComputerUseWireRoute: typeof ApiAnthropicComputerUseWireRoute ApiAnthropicHaiku55WireRoute: typeof ApiAnthropicHaiku55WireRoute ApiAnthropicMaxTokensUsageRoute: typeof ApiAnthropicMaxTokensUsageRoute ApiAnthropicMultiTurnStructuredWireRoute: typeof ApiAnthropicMultiTurnStructuredWireRoute @@ -2647,6 +2661,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiAnthropicBugTestRouteImport parentRoute: typeof rootRouteImport } + '/api/anthropic-computer-use-wire': { + id: '/api/anthropic-computer-use-wire' + path: '/api/anthropic-computer-use-wire' + fullPath: '/api/anthropic-computer-use-wire' + preLoaderRoute: typeof ApiAnthropicComputerUseWireRouteImport + parentRoute: typeof rootRouteImport + } '/api/anthropic-haiku-5-5-wire': { id: '/api/anthropic-haiku-5-5-wire' path: '/api/anthropic-haiku-5-5-wire' @@ -3656,6 +3677,7 @@ const rootRouteChildren: RootRouteChildren = { ApiAdapterInputModalitiesRoute: ApiAdapterInputModalitiesRoute, ApiAnthropicAuthWireRoute: ApiAnthropicAuthWireRoute, ApiAnthropicBugTestRoute: ApiAnthropicBugTestRoute, + ApiAnthropicComputerUseWireRoute: ApiAnthropicComputerUseWireRoute, ApiAnthropicHaiku55WireRoute: ApiAnthropicHaiku55WireRoute, ApiAnthropicMaxTokensUsageRoute: ApiAnthropicMaxTokensUsageRoute, ApiAnthropicMultiTurnStructuredWireRoute: diff --git a/testing/e2e/src/routes/api.anthropic-computer-use-wire.ts b/testing/e2e/src/routes/api.anthropic-computer-use-wire.ts new file mode 100644 index 0000000000..2b20b584e9 --- /dev/null +++ b/testing/e2e/src/routes/api.anthropic-computer-use-wire.ts @@ -0,0 +1,96 @@ +import { createFileRoute } from '@tanstack/react-router' +import { chat, createChatOptions } from '@tanstack/ai' +import { createAnthropicChat } from '@tanstack/ai-anthropic' +import { computerUseTool } from '@tanstack/ai-anthropic/tools' + +const DUMMY_KEY = 'sk-ant-e2e-test-dummy-key' + +/** + * Drives the Anthropic chat adapter with `computerUseTool()`. A custom + * `fetch` captures the headers of the outgoing request and answers with a + * short Claude SSE stream, so the companion spec can read the + * `anthropic-beta` header without an API key. + */ +function endTurnStream(): ReadableStream { + const encoder = new TextEncoder() + const events = [ + { + type: 'message_start', + message: { + id: 'msg_computer_use_wire', + type: 'message', + role: 'assistant', + content: [], + model: 'claude-haiku-4-5', + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 5, output_tokens: 0 }, + }, + }, + { + type: 'message_delta', + delta: { stop_reason: 'end_turn', stop_sequence: null }, + usage: { output_tokens: 1 }, + }, + { type: 'message_stop' }, + ] + return new ReadableStream({ + start(controller) { + for (const event of events) { + controller.enqueue( + encoder.encode( + `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`, + ), + ) + } + controller.close() + }, + }) +} + +export const Route = createFileRoute('/api/anthropic-computer-use-wire')({ + server: { + handlers: { + POST: async () => { + let anthropicBeta: string | null = null + const capturingFetch: typeof fetch = async (input, init) => { + const req = + input instanceof Request ? input : new Request(input, init) + anthropicBeta = req.headers.get('anthropic-beta') + return new Response(endTurnStream(), { + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + }) + } + + const adapter = createAnthropicChat('claude-haiku-4-5', DUMMY_KEY, { + fetch: capturingFetch, + }) + + try { + for await (const _ of chat({ + ...createChatOptions({ adapter }), + messages: [{ role: 'user', content: 'Open the browser.' }], + tools: [ + computerUseTool({ + type: 'computer_20250124', + name: 'computer', + display_width_px: 1024, + display_height_px: 768, + }), + ], + })) { + // Drain the stream. + } + } catch (error) { + return Response.json({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }) + } + + return Response.json({ ok: true, anthropicBeta }) + }, + }, + }, +}) diff --git a/testing/e2e/tests/anthropic-computer-use-wire.spec.ts b/testing/e2e/tests/anthropic-computer-use-wire.spec.ts new file mode 100644 index 0000000000..7981f8d481 --- /dev/null +++ b/testing/e2e/tests/anthropic-computer-use-wire.spec.ts @@ -0,0 +1,28 @@ +import { test, expect } from './fixtures' + +/** + * Wire format of Anthropic computer use. `computerUseTool()` is a beta tool, + * so the request must carry its `anthropic-beta` header. + * + * `/api/anthropic-computer-use-wire` captures the outgoing request with a + * custom `fetch` and returns its `anthropic-beta` header as JSON. + */ +test.describe('anthropic — computer use wire format', () => { + test('computer_20250124 sends the computer-use-2025-01-24 beta', async ({ + request, + }) => { + const res = await request.post('/api/anthropic-computer-use-wire') + expect(res.ok()).toBe(true) + const { ok, error, anthropicBeta } = (await res.json()) as { + ok: boolean + error?: string + anthropicBeta: string | null + } + + if (!ok) { + throw new Error(`Route failed: ${error}`) + } + + expect(anthropicBeta ?? '').toContain('computer-use-2025-01-24') + }) +}) From 8384355ab9497c27b603f0e5ef4b65241d679d8a Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 19 Oct 2026 18:43:10 +0200 Subject: [PATCH 05/32] feat(ai-anthropic): computer use on every Claude model Upgrade @anthropic-ai/sdk to 0.135.0. Add computer_20251124 with its computer-use-2025-11-24 beta, and computer_toolset_20260801 for the Claude 5.5 models. A toolset call runs as one computer call with the action in input.action, and goes back to Claude in the form that it sent. Each model now takes only the computer use versions that it supports. --- docs/adapters/anthropic.md | 82 +++++-- docs/config.json | 4 +- docs/tools/provider-tools.md | 8 +- examples/ts-react-chat/package.json | 2 +- packages/ai-anthropic/package.json | 3 +- packages/ai-anthropic/src/adapters/files.ts | 4 +- packages/ai-anthropic/src/adapters/text.ts | 132 ++++++++--- packages/ai-anthropic/src/model-meta.ts | 43 ++-- packages/ai-anthropic/src/prompt-cache.ts | 4 +- .../src/tools/computer-use-tool.ts | 101 ++++++-- .../client-injection-type-safety.test.ts | 11 - .../tests/computer-toolset.test.ts | 222 ++++++++++++++++++ .../tests/tools-per-model-type-safety.test.ts | 76 ++++-- pnpm-lock.yaml | 45 ++-- scripts/model-sync/provider-supports.test.ts | 3 +- scripts/model-sync/provider-supports.ts | 3 +- testing/e2e/package.json | 2 +- 17 files changed, 594 insertions(+), 151 deletions(-) create mode 100644 packages/ai-anthropic/tests/computer-toolset.test.ts diff --git a/docs/adapters/anthropic.md b/docs/adapters/anthropic.md index 8b8b2b73a8..579f998c51 100644 --- a/docs/adapters/anthropic.md +++ b/docs/adapters/anthropic.md @@ -618,30 +618,84 @@ equivalent — see [Provider Skills](../tools/provider-skills.md). ### `computerUseTool` -Allows Claude to observe a virtual desktop (screenshots) and interact with it -via keyboard and mouse events. Provide the screen resolution so Claude can -calculate accurate coordinates. +Claude can look at a screen and use the mouse and the keyboard. Your app owns the screen: it runs each action that Claude asks for and sends back a screenshot. + +1. Pick the version that your model takes: + + | Version | Models | + | --- | --- | + | `computer_toolset_20260801` | Claude Opus 5.5, Sonnet 5.5, Haiku 5.5, Fable 5.1, Fable 5, Opus 5, Sonnet 5, Opus 4.8 | + | `computer_20251124` | Claude Fable 5.1, Fable 5, Opus 5, Sonnet 5, Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 4.6, Opus 4.5 | + | `computer_20250124` | Claude Sonnet 4.5, Haiku 4.5, Opus 4.1 | + +2. Give the tool an `execute` that runs each action on your screen. +3. Return a screenshot for the `screenshot` action, and `"OK"` for the other actions. ```typescript import { chat } from "@tanstack/ai"; import { anthropicText } from "@tanstack/ai-anthropic"; import { computerUseTool } from "@tanstack/ai-anthropic/tools"; +// Your screen, for example a virtual machine. Replace these two functions. +async function runAction(input: { action: string }): Promise { + console.log("run", input); +} +async function takeScreenshot(): Promise { + return ""; // A PNG, as base64. +} + +const computer = { + ...computerUseTool({ + type: "computer_toolset_20260801", + configs: { zoom: { enabled: false } }, + }), + // Every version calls `execute` with the action in `input.action`. + execute: async (input: { action: string }) => { + if (input.action === "screenshot") { + const value = await takeScreenshot(); + return [ + { + type: "image" as const, + source: { type: "data" as const, value, mimeType: "image/png" }, + }, + ]; + } + await runAction(input); + return "OK"; + }, +}; + const stream = chat({ - adapter: anthropicText("claude-sonnet-4-6"), - messages: [{ role: "user", content: "Open the browser and go to example.com" }], - tools: [ - computerUseTool({ - type: "computer_20250124", - name: "computer", - display_width_px: 1024, - display_height_px: 768, - }), - ], + adapter: anthropicText("claude-opus-5-5"), + messages: [{ role: "user", content: "Open the display settings." }], + tools: [computer], + // Claude can ask for several actions in one turn. Run them in order. + toolExecution: "sequential", }); ``` -**Supported models:** Claude Sonnet 3.5 and above, with two exceptions. `claude-opus-5-fast` takes no provider tools. Claude Opus 5.5, Claude Sonnet 5.5, and Claude Haiku 5.5 accept only the `computer_toolset_20260801` toolset, which the adapter does not offer yet. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). +The earlier versions also take a name and the screen size: + +```typescript +import { computerUseTool } from "@tanstack/ai-anthropic/tools"; + +const computer = computerUseTool({ + type: "computer_20251124", + name: "computer", + display_width_px: 1280, + display_height_px: 800, +}); +``` + +Good to know: + +- The adapter sends the beta header that an earlier version needs. The toolset needs no beta header. +- With the toolset, Claude calls each action as its own tool. The adapter gives you each call as one `computer` call, and sends it back in the form that Claude sent. So one `execute` works for every version. +- The toolset turns on `zoom` by default. To let Claude zoom, remove `configs` and return an image for the `zoom` action too. +- Send screenshots that fit the image limits of the model. The toolset does not make a large screenshot smaller. It refuses it. Claude gives coordinates in the pixels of your screenshots. +- If an action fails, answer each later action of the same turn with `Not executed: an earlier computer action in this turn failed.` + +A version that your model does not take is a type error. `claude-opus-5-fast` takes no provider tools. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). ### `bashTool` diff --git a/docs/config.json b/docs/config.json index 5fed936ceb..c235f5f524 100644 --- a/docs/config.json +++ b/docs/config.json @@ -323,7 +323,7 @@ "label": "Provider Tools", "to": "tools/provider-tools", "addedAt": "2026-04-21", - "updatedAt": "2026-09-24" + "updatedAt": "2026-10-19" }, { "label": "Provider Skills", @@ -1215,7 +1215,7 @@ "label": "Anthropic", "to": "adapters/anthropic", "addedAt": "2026-04-15", - "updatedAt": "2026-10-09" + "updatedAt": "2026-10-19" }, { "label": "Google Gemini", diff --git a/docs/tools/provider-tools.md b/docs/tools/provider-tools.md index 25da0275c4..64504bf5ac 100644 --- a/docs/tools/provider-tools.md +++ b/docs/tools/provider-tools.md @@ -149,10 +149,10 @@ Each adapter's `supports.tools` array is the source of truth. The comparison matrix is maintained alongside `model-meta.ts` and reflected here: - **Anthropic**: every registered model supports the full tool superset, - except four. `claude-opus-5-5`, `claude-sonnet-5-5`, and - `claude-haiku-5-5` support every tool but `computerUseTool`, and - `claude-opus-5-fast` supports none. The - retired Claude 3.x models with narrower support were removed. + except `claude-opus-5-fast`, which supports none. For `computerUseTool`, + each model takes only some versions. The Claude 5.5 models take only + `computer_toolset_20260801`. See the version table in + [`computerUseTool`](../adapters/anthropic.md#computerusetool). - **OpenAI**: GPT-5 family and reasoning models (O-series) support the full superset. GPT-6 Astra, Sol, and Luna support every tool except `webSearchPreviewTool` and `localShellTool`. GPT-4-series supports diff --git a/examples/ts-react-chat/package.json b/examples/ts-react-chat/package.json index dace1d669a..13ddbcc526 100644 --- a/examples/ts-react-chat/package.json +++ b/examples/ts-react-chat/package.json @@ -12,7 +12,7 @@ "test:types": "tsc --noEmit" }, "dependencies": { - "@anthropic-ai/sdk": "^0.97.1", + "@anthropic-ai/sdk": "^0.135.0", "@elevenlabs/elevenlabs-js": "^2.68.0", "@mcp-ui/client": "^7.1.1", "@modelcontextprotocol/sdk": "^1.29.0", diff --git a/packages/ai-anthropic/package.json b/packages/ai-anthropic/package.json index 1afd093b13..ff946b18c2 100644 --- a/packages/ai-anthropic/package.json +++ b/packages/ai-anthropic/package.json @@ -67,7 +67,7 @@ "test:types": "tsc" }, "dependencies": { - "@anthropic-ai/sdk": "^0.97.1", + "@anthropic-ai/sdk": "^0.135.0", "@tanstack/ai-utils": "workspace:^" }, "peerDependencies": { @@ -80,7 +80,6 @@ } }, "devDependencies": { - "@anthropic-ai/sdk-v112": "npm:@anthropic-ai/sdk@0.112.4", "@anthropic-ai/vertex-sdk": "^0.19.0", "@tanstack/ai": "workspace:*", "@vitest/coverage-v8": "4.1.10", diff --git a/packages/ai-anthropic/src/adapters/files.ts b/packages/ai-anthropic/src/adapters/files.ts index 37394d5fb2..133859ddad 100644 --- a/packages/ai-anthropic/src/adapters/files.ts +++ b/packages/ai-anthropic/src/adapters/files.ts @@ -8,7 +8,7 @@ import { getAnthropicCredentialFromEnv, } from '../utils/client' import type Anthropic_SDK from '@anthropic-ai/sdk' -import type { FileMetadata } from '@anthropic-ai/sdk/resources/beta/files' +import type { BetaFileMetadata } from '@anthropic-ai/sdk/resources/beta/files' import type { FileHandle, FileUploadInput } from '@tanstack/ai/adapters' import type { AnthropicClientConfig } from '../utils/client' @@ -55,7 +55,7 @@ export class AnthropicFilesAdapter extends BaseFilesAdapter<'anthropic'> { } } -function toFileHandle(file: FileMetadata): FileHandle<'anthropic'> { +function toFileHandle(file: BetaFileMetadata): FileHandle<'anthropic'> { return { id: file.id, provider: 'anthropic', diff --git a/packages/ai-anthropic/src/adapters/text.ts b/packages/ai-anthropic/src/adapters/text.ts index dd3dec4dbc..1e5f32c3c6 100644 --- a/packages/ai-anthropic/src/adapters/text.ts +++ b/packages/ai-anthropic/src/adapters/text.ts @@ -16,7 +16,13 @@ import { import { BaseTextAdapter } from '@tanstack/ai/adapters' import { convertToolsToProviderFormat } from '../tools/tool-converter' import { getAnthropicProviderToolKind } from '../tools/anthropic-provider-tool' -import { convertComputerUseToolToAdapterFormat } from '../tools/computer-use-tool' +import { + COMPUTER_TOOL_NAME, + convertComputerUseToolToAdapterFormat, + isComputerToolset, + joinComputerToolsetCall, + splitComputerToolsetCall, +} from '../tools/computer-use-tool' import { readCodeExecutionConfig, readCodeExecutionSkills, @@ -106,6 +112,7 @@ import type { AnthropicClientConfig, AnthropicMessagesClient, } from '../utils/client' +import type { ComputerUseSingleToolConfig } from '../tools/computer-use-tool' /** * The block type carried by an Anthropic provider-executed (server) tool's @@ -212,13 +219,20 @@ function toAnthropicToolChoice( return { type: 'tool' as const, name: choice.name } } +/** The beta header of each single `computer` tool version. */ +const COMPUTER_USE_BETAS = { + computer_20241022: 'computer-use-2024-10-22', + computer_20250124: 'computer-use-2025-01-24', + computer_20251124: 'computer-use-2025-11-24', +} as const satisfies Record + /** * Computes the `betas` array for a Messages request. Unions: * - `interleaved-thinking-2025-05-14` when interleaved thinking is enabled, * - `code-execution-2025-08-25` when a `code_execution` tool is present, * - `skills-2025-10-02` when that tool carries skills, - * - `computer-use-2024-10-22` or `computer-use-2025-01-24` for the version - * of a `computer` tool, + * - the `computer-use-*` beta of a single `computer` tool version (the + * computer toolset needs none), * - `context-management-2025-06-27` when `context_management` is set, * - `files-api-2025-04-14` when a message references an uploaded file handle, * - `mid-conversation-tool-changes-2026-07-01` in mid-conversation tool mode. @@ -299,12 +313,10 @@ export function computeAnthropicBetas( (tool) => getAnthropicProviderToolKind(tool) === 'computer_use', ) if (computerTool) { - betas.add( - convertComputerUseToolToAdapterFormat(computerTool).type === - 'computer_20241022' - ? 'computer-use-2024-10-22' - : 'computer-use-2025-01-24', - ) + const { type } = convertComputerUseToolToAdapterFormat(computerTool) + if (type !== 'computer_toolset_20260801') { + betas.add(COMPUTER_USE_BETAS[type]) + } } return betas.size > 0 ? Array.from(betas) : undefined @@ -323,18 +335,6 @@ const DEFERRED_TOOL_PLACEHOLDER: BetaTool = { defer_loading: true, } -/** - * A mid-conversation `system` message. SDK 0.97.1 types only `user` and - * `assistant` messages and has no `tool_addition` block, so this type is local. - */ -interface AnthropicMidConversationSystemMessage { - role: 'system' - content: Array< - | TextBlockParam - | { type: 'tool_addition'; tool: { type: 'tool_reference'; name: string } } - > -} - /** * Configuration for Anthropic text adapter */ @@ -607,7 +607,8 @@ export class AnthropicTextAdapter< messagesHaveFileSource(options.messages), // Mid-conversation tool mode puts the deferred placeholder in `tools`. requestParams.tools?.some( - (tool) => tool.name === DEFERRED_TOOL_PLACEHOLDER.name, + (tool) => + 'name' in tool && tool.name === DEFERRED_TOOL_PLACEHOLDER.name, ), ), ) @@ -821,6 +822,7 @@ export class AnthropicTextAdapter< const formattedMessages = this.formatMessages( replay.messages, mid?.systemMessages, + options.tools?.some(isComputerToolset) ?? false, ) const tools = options.tools ? (mid?.tools ?? convertToolsToProviderFormat(options.tools)) @@ -1064,7 +1066,7 @@ export class AnthropicTextAdapter< ) const systemMessages = new Map() for (const [before, change] of split.at) { - const content: AnthropicMidConversationSystemMessage['content'] = [ + const content: Array = [ ...(channels.systemPrompts ? change.systemPrompts.map( (p): TextBlockParam => ({ type: 'text', text: p.content }), @@ -1078,12 +1080,7 @@ export class AnthropicTextAdapter< : []), ] if (content.length === 0) continue - const message: AnthropicMidConversationSystemMessage = { - role: 'system', - content, - } - // oxlint-disable-next-line eslint-js/no-restricted-syntax -- SDK 0.97.1 types no `system` message in `messages`; the models in the channel map accept one (pi 0.87.1). - systemMessages.set(before, message as unknown as BetaMessageParam) + systemMessages.set(before, { role: 'system', content }) } return { ...(channels.systemPrompts && { @@ -1218,8 +1215,12 @@ export class AnthropicTextAdapter< messages: Array, /** Mid-conversation `system` messages by message index (`before`). */ systemMessages?: ReadonlyMap, + /** The tools have the computer toolset, so `computer` calls are its calls. */ + computerToolset = false, ): InternalTextProviderOptions['messages'] { const formattedMessages: InternalTextProviderOptions['messages'] = [] + // The ids of the computer toolset calls. Their results name the toolset. + const toolsetCalls = computerToolset ? new Set() : undefined // A system message waits for the next assistant message (or the end), // so a `tool_result` still follows its `tool_use` directly. const pendingSystemMessages: InternalTextProviderOptions['messages'] = [] @@ -1240,6 +1241,9 @@ export class AnthropicTextAdapter< { type: 'tool_result', tool_use_id: message.toolCallId, + ...(toolsetCalls?.has(message.toolCallId) && { + toolset_name: COMPUTER_TOOL_NAME, + }), content: Array.isArray(toolContent) ? toolContent.map((part) => this.convertContentPartToAnthropic(part), @@ -1264,7 +1268,7 @@ export class AnthropicTextAdapter< if (block.type === 'thinking') { this.appendThinkingBlocks(contentBlocks, [block.thinking]) } else if (block.type === 'tool-call') { - this.appendToolCallBlocks(contentBlocks, block.toolCall) + this.appendToolCallBlocks(contentBlocks, block.toolCall, toolsetCalls) } else { contentBlocks.push({ type: 'text', text: block.text }) } @@ -1292,7 +1296,7 @@ export class AnthropicTextAdapter< } for (const toolCall of message.toolCalls) { - this.appendToolCallBlocks(contentBlocks, toolCall) + this.appendToolCallBlocks(contentBlocks, toolCall, toolsetCalls) } formattedMessages.push({ @@ -1389,10 +1393,15 @@ export class AnthropicTextAdapter< } } - /** A tool call as a `tool_use` block, or a server tool as its two blocks. */ + /** + * A tool call as a `tool_use` block, or a server tool as its two blocks. + * With `toolsetCalls`, a `computer` call goes back as the computer toolset + * call that the model sent, and its id goes in `toolsetCalls`. + */ private appendToolCallBlocks( contentBlocks: Array, toolCall: ToolCall, + toolsetCalls?: Set, ): void { let parsedInput: unknown = {} try { @@ -1421,6 +1430,25 @@ export class AnthropicTextAdapter< return } + const member = + toolsetCalls && + toolCall.function.name === COMPUTER_TOOL_NAME && + typeof parsedInput === 'object' && + parsedInput !== null + ? splitComputerToolsetCall(parsedInput) + : undefined + if (member) { + toolsetCalls?.add(toolCall.id) + contentBlocks.push({ + type: 'tool_use', + id: toolCall.id, + name: member.name, + toolset_name: COMPUTER_TOOL_NAME, + input: member.input, + }) + return + } + const toolUseBlock: ToolUseBlockParam = { type: 'tool_use', id: toolCall.id, @@ -1534,6 +1562,8 @@ export class AnthropicTextAdapter< input: string started: boolean hasInputDelta: boolean + /** The action of a computer toolset call (`left_click`, …). */ + member?: string } >() let currentToolIndex = -1 @@ -1596,9 +1626,16 @@ export class AnthropicTextAdapter< currentBlockType = event.content_block.type if (event.content_block.type === 'tool_use') { currentToolIndex++ + // A computer toolset call names its action. It runs as one + // `computer` call with the action in `input.action`. + const isComputerCall = + event.content_block.toolset_name === COMPUTER_TOOL_NAME toolCallsMap.set(currentToolIndex, { id: event.content_block.id, - name: event.content_block.name, + name: isComputerCall + ? COMPUTER_TOOL_NAME + : event.content_block.name, + ...(isComputerCall && { member: event.content_block.name }), // The initial input snapshot. The first `input_json_delta` // replaces it. input: JSON.stringify(event.content_block.input) ?? '', @@ -1860,7 +1897,14 @@ export class AnthropicTextAdapter< // never appends onto the prior client tool's buffer. if (currentBlockType === 'tool_use') { const existing = toolCallsMap.get(currentToolIndex) - if (existing) { + // A computer toolset call sends its joined arguments at the end. + if (existing?.member !== undefined) { + if (!existing.hasInputDelta) { + existing.input = '' + existing.hasInputDelta = true + } + existing.input += event.delta.partial_json + } else if (existing) { // Emit TOOL_CALL_START on first args delta if (!existing.started) { existing.started = true @@ -1971,6 +2015,26 @@ export class AnthropicTextAdapter< // Keep `parsedInput` undefined for the tool error. } + if ( + existing.member !== undefined && + typeof parsedInput === 'object' && + parsedInput !== null + ) { + parsedInput = joinComputerToolsetCall( + existing.member, + parsedInput, + ) + existing.input = JSON.stringify(parsedInput) + yield { + type: EventType.TOOL_CALL_ARGS, + toolCallId: existing.id, + model, + timestamp: Date.now(), + delta: existing.input, + args: existing.input, + } + } + yield { type: EventType.TOOL_CALL_END, toolCallId: existing.id, diff --git a/packages/ai-anthropic/src/model-meta.ts b/packages/ai-anthropic/src/model-meta.ts index 4534d13c97..80dc5e69ce 100644 --- a/packages/ai-anthropic/src/model-meta.ts +++ b/packages/ai-anthropic/src/model-meta.ts @@ -92,11 +92,19 @@ interface ModelMeta< * every request, with or without thinking. Absent: it accepts one. */ forced_tool_choice?: false + /** + * Computer use has three kinds, one for each version that a model + * accepts: `computer_use` (`computer_20241022` and `computer_20250124`), + * `computer_use_20251124`, and `computer_toolset` + * (`computer_toolset_20260801`). + */ tools?: Array< | 'web_search' | 'web_fetch' | 'code_execution' | 'computer_use' + | 'computer_use_20251124' + | 'computer_toolset' | 'bash' | 'text_editor' | 'memory' @@ -155,7 +163,7 @@ const CLAUDE_OPUS_4_6 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', 'bash', 'text_editor', 'memory', @@ -195,7 +203,7 @@ const CLAUDE_OPUS_4_5 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', 'bash', 'text_editor', 'memory', @@ -238,7 +246,7 @@ const CLAUDE_SONNET_4_6 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', 'bash', 'text_editor', 'memory', @@ -393,7 +401,7 @@ const CLAUDE_OPUS_4_7 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', 'bash', 'text_editor', 'memory', @@ -437,7 +445,8 @@ const CLAUDE_OPUS_4_8 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -484,7 +493,8 @@ const CLAUDE_FABLE_5 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -529,7 +539,8 @@ const CLAUDE_SONNET_5 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -577,7 +588,8 @@ const CLAUDE_OPUS_5 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -645,7 +657,8 @@ const CLAUDE_FABLE_5_1 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -681,12 +694,12 @@ const CLAUDE_OPUS_5_5 = { input: ['text', 'image', 'document'], forced_tool_choice: false, mid_conversation_channels: { tools: true, systemPrompts: true }, - // No 'computer_use': this model accepts only `computer_toolset_20260801`, - // and `computerUseTool()` sends the older versions, which return a 400. + // Computer use only through `computer_toolset_20260801`. tools: [ 'web_search', 'web_fetch', 'code_execution', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -728,12 +741,12 @@ const CLAUDE_SONNET_5_5 = { forced_tool_choice: false, extended_thinking: false, adaptive_thinking: true, - // No 'computer_use': this model accepts only `computer_toolset_20260801`, - // and `computerUseTool()` sends the older versions, which return a 400. + // Computer use only through `computer_toolset_20260801`. tools: [ 'web_search', 'web_fetch', 'code_execution', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -771,12 +784,12 @@ const CLAUDE_HAIKU_5_5 = { input: ['text', 'image', 'document'], extended_thinking: false, adaptive_thinking: true, - // No 'computer_use': this model accepts only `computer_toolset_20260801`, - // and `computerUseTool()` sends the older versions, which return a 400. + // Computer use only through `computer_toolset_20260801`. tools: [ 'web_search', 'web_fetch', 'code_execution', + 'computer_toolset', 'bash', 'text_editor', 'memory', diff --git a/packages/ai-anthropic/src/prompt-cache.ts b/packages/ai-anthropic/src/prompt-cache.ts index 60f9fde70c..dee08e3bb7 100644 --- a/packages/ai-anthropic/src/prompt-cache.ts +++ b/packages/ai-anthropic/src/prompt-cache.ts @@ -153,7 +153,9 @@ export function applyAnthropicPromptCache( // last tool, as before. const placeholder = request.tools?.findIndex( - (tool) => tool.name === ANTHROPIC_DEFERRED_TOOL_PLACEHOLDER_NAME, + // The computer toolset entry has no name. + (tool) => + 'name' in tool && tool.name === ANTHROPIC_DEFERRED_TOOL_PLACEHOLDER_NAME, ) ?? -1 const tools = request.tools?.map((tool, index, all) => index === (placeholder > 0 ? placeholder - 1 : all.length - 1) diff --git a/packages/ai-anthropic/src/tools/computer-use-tool.ts b/packages/ai-anthropic/src/tools/computer-use-tool.ts index d37f5ce07b..8959ca6546 100644 --- a/packages/ai-anthropic/src/tools/computer-use-tool.ts +++ b/packages/ai-anthropic/src/tools/computer-use-tool.ts @@ -1,22 +1,50 @@ -import { - brandAnthropicProviderTool, - getAnthropicProviderToolMetadata, -} from './anthropic-provider-tool' +import { brandProviderTool } from '@tanstack/ai' +import { getAnthropicProviderToolMetadata } from './anthropic-provider-tool' import type { + BetaComputerToolset20260801, BetaToolComputerUse20241022, BetaToolComputerUse20250124, + BetaToolComputerUse20251124, } from '@anthropic-ai/sdk/resources/beta' import type { ProviderTool, Tool } from '@tanstack/ai' -export type ComputerUseToolConfig = +/** The single `computer` tool that sends its action in `input.action`. */ +export type ComputerUseSingleToolConfig = | BetaToolComputerUse20241022 | BetaToolComputerUse20250124 + | BetaToolComputerUse20251124 + +/** + * The computer toolset: one entry with no name. The model calls each action + * as its own tool (`left_click`, `screenshot`, …). + */ +export type ComputerToolsetConfig = BetaComputerToolset20260801 + +export type ComputerUseToolConfig = + | ComputerUseSingleToolConfig + | ComputerToolsetConfig /** @deprecated Renamed to `ComputerUseToolConfig`. Will be removed in a future release. */ export type ComputerUseTool = ComputerUseToolConfig +/** `computer_20241022` or `computer_20250124`. */ export type AnthropicComputerUseTool = ProviderTool<'anthropic', 'computer_use'> +/** `computer_20251124`. */ +export type AnthropicComputerUse20251124Tool = ProviderTool< + 'anthropic', + 'computer_use_20251124' +> + +/** `computer_toolset_20260801`. */ +export type AnthropicComputerToolset = ProviderTool< + 'anthropic', + 'computer_toolset' +> + +/** The name of the tool that every computer use version runs as. */ +export const COMPUTER_TOOL_NAME = 'computer' + export function convertComputerUseToolToAdapterFormat( tool: Tool, ): ComputerUseToolConfig { @@ -26,15 +54,58 @@ export function convertComputerUseToolToAdapterFormat( return metadata } +/** True for the `computer_toolset_20260801` entry. */ +export function isComputerToolset(tool: Tool) { + return tool.metadata?.['type'] === 'computer_toolset_20260801' +} + +/** + * One toolset call (`left_click` with `{ coordinate }`) as one `computer` + * call: `{ action: 'left_click', coordinate }`. So one `execute` works for + * every version. + */ +export function joinComputerToolsetCall(member: string, input: object) { + return { action: member, ...input } +} + +/** A `computer` call as the toolset call that the model sent. */ +export function splitComputerToolsetCall(input: object) { + if (!('action' in input) || typeof input.action !== 'string') { + return undefined + } + const { action, ...memberInput } = input + return { name: action, input: memberInput } +} + +/** + * Computer use. Every version runs as one tool named `computer`, with the + * action in `input.action`: + * + * - `computer_toolset_20260801`: Claude Opus 5.5, Sonnet 5.5, Haiku 5.5, and + * the other models that support the toolset. No beta header. + * - `computer_20251124`: the earlier tool of Claude Opus 4.5 to Claude Opus 5, + * Sonnet 4.6, and Sonnet 5. + * - `computer_20250124`: Claude Sonnet 4.5, Haiku 4.5, and Opus 4.1. + */ +export function computerUseTool( + config: ComputerToolsetConfig, +): AnthropicComputerToolset +export function computerUseTool( + config: BetaToolComputerUse20251124, +): AnthropicComputerUse20251124Tool export function computerUseTool( - config: ComputerUseToolConfig, -): AnthropicComputerUseTool { - return brandAnthropicProviderTool( - { - name: 'computer', - description: '', - metadata: config, - }, - 'computer_use', - ) + config: BetaToolComputerUse20241022 | BetaToolComputerUse20250124, +): AnthropicComputerUseTool +export function computerUseTool(config: ComputerUseToolConfig) { + return brandProviderTool< + | AnthropicComputerToolset + | AnthropicComputerUse20251124Tool + | AnthropicComputerUseTool + >({ + name: COMPUTER_TOOL_NAME, + description: '', + // Every version has the same runtime kind. The type kind picks the + // models that accept it. + metadata: { ...config, __kind: 'anthropic.computer_use' }, + }) } diff --git a/packages/ai-anthropic/tests/client-injection-type-safety.test.ts b/packages/ai-anthropic/tests/client-injection-type-safety.test.ts index 57392a90c4..18d3f73fcf 100644 --- a/packages/ai-anthropic/tests/client-injection-type-safety.test.ts +++ b/packages/ai-anthropic/tests/client-injection-type-safety.test.ts @@ -1,18 +1,7 @@ import { expectTypeOf, it } from 'vitest' -import type AnthropicSdkV112 from '@anthropic-ai/sdk-v112' import type { AnthropicVertex } from '@anthropic-ai/vertex-sdk' import type { AnthropicMessagesClient } from '../src' -type V112MessagesClient = { - readonly beta: { - readonly messages: Pick - } -} - it('accepts the official Vertex client', () => { expectTypeOf().toExtend() }) - -it('accepts clients backed by a newer Anthropic SDK version', () => { - expectTypeOf().toExtend() -}) diff --git a/packages/ai-anthropic/tests/computer-toolset.test.ts b/packages/ai-anthropic/tests/computer-toolset.test.ts new file mode 100644 index 0000000000..9edfbe1fc9 --- /dev/null +++ b/packages/ai-anthropic/tests/computer-toolset.test.ts @@ -0,0 +1,222 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { chat } from '@tanstack/ai' +import { AnthropicTextAdapter } from '../src/adapters/text' +import { computerUseTool } from '../src/tools' + +const mocks = vi.hoisted(() => { + const betaMessagesCreate = vi.fn() + const client = { beta: { messages: { create: betaMessagesCreate } } } + return { betaMessagesCreate, client } +}) + +vi.mock('@anthropic-ai/sdk', () => { + const { client } = mocks + class MockAnthropic { + beta = client.beta + constructor(_: { apiKey: string }) {} + } + return { default: MockAnthropic } +}) + +function stream(events: Array) { + mocks.betaMessagesCreate.mockResolvedValueOnce( + (async function* () { + yield* events + })(), + ) +} + +/** A computer toolset call, as the model streams it. */ +function memberCall(index: number, id: string, name: string, json?: string) { + return [ + { + type: 'content_block_start', + index, + content_block: { + type: 'tool_use', + id, + name, + toolset_name: 'computer', + input: {}, + }, + }, + ...(json === undefined + ? [] + : [ + { + type: 'content_block_delta', + index, + delta: { type: 'input_json_delta', partial_json: json }, + }, + ]), + { type: 'content_block_stop', index }, + ] +} + +const end = (stopReason: string) => [ + { + type: 'message_delta', + delta: { stop_reason: stopReason }, + usage: { output_tokens: 1 }, + }, + { type: 'message_stop' }, +] + +const screenshot = [ + { + type: 'image' as const, + source: { type: 'data' as const, value: 'AAAA', mimeType: 'image/png' }, + }, +] + +beforeEach(() => { + mocks.betaMessagesCreate.mockReset() +}) + +describe('anthropic computer toolset', () => { + it('sends the toolset entry with no name and no beta', async () => { + stream(end('end_turn')) + const adapter = new AnthropicTextAdapter({ apiKey: 'k' }, 'claude-opus-5-5') + + for await (const _ of chat({ + adapter, + messages: [{ role: 'user', content: 'Open the settings.' }], + tools: [ + computerUseTool({ + type: 'computer_toolset_20260801', + configs: { zoom: { enabled: false } }, + }), + ], + })) { + // consume + } + + const [payload] = mocks.betaMessagesCreate.mock.calls[0]! + // The prompt cache marker can also sit on the entry. + expect(payload.tools).toMatchObject([ + { type: 'computer_toolset_20260801', configs: { zoom: { enabled: false } } }, + ]) + expect(payload.tools[0]).not.toHaveProperty('name') + expect(payload.tools[0]).not.toHaveProperty('__kind') + expect(payload.betas).toBeUndefined() + }) + + it('runs each toolset call as one computer call and sends it back', async () => { + stream([ + ...memberCall(0, 'toolu_click', 'left_click', '{"coordinate":[500,300]}'), + ...memberCall(1, 'toolu_shot', 'screenshot'), + ...end('tool_use'), + ]) + stream(end('end_turn')) + const calls: Array = [] + const tool = { + ...computerUseTool({ type: 'computer_toolset_20260801' }), + execute: (input: unknown) => { + calls.push(input) + return calls.length === 1 ? 'OK' : screenshot + }, + } + const adapter = new AnthropicTextAdapter({ apiKey: 'k' }, 'claude-opus-5-5') + + for await (const _ of chat({ + adapter, + messages: [{ role: 'user', content: 'Click the button.' }], + tools: [tool], + toolExecution: 'sequential', + })) { + // consume + } + + expect(calls).toEqual([ + { action: 'left_click', coordinate: [500, 300] }, + { action: 'screenshot' }, + ]) + const [payload] = mocks.betaMessagesCreate.mock.calls[1]! + const [, assistant, results] = payload.messages + expect(assistant.content).toEqual([ + { + type: 'tool_use', + id: 'toolu_click', + name: 'left_click', + toolset_name: 'computer', + input: { coordinate: [500, 300] }, + }, + { + type: 'tool_use', + id: 'toolu_shot', + name: 'screenshot', + toolset_name: 'computer', + input: {}, + }, + ]) + expect(results.content).toMatchObject([ + { + type: 'tool_result', + tool_use_id: 'toolu_click', + toolset_name: 'computer', + content: 'OK', + }, + { + type: 'tool_result', + tool_use_id: 'toolu_shot', + toolset_name: 'computer', + content: [ + { + type: 'image', + source: { type: 'base64', data: 'AAAA', media_type: 'image/png' }, + }, + ], + }, + ]) + }) + + it('sends a computer call back as the single tool when the toolset is not in tools', async () => { + stream(end('end_turn')) + const adapter = new AnthropicTextAdapter({ apiKey: 'k' }, 'claude-opus-4-6') + + for await (const _ of chat({ + adapter, + messages: [ + { role: 'user', content: 'Click the button.' }, + { + role: 'assistant', + content: '', + toolCalls: [ + { + id: 'toolu_click', + type: 'function', + function: { + name: 'computer', + arguments: '{"action":"left_click","coordinate":[1,2]}', + }, + }, + ], + }, + { role: 'tool', toolCallId: 'toolu_click', content: 'OK' }, + ], + tools: [ + computerUseTool({ + type: 'computer_20251124', + name: 'computer', + display_width_px: 1024, + display_height_px: 768, + }), + ], + })) { + // consume + } + + const [payload] = mocks.betaMessagesCreate.mock.calls[0]! + expect(payload.betas).toContain('computer-use-2025-11-24') + const [, assistant, results] = payload.messages + expect(assistant.content).toEqual([ + { + type: 'tool_use', + id: 'toolu_click', + name: 'computer', + input: { action: 'left_click', coordinate: [1, 2] }, + }, + ]) + expect(results.content[0]).not.toHaveProperty('toolset_name') + }) +}) diff --git a/packages/ai-anthropic/tests/tools-per-model-type-safety.test.ts b/packages/ai-anthropic/tests/tools-per-model-type-safety.test.ts index d46b14bf07..abfca22c0f 100644 --- a/packages/ai-anthropic/tests/tools-per-model-type-safety.test.ts +++ b/packages/ai-anthropic/tests/tools-per-model-type-safety.test.ts @@ -58,7 +58,7 @@ describe('Anthropic per-model tool gating', () => { type: 'code_execution_20250825', }), computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -83,7 +83,7 @@ describe('Anthropic per-model tool gating', () => { type: 'code_execution_20250825', }), computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -108,7 +108,7 @@ describe('Anthropic per-model tool gating', () => { type: 'code_execution_20250825', }), computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -133,7 +133,7 @@ describe('Anthropic per-model tool gating', () => { type: 'code_execution_20250825', }), computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -158,7 +158,7 @@ describe('Anthropic per-model tool gating', () => { type: 'code_execution_20250825', }), computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -172,7 +172,7 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-opus-5-5 accepts every provider tool except computer use', () => { + it('claude-opus-5-5 accepts the full tool superset with the computer toolset', () => { const adapter = anthropicText('claude-opus-5-5') typedTools(adapter, [ userTool, @@ -182,6 +182,7 @@ describe('Anthropic per-model tool gating', () => { name: 'code_execution', type: 'code_execution_20250825', }), + computerUseTool({ type: 'computer_toolset_20260801' }), bashTool({ name: 'bash', type: 'bash_20250124' }), textEditorTool({ type: 'text_editor_20250124', @@ -191,12 +192,12 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-opus-5-5 rejects computerUseTool (only computer_toolset_20260801 works)', () => { + it('claude-opus-5-5 rejects computer_20251124 (only the toolset works)', () => { const adapter = anthropicText('claude-opus-5-5') typedTools(adapter, [ - // @ts-expect-error - computer_20250124 returns a 400 on claude-opus-5-5 + // @ts-expect-error - computer_20251124 returns a 400 on claude-opus-5-5 computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -204,7 +205,7 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-sonnet-5-5 accepts every provider tool except computer use', () => { + it('claude-sonnet-5-5 accepts the full tool superset with the computer toolset', () => { const adapter = anthropicText('claude-sonnet-5-5') typedTools(adapter, [ userTool, @@ -214,6 +215,7 @@ describe('Anthropic per-model tool gating', () => { name: 'code_execution', type: 'code_execution_20250825', }), + computerUseTool({ type: 'computer_toolset_20260801' }), bashTool({ name: 'bash', type: 'bash_20250124' }), textEditorTool({ type: 'text_editor_20250124', @@ -223,12 +225,12 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-sonnet-5-5 rejects computerUseTool (only computer_toolset_20260801 works)', () => { + it('claude-sonnet-5-5 rejects computer_20251124 (only the toolset works)', () => { const adapter = anthropicText('claude-sonnet-5-5') typedTools(adapter, [ - // @ts-expect-error - computer_20250124 returns a 400 on claude-sonnet-5-5 + // @ts-expect-error - computer_20251124 returns a 400 on claude-sonnet-5-5 computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -236,7 +238,7 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-haiku-5-5 accepts every provider tool except computer use', () => { + it('claude-haiku-5-5 accepts the full tool superset with the computer toolset', () => { const adapter = anthropicText('claude-haiku-5-5') typedTools(adapter, [ userTool, @@ -246,6 +248,7 @@ describe('Anthropic per-model tool gating', () => { name: 'code_execution', type: 'code_execution_20250825', }), + computerUseTool({ type: 'computer_toolset_20260801' }), bashTool({ name: 'bash', type: 'bash_20250124' }), textEditorTool({ type: 'text_editor_20250124', @@ -255,16 +258,57 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-haiku-5-5 rejects computerUseTool (only computer_toolset_20260801 works)', () => { + it('claude-haiku-5-5 rejects computer_20251124 (only the toolset works)', () => { const adapter = anthropicText('claude-haiku-5-5') typedTools(adapter, [ - // @ts-expect-error - computer_20250124 returns a 400 on claude-haiku-5-5 + // @ts-expect-error - computer_20251124 returns a 400 on claude-haiku-5-5 + computerUseTool({ + type: 'computer_20251124', + name: 'computer', + display_width_px: 1024, + display_height_px: 768, + }), + ]) + }) + + it('claude-opus-4-6 rejects computer_20250124 and the computer toolset', () => { + const adapter = anthropicText('claude-opus-4-6') + typedTools(adapter, [ + // @ts-expect-error - claude-opus-4-6 takes only computer_20251124 computerUseTool({ type: 'computer_20250124', name: 'computer', display_width_px: 1024, display_height_px: 768, }), + // @ts-expect-error - claude-opus-4-6 takes only computer_20251124 + computerUseTool({ type: 'computer_toolset_20260801' }), + ]) + }) + + it('claude-haiku-4-5 accepts computer_20250124 and rejects computer_20251124', () => { + const adapter = anthropicText('claude-haiku-4-5') + typedTools(adapter, [ + computerUseTool({ + type: 'computer_20250124', + name: 'computer', + display_width_px: 1024, + display_height_px: 768, + }), + // @ts-expect-error - claude-haiku-4-5 takes only computer_20250124 + computerUseTool({ + type: 'computer_20251124', + name: 'computer', + display_width_px: 1024, + display_height_px: 768, + }), + ]) + }) + + it('claude-opus-5 accepts the computer toolset', () => { + const adapter = anthropicText('claude-opus-5') + typedTools(adapter, [ + computerUseTool({ type: 'computer_toolset_20260801' }), ]) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bb0a8207db..7f90ae25c0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1065,8 +1065,8 @@ importers: examples/ts-react-chat: dependencies: '@anthropic-ai/sdk': - specifier: ^0.97.1 - version: 0.97.1(zod@4.2.1) + specifier: ^0.135.0 + version: 0.135.0(zod@4.2.1) '@elevenlabs/elevenlabs-js': specifier: ^2.68.0 version: 2.68.0 @@ -2120,15 +2120,12 @@ importers: packages/ai-anthropic: dependencies: '@anthropic-ai/sdk': - specifier: ^0.97.1 - version: 0.97.1(zod@4.2.1) + specifier: ^0.135.0 + version: 0.135.0(zod@4.2.1) '@tanstack/ai-utils': specifier: workspace:^ version: link:../ai-utils devDependencies: - '@anthropic-ai/sdk-v112': - specifier: npm:@anthropic-ai/sdk@0.112.4 - version: '@anthropic-ai/sdk@0.112.4(zod@4.2.1)' '@anthropic-ai/vertex-sdk': specifier: ^0.19.0 version: 0.19.0(zod@4.2.1) @@ -3456,7 +3453,7 @@ importers: devDependencies: '@google/genai': specifier: ^2.10.0 - version: 2.10.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6)) + version: 2.10.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.6.5)) '@tanstack/ai': specifier: workspace:* version: link:../ai @@ -3640,8 +3637,8 @@ importers: testing/e2e: dependencies: '@anthropic-ai/sdk': - specifier: ^0.97.1 - version: 0.97.1(zod@4.3.6) + specifier: ^0.135.0 + version: 0.135.0(zod@4.3.6) '@copilotkit/aimock': specifier: ^1.34.0 version: 1.34.0(vitest@4.1.11(@opentelemetry/api@1.9.1)(@types/node@24.10.3)(@vitest/coverage-v8@4.1.10(vitest@4.1.10))(happy-dom@20.11.2)(jsdom@27.4.0(@noble/hashes@2.3.0))(vite@8.2.1(@types/node@24.10.3)(esbuild@0.28.1)(jiti@2.7.0)(less@4.6.6)(sass@1.101.0)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0))) @@ -4155,17 +4152,8 @@ packages: '@angular/animations': optional: true - '@anthropic-ai/sdk@0.112.4': - resolution: {integrity: sha512-7eXJJnrmBI5GMC6drrCiSkycVsT7crRZX3qv5HusLSm+qiILjmtqP7gf+UiT7ASu/7Gdj+Zfl4f2haV8wATKUg==} - hasBin: true - peerDependencies: - zod: ^3.25.0 || ^4.0.0 - peerDependenciesMeta: - zod: - optional: true - - '@anthropic-ai/sdk@0.97.1': - resolution: {integrity: sha512-wOf7AUeJPitcVpvKO4UMu63mWH5SaVipkGd7OOQJt/G6VYGlV8D2Gp9dLxOrttDJh/9gqPqdaBwDGcBevumeAg==} + '@anthropic-ai/sdk@0.135.0': + resolution: {integrity: sha512-WWrdL57JhU5KoynpTjt9GXK6k7ud9nPw2uw/Trn+oBXh6s0KpjhAccWZm2Qxw08ARYG1UDQj5iC+JO42j1rt/g==} hasBin: true peerDependencies: zod: ^3.25.0 || ^4.0.0 @@ -19221,22 +19209,17 @@ snapshots: '@angular/core': 21.2.20(@angular/compiler@21.2.20)(rxjs@7.8.2)(zone.js@0.15.1) tslib: 2.8.1 - '@anthropic-ai/sdk@0.112.4(zod@4.2.1)': - dependencies: - json-schema-to-ts: 3.1.1 - standardwebhooks: 1.0.0 - optionalDependencies: - zod: 4.2.1 - - '@anthropic-ai/sdk@0.97.1(zod@4.2.1)': + '@anthropic-ai/sdk@0.135.0(zod@4.2.1)': dependencies: + '@opentelemetry/api': 1.9.1 json-schema-to-ts: 3.1.1 standardwebhooks: 1.0.0 optionalDependencies: zod: 4.2.1 - '@anthropic-ai/sdk@0.97.1(zod@4.3.6)': + '@anthropic-ai/sdk@0.135.0(zod@4.3.6)': dependencies: + '@opentelemetry/api': 1.9.1 json-schema-to-ts: 3.1.1 standardwebhooks: 1.0.0 optionalDependencies: @@ -19244,7 +19227,7 @@ snapshots: '@anthropic-ai/vertex-sdk@0.19.0(zod@4.2.1)': dependencies: - '@anthropic-ai/sdk': 0.112.4(zod@4.2.1) + '@anthropic-ai/sdk': 0.135.0(zod@4.2.1) google-auth-library: 10.5.0 transitivePeerDependencies: - supports-color diff --git a/scripts/model-sync/provider-supports.test.ts b/scripts/model-sync/provider-supports.test.ts index 355c75eae4..c311313371 100644 --- a/scripts/model-sync/provider-supports.test.ts +++ b/scripts/model-sync/provider-supports.test.ts @@ -38,7 +38,8 @@ describe('buildProviderSupportsBody', () => { expect(body).toContain("'web_search'") expect(body).toContain("'web_fetch'") expect(body).toContain("'code_execution'") - expect(body).toContain("'computer_use'") + expect(body).toContain("'computer_toolset'") + expect(body).not.toContain("'computer_use'") expect(body).toContain('extended_thinking: true') expect(body).toContain('priority_tier: true') expect(body).not.toContain('adaptive_thinking') diff --git a/scripts/model-sync/provider-supports.ts b/scripts/model-sync/provider-supports.ts index d27c86b0af..127b9c2c8d 100644 --- a/scripts/model-sync/provider-supports.ts +++ b/scripts/model-sync/provider-supports.ts @@ -17,11 +17,12 @@ export interface ProviderSupportsInput { reasoningMandatory?: boolean } +// New Claude models take computer use only through the computer toolset. const ANTHROPIC_SERVER_TOOLS = [ 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_toolset', 'bash', 'text_editor', 'memory', diff --git a/testing/e2e/package.json b/testing/e2e/package.json index 7001fbcc32..78f353b467 100644 --- a/testing/e2e/package.json +++ b/testing/e2e/package.json @@ -12,7 +12,7 @@ "postinstall": "playwright install chromium" }, "dependencies": { - "@anthropic-ai/sdk": "^0.97.1", + "@anthropic-ai/sdk": "^0.135.0", "@copilotkit/aimock": "^1.34.0", "@modelcontextprotocol/sdk": "^1.29.0", "@openrouter/sdk": "0.13.20", From 575855a7ebc50142af0511347bb8a98c96bb7894 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 19 Oct 2026 18:53:20 +0200 Subject: [PATCH 06/32] test(e2e): check the computer toolset loop on the wire Also add the changeset and format the changed files. --- .changeset/anthropic-computer-toolset.md | 10 ++ packages/ai-anthropic/src/adapters/text.ts | 6 +- packages/ai-anthropic/src/prompt-cache.ts | 3 +- .../tests/computer-toolset.test.ts | 5 +- testing/e2e/src/routeTree.gen.ts | 22 +++ .../api.anthropic-computer-toolset-wire.ts | 162 ++++++++++++++++++ .../anthropic-computer-toolset-wire.spec.ts | 97 +++++++++++ 7 files changed, 302 insertions(+), 3 deletions(-) create mode 100644 .changeset/anthropic-computer-toolset.md create mode 100644 testing/e2e/src/routes/api.anthropic-computer-toolset-wire.ts create mode 100644 testing/e2e/tests/anthropic-computer-toolset-wire.spec.ts diff --git a/.changeset/anthropic-computer-toolset.md b/.changeset/anthropic-computer-toolset.md new file mode 100644 index 0000000000..94212459f0 --- /dev/null +++ b/.changeset/anthropic-computer-toolset.md @@ -0,0 +1,10 @@ +--- +'@tanstack/ai-anthropic': minor +--- + +Run computer use on every Claude model that supports it. + +- `computerUseTool()` takes `computer_toolset_20260801`, the only computer use version of Claude Opus 5.5, Sonnet 5.5, and Haiku 5.5. The model calls each action as its own tool. The adapter runs each call as one `computer` call with the action in `input.action`, and sends it back to Claude in the form that Claude sent. So one `execute` works for every version. +- `computerUseTool()` takes `computer_20251124`, with its `computer-use-2025-11-24` beta header. +- Each model now takes only the computer use versions that it supports. For example, `computer_20250124` on `claude-opus-4-6` is a type error, because Anthropic rejects that pair. +- `@anthropic-ai/sdk` is now `^0.135.0`. diff --git a/packages/ai-anthropic/src/adapters/text.ts b/packages/ai-anthropic/src/adapters/text.ts index 1e5f32c3c6..4c906aedef 100644 --- a/packages/ai-anthropic/src/adapters/text.ts +++ b/packages/ai-anthropic/src/adapters/text.ts @@ -1268,7 +1268,11 @@ export class AnthropicTextAdapter< if (block.type === 'thinking') { this.appendThinkingBlocks(contentBlocks, [block.thinking]) } else if (block.type === 'tool-call') { - this.appendToolCallBlocks(contentBlocks, block.toolCall, toolsetCalls) + this.appendToolCallBlocks( + contentBlocks, + block.toolCall, + toolsetCalls, + ) } else { contentBlocks.push({ type: 'text', text: block.text }) } diff --git a/packages/ai-anthropic/src/prompt-cache.ts b/packages/ai-anthropic/src/prompt-cache.ts index dee08e3bb7..6874d4cb3a 100644 --- a/packages/ai-anthropic/src/prompt-cache.ts +++ b/packages/ai-anthropic/src/prompt-cache.ts @@ -155,7 +155,8 @@ export function applyAnthropicPromptCache( request.tools?.findIndex( // The computer toolset entry has no name. (tool) => - 'name' in tool && tool.name === ANTHROPIC_DEFERRED_TOOL_PLACEHOLDER_NAME, + 'name' in tool && + tool.name === ANTHROPIC_DEFERRED_TOOL_PLACEHOLDER_NAME, ) ?? -1 const tools = request.tools?.map((tool, index, all) => index === (placeholder > 0 ? placeholder - 1 : all.length - 1) diff --git a/packages/ai-anthropic/tests/computer-toolset.test.ts b/packages/ai-anthropic/tests/computer-toolset.test.ts index 9edfbe1fc9..1f952f1507 100644 --- a/packages/ai-anthropic/tests/computer-toolset.test.ts +++ b/packages/ai-anthropic/tests/computer-toolset.test.ts @@ -94,7 +94,10 @@ describe('anthropic computer toolset', () => { const [payload] = mocks.betaMessagesCreate.mock.calls[0]! // The prompt cache marker can also sit on the entry. expect(payload.tools).toMatchObject([ - { type: 'computer_toolset_20260801', configs: { zoom: { enabled: false } } }, + { + type: 'computer_toolset_20260801', + configs: { zoom: { enabled: false } }, + }, ]) expect(payload.tools[0]).not.toHaveProperty('name') expect(payload.tools[0]).not.toHaveProperty('__kind') diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index 70bc3a477a..3d2109a65d 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -54,6 +54,7 @@ import { Route as ApiActivityTestRouteImport } from './routes/api.activity-test' import { Route as ApiAdapterInputModalitiesRouteImport } from './routes/api.adapter-input-modalities' import { Route as ApiAnthropicAuthWireRouteImport } from './routes/api.anthropic-auth-wire' import { Route as ApiAnthropicBugTestRouteImport } from './routes/api.anthropic-bug-test' +import { Route as ApiAnthropicComputerToolsetWireRouteImport } from './routes/api.anthropic-computer-toolset-wire' import { Route as ApiAnthropicComputerUseWireRouteImport } from './routes/api.anthropic-computer-use-wire' import { Route as ApiAnthropicHaiku55WireRouteImport } from './routes/api.anthropic-haiku-5-5-wire' import { Route as ApiAnthropicMaxTokensUsageRouteImport } from './routes/api.anthropic-max-tokens-usage' @@ -414,6 +415,12 @@ const ApiAnthropicBugTestRoute = ApiAnthropicBugTestRouteImport.update({ path: '/api/anthropic-bug-test', getParentRoute: () => rootRouteImport, } as any) +const ApiAnthropicComputerToolsetWireRoute = + ApiAnthropicComputerToolsetWireRouteImport.update({ + id: '/api/anthropic-computer-toolset-wire', + path: '/api/anthropic-computer-toolset-wire', + getParentRoute: () => rootRouteImport, + } as any) const ApiAnthropicComputerUseWireRoute = ApiAnthropicComputerUseWireRouteImport.update({ id: '/api/anthropic-computer-use-wire', @@ -1150,6 +1157,7 @@ export interface FileRoutesByFullPath { '/api/adapter-input-modalities': typeof ApiAdapterInputModalitiesRoute '/api/anthropic-auth-wire': typeof ApiAnthropicAuthWireRoute '/api/anthropic-bug-test': typeof ApiAnthropicBugTestRoute + '/api/anthropic-computer-toolset-wire': typeof ApiAnthropicComputerToolsetWireRoute '/api/anthropic-computer-use-wire': typeof ApiAnthropicComputerUseWireRoute '/api/anthropic-haiku-5-5-wire': typeof ApiAnthropicHaiku55WireRoute '/api/anthropic-max-tokens-usage': typeof ApiAnthropicMaxTokensUsageRoute @@ -1327,6 +1335,7 @@ export interface FileRoutesByTo { '/api/adapter-input-modalities': typeof ApiAdapterInputModalitiesRoute '/api/anthropic-auth-wire': typeof ApiAnthropicAuthWireRoute '/api/anthropic-bug-test': typeof ApiAnthropicBugTestRoute + '/api/anthropic-computer-toolset-wire': typeof ApiAnthropicComputerToolsetWireRoute '/api/anthropic-computer-use-wire': typeof ApiAnthropicComputerUseWireRoute '/api/anthropic-haiku-5-5-wire': typeof ApiAnthropicHaiku55WireRoute '/api/anthropic-max-tokens-usage': typeof ApiAnthropicMaxTokensUsageRoute @@ -1505,6 +1514,7 @@ export interface FileRoutesById { '/api/adapter-input-modalities': typeof ApiAdapterInputModalitiesRoute '/api/anthropic-auth-wire': typeof ApiAnthropicAuthWireRoute '/api/anthropic-bug-test': typeof ApiAnthropicBugTestRoute + '/api/anthropic-computer-toolset-wire': typeof ApiAnthropicComputerToolsetWireRoute '/api/anthropic-computer-use-wire': typeof ApiAnthropicComputerUseWireRoute '/api/anthropic-haiku-5-5-wire': typeof ApiAnthropicHaiku55WireRoute '/api/anthropic-max-tokens-usage': typeof ApiAnthropicMaxTokensUsageRoute @@ -1684,6 +1694,7 @@ export interface FileRouteTypes { | '/api/adapter-input-modalities' | '/api/anthropic-auth-wire' | '/api/anthropic-bug-test' + | '/api/anthropic-computer-toolset-wire' | '/api/anthropic-computer-use-wire' | '/api/anthropic-haiku-5-5-wire' | '/api/anthropic-max-tokens-usage' @@ -1861,6 +1872,7 @@ export interface FileRouteTypes { | '/api/adapter-input-modalities' | '/api/anthropic-auth-wire' | '/api/anthropic-bug-test' + | '/api/anthropic-computer-toolset-wire' | '/api/anthropic-computer-use-wire' | '/api/anthropic-haiku-5-5-wire' | '/api/anthropic-max-tokens-usage' @@ -2038,6 +2050,7 @@ export interface FileRouteTypes { | '/api/adapter-input-modalities' | '/api/anthropic-auth-wire' | '/api/anthropic-bug-test' + | '/api/anthropic-computer-toolset-wire' | '/api/anthropic-computer-use-wire' | '/api/anthropic-haiku-5-5-wire' | '/api/anthropic-max-tokens-usage' @@ -2216,6 +2229,7 @@ export interface RootRouteChildren { ApiAdapterInputModalitiesRoute: typeof ApiAdapterInputModalitiesRoute ApiAnthropicAuthWireRoute: typeof ApiAnthropicAuthWireRoute ApiAnthropicBugTestRoute: typeof ApiAnthropicBugTestRoute + ApiAnthropicComputerToolsetWireRoute: typeof ApiAnthropicComputerToolsetWireRoute ApiAnthropicComputerUseWireRoute: typeof ApiAnthropicComputerUseWireRoute ApiAnthropicHaiku55WireRoute: typeof ApiAnthropicHaiku55WireRoute ApiAnthropicMaxTokensUsageRoute: typeof ApiAnthropicMaxTokensUsageRoute @@ -2661,6 +2675,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiAnthropicBugTestRouteImport parentRoute: typeof rootRouteImport } + '/api/anthropic-computer-toolset-wire': { + id: '/api/anthropic-computer-toolset-wire' + path: '/api/anthropic-computer-toolset-wire' + fullPath: '/api/anthropic-computer-toolset-wire' + preLoaderRoute: typeof ApiAnthropicComputerToolsetWireRouteImport + parentRoute: typeof rootRouteImport + } '/api/anthropic-computer-use-wire': { id: '/api/anthropic-computer-use-wire' path: '/api/anthropic-computer-use-wire' @@ -3677,6 +3698,7 @@ const rootRouteChildren: RootRouteChildren = { ApiAdapterInputModalitiesRoute: ApiAdapterInputModalitiesRoute, ApiAnthropicAuthWireRoute: ApiAnthropicAuthWireRoute, ApiAnthropicBugTestRoute: ApiAnthropicBugTestRoute, + ApiAnthropicComputerToolsetWireRoute: ApiAnthropicComputerToolsetWireRoute, ApiAnthropicComputerUseWireRoute: ApiAnthropicComputerUseWireRoute, ApiAnthropicHaiku55WireRoute: ApiAnthropicHaiku55WireRoute, ApiAnthropicMaxTokensUsageRoute: ApiAnthropicMaxTokensUsageRoute, diff --git a/testing/e2e/src/routes/api.anthropic-computer-toolset-wire.ts b/testing/e2e/src/routes/api.anthropic-computer-toolset-wire.ts new file mode 100644 index 0000000000..0fbba1e94b --- /dev/null +++ b/testing/e2e/src/routes/api.anthropic-computer-toolset-wire.ts @@ -0,0 +1,162 @@ +import { createFileRoute } from '@tanstack/react-router' +import { chat, createChatOptions } from '@tanstack/ai' +import { createAnthropicChat } from '@tanstack/ai-anthropic' +import { computerUseTool } from '@tanstack/ai-anthropic/tools' + +const DUMMY_KEY = 'sk-ant-e2e-test-dummy-key' + +/** + * Drives one computer toolset loop on `claude-opus-5-5`. A custom `fetch` + * captures each outgoing request. The first answer has two toolset calls + * (`left_click` and `screenshot`). The second answer ends the turn. + * + * The companion spec reads the captured requests and the inputs that + * `execute` got, so it can check the toolset wire format without an API key. + */ +function sseStream(events: Array<{ type: string }>) { + const encoder = new TextEncoder() + return new ReadableStream({ + start(controller) { + for (const event of events) { + controller.enqueue( + encoder.encode( + `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`, + ), + ) + } + controller.close() + }, + }) +} + +const messageStart = { + type: 'message_start', + message: { + id: 'msg_computer_toolset_wire', + type: 'message', + role: 'assistant', + content: [], + model: 'claude-opus-5-5', + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 5, output_tokens: 0 }, + }, +} + +const end = (stopReason: string) => [ + { + type: 'message_delta', + delta: { stop_reason: stopReason, stop_sequence: null }, + usage: { output_tokens: 1 }, + }, + { type: 'message_stop' }, +] + +const toolsetCalls = [ + messageStart, + { + type: 'content_block_start', + index: 0, + content_block: { + type: 'tool_use', + id: 'toolu_click', + name: 'left_click', + toolset_name: 'computer', + input: {}, + }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { + type: 'input_json_delta', + partial_json: '{"coordinate":[500,300]}', + }, + }, + { type: 'content_block_stop', index: 0 }, + { + type: 'content_block_start', + index: 1, + content_block: { + type: 'tool_use', + id: 'toolu_shot', + name: 'screenshot', + toolset_name: 'computer', + input: {}, + }, + }, + { type: 'content_block_stop', index: 1 }, + ...end('tool_use'), +] + +export const Route = createFileRoute('/api/anthropic-computer-toolset-wire')({ + server: { + handlers: { + POST: async () => { + const requests: Array<{ anthropicBeta: string | null; body: unknown }> = + [] + const capturingFetch: typeof fetch = async (input, init) => { + const req = + input instanceof Request ? input : new Request(input, init) + const rawBody = await req.text() + requests.push({ + anthropicBeta: req.headers.get('anthropic-beta'), + body: rawBody ? JSON.parse(rawBody) : null, + }) + const events = + requests.length === 1 + ? toolsetCalls + : [messageStart, ...end('end_turn')] + return new Response(sseStream(events), { + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + }) + } + + const calls: Array = [] + const computer = { + ...computerUseTool({ + type: 'computer_toolset_20260801', + configs: { zoom: { enabled: false } }, + }), + execute: (input: { action: string }) => { + calls.push(input) + if (input.action !== 'screenshot') return 'OK' + return [ + { + type: 'image' as const, + source: { + type: 'data' as const, + value: 'AAAA', + mimeType: 'image/png', + }, + }, + ] + }, + } + + const adapter = createAnthropicChat('claude-opus-5-5', DUMMY_KEY, { + fetch: capturingFetch, + }) + + try { + for await (const _ of chat({ + ...createChatOptions({ adapter }), + messages: [{ role: 'user', content: 'Click the button.' }], + tools: [computer], + toolExecution: 'sequential', + })) { + // Drain the stream. + } + } catch (error) { + return Response.json({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }) + } + + return Response.json({ ok: true, calls, requests }) + }, + }, + }, +}) diff --git a/testing/e2e/tests/anthropic-computer-toolset-wire.spec.ts b/testing/e2e/tests/anthropic-computer-toolset-wire.spec.ts new file mode 100644 index 0000000000..3a982a686b --- /dev/null +++ b/testing/e2e/tests/anthropic-computer-toolset-wire.spec.ts @@ -0,0 +1,97 @@ +import { test, expect } from './fixtures' +import type { APIRequestContext } from '@playwright/test' + +/** + * Wire format of the Anthropic computer toolset (`computer_toolset_20260801`). + * + * The model calls each action as its own tool (`left_click`, `screenshot`) + * with `toolset_name: 'computer'`. The adapter runs each call as one + * `computer` call with the action in `input.action`. On the next request it + * sends each call back in the form that the model sent, and each result + * names the toolset. + * + * `/api/anthropic-computer-toolset-wire` runs one loop against a custom + * `fetch`, and returns the captured requests and the inputs that `execute` + * got. + */ +interface WireResult { + ok: boolean + error?: string + calls: Array + requests: Array<{ + anthropicBeta: string | null + body: { + tools: Array> + messages: Array<{ role: string; content: Array> }> + } + }> +} + +async function runLoop(request: APIRequestContext) { + const res = await request.post('/api/anthropic-computer-toolset-wire') + expect(res.ok()).toBe(true) + const result = (await res.json()) as WireResult + if (!result.ok) { + throw new Error(`Route failed: ${result.error}`) + } + return result +} + +test.describe('anthropic — computer toolset wire format', () => { + test('sends the toolset entry with no name and no computer use beta', async ({ + request, + }) => { + const { requests } = await runLoop(request) + const [first] = requests + expect(first?.body.tools[0]).toMatchObject({ + type: 'computer_toolset_20260801', + configs: { zoom: { enabled: false } }, + }) + expect(first?.body.tools[0]).not.toHaveProperty('name') + expect(first?.anthropicBeta ?? '').not.toContain('computer-use') + }) + + test('runs each toolset call as one computer call', async ({ request }) => { + const { calls } = await runLoop(request) + expect(calls).toEqual([ + { action: 'left_click', coordinate: [500, 300] }, + { action: 'screenshot' }, + ]) + }) + + test('sends the calls back as toolset calls, and the results name the toolset', async ({ + request, + }) => { + const { requests } = await runLoop(request) + const second = requests[1] + expect(second).toBeDefined() + const assistant = second?.body.messages.find((m) => m.role === 'assistant') + expect(assistant?.content).toMatchObject([ + { + type: 'tool_use', + id: 'toolu_click', + name: 'left_click', + toolset_name: 'computer', + input: { coordinate: [500, 300] }, + }, + { + type: 'tool_use', + id: 'toolu_shot', + name: 'screenshot', + toolset_name: 'computer', + input: {}, + }, + ]) + const results = second?.body.messages + .flatMap((m) => m.content) + .filter((block) => block['type'] === 'tool_result') + expect(results).toMatchObject([ + { tool_use_id: 'toolu_click', toolset_name: 'computer', content: 'OK' }, + { + tool_use_id: 'toolu_shot', + toolset_name: 'computer', + content: [{ type: 'image', source: { type: 'base64', data: 'AAAA' } }], + }, + ]) + }) +}) From 854320e01a956d25e671c03b296716912758bf5d Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 19 Oct 2026 18:54:56 +0200 Subject: [PATCH 07/32] feat(ai-anthropic): export the computer use config and tool types --- packages/ai-anthropic/src/tools/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/ai-anthropic/src/tools/index.ts b/packages/ai-anthropic/src/tools/index.ts index 9ad77c0431..bc6d3d9354 100644 --- a/packages/ai-anthropic/src/tools/index.ts +++ b/packages/ai-anthropic/src/tools/index.ts @@ -26,7 +26,11 @@ export { } from './code-execution-tool' export { computerUseTool, + type AnthropicComputerToolset, + type AnthropicComputerUse20251124Tool, type AnthropicComputerUseTool, + type ComputerToolsetConfig, + type ComputerUseSingleToolConfig, type ComputerUseToolConfig, type ComputerUseTool, } from './computer-use-tool' From becc71f594a783a66c589666b5586edd5ee27164 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 19 Oct 2026 19:05:44 +0200 Subject: [PATCH 08/32] feat(openai-base): send computer screenshots with the original detail OpenAI's computer use guide sends each screenshot with detail: 'original', so the model sees the screen at full size. The SDK type has no detail field yet, so the adapter adds it to its own screenshot type. --- .changeset/openai-computer-tool.md | 2 +- packages/openai-base/src/adapters/responses-text.ts | 9 +++++++-- packages/openai-base/tests/computer-use-loop.test.ts | 2 ++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.changeset/openai-computer-tool.md b/.changeset/openai-computer-tool.md index a2c1d63e9c..40fbc8cab4 100644 --- a/.changeset/openai-computer-tool.md +++ b/.changeset/openai-computer-tool.md @@ -3,4 +3,4 @@ '@tanstack/ai-openai': minor --- -`computerUseTool({ type: 'computer' })` adds the computer tool of the newer OpenAI models, for example `gpt-6.1-sol`. The tool name is `computer`. Each call has the batched `actions` of the model and the `pending_safety_checks`, and the replay sends the actions back as they came. `computerUseTool({ type: 'computer_use_preview', ... })` for the `computer-use-preview` model works as before. The Responses tool choice type also accepts `computer`. +`computerUseTool({ type: 'computer' })` adds the computer tool of the newer OpenAI models, for example `gpt-6.1-sol`. The tool name is `computer`. Each call has the batched `actions` of the model and the `pending_safety_checks`, and the replay sends the actions back as they came. `computerUseTool({ type: 'computer_use_preview', ... })` for the `computer-use-preview` model works as before. The Responses tool choice type also accepts `computer`. Computer screenshots now go out with `detail: 'original'`, as in OpenAI's computer use guide. diff --git a/packages/openai-base/src/adapters/responses-text.ts b/packages/openai-base/src/adapters/responses-text.ts index 4e8d55794d..49f14ed0c1 100644 --- a/packages/openai-base/src/adapters/responses-text.ts +++ b/packages/openai-base/src/adapters/responses-text.ts @@ -3164,10 +3164,14 @@ export abstract class OpenAIBaseResponsesTextAdapter< return result } - /** The first image of a tool result, as a computer screenshot. */ + /** + * The first image of a tool result, as a computer screenshot. It goes out + * with `detail: 'original'`, as in OpenAI's computer use guide, so the + * model sees the screen at full size. The SDK type has no `detail` yet. + */ private screenshotOf( content: ModelMessage['content'], - ): ResponseComputerToolCallOutputScreenshot { + ): ResponseComputerToolCallOutputScreenshot & { detail?: 'original' } { const image = Array.isArray(content) ? content.find((part) => part.type === 'image') : undefined @@ -3177,6 +3181,7 @@ export abstract class OpenAIBaseResponsesTextAdapter< type: 'computer_screenshot', ...(input.image_url ? { image_url: input.image_url } : {}), ...(input.file_id ? { file_id: input.file_id } : {}), + detail: 'original', } } diff --git a/packages/openai-base/tests/computer-use-loop.test.ts b/packages/openai-base/tests/computer-use-loop.test.ts index e8950ea1ac..0df6b47bfb 100644 --- a/packages/openai-base/tests/computer-use-loop.test.ts +++ b/packages/openai-base/tests/computer-use-loop.test.ts @@ -126,6 +126,7 @@ describe('computer use loop', () => { output: { type: 'computer_screenshot', image_url: 'data:image/png;base64,iVBORw0KGgo=', + detail: 'original', }, acknowledged_safety_checks: [safetyCheck], }, @@ -172,6 +173,7 @@ describe('computer use loop', () => { output: { type: 'computer_screenshot', image_url: 'data:image/png;base64,iVBORw0KGgo=', + detail: 'original', }, }, ]), From 0eec128a8e2aa7a753f61256499a37903ffceca5 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 19 Oct 2026 19:35:12 +0200 Subject: [PATCH 09/32] fix(ai-anthropic): keep the vertex genai lock entry and expect one files beta header The SDK install moved the @google/genai of ai-vertex to a copy with zod 4.6.5, so its test mock missed the copy that ai-gemini loads. SDK 0.135.0 sends the files-api beta header once, where 0.97.1 sent it twice. --- packages/ai-anthropic/tests/auth-parity.test.ts | 2 +- pnpm-lock.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ai-anthropic/tests/auth-parity.test.ts b/packages/ai-anthropic/tests/auth-parity.test.ts index da1f951822..25af71ff17 100644 --- a/packages/ai-anthropic/tests/auth-parity.test.ts +++ b/packages/ai-anthropic/tests/auth-parity.test.ts @@ -473,7 +473,7 @@ describe('files factories', () => { authorization: null, xApiKey: 'files-key', xApp: null, - anthropicBeta: 'files-api-2025-04-14,files-api-2025-04-14', + anthropicBeta: 'files-api-2025-04-14', }) expect(file).toEqual(FILE) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7f90ae25c0..1e072c4214 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3453,7 +3453,7 @@ importers: devDependencies: '@google/genai': specifier: ^2.10.0 - version: 2.10.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.6.5)) + version: 2.10.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6)) '@tanstack/ai': specifier: workspace:* version: link:../ai From d43c6ecb62cffa916890bc6a0c9cb768d0f5989a Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 19 Oct 2026 19:46:02 +0200 Subject: [PATCH 10/32] fix(openai-base): read a screenshot only for a computer result The result of a shell, local_shell, or apply_patch call no longer goes through the screenshot read. A file from another provider in such a result threw there, but the result never sends an image. --- .../src/adapters/responses-text.ts | 2 +- .../src/adapters/responses-user-tools.ts | 9 ++-- .../tests/computer-use-loop.test.ts | 50 +++++++++++++++++++ 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/packages/openai-base/src/adapters/responses-text.ts b/packages/openai-base/src/adapters/responses-text.ts index 1628d75202..fbaf2720e0 100644 --- a/packages/openai-base/src/adapters/responses-text.ts +++ b/packages/openai-base/src/adapters/responses-text.ts @@ -2918,7 +2918,7 @@ export abstract class OpenAIBaseResponsesTextAdapter< : undefined const userOutput = owner ? userToolResultItem(owner, message.content, { - screenshot: this.screenshotOf(message.content), + screenshot: () => this.screenshotOf(message.content), failed: message.error !== undefined, }) : null diff --git a/packages/openai-base/src/adapters/responses-user-tools.ts b/packages/openai-base/src/adapters/responses-user-tools.ts index 01a02b2a8e..94e8294e97 100644 --- a/packages/openai-base/src/adapters/responses-user-tools.ts +++ b/packages/openai-base/src/adapters/responses-user-tools.ts @@ -366,14 +366,15 @@ function shellOutputList(content: unknown): Array<{ /** * Replay the app's tool result as the matching Responses output item. * `computer` is what a `computer_use_preview` result needs: the screenshot, - * and whether the call failed. The output of a call that did not fail - * acknowledges the safety checks of the call. + * and whether the call failed. Only a computer result reads the screenshot. + * The output of a call that did not fail acknowledges the safety checks of + * the call. */ export function userToolResultItem( toolCall: ToolCallLike, content: unknown, computer?: { - screenshot: ResponseComputerToolCallOutputScreenshot + screenshot: () => ResponseComputerToolCallOutputScreenshot failed: boolean }, ): ResponseInputItem | null { @@ -389,7 +390,7 @@ export function userToolResultItem( return { type: 'computer_call_output', call_id: toolCall.id, - output: computer?.screenshot ?? { type: 'computer_screenshot' }, + output: computer?.screenshot() ?? { type: 'computer_screenshot' }, ...(checks.length > 0 ? { acknowledged_safety_checks: checks } : {}), } } diff --git a/packages/openai-base/tests/computer-use-loop.test.ts b/packages/openai-base/tests/computer-use-loop.test.ts index f15e0674c6..b2566dab58 100644 --- a/packages/openai-base/tests/computer-use-loop.test.ts +++ b/packages/openai-base/tests/computer-use-loop.test.ts @@ -53,6 +53,10 @@ const answer = { class TestAdapter extends OpenAIBaseResponsesTextAdapter {} +class FileAdapter extends TestAdapter { + override readonly supportsFileSources = true +} + async function runLoop(execute: (input: unknown) => Promise) { const create = vi .fn() @@ -120,6 +124,52 @@ describe('computer use loop', () => { ) }) + it('reads no screenshot from the result of another app tool', async () => { + const create = vi.fn().mockResolvedValueOnce(completed([answer])) + const client = { + responses: { create: (params: unknown) => create(params) }, + } as unknown as OpenAI + // OpenAI takes file sources, but not a file from another provider, so a + // screenshot read of this result would throw. + const adapter = new FileAdapter('gpt-5.5', 'openai', client) + for await (const _ of chat({ + adapter, + messages: [ + { + role: 'assistant', + content: null, + toolCalls: [ + { + id: 'call_sh', + type: 'function', + function: { + name: 'local_shell', + arguments: JSON.stringify({ command: ['ls'] }), + }, + metadata: { openaiUserTool: 'local_shell', itemId: 'ls_1' }, + }, + ], + }, + { + role: 'tool', + toolCallId: 'call_sh', + content: [ + { + type: 'image', + source: { type: 'file', value: 'file_1', provider: 'gemini' }, + }, + ], + }, + ], + })) { + // Drain the stream. + } + + expect(create.mock.calls[0]?.[0].input).toContainEqual( + expect.objectContaining({ type: 'local_shell_call_output' }), + ) + }) + it('does not acknowledge the safety checks of a failed call', async () => { const { create } = await runLoop(async () => { throw new Error('A person did not allow the click.') From 98cb24ccd0b15ec2259398097890d05eb7c5bda2 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Tue, 20 Oct 2026 08:22:45 +0200 Subject: [PATCH 11/32] test(e2e): check the OpenAI computer use loop on the wire aimock cannot send a computer_call, so the route answers with a custom fetch. The spec checks the tool input, and that the next request sends the computer_call and its computer_call_output with the screenshot and the acknowledged safety check. --- testing/e2e/src/routeTree.gen.ts | 22 +++ .../routes/api.openai-computer-use-wire.ts | 135 ++++++++++++++++++ .../tests/openai-computer-use-wire.spec.ts | 59 ++++++++ 3 files changed, 216 insertions(+) create mode 100644 testing/e2e/src/routes/api.openai-computer-use-wire.ts create mode 100644 testing/e2e/tests/openai-computer-use-wire.spec.ts diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index 98d5716a87..c28f175ac5 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -130,6 +130,7 @@ import { Route as ApiMoonshotUsageDetailsRouteImport } from './routes/api.moonsh import { Route as ApiMultimodalToolResultWireRouteImport } from './routes/api.multimodal-tool-result-wire' import { Route as ApiNonStreamingRunErrorRouteImport } from './routes/api.non-streaming-run-error' import { Route as ApiOpenaiCompletedResponseTextRouteImport } from './routes/api.openai-completed-response-text' +import { Route as ApiOpenaiComputerUseWireRouteImport } from './routes/api.openai-computer-use-wire' import { Route as ApiOpenaiImage25ModelsRouteImport } from './routes/api.openai-image-2-5-models' import { Route as ApiOpenaiMalformedToolArgumentsRouteImport } from './routes/api.openai-malformed-tool-arguments' import { Route as ApiOpenaiResponsesUsageRouteImport } from './routes/api.openai-responses-usage' @@ -815,6 +816,12 @@ const ApiOpenaiCompletedResponseTextRoute = path: '/api/openai-completed-response-text', getParentRoute: () => rootRouteImport, } as any) +const ApiOpenaiComputerUseWireRoute = + ApiOpenaiComputerUseWireRouteImport.update({ + id: '/api/openai-computer-use-wire', + path: '/api/openai-computer-use-wire', + getParentRoute: () => rootRouteImport, + } as any) const ApiOpenaiImage25ModelsRoute = ApiOpenaiImage25ModelsRouteImport.update({ id: '/api/openai-image-2-5-models', path: '/api/openai-image-2-5-models', @@ -1219,6 +1226,7 @@ export interface FileRoutesByFullPath { '/api/multimodal-tool-result-wire': typeof ApiMultimodalToolResultWireRoute '/api/non-streaming-run-error': typeof ApiNonStreamingRunErrorRoute '/api/openai-completed-response-text': typeof ApiOpenaiCompletedResponseTextRoute + '/api/openai-computer-use-wire': typeof ApiOpenaiComputerUseWireRoute '/api/openai-image-2-5-models': typeof ApiOpenaiImage25ModelsRoute '/api/openai-malformed-tool-arguments': typeof ApiOpenaiMalformedToolArgumentsRoute '/api/openai-responses-usage': typeof ApiOpenaiResponsesUsageRoute @@ -1395,6 +1403,7 @@ export interface FileRoutesByTo { '/api/multimodal-tool-result-wire': typeof ApiMultimodalToolResultWireRoute '/api/non-streaming-run-error': typeof ApiNonStreamingRunErrorRoute '/api/openai-completed-response-text': typeof ApiOpenaiCompletedResponseTextRoute + '/api/openai-computer-use-wire': typeof ApiOpenaiComputerUseWireRoute '/api/openai-image-2-5-models': typeof ApiOpenaiImage25ModelsRoute '/api/openai-malformed-tool-arguments': typeof ApiOpenaiMalformedToolArgumentsRoute '/api/openai-responses-usage': typeof ApiOpenaiResponsesUsageRoute @@ -1572,6 +1581,7 @@ export interface FileRoutesById { '/api/multimodal-tool-result-wire': typeof ApiMultimodalToolResultWireRoute '/api/non-streaming-run-error': typeof ApiNonStreamingRunErrorRoute '/api/openai-completed-response-text': typeof ApiOpenaiCompletedResponseTextRoute + '/api/openai-computer-use-wire': typeof ApiOpenaiComputerUseWireRoute '/api/openai-image-2-5-models': typeof ApiOpenaiImage25ModelsRoute '/api/openai-malformed-tool-arguments': typeof ApiOpenaiMalformedToolArgumentsRoute '/api/openai-responses-usage': typeof ApiOpenaiResponsesUsageRoute @@ -1750,6 +1760,7 @@ export interface FileRouteTypes { | '/api/multimodal-tool-result-wire' | '/api/non-streaming-run-error' | '/api/openai-completed-response-text' + | '/api/openai-computer-use-wire' | '/api/openai-image-2-5-models' | '/api/openai-malformed-tool-arguments' | '/api/openai-responses-usage' @@ -1926,6 +1937,7 @@ export interface FileRouteTypes { | '/api/multimodal-tool-result-wire' | '/api/non-streaming-run-error' | '/api/openai-completed-response-text' + | '/api/openai-computer-use-wire' | '/api/openai-image-2-5-models' | '/api/openai-malformed-tool-arguments' | '/api/openai-responses-usage' @@ -2102,6 +2114,7 @@ export interface FileRouteTypes { | '/api/multimodal-tool-result-wire' | '/api/non-streaming-run-error' | '/api/openai-completed-response-text' + | '/api/openai-computer-use-wire' | '/api/openai-image-2-5-models' | '/api/openai-malformed-tool-arguments' | '/api/openai-responses-usage' @@ -2279,6 +2292,7 @@ export interface RootRouteChildren { ApiMultimodalToolResultWireRoute: typeof ApiMultimodalToolResultWireRoute ApiNonStreamingRunErrorRoute: typeof ApiNonStreamingRunErrorRoute ApiOpenaiCompletedResponseTextRoute: typeof ApiOpenaiCompletedResponseTextRoute + ApiOpenaiComputerUseWireRoute: typeof ApiOpenaiComputerUseWireRoute ApiOpenaiImage25ModelsRoute: typeof ApiOpenaiImage25ModelsRoute ApiOpenaiMalformedToolArgumentsRoute: typeof ApiOpenaiMalformedToolArgumentsRoute ApiOpenaiResponsesUsageRoute: typeof ApiOpenaiResponsesUsageRoute @@ -3179,6 +3193,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiOpenaiCompletedResponseTextRouteImport parentRoute: typeof rootRouteImport } + '/api/openai-computer-use-wire': { + id: '/api/openai-computer-use-wire' + path: '/api/openai-computer-use-wire' + fullPath: '/api/openai-computer-use-wire' + preLoaderRoute: typeof ApiOpenaiComputerUseWireRouteImport + parentRoute: typeof rootRouteImport + } '/api/openai-image-2-5-models': { id: '/api/openai-image-2-5-models' path: '/api/openai-image-2-5-models' @@ -3737,6 +3758,7 @@ const rootRouteChildren: RootRouteChildren = { ApiMultimodalToolResultWireRoute: ApiMultimodalToolResultWireRoute, ApiNonStreamingRunErrorRoute: ApiNonStreamingRunErrorRoute, ApiOpenaiCompletedResponseTextRoute: ApiOpenaiCompletedResponseTextRoute, + ApiOpenaiComputerUseWireRoute: ApiOpenaiComputerUseWireRoute, ApiOpenaiImage25ModelsRoute: ApiOpenaiImage25ModelsRoute, ApiOpenaiMalformedToolArgumentsRoute: ApiOpenaiMalformedToolArgumentsRoute, ApiOpenaiResponsesUsageRoute: ApiOpenaiResponsesUsageRoute, diff --git a/testing/e2e/src/routes/api.openai-computer-use-wire.ts b/testing/e2e/src/routes/api.openai-computer-use-wire.ts new file mode 100644 index 0000000000..8e4df804a1 --- /dev/null +++ b/testing/e2e/src/routes/api.openai-computer-use-wire.ts @@ -0,0 +1,135 @@ +import { createFileRoute } from '@tanstack/react-router' +import { chat, createChatOptions } from '@tanstack/ai' +import { createOpenaiChat } from '@tanstack/ai-openai' +import { computerUseTool } from '@tanstack/ai-openai/tools' + +const DUMMY_KEY = 'sk-e2e-test-dummy-key' +const SCREENSHOT = 'iVBORw0KGgo=' + +const safetyCheck = { + id: 'sc_e2e', + code: 'malicious_instructions', + message: 'The page asks the model to do something else.', +} + +const computerCall = { + type: 'computer_call', + id: 'cu_e2e', + call_id: 'call_cu_e2e', + status: 'completed', + action: { type: 'click', button: 'left', x: 10, y: 20 }, + pending_safety_checks: [safetyCheck], +} + +const answer = { + type: 'message', + id: 'msg_cu_e2e', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: 'Done.', annotations: [] }], +} + +function responseStream( + id: string, + output: Array, +): ReadableStream { + const encoder = new TextEncoder() + const events = [ + { + type: 'response.created', + response: { id, object: 'response', status: 'in_progress', output: [] }, + }, + { + type: 'response.completed', + response: { + id, + object: 'response', + status: 'completed', + output, + usage: { input_tokens: 5, output_tokens: 2, total_tokens: 7 }, + }, + }, + ] + return new ReadableStream({ + start(controller) { + for (const event of events) { + controller.enqueue(encoder.encode(`data: ${JSON.stringify(event)}\n\n`)) + } + controller.enqueue(encoder.encode('data: [DONE]\n\n')) + controller.close() + }, + }) +} + +/** + * Drives the OpenAI Responses adapter through one computer use step. A + * custom `fetch` answers the first request with a `computer_call` and the + * second with text, and records each request body. aimock cannot send a + * `computer_call`, so this route serves the stream itself. + */ +export const Route = createFileRoute('/api/openai-computer-use-wire')({ + server: { + handlers: { + POST: async () => { + const requestBodies: Array = [] + const mockFetch: typeof fetch = async (input, init) => { + const req = + input instanceof Request ? input : new Request(input, init) + requestBodies.push(JSON.parse(await req.text())) + const stream = + requestBodies.length === 1 + ? responseStream('resp_cu_e2e_1', [computerCall]) + : responseStream('resp_cu_e2e_2', [answer]) + return new Response(stream, { + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + }) + } + + const adapter = createOpenaiChat('computer-use-preview', DUMMY_KEY, { + fetch: mockFetch, + }) + + const toolInputs: Array = [] + try { + for await (const _ of chat({ + ...createChatOptions({ adapter }), + messages: [{ role: 'user', content: 'Click the button.' }], + tools: [ + { + ...computerUseTool({ + type: 'computer_use_preview', + display_width: 1024, + display_height: 768, + environment: 'browser', + }), + execute: async (input: unknown) => { + toolInputs.push(input) + return [ + { + type: 'image' as const, + source: { + type: 'data' as const, + value: SCREENSHOT, + mimeType: 'image/png', + }, + }, + ] + }, + }, + ], + })) { + // Drain the stream. + } + } catch (error) { + return Response.json({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }) + } + + return Response.json({ ok: true, toolInputs, requestBodies }) + }, + }, + }, +}) diff --git a/testing/e2e/tests/openai-computer-use-wire.spec.ts b/testing/e2e/tests/openai-computer-use-wire.spec.ts new file mode 100644 index 0000000000..d0db59ee87 --- /dev/null +++ b/testing/e2e/tests/openai-computer-use-wire.spec.ts @@ -0,0 +1,59 @@ +import { test, expect } from './fixtures' + +/** + * Wire format of OpenAI computer use. The model sends a `computer_call`, the + * app runs it, and the next request sends the call back with its + * `computer_call_output`. + * + * `/api/openai-computer-use-wire` serves the model responses with a custom + * `fetch` and returns the tool inputs and the request bodies as JSON. + */ +test.describe('openai — computer use wire format', () => { + test('runs the computer call and sends the screenshot back', async ({ + request, + }) => { + const res = await request.post('/api/openai-computer-use-wire') + expect(res.ok()).toBe(true) + const { ok, error, toolInputs, requestBodies } = (await res.json()) as { + ok: boolean + error?: string + toolInputs: Array + requestBodies: Array<{ input: Array }> + } + + if (!ok) { + throw new Error(`Route failed: ${error}`) + } + + const safetyCheck = { + id: 'sc_e2e', + code: 'malicious_instructions', + message: 'The page asks the model to do something else.', + } + expect(toolInputs).toEqual([ + { + action: { type: 'click', button: 'left', x: 10, y: 20 }, + pending_safety_checks: [safetyCheck], + }, + ]) + expect(requestBodies).toHaveLength(2) + expect(requestBodies[1]!.input).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'computer_call', + call_id: 'call_cu_e2e', + action: { type: 'click', button: 'left', x: 10, y: 20 }, + }), + { + type: 'computer_call_output', + call_id: 'call_cu_e2e', + output: { + type: 'computer_screenshot', + image_url: 'data:image/png;base64,iVBORw0KGgo=', + }, + acknowledged_safety_checks: [safetyCheck], + }, + ]), + ) + }) +}) From 7d482c80e8c4e664446b8b5df8a1fea2969e2f94 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Tue, 20 Oct 2026 08:24:03 +0200 Subject: [PATCH 12/32] test(e2e): check the computer tool batch and the screenshot detail on the wire The route takes ?tool=computer for the computer tool of gpt-6.1-sol. The spec checks the batched actions, the computer tool entry, and detail: 'original' on each screenshot. --- .../routes/api.openai-computer-use-wire.ts | 97 +++++++++++++------ .../tests/openai-computer-use-wire.spec.ts | 44 +++++++++ 2 files changed, 109 insertions(+), 32 deletions(-) diff --git a/testing/e2e/src/routes/api.openai-computer-use-wire.ts b/testing/e2e/src/routes/api.openai-computer-use-wire.ts index 8e4df804a1..8461857f5f 100644 --- a/testing/e2e/src/routes/api.openai-computer-use-wire.ts +++ b/testing/e2e/src/routes/api.openai-computer-use-wire.ts @@ -21,6 +21,26 @@ const computerCall = { pending_safety_checks: [safetyCheck], } +// The computer tool of the newer models sends a batch of actions. +const batchCall = { + type: 'computer_call', + id: 'cu_e2e_batch', + call_id: 'call_cu_e2e_batch', + status: 'completed', + actions: [ + { type: 'click', button: 'left', x: 10, y: 20 }, + { type: 'type', text: 'hello' }, + ], + pending_safety_checks: [], +} + +const screenshot = [ + { + type: 'image' as const, + source: { type: 'data' as const, value: SCREENSHOT, mimeType: 'image/png' }, + }, +] + const answer = { type: 'message', id: 'msg_cu_e2e', @@ -66,11 +86,16 @@ function responseStream( * custom `fetch` answers the first request with a `computer_call` and the * second with text, and records each request body. aimock cannot send a * `computer_call`, so this route serves the stream itself. + * + * `?tool=computer` uses the `computer` tool of `gpt-6.1-sol`, with a batch + * of actions. Anything else uses `computer_use_preview`. */ export const Route = createFileRoute('/api/openai-computer-use-wire')({ server: { handlers: { - POST: async () => { + POST: async ({ request }) => { + const batch = + new URL(request.url).searchParams.get('tool') === 'computer' const requestBodies: Array = [] const mockFetch: typeof fetch = async (input, init) => { const req = @@ -78,7 +103,9 @@ export const Route = createFileRoute('/api/openai-computer-use-wire')({ requestBodies.push(JSON.parse(await req.text())) const stream = requestBodies.length === 1 - ? responseStream('resp_cu_e2e_1', [computerCall]) + ? responseStream('resp_cu_e2e_1', [ + batch ? batchCall : computerCall, + ]) : responseStream('resp_cu_e2e_2', [answer]) return new Response(stream, { status: 200, @@ -86,39 +113,45 @@ export const Route = createFileRoute('/api/openai-computer-use-wire')({ }) } - const adapter = createOpenaiChat('computer-use-preview', DUMMY_KEY, { - fetch: mockFetch, - }) - const toolInputs: Array = [] + const execute = async (input: unknown) => { + toolInputs.push(input) + return screenshot + } + const messages = [ + { role: 'user' as const, content: 'Click the button.' }, + ] try { - for await (const _ of chat({ - ...createChatOptions({ adapter }), - messages: [{ role: 'user', content: 'Click the button.' }], - tools: [ - { - ...computerUseTool({ - type: 'computer_use_preview', - display_width: 1024, - display_height: 768, - environment: 'browser', + const stream = batch + ? chat({ + ...createChatOptions({ + adapter: createOpenaiChat('gpt-6.1-sol', DUMMY_KEY, { + fetch: mockFetch, + }), + }), + messages, + tools: [{ ...computerUseTool({ type: 'computer' }), execute }], + }) + : chat({ + ...createChatOptions({ + adapter: createOpenaiChat('computer-use-preview', DUMMY_KEY, { + fetch: mockFetch, + }), }), - execute: async (input: unknown) => { - toolInputs.push(input) - return [ - { - type: 'image' as const, - source: { - type: 'data' as const, - value: SCREENSHOT, - mimeType: 'image/png', - }, - }, - ] - }, - }, - ], - })) { + messages, + tools: [ + { + ...computerUseTool({ + type: 'computer_use_preview', + display_width: 1024, + display_height: 768, + environment: 'browser', + }), + execute, + }, + ], + }) + for await (const _ of stream) { // Drain the stream. } } catch (error) { diff --git a/testing/e2e/tests/openai-computer-use-wire.spec.ts b/testing/e2e/tests/openai-computer-use-wire.spec.ts index d0db59ee87..b700e7023f 100644 --- a/testing/e2e/tests/openai-computer-use-wire.spec.ts +++ b/testing/e2e/tests/openai-computer-use-wire.spec.ts @@ -50,10 +50,54 @@ test.describe('openai — computer use wire format', () => { output: { type: 'computer_screenshot', image_url: 'data:image/png;base64,iVBORw0KGgo=', + detail: 'original', }, acknowledged_safety_checks: [safetyCheck], }, ]), ) }) + + test('runs the batched actions of the computer tool', async ({ request }) => { + const res = await request.post( + '/api/openai-computer-use-wire?tool=computer', + ) + expect(res.ok()).toBe(true) + const { ok, error, toolInputs, requestBodies } = (await res.json()) as { + ok: boolean + error?: string + toolInputs: Array + requestBodies: Array<{ tools?: Array; input: Array }> + } + + if (!ok) { + throw new Error(`Route failed: ${error}`) + } + + const actions = [ + { type: 'click', button: 'left', x: 10, y: 20 }, + { type: 'type', text: 'hello' }, + ] + expect(toolInputs).toEqual([{ actions, pending_safety_checks: [] }]) + expect(requestBodies).toHaveLength(2) + expect(requestBodies[0]!.tools).toEqual([{ type: 'computer' }]) + expect(requestBodies[1]!.input).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'computer_call', + call_id: 'call_cu_e2e_batch', + actions, + }), + { + type: 'computer_call_output', + call_id: 'call_cu_e2e_batch', + output: { + type: 'computer_screenshot', + image_url: 'data:image/png;base64,iVBORw0KGgo=', + detail: 'original', + }, + }, + ]), + ) + }) }) From 04cf74d9cf6cad0b925822fe098674c513be823b Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Tue, 20 Oct 2026 10:21:48 +0200 Subject: [PATCH 13/32] docs(ai-openai): drop the as const in the computer use sample --- docs/adapters/openai.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/adapters/openai.md b/docs/adapters/openai.md index 6646c49201..1c48a41df5 100644 --- a/docs/adapters/openai.md +++ b/docs/adapters/openai.md @@ -1036,8 +1036,8 @@ const stream = chat({ const screenshot = await runAction(input.action); return [ { - type: "image" as const, - source: { type: "data" as const, value: screenshot, mimeType: "image/png" }, + type: "image", + source: { type: "data", value: screenshot, mimeType: "image/png" }, }, ]; }, From 7fac2ac4ca5e884868448bb0c57e192f1f9e845d Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Tue, 20 Oct 2026 10:21:57 +0200 Subject: [PATCH 14/32] docs(ai-anthropic): drop the as const in the computer use sample --- docs/adapters/anthropic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/adapters/anthropic.md b/docs/adapters/anthropic.md index 579f998c51..f603d0f027 100644 --- a/docs/adapters/anthropic.md +++ b/docs/adapters/anthropic.md @@ -655,8 +655,8 @@ const computer = { const value = await takeScreenshot(); return [ { - type: "image" as const, - source: { type: "data" as const, value, mimeType: "image/png" }, + type: "image", + source: { type: "data", value, mimeType: "image/png" }, }, ]; } From ce9b4ab21cbb073e64a6bd6d57291e147d379b30 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Tue, 20 Oct 2026 13:48:39 +0200 Subject: [PATCH 15/32] feat(ai-computer-use): add computer use with Playwright and E2B drivers createComputerUse({ driver }) returns a tool, a middleware, and a safety-check interrupt for chat(). The same code runs on OpenAI and on Anthropic. Each adapter swaps the generic tool for its native computer tool and picks the newest version the model supports. - @tanstack/ai-computer-use: the tool, the action mapping, the key names, the safety-check interrupt, and the failed-turn rule. - @tanstack/ai-computer-playwright: a driver for a Playwright page. - @tanstack/ai-computer-e2b: a driver for an E2B Desktop sandbox. - @tanstack/ai: the shared tool metadata in adapter-internals. - ai-anthropic and openai-base: resolve the generic tool per model. - ai-openai, ai-anthropic, model-sync: the new 'computer' tool kind. - Docs: a new Computer Use section. --- .agent/domain-map.json | 197 ++++++++ .changeset/ai-computer-use.md | 31 ++ docs/adapters/anthropic.md | 2 + docs/adapters/openai.md | 2 + docs/computer-use/computer-use.md | 188 ++++++++ docs/computer-use/custom-driver.md | 120 +++++ docs/computer-use/drivers.md | 124 +++++ docs/computer-use/safety-checks.md | 121 +++++ docs/config.json | 32 +- docs/tools/provider-tools.md | 25 + packages/ai-anthropic/src/adapters/text.ts | 20 +- packages/ai-anthropic/src/model-meta.ts | 45 +- .../src/tools/computer-use-tool.ts | 66 +++ .../tests/computer-use-generic.test.ts | 238 ++++++++++ packages/ai-computer-e2b/README.md | 73 +++ packages/ai-computer-e2b/package.json | 55 +++ packages/ai-computer-e2b/src/driver.ts | 152 ++++++ packages/ai-computer-e2b/src/index.ts | 5 + packages/ai-computer-e2b/tests/driver.test.ts | 145 ++++++ packages/ai-computer-e2b/tests/types.test.ts | 9 + packages/ai-computer-e2b/tsconfig.json | 8 + packages/ai-computer-e2b/vite.config.ts | 36 ++ packages/ai-computer-playwright/README.md | 72 +++ packages/ai-computer-playwright/package.json | 55 +++ packages/ai-computer-playwright/src/driver.ts | 93 ++++ packages/ai-computer-playwright/src/index.ts | 5 + .../tests/driver.test.ts | 132 ++++++ .../tests/types.test.ts | 9 + packages/ai-computer-playwright/tsconfig.json | 8 + .../ai-computer-playwright/vite.config.ts | 36 ++ packages/ai-computer-use/README.md | 91 ++++ packages/ai-computer-use/package.json | 58 +++ packages/ai-computer-use/src/actions.ts | 372 +++++++++++++++ .../src/create-computer-use.ts | 219 +++++++++ packages/ai-computer-use/src/index.ts | 4 + packages/ai-computer-use/src/keys.ts | 84 ++++ packages/ai-computer-use/src/safety-check.ts | 29 ++ packages/ai-computer-use/src/types.ts | 53 +++ .../ai-computer-use/tests/actions.test.ts | 413 ++++++++++++++++ .../tests/create-computer-use.test.ts | 429 +++++++++++++++++ packages/ai-computer-use/tests/keys.test.ts | 63 +++ .../tests/safety-check.test.ts | 65 +++ packages/ai-computer-use/tsconfig.json | 8 + packages/ai-computer-use/vite.config.ts | 36 ++ packages/ai-openai/src/model-meta.ts | 34 +- .../tests/computer-tool-kind.test.ts | 46 ++ packages/ai/src/adapter-internals.ts | 2 + packages/ai/src/tools/computer-use.ts | 59 +++ .../ai/tests/computer-use-metadata.test.ts | 53 +++ .../src/adapters/chat-completions-text.ts | 9 + .../src/adapters/responses-text.ts | 13 +- .../src/tools/computer-use-tool.ts | 29 ++ .../tests/computer-use-generic.test.ts | 186 ++++++++ pnpm-lock.yaml | 148 ++++-- scripts/model-sync/provider-supports.test.ts | 4 +- scripts/model-sync/provider-supports.ts | 5 + testing/e2e/package.json | 2 + testing/e2e/tests/computer-use.spec.ts | 445 ++++++++++++++++++ 58 files changed, 5027 insertions(+), 36 deletions(-) create mode 100644 .agent/domain-map.json create mode 100644 .changeset/ai-computer-use.md create mode 100644 docs/computer-use/computer-use.md create mode 100644 docs/computer-use/custom-driver.md create mode 100644 docs/computer-use/drivers.md create mode 100644 docs/computer-use/safety-checks.md create mode 100644 packages/ai-anthropic/tests/computer-use-generic.test.ts create mode 100644 packages/ai-computer-e2b/README.md create mode 100644 packages/ai-computer-e2b/package.json create mode 100644 packages/ai-computer-e2b/src/driver.ts create mode 100644 packages/ai-computer-e2b/src/index.ts create mode 100644 packages/ai-computer-e2b/tests/driver.test.ts create mode 100644 packages/ai-computer-e2b/tests/types.test.ts create mode 100644 packages/ai-computer-e2b/tsconfig.json create mode 100644 packages/ai-computer-e2b/vite.config.ts create mode 100644 packages/ai-computer-playwright/README.md create mode 100644 packages/ai-computer-playwright/package.json create mode 100644 packages/ai-computer-playwright/src/driver.ts create mode 100644 packages/ai-computer-playwright/src/index.ts create mode 100644 packages/ai-computer-playwright/tests/driver.test.ts create mode 100644 packages/ai-computer-playwright/tests/types.test.ts create mode 100644 packages/ai-computer-playwright/tsconfig.json create mode 100644 packages/ai-computer-playwright/vite.config.ts create mode 100644 packages/ai-computer-use/README.md create mode 100644 packages/ai-computer-use/package.json create mode 100644 packages/ai-computer-use/src/actions.ts create mode 100644 packages/ai-computer-use/src/create-computer-use.ts create mode 100644 packages/ai-computer-use/src/index.ts create mode 100644 packages/ai-computer-use/src/keys.ts create mode 100644 packages/ai-computer-use/src/safety-check.ts create mode 100644 packages/ai-computer-use/src/types.ts create mode 100644 packages/ai-computer-use/tests/actions.test.ts create mode 100644 packages/ai-computer-use/tests/create-computer-use.test.ts create mode 100644 packages/ai-computer-use/tests/keys.test.ts create mode 100644 packages/ai-computer-use/tests/safety-check.test.ts create mode 100644 packages/ai-computer-use/tsconfig.json create mode 100644 packages/ai-computer-use/vite.config.ts create mode 100644 packages/ai-openai/tests/computer-tool-kind.test.ts create mode 100644 packages/ai/src/tools/computer-use.ts create mode 100644 packages/ai/tests/computer-use-metadata.test.ts create mode 100644 packages/openai-base/tests/computer-use-generic.test.ts create mode 100644 testing/e2e/tests/computer-use.spec.ts diff --git a/.agent/domain-map.json b/.agent/domain-map.json new file mode 100644 index 0000000000..c64a3c996c --- /dev/null +++ b/.agent/domain-map.json @@ -0,0 +1,197 @@ +{ + "version": 1, + "domains": [ + { + "id": "core", + "owns": [ + "packages/ai", + "packages/ai-utils", + "packages/ai-event-client" + ], + "talksTo": [] + }, + { + "id": "provider-adapters", + "owns": [ + "packages/openai-base", + "packages/ai-openai", + "packages/ai-anthropic", + "packages/ai-gemini", + "packages/ai-ollama", + "packages/ai-bedrock", + "packages/ai-byteplus", + "packages/ai-cloudflare", + "packages/ai-cohere", + "packages/ai-elevenlabs", + "packages/ai-fal", + "packages/ai-grok", + "packages/ai-groq", + "packages/ai-llmgateway", + "packages/ai-lovable", + "packages/ai-mistral", + "packages/ai-ollaya", + "packages/ai-openrouter", + "packages/ai-perplexity", + "packages/ai-reactor", + "packages/ai-typesafe", + "packages/ai-vercel-gateway", + "packages/ai-vertex", + "packages/ai-worldlabs" + ], + "talksTo": [ + "core" + ] + }, + { + "id": "harness", + "owns": [ + "packages/ai-acp", + "packages/ai-claude-code", + "packages/ai-codex", + "packages/ai-opencode", + "packages/ai-grok-build" + ], + "talksTo": [ + "core", + "sandbox" + ] + }, + { + "id": "client", + "owns": [ + "packages/ai-client", + "packages/ai-react", + "packages/ai-solid", + "packages/ai-vue", + "packages/ai-svelte", + "packages/ai-angular", + "packages/ai-preact", + "packages/ai-remix", + "packages/ai-octane", + "packages/ai-react-ui", + "packages/ai-solid-ui", + "packages/ai-vue-ui" + ], + "talksTo": [ + "core" + ] + }, + { + "id": "code-mode", + "owns": [ + "packages/ai-code-mode", + "packages/ai-code-mode-snippets", + "packages/ai-isolate-cloudflare", + "packages/ai-isolate-daytona", + "packages/ai-isolate-e2b", + "packages/ai-isolate-node", + "packages/ai-isolate-quickjs", + "packages/ai-isolate-quickjs-bun" + ], + "talksTo": [ + "core" + ] + }, + { + "id": "sandbox", + "owns": [ + "packages/ai-sandbox", + "packages/ai-sandbox-blaxel", + "packages/ai-sandbox-boxd", + "packages/ai-sandbox-cloudflare", + "packages/ai-sandbox-daytona", + "packages/ai-sandbox-docker", + "packages/ai-sandbox-e2b", + "packages/ai-sandbox-local-process", + "packages/ai-sandbox-railway", + "packages/ai-sandbox-sprites", + "packages/ai-sandbox-upstash-box", + "packages/ai-sandbox-vercel" + ], + "talksTo": [ + "core", + "middleware", + "persistence" + ] + }, + { + "id": "middleware", + "owns": [ + "packages/ai-compaction", + "packages/ai-memory", + "packages/ai-skills", + "packages/ai-mcp" + ], + "talksTo": [ + "core" + ] + }, + { + "id": "persistence", + "owns": [ + "packages/ai-persistence", + "packages/ai-durable-stream" + ], + "talksTo": [ + "core" + ] + }, + { + "id": "devtools", + "owns": [ + "packages/ai-devtools", + "packages/react-ai-devtools", + "packages/solid-ai-devtools", + "packages/svelte-ai-devtools", + "packages/preact-ai-devtools" + ], + "talksTo": [ + "core" + ] + }, + { + "id": "computer-use", + "owns": [ + "packages/ai-computer-use", + "packages/ai-computer-playwright", + "packages/ai-computer-e2b" + ], + "talksTo": [ + "core" + ] + }, + { + "id": "e2e", + "owns": [ + "testing/e2e" + ], + "talksTo": [ + "client", + "code-mode", + "computer-use", + "core", + "harness", + "middleware", + "persistence", + "provider-adapters", + "sandbox" + ] + }, + { + "id": "docs", + "owns": [ + "docs" + ], + "talksTo": [] + }, + { + "id": "model-sync", + "owns": [ + "scripts/model-sync" + ], + "talksTo": [ + "provider-adapters" + ] + } + ] +} diff --git a/.changeset/ai-computer-use.md b/.changeset/ai-computer-use.md new file mode 100644 index 0000000000..b08bc7c173 --- /dev/null +++ b/.changeset/ai-computer-use.md @@ -0,0 +1,31 @@ +--- +'@tanstack/ai-computer-use': minor +'@tanstack/ai-computer-playwright': minor +'@tanstack/ai-computer-e2b': minor +'@tanstack/ai': minor +'@tanstack/openai-base': minor +'@tanstack/ai-openai': minor +'@tanstack/ai-anthropic': minor +--- + +Add computer use that runs the same code on OpenAI and on Anthropic. A driver controls the machine. + +- `@tanstack/ai-computer-use` adds `createComputerUse()`. It returns a tool, a middleware, and a safety-check interrupt for `chat()`. +- `@tanstack/ai-computer-playwright` adds `createPlaywrightComputerDriver()`. It controls a Playwright page. +- `@tanstack/ai-computer-e2b` adds `createE2BComputerDriver()`. It controls a Linux desktop in an E2B sandbox. +- `@tanstack/ai-anthropic` and `@tanstack/openai-base` send the generic computer tool to the model as the native computer tool of the provider. `@tanstack/ai` adds the metadata reader for this in `@tanstack/ai/adapter-internals`. +- `@tanstack/ai-openai` marks the models that support the `computer` tool. + +```ts +const { tool, middleware, interrupt } = createComputerUse({ + driver: createPlaywrightComputerDriver({ page }), +}) + +chat({ + adapter, + messages, + tools: [tool], + middleware: [middleware], + interrupts: [interrupt], +}) +``` diff --git a/docs/adapters/anthropic.md b/docs/adapters/anthropic.md index f603d0f027..a24edc98e9 100644 --- a/docs/adapters/anthropic.md +++ b/docs/adapters/anthropic.md @@ -620,6 +620,8 @@ equivalent — see [Provider Skills](../tools/provider-skills.md). Claude can look at a screen and use the mouse and the keyboard. Your app owns the screen: it runs each action that Claude asks for and sends back a screenshot. +To run the actions on a browser or a desktop without your own action code, use `createComputerUse` from `@tanstack/ai-computer-use`. It runs each action on a Playwright page or an E2B desktop, and the adapter picks the tool version for your model. See [Computer use](../computer-use/computer-use.md). The rest of this section shows how to run the actions yourself. + 1. Pick the version that your model takes: | Version | Models | diff --git a/docs/adapters/openai.md b/docs/adapters/openai.md index 7a053ba05c..277c49ea69 100644 --- a/docs/adapters/openai.md +++ b/docs/adapters/openai.md @@ -995,6 +995,8 @@ const stream = chat({ Lets the model observe a virtual desktop via screenshots and interact with it using keyboard and mouse events. +To run the actions on a browser or a desktop without your own action code, use `createComputerUse` from `@tanstack/ai-computer-use`. It runs each action on a Playwright page or an E2B desktop. See [Computer use](../computer-use/computer-use.md). For the approval of safety checks, see [Safety checks](../computer-use/safety-checks.md). The rest of this section shows how to run the actions yourself. + The model asks for actions, for example a click and then some typing. Your app runs the actions, then returns a screenshot of the screen. Add an `execute` to the tool that does this: ```typescript diff --git a/docs/computer-use/computer-use.md b/docs/computer-use/computer-use.md new file mode 100644 index 0000000000..5e2e623890 --- /dev/null +++ b/docs/computer-use/computer-use.md @@ -0,0 +1,188 @@ +--- +title: Computer Use +id: computer-use +order: 1 +description: "Let the model use a web page for the user. It sees screenshots and sends clicks and key presses. The same code runs on OpenAI and Anthropic." +keywords: + - tanstack ai + - computer use + - createComputerUse + - browser agent + - playwright + - openai + - anthropic +--- + +You have a chat app. You want the model to do a task on a web page for the user, for example fill in a form or find a price. Computer use gives the model a screen, a mouse, and a keyboard. The same code runs on OpenAI and on Anthropic. + +## Install + + + +react: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod +vue: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod +solid: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod +svelte: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod +preact: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod +angular: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod +vanilla: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod +octane: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod + + + +## Give the model a browser page + +1. Open a Playwright page that has a viewport size. +2. Create the computer from a driver for that page. +3. Give `tool`, `middleware`, and `interrupt` to `chat()`. + +```ts +// app/api/chat/computers.ts +import { createComputerUse } from '@tanstack/ai-computer-use' +import { createPlaywrightComputerDriver } from '@tanstack/ai-computer-playwright' +import { chromium } from 'playwright' + +const browser = await chromium.launch() +const computers = new Map>() + +export async function computerFor(threadId: string) { + const existing = computers.get(threadId) + if (existing) return existing + const page = await browser.newPage({ viewport: { width: 1280, height: 720 } }) + await page.goto('https://example.com') + const computer = createComputerUse({ + driver: createPlaywrightComputerDriver({ page }), + }) + computers.set(threadId, computer) + return computer +} +``` + +```ts +// app/api/chat/route.ts +import { chat, chatParamsFromRequest, toServerSentEventsResponse } from '@tanstack/ai' +import { openaiText } from '@tanstack/ai-openai' +import { computerFor } from './computers' + +export async function POST(request: Request) { + const params = await chatParamsFromRequest(request) + const { tool, middleware, interrupt } = await computerFor(params.threadId) + const stream = chat({ + adapter: openaiText('gpt-6.1-sol'), + messages: params.messages, + threadId: params.threadId, + runId: params.runId, + ...(params.parentRunId ? { parentRunId: params.parentRunId } : {}), + ...(params.resume ? { resume: params.resume } : {}), + tools: [tool], + middleware: [middleware], + interrupts: [interrupt], + }) + return toServerSentEventsResponse(stream) +} +``` + +One `createComputerUse` drives one machine for one conversation at a time. The map keeps one computer for each thread. Your app owns the browser. When the conversation ends, close the page. + +The `middleware` stops the rest of a turn after a failed action. It also runs the OpenAI safety checks. The `interrupt` asks a person to approve an action that has a safety check. + +To use Anthropic, change only the adapter: + +```ts +import { anthropicText } from '@tanstack/ai-anthropic' + +const adapter = anthropicText('claude-opus-5-5') +``` + +Need a full Linux desktop and not a browser page? See [Drivers](./drivers). + +## Register the interrupt on the client + +The client imports `computerSafetyCheck`. It is the same object as `interrupt`, so the client does not load the driver or Playwright. + +```tsx +// app/chat.tsx +import { useState } from 'react' +import { fetchServerSentEvents, useChat } from '@tanstack/ai-react' +import { computerSafetyCheck } from '@tanstack/ai-computer-use' + +export function Chat() { + const [input, setInput] = useState('') + const { messages, sendMessage } = useChat({ + connection: fetchServerSentEvents('/api/chat'), + interrupts: [computerSafetyCheck], + }) + + return ( + <> + {messages.map((message) => ( +
+ {message.parts.map((part, index) => + part.type === 'text' ?

{part.content}

: null, + )} +
+ ))} +
{ + event.preventDefault() + sendMessage(input) + setInput('') + }} + > + setInput(event.target.value)} /> + +
+ + ) +} +``` + +To show the approval card for a safety check, see [Safety checks](./safety-checks). + +## What the model can do + +The model clicks, double-clicks, drags, moves the mouse, scrolls, types text, and presses key combos. It can also wait. + +- Every action returns a screenshot of the screen. +- The screenshot comes 500 ms after the action. This time lets the page settle. +- `zoom` returns an image of one region at full size. +- `cursor_position` returns the mouse position as text. + +To change the delay, set `screenshotDelay` in ms: + +```ts +const computer = createComputerUse({ + driver: createPlaywrightComputerDriver({ page }), + screenshotDelay: 1000, +}) +``` + +## Order and failures + +Actions run one at a time, in the order that the model calls them. + +If one action fails, the other actions of that model turn do not run. The model gets this result for each of them: + +```text +Not executed: an earlier computer action in this turn failed. +``` + +The next model turn starts clean. The model sees the error and can try again. + +## Model support + +- A model with no computer use is a type error at `chat()`. +- The Anthropic adapter picks the newest computer tool version that the model supports. If the model supports no version, the adapter throws an error. +- `openaiText` uses the Responses API, which has the computer tool. The Chat Completions adapters throw an error, because that API has no computer tool. + +## Try it + +Start the app and send this message: + +```text +Open example.com and tell me the title of the page. +``` + +The model takes a screenshot, reads the page, and answers in the chat. You now have a browser agent. + +To control your own machine with no driver package, see [Custom driver](./custom-driver). diff --git a/docs/computer-use/custom-driver.md b/docs/computer-use/custom-driver.md new file mode 100644 index 0000000000..972aa36eff --- /dev/null +++ b/docs/computer-use/custom-driver.md @@ -0,0 +1,120 @@ +--- +title: Custom Computer Use Driver +id: computer-use-custom-driver +order: 4 +description: "Let the model use your own machine. Write a ComputerDriver with a small set of mouse, keyboard, and screenshot methods." +keywords: + - tanstack ai + - computer use + - ComputerDriver + - custom driver + - virtual machine +--- + +You have your own machine: a VM, a remote desktop, or a device in a lab. No driver package controls it. Write a `ComputerDriver`, and [computer use](./computer-use) runs on it with no other change. + +## Write the driver + +A driver is an object with a screen size and a small set of methods. This example sends each method to an HTTP agent that runs on the machine. + +```ts +// app/my-driver.ts +import type { ComputerDriver } from '@tanstack/ai-computer-use' + +const base = 'http://localhost:8000' + +async function send(path: string, body: object) { + const response = await fetch(`${base}${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + if (!response.ok) throw new Error(`${path} failed: ${response.status}`) +} + +export const myDriver: ComputerDriver = { + display: { width: 1920, height: 1080 }, + environment: 'windows', + async screenshot() { + const response = await fetch(`${base}/screenshot`) + if (!response.ok) throw new Error(`/screenshot failed: ${response.status}`) + return new Uint8Array(await response.arrayBuffer()) + }, + click: (x, y, { button, count }) => send('/click', { x, y, button, count }), + move: (x, y) => send('/move', { x, y }), + mouseDown: (button) => send('/mouse-down', { button }), + mouseUp: (button) => send('/mouse-up', { button }), + scroll: (x, y, deltaX, deltaY) => send('/scroll', { x, y, deltaX, deltaY }), + keyDown: (key) => send('/key-down', { key }), + keyUp: (key) => send('/key-up', { key }), + press: (keys) => send('/press', { keys }), + type: (text) => send('/type', { text }), +} +``` + +Then pass it to `createComputerUse`: + +```ts +import { createComputerUse } from '@tanstack/ai-computer-use' +import { myDriver } from './my-driver' + +const { tool, middleware, interrupt } = createComputerUse({ driver: myDriver }) +``` + +## The contract + +Coordinates are in pixels, in `display` space. Key names are `KeyboardEvent.key` names, for example `'Enter'`, `'Control'`, or `'a'`. + +| Member | What it does | +| --- | --- | +| `display` | The screen size in pixels. Screenshots have this size. | +| `environment` | `'browser'`, `'linux'`, `'mac'`, or `'windows'`. | +| `screenshot()` | Returns PNG bytes as a `Uint8Array`. | +| `click(x, y, { button, count })` | Clicks `count` times. `button` is `left`, `right`, `middle`, `back`, or `forward`. | +| `move(x, y)` | Moves the mouse. | +| `mouseDown(button)`, `mouseUp(button)` | Presses or releases a button at the mouse position. | +| `scroll(x, y, deltaX, deltaY)` | Scrolls at the point. The deltas are in pixels. | +| `keyDown(key)`, `keyUp(key)` | Presses or releases one key. | +| `press(keys)` | Presses one combo, like `['Control', 's']`. | +| `type(text)` | Types text as a user does. | + +## Add the optional methods + +Two methods are optional. When the driver has one, the Anthropic adapter turns on that action for the model. + +- `zoom(region)`: returns PNG bytes of one region at full size. The region has `x`, `y`, `width`, and `height`. +- `cursorPosition()`: returns the mouse position as `{ x, y }`. + +Add this function to `app/my-driver.ts`. It narrows the JSON from your agent before it returns it: + +```ts +async function cursorPosition() { + const response = await fetch(`${base}/cursor`) + const data: unknown = await response.json() + const isPoint = + typeof data === 'object' && + data !== null && + 'x' in data && + 'y' in data && + typeof data.x === 'number' && + typeof data.y === 'number' + if (!isPoint) throw new Error('The agent sent no cursor position.') + return { x: data.x, y: data.y } +} +``` + +Then add `cursorPosition` to `myDriver`. + +## What the core builds for you + +Your driver does only the small steps. The core builds the rest from them: + +- **Drags**: `move`, `mouseDown`, more `move` calls, then `mouseUp`. +- **Modifier clicks and scrolls**: `keyDown` for each modifier, the action, then `keyUp` in reverse order. +- **`hold_key` and `wait`**: the core holds the keys or waits, then continues. +- **Key names**: the core changes OpenAI and Anthropic key names to `KeyboardEvent.key` names. +- **Screenshots**: after each action, the core waits `screenshotDelay` ms and calls `screenshot()`. + +## Try it + +Start your agent on the machine, and send a task in the chat. Your agent gets the first `/screenshot` request, then the clicks and the key presses. The model now uses your own machine. diff --git a/docs/computer-use/drivers.md b/docs/computer-use/drivers.md new file mode 100644 index 0000000000..287d1f51cc --- /dev/null +++ b/docs/computer-use/drivers.md @@ -0,0 +1,124 @@ +--- +title: Computer Use Drivers +id: computer-use-drivers +order: 2 +description: "Pick the machine that the model uses: a Playwright browser page or a full Linux desktop in an E2B sandbox." +keywords: + - tanstack ai + - computer use + - driver + - playwright + - e2b + - linux desktop + - sandbox +--- + +A browser page is not always enough. Sometimes the model must use a desktop app, a file manager, or a terminal. A driver decides which machine the model uses. Change the driver, and the rest of your [computer use](./computer-use) code stays the same. + +## Pick a driver + +| | Playwright | E2B | +| ------------------- | ----------------------------------- | -------------------------------- | +| **Machine** | One browser page | A full Linux desktop in a sandbox | +| **Environment** | `browser` | `linux` | +| **Runs on** | Your server | E2B cloud | +| **`zoom`** | Yes | No | +| **`cursor_position`** | No | Yes | +| **Setup** | `pnpm add`, then launch a browser | An E2B API key, then create a sandbox | + +When a driver has `zoom` or `cursorPosition`, the Anthropic adapter turns on that action for the model. + +## Playwright + +If the task stays inside a web page, use Playwright. + + + +react: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod +vue: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod +solid: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod +svelte: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod +preact: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod +angular: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod +vanilla: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod +octane: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod + + + +```ts +import { chromium } from 'playwright' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { createPlaywrightComputerDriver } from '@tanstack/ai-computer-playwright' + +const browser = await chromium.launch() +const page = await browser.newPage({ viewport: { width: 1280, height: 720 } }) + +const { tool, middleware, interrupt } = createComputerUse({ + driver: createPlaywrightComputerDriver({ page }), +}) +``` + +- The page must have a viewport size. If it has none, the driver throws an error. +- The driver reads the viewport size one time, when you create it. +- The `back` and `forward` mouse buttons go back and forward in the page history. +- A page from `playwright-core` or `@playwright/test` also works. + +Your app owns the browser. The driver does not launch it or close it. + +## E2B + +If the model must use any app on a Linux desktop, use E2B. The desktop runs in a sandbox, away from your server. + + + +react: @tanstack/ai-computer-use @tanstack/ai-computer-e2b @e2b/desktop zod +vue: @tanstack/ai-computer-use @tanstack/ai-computer-e2b @e2b/desktop zod +solid: @tanstack/ai-computer-use @tanstack/ai-computer-e2b @e2b/desktop zod +svelte: @tanstack/ai-computer-use @tanstack/ai-computer-e2b @e2b/desktop zod +preact: @tanstack/ai-computer-use @tanstack/ai-computer-e2b @e2b/desktop zod +angular: @tanstack/ai-computer-use @tanstack/ai-computer-e2b @e2b/desktop zod +vanilla: @tanstack/ai-computer-use @tanstack/ai-computer-e2b @e2b/desktop zod +octane: @tanstack/ai-computer-use @tanstack/ai-computer-e2b @e2b/desktop zod + + + +1. Set `E2B_API_KEY` in your server environment. +2. Create a desktop sandbox. +3. Pass the sandbox to the driver. + +```ts +import { Sandbox } from '@e2b/desktop' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { createE2BComputerDriver } from '@tanstack/ai-computer-e2b' + +const desktop = await Sandbox.create() + +const { tool, middleware, interrupt } = createComputerUse({ + driver: await createE2BComputerDriver({ desktop }), +}) +``` + +- `createE2BComputerDriver` is async. It reads the screen size one time, so call it after the desktop is ready. +- One scroll step is 100 pixels. +- The driver quotes every key name before it goes to the shell. + +Your app owns the sandbox. The driver does not create it or kill it. When the conversation ends, call `await desktop.kill()`. + +## Use it in chat + +Both drivers give the same three values. Pass them to `chat()`: + +```ts +import { chat } from '@tanstack/ai' +import { anthropicText } from '@tanstack/ai-anthropic' + +const stream = chat({ + adapter: anthropicText('claude-opus-5-5'), + messages, + tools: [tool], + middleware: [middleware], + interrupts: [interrupt], +}) +``` + +The model now uses a browser page or a full Linux desktop. For a machine that neither driver controls, write a [custom driver](./custom-driver). diff --git a/docs/computer-use/safety-checks.md b/docs/computer-use/safety-checks.md new file mode 100644 index 0000000000..bae65c2d03 --- /dev/null +++ b/docs/computer-use/safety-checks.md @@ -0,0 +1,121 @@ +--- +title: Computer Use Safety Checks +id: computer-use-safety-checks +order: 3 +description: "Pause a computer action until a person approves it. OpenAI attaches safety checks to risky actions, and TanStack AI turns each one into an interrupt." +keywords: + - tanstack ai + - computer use + - safety checks + - pending_safety_checks + - computerSafetyCheck + - interrupt + - approval +--- + +Some computer actions are risky. The page can ask the model to do a thing the user did not ask for, or the action can send data. OpenAI attaches safety checks to these actions. The run pauses, and a person approves or declines the action. + +## How it works + +1. The model sends a computer call with `pending_safety_checks`. +2. The `middleware` pauses the run with one interrupt for each call that has checks. +3. A person approves or declines on the client. + +- If the person approves all calls, the calls run. The checks go back to OpenAI as acknowledged. +- If the person declines one call, no call of that turn runs. +- A call with checks that nobody approved does not run. The tool refuses it. + +Safety checks are an OpenAI feature. Anthropic models do not send them, so this code does nothing on Anthropic. + +## Server + +Give `middleware` and `interrupt` to `chat()`. Also pass `parentRunId` and `resume`, so that the approval can continue the run. + +```ts +// app/api/chat/route.ts +import { chat, chatParamsFromRequest, toServerSentEventsResponse } from '@tanstack/ai' +import { openaiText } from '@tanstack/ai-openai' +import { computerFor } from './computers' + +export async function POST(request: Request) { + const params = await chatParamsFromRequest(request) + const { tool, middleware, interrupt } = await computerFor(params.threadId) + const stream = chat({ + adapter: openaiText('gpt-6.1-sol'), + messages: params.messages, + threadId: params.threadId, + runId: params.runId, + ...(params.parentRunId ? { parentRunId: params.parentRunId } : {}), + ...(params.resume ? { resume: params.resume } : {}), + tools: [tool], + middleware: [middleware], + interrupts: [interrupt], + }) + return toServerSentEventsResponse(stream) +} +``` + +`computerFor` returns the same `createComputerUse` result for each thread. The approval is stored on that result, so the continuation must use the same one. [Computer Use](./computer-use) shows `computerFor`. + +## Client + +Register `computerSafetyCheck` with `useChat`. Each item has these fields: + +- `payload.checks`: the safety checks. Each check has an `id`, and can have a `code` and a `message`. +- `payload.action`: the action that waits. For a batch, this is the list of actions. + +```tsx +// app/chat.tsx +import { fetchServerSentEvents, useChat } from '@tanstack/ai-react' +import type { GenericInterrupt } from '@tanstack/ai-react' +import { computerSafetyCheck } from '@tanstack/ai-computer-use' + +function SafetyCheckCard({ + interrupt, +}: { + interrupt: GenericInterrupt +}) { + const checks = interrupt.payload?.checks ?? [] + return ( +
+

Approve this computer action?

+
    + {checks.map((check) => ( +
  • {check.message ?? check.code ?? check.id}
    • + ))} +
+ 
{JSON.stringify(interrupt.payload?.action, null, 2)}
+ + +
+ ) +} + +export function Chat() { + const { interrupts } = useChat({ + connection: fetchServerSentEvents('/api/chat'), + interrupts: [computerSafetyCheck], + }) + + return ( + <> + {interrupts.map((interrupt) => { + if (interrupt.kind !== 'generic') return null + if (!('definitionId' in interrupt)) return null + if (interrupt.definitionId !== computerSafetyCheck.id) return null + return + })} + + ) +} +``` + +`interrupt.cancel()` also declines the action. When every interrupt of the batch has an answer, the client sends the answers. For more about generic interrupts, see [Generic Interrupts](../interrupts/generic). + +## Try it + +Ask the model to do a task that sends data, for example a form submit. When OpenAI adds a safety check, the card shows. Click **Approve**, and the action runs. Click **Decline**, and the page stays as it was. diff --git a/docs/config.json b/docs/config.json index 1c4e5fca3f..ded4edfa89 100644 --- a/docs/config.json +++ b/docs/config.json @@ -323,7 +323,7 @@ "label": "Provider Tools", "to": "tools/provider-tools", "addedAt": "2026-04-21", - "updatedAt": "2026-10-19" + "updatedAt": "2026-10-20" }, { "label": "Provider Skills", @@ -919,6 +919,32 @@ ], "tab": "guides" }, + { + "label": "Computer Use", + "children": [ + { + "label": "Computer Use", + "to": "computer-use/computer-use", + "addedAt": "2026-10-20" + }, + { + "label": "Drivers", + "to": "computer-use/drivers", + "addedAt": "2026-10-20" + }, + { + "label": "Safety Checks", + "to": "computer-use/safety-checks", + "addedAt": "2026-10-20" + }, + { + "label": "Custom Driver", + "to": "computer-use/custom-driver", + "addedAt": "2026-10-20" + } + ], + "tab": "guides" + }, { "label": "Sandboxes", "tab": "guides", @@ -1209,13 +1235,13 @@ "label": "OpenAI", "to": "adapters/openai", "addedAt": "2026-04-15", - "updatedAt": "2026-10-19" + "updatedAt": "2026-10-20" }, { "label": "Anthropic", "to": "adapters/anthropic", "addedAt": "2026-04-15", - "updatedAt": "2026-10-19" + "updatedAt": "2026-10-20" }, { "label": "Google Gemini", diff --git a/docs/tools/provider-tools.md b/docs/tools/provider-tools.md index 117eed7b44..7be0949fec 100644 --- a/docs/tools/provider-tools.md +++ b/docs/tools/provider-tools.md @@ -92,6 +92,31 @@ assistant message; the agent loop never tries to run it client-side. OpenAI `applyPatchTool`, `localShellTool`, `computerUseTool`, and `shellTool({ environment: { type: "local" } })` are different. The model returns the call, and your app runs it. See [OpenAI adapter](../adapters/openai.md#applypatchtool) and [`computerUseTool`](../adapters/openai.md#computerusetool). +To run computer use actions on a browser or a desktop without your own action code, use `createComputerUse` from `@tanstack/ai-computer-use`. The same `tool` works with OpenAI and Anthropic. See [Computer use](../computer-use/computer-use.md). + +```typescript +import { chat } from '@tanstack/ai' +import { openaiText } from '@tanstack/ai-openai' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { createPlaywrightComputerDriver } from '@tanstack/ai-computer-playwright' +import { chromium } from 'playwright' + +const browser = await chromium.launch() +const page = await browser.newPage({ viewport: { width: 1280, height: 800 } }) + +const { tool, middleware, interrupt } = createComputerUse({ + driver: createPlaywrightComputerDriver({ page }), +}) + +const stream = chat({ + adapter: openaiText('gpt-6.1-sol'), + messages: [{ role: 'user', content: 'Open example.com' }], + tools: [tool], + middleware: [middleware], + interrupts: [interrupt], +}) +``` + ### Read web search sources OpenAI and Gemini web search calls expose a common `metadata.sources` array on diff --git a/packages/ai-anthropic/src/adapters/text.ts b/packages/ai-anthropic/src/adapters/text.ts index 4c906aedef..8bb54439f8 100644 --- a/packages/ai-anthropic/src/adapters/text.ts +++ b/packages/ai-anthropic/src/adapters/text.ts @@ -21,6 +21,7 @@ import { convertComputerUseToolToAdapterFormat, isComputerToolset, joinComputerToolsetCall, + resolveComputerUseTool, splitComputerToolsetCall, } from '../tools/computer-use-tool' import { @@ -585,10 +586,18 @@ export class AnthropicTextAdapter< } async *chatStream( - options: TextOptions, + chatOptions: TextOptions, ): AsyncIterable { + let options = chatOptions const { logger } = options try { + // The rest of the request sees the native computer tool. + options = { + ...options, + tools: options.tools?.map((tool) => + resolveComputerUseTool(tool, this.model), + ), + } const requestParams = this.mapCommonOptionsToAnthropic(options) logger.request( @@ -684,7 +693,14 @@ export class AnthropicTextAdapter< async structuredOutput( options: StructuredOutputOptions, ): Promise> { - const { chatOptions, outputSchema } = options + const { outputSchema } = options + // The betas and the history see the native computer tool. + const chatOptions = { + ...options.chatOptions, + tools: options.chatOptions.tools?.map((tool) => + resolveComputerUseTool(tool, this.model), + ), + } const { logger } = chatOptions // `structuredOutput()` issues a non-streaming `messages.create({ stream: diff --git a/packages/ai-anthropic/src/model-meta.ts b/packages/ai-anthropic/src/model-meta.ts index 80dc5e69ce..748d798ab4 100644 --- a/packages/ai-anthropic/src/model-meta.ts +++ b/packages/ai-anthropic/src/model-meta.ts @@ -96,12 +96,14 @@ interface ModelMeta< * Computer use has three kinds, one for each version that a model * accepts: `computer_use` (`computer_20241022` and `computer_20250124`), * `computer_use_20251124`, and `computer_toolset` - * (`computer_toolset_20260801`). + * (`computer_toolset_20260801`). `computer` is the generic tool from + * `@tanstack/ai-computer-use`. Every model with a computer version has it. */ tools?: Array< | 'web_search' | 'web_fetch' | 'code_execution' + | 'computer' | 'computer_use' | 'computer_use_20251124' | 'computer_toolset' @@ -163,6 +165,7 @@ const CLAUDE_OPUS_4_6 = { 'web_search', 'web_fetch', 'code_execution', + 'computer', 'computer_use_20251124', 'bash', 'text_editor', @@ -203,6 +206,7 @@ const CLAUDE_OPUS_4_5 = { 'web_search', 'web_fetch', 'code_execution', + 'computer', 'computer_use_20251124', 'bash', 'text_editor', @@ -246,6 +250,7 @@ const CLAUDE_SONNET_4_6 = { 'web_search', 'web_fetch', 'code_execution', + 'computer', 'computer_use_20251124', 'bash', 'text_editor', @@ -286,6 +291,7 @@ const CLAUDE_SONNET_4_5 = { 'web_search', 'web_fetch', 'code_execution', + 'computer', 'computer_use', 'bash', 'text_editor', @@ -326,6 +332,7 @@ const CLAUDE_HAIKU_4_5 = { 'web_search', 'web_fetch', 'code_execution', + 'computer', 'computer_use', 'bash', 'text_editor', @@ -366,6 +373,7 @@ const CLAUDE_OPUS_4_1 = { 'web_search', 'web_fetch', 'code_execution', + 'computer', 'computer_use', 'bash', 'text_editor', @@ -401,6 +409,7 @@ const CLAUDE_OPUS_4_7 = { 'web_search', 'web_fetch', 'code_execution', + 'computer', 'computer_use_20251124', 'bash', 'text_editor', @@ -445,6 +454,7 @@ const CLAUDE_OPUS_4_8 = { 'web_search', 'web_fetch', 'code_execution', + 'computer', 'computer_use_20251124', 'computer_toolset', 'bash', @@ -493,6 +503,7 @@ const CLAUDE_FABLE_5 = { 'web_search', 'web_fetch', 'code_execution', + 'computer', 'computer_use_20251124', 'computer_toolset', 'bash', @@ -539,6 +550,7 @@ const CLAUDE_SONNET_5 = { 'web_search', 'web_fetch', 'code_execution', + 'computer', 'computer_use_20251124', 'computer_toolset', 'bash', @@ -588,6 +600,7 @@ const CLAUDE_OPUS_5 = { 'web_search', 'web_fetch', 'code_execution', + 'computer', 'computer_use_20251124', 'computer_toolset', 'bash', @@ -657,6 +670,7 @@ const CLAUDE_FABLE_5_1 = { 'web_search', 'web_fetch', 'code_execution', + 'computer', 'computer_use_20251124', 'computer_toolset', 'bash', @@ -699,6 +713,7 @@ const CLAUDE_OPUS_5_5 = { 'web_search', 'web_fetch', 'code_execution', + 'computer', 'computer_toolset', 'bash', 'text_editor', @@ -746,6 +761,7 @@ const CLAUDE_SONNET_5_5 = { 'web_search', 'web_fetch', 'code_execution', + 'computer', 'computer_toolset', 'bash', 'text_editor', @@ -789,6 +805,7 @@ const CLAUDE_HAIKU_5_5 = { 'web_search', 'web_fetch', 'code_execution', + 'computer', 'computer_toolset', 'bash', 'text_editor', @@ -1153,6 +1170,32 @@ export type AnthropicChatModelToolCapabilitiesByName = { [CLAUDE_HAIKU_5_5.id]: typeof CLAUDE_HAIKU_5_5.supports.tools } +/** + * Runtime map from chat model name to its tool kinds, for the text adapter. + * `satisfies` keeps it equal to {@link AnthropicChatModelToolCapabilitiesByName}. + * An unknown name gives `undefined`. + */ +export const ANTHROPIC_MODEL_TOOLS: Readonly< + Record> +> = { + [CLAUDE_OPUS_4_6.id]: CLAUDE_OPUS_4_6.supports.tools, + [CLAUDE_OPUS_4_5.id]: CLAUDE_OPUS_4_5.supports.tools, + [CLAUDE_SONNET_4_6.id]: CLAUDE_SONNET_4_6.supports.tools, + [CLAUDE_SONNET_4_5.id]: CLAUDE_SONNET_4_5.supports.tools, + [CLAUDE_HAIKU_4_5.id]: CLAUDE_HAIKU_4_5.supports.tools, + [CLAUDE_OPUS_4_1.id]: CLAUDE_OPUS_4_1.supports.tools, + [CLAUDE_OPUS_4_7.id]: CLAUDE_OPUS_4_7.supports.tools, + [CLAUDE_OPUS_4_8.id]: CLAUDE_OPUS_4_8.supports.tools, + [CLAUDE_FABLE_5.id]: CLAUDE_FABLE_5.supports.tools, + [CLAUDE_SONNET_5.id]: CLAUDE_SONNET_5.supports.tools, + [CLAUDE_OPUS_5.id]: CLAUDE_OPUS_5.supports.tools, + [CLAUDE_OPUS_5_FAST.id]: CLAUDE_OPUS_5_FAST.supports.tools, + [CLAUDE_FABLE_5_1.id]: CLAUDE_FABLE_5_1.supports.tools, + [CLAUDE_OPUS_5_5.id]: CLAUDE_OPUS_5_5.supports.tools, + [CLAUDE_SONNET_5_5.id]: CLAUDE_SONNET_5_5.supports.tools, + [CLAUDE_HAIKU_5_5.id]: CLAUDE_HAIKU_5_5.supports.tools, +} satisfies AnthropicChatModelToolCapabilitiesByName + /** * Type-only map from chat model name to its supported input modalities. * All Anthropic Claude models support text, image, and document (PDF) input. diff --git a/packages/ai-anthropic/src/tools/computer-use-tool.ts b/packages/ai-anthropic/src/tools/computer-use-tool.ts index 8959ca6546..8c6bfb3c45 100644 --- a/packages/ai-anthropic/src/tools/computer-use-tool.ts +++ b/packages/ai-anthropic/src/tools/computer-use-tool.ts @@ -1,5 +1,8 @@ import { brandProviderTool } from '@tanstack/ai' +import { getComputerUseToolMetadata } from '@tanstack/ai/adapter-internals' +import { ANTHROPIC_MODEL_TOOLS } from '../model-meta' import { getAnthropicProviderToolMetadata } from './anthropic-provider-tool' +import type { ComputerUseToolMetadata } from '@tanstack/ai/adapter-internals' import type { BetaComputerToolset20260801, BetaToolComputerUse20241022, @@ -109,3 +112,66 @@ export function computerUseTool(config: ComputerUseToolConfig) { metadata: { ...config, __kind: 'anthropic.computer_use' }, }) } + +/** The native tool for the newest computer version in the model's kinds. */ +function nativeComputerTool( + { display, capabilities }: ComputerUseToolMetadata, + kinds: ReadonlyArray, +) { + if (kinds.includes('computer_toolset')) { + // The toolset type has no display size field. + return computerUseTool({ + type: 'computer_toolset_20260801', + configs: { + zoom: { enabled: capabilities.zoom }, + cursor_position: { enabled: capabilities.cursorPosition }, + }, + }) + } + const size = { + display_width_px: display.width, + display_height_px: display.height, + } + if (kinds.includes('computer_use_20251124')) { + return computerUseTool({ + type: 'computer_20251124', + name: COMPUTER_TOOL_NAME, + ...size, + enable_zoom: capabilities.zoom, + }) + } + if (kinds.includes('computer_use')) { + return computerUseTool({ + type: 'computer_20250124', + name: COMPUTER_TOOL_NAME, + ...size, + }) + } + return undefined +} + +/** + * Turn the generic computer tool from `@tanstack/ai-computer-use` into the + * native computer tool of the model. The tool keeps its name `computer` and + * its `execute`. Every other tool comes back unchanged. + * + * The version is the newest one in the model meta: `computer_toolset_20260801`, + * then `computer_20251124`, then `computer_20250124`. + * + * @param tool - A tool from the chat tool list. + * @param model - The Anthropic model id of the request. + * @throws Error when the model has no computer version, or the model id is + * not in the model meta. + */ +export function resolveComputerUseTool(tool: Tool, model: string) { + const computer = getComputerUseToolMetadata(tool) + if (!computer) return tool + const native = nativeComputerTool( + computer, + ANTHROPIC_MODEL_TOOLS[model] ?? [], + ) + if (!native) { + throw new Error(`Model "${model}" does not support computer use.`) + } + return { ...tool, metadata: native.metadata } +} diff --git a/packages/ai-anthropic/tests/computer-use-generic.test.ts b/packages/ai-anthropic/tests/computer-use-generic.test.ts new file mode 100644 index 0000000000..86c6ebaa24 --- /dev/null +++ b/packages/ai-anthropic/tests/computer-use-generic.test.ts @@ -0,0 +1,238 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { brandProviderTool, chat } from '@tanstack/ai' +import { createAnthropicChat } from '../src/adapters/text' +import { resolveComputerUseTool } from '../src/tools/computer-use-tool' +import type { ProviderTool } from '@tanstack/ai' +import type { ComputerUseToolMetadata } from '@tanstack/ai/adapter-internals' +import type { TextActivityOptions } from '@tanstack/ai/adapters' + +const mocks = vi.hoisted(() => { + const betaMessagesCreate = vi.fn() + const client = { beta: { messages: { create: betaMessagesCreate } } } + return { betaMessagesCreate, client } +}) + +vi.mock('@anthropic-ai/sdk', () => { + const { client } = mocks + class MockAnthropic { + beta = client.beta + constructor(_: { apiKey: string }) {} + } + return { default: MockAnthropic } +}) + +function stream(events: Array) { + mocks.betaMessagesCreate.mockResolvedValueOnce( + (async function* () { + yield* events + })(), + ) +} + +/** A computer toolset call, as the model streams it. */ +function memberCall(index: number, id: string, name: string, json: string) { + return [ + { + type: 'content_block_start', + index, + content_block: { + type: 'tool_use', + id, + name, + toolset_name: 'computer', + input: {}, + }, + }, + { + type: 'content_block_delta', + index, + delta: { type: 'input_json_delta', partial_json: json }, + }, + { type: 'content_block_stop', index }, + ] +} + +const end = (stopReason: string) => [ + { + type: 'message_delta', + delta: { stop_reason: stopReason }, + usage: { output_tokens: 1 }, + }, + { type: 'message_stop' }, +] + +type Capabilities = ComputerUseToolMetadata['capabilities'] + +const NO_EXTRAS: Capabilities = { zoom: false, cursorPosition: false } + +/** The generic tool, as `@tanstack/ai-computer-use` builds it. */ +function computerTool( + capabilities: Capabilities = NO_EXTRAS, + execute: (input: unknown) => unknown = () => 'OK', +) { + return brandProviderTool>({ + name: 'computer', + description: 'Use the computer.', + metadata: { + __kind: 'computer_use', + display: { width: 1280, height: 800 }, + environment: 'browser', + capabilities, + }, + execute, + }) +} + +type ComputerModel = 'claude-opus-5-5' | 'claude-opus-4-6' | 'claude-sonnet-4-5' + +/** The `tools` and `betas` of the first request for one computer tool. */ +async function firstRequest(model: ComputerModel, capabilities: Capabilities) { + stream(end('end_turn')) + for await (const _ of chat({ + adapter: createAnthropicChat(model, 'k'), + messages: [{ role: 'user', content: 'Open the settings.' }], + tools: [computerTool(capabilities)], + })) { + // consume + } + const [payload] = mocks.betaMessagesCreate.mock.calls[0]! + return { tools: payload.tools, betas: payload.betas } +} + +/** Type helper: the `tools` that `chat()` accepts for this adapter. */ +function typedTools>( + adapter: TAdapter, + tools: TextActivityOptions['tools'], +) { + return { adapter, tools } +} + +beforeEach(() => { + mocks.betaMessagesCreate.mockReset() +}) + +describe('anthropic generic computer tool', () => { + it.each([ + { + capabilities: { zoom: true, cursorPosition: false }, + configs: { zoom: { enabled: true }, cursor_position: { enabled: false } }, + }, + { + capabilities: { zoom: false, cursorPosition: true }, + configs: { zoom: { enabled: false }, cursor_position: { enabled: true } }, + }, + ])( + 'sends the toolset with configs $configs on claude-opus-5-5', + async ({ capabilities, configs }) => { + const { tools, betas } = await firstRequest( + 'claude-opus-5-5', + capabilities, + ) + + expect(tools).toHaveLength(1) + expect(tools[0]).toMatchObject({ + type: 'computer_toolset_20260801', + configs, + }) + expect(tools[0].configs).toEqual(configs) + expect(tools[0]).not.toHaveProperty('name') + expect(betas).toBeUndefined() + }, + ) + + it('sends computer_20251124 with enable_zoom on claude-opus-4-6', async () => { + const { tools, betas } = await firstRequest('claude-opus-4-6', { + zoom: true, + cursorPosition: false, + }) + + expect(tools).toHaveLength(1) + expect(tools[0]).toMatchObject({ + type: 'computer_20251124', + name: 'computer', + display_width_px: 1280, + display_height_px: 800, + enable_zoom: true, + }) + expect(betas).toContain('computer-use-2025-11-24') + }) + + it('sends computer_20250124 on claude-sonnet-4-5', async () => { + const { tools, betas } = await firstRequest('claude-sonnet-4-5', NO_EXTRAS) + + expect(tools).toHaveLength(1) + expect(tools[0]).toMatchObject({ + type: 'computer_20250124', + name: 'computer', + display_width_px: 1280, + display_height_px: 800, + }) + expect(tools[0]).not.toHaveProperty('enable_zoom') + expect(betas).toContain('computer-use-2025-01-24') + }) + + it('runs the generic execute for each toolset call and sends the results back', async () => { + stream([ + ...memberCall(0, 'toolu_click', 'left_click', '{"coordinate":[500,300]}'), + ...end('tool_use'), + ]) + stream(end('end_turn')) + const calls: Array = [] + const tool = computerTool(NO_EXTRAS, (input) => { + calls.push(input) + return 'OK' + }) + + for await (const _ of chat({ + adapter: createAnthropicChat('claude-opus-5-5', 'k'), + messages: [{ role: 'user', content: 'Click the button.' }], + tools: [tool], + })) { + // consume + } + + expect(calls).toEqual([{ action: 'left_click', coordinate: [500, 300] }]) + const [payload] = mocks.betaMessagesCreate.mock.calls[1]! + const [, assistant, results] = payload.messages + expect(assistant.content).toEqual([ + { + type: 'tool_use', + id: 'toolu_click', + name: 'left_click', + toolset_name: 'computer', + input: { coordinate: [500, 300] }, + }, + ]) + expect(results.content).toMatchObject([ + { + type: 'tool_result', + tool_use_id: 'toolu_click', + toolset_name: 'computer', + content: 'OK', + }, + ]) + }) + + it.each(['claude-opus-5-fast', 'claude-opus-4-5@20251101'])( + 'throws for %s, which has no computer version in the model meta', + (model) => { + expect(() => resolveComputerUseTool(computerTool(), model)).toThrow( + `Model "${model}" does not support computer use.`, + ) + }, + ) + + it('returns a tool with no computer metadata unchanged', () => { + const tool = { name: 'echo', description: 'Echo the input.' } + + expect(resolveComputerUseTool(tool, 'claude-opus-5-fast')).toBe(tool) + }) + + it('types: chat() takes the generic tool only on a model with a computer version', () => { + typedTools(createAnthropicChat('claude-opus-5-5', 'k'), [computerTool()]) + typedTools(createAnthropicChat('claude-opus-5-fast', 'k'), [ + // @ts-expect-error - claude-opus-5-fast has no computer version + computerTool(), + ]) + }) +}) diff --git a/packages/ai-computer-e2b/README.md b/packages/ai-computer-e2b/README.md new file mode 100644 index 0000000000..3de53c87e9 --- /dev/null +++ b/packages/ai-computer-e2b/README.md @@ -0,0 +1,73 @@ +
+ + + + TanStack AI + +
+ +
+ +# @tanstack/ai-computer-e2b + +E2B desktop driver for TanStack AI computer use. + +Let the model use a Linux desktop in an E2B sandbox. The model sees screenshots and sends clicks, scrolls, and key presses. The driver runs them on the desktop. + +## Installation + +```bash +pnpm add @tanstack/ai-computer-e2b @tanstack/ai-computer-use @e2b/desktop +``` + +## Usage + +1. Set `E2B_API_KEY` in your server environment. +2. Create a desktop sandbox and pass it to the driver: + +```typescript +import { Sandbox } from '@e2b/desktop' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { createE2BComputerDriver } from '@tanstack/ai-computer-e2b' + +const desktop = await Sandbox.create() + +const { tool, middleware, interrupt } = createComputerUse({ + driver: await createE2BComputerDriver({ desktop }), +}) +``` + +3. Give `tool`, `middleware`, and `interrupt` to `chat()`. The model can now use the desktop. + +The driver accepts any object with the shape of an `@e2b/desktop` `Sandbox`. + +## What your application owns + +- **The sandbox.** The driver does not create, pause, or kill it. Kill it when you are done. +- **The screen size.** The driver reads it once, when you create the driver. + +## What the driver does + +- It sends the actions to the desktop through the E2B SDK and `xdotool`. +- It maps `KeyboardEvent.key` names, like `Control` and `Enter`, to `xdotool` names. +- It quotes every key name before it goes to the shell. +- One scroll step is 100 pixels. +- It supports `cursorPosition`. It does not support `zoom`. + +See the [drivers guide](https://tanstack.com/ai/latest/docs/computer-use/drivers) for more. + +## Tests + +```bash +pnpm --filter @tanstack/ai-computer-e2b test:lib +``` diff --git a/packages/ai-computer-e2b/package.json b/packages/ai-computer-e2b/package.json new file mode 100644 index 0000000000..9f6cb3d457 --- /dev/null +++ b/packages/ai-computer-e2b/package.json @@ -0,0 +1,55 @@ +{ + "name": "@tanstack/ai-computer-e2b", + "version": "0.0.0", + "description": "E2B desktop sandbox driver for TanStack AI computer use.", + "author": "", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/TanStack/ai.git", + "directory": "packages/ai-computer-e2b" + }, + "type": "module", + "module": "./dist/esm/index.js", + "types": "./dist/esm/index.d.ts", + "exports": { + ".": { + "types": "./dist/esm/index.d.ts", + "import": "./dist/esm/index.js" + } + }, + "sideEffects": false, + "engines": { + "node": ">=18" + }, + "files": [ + "dist", + "src" + ], + "scripts": { + "build": "vite build", + "clean": "premove ./build ./dist", + "lint:fix": "oxlint src --type-aware --fix", + "test:build": "publint --strict", + "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:lib": "vitest --passWithNoTests", + "test:lib:dev": "pnpm test:lib --watch", + "test:types": "tsc" + }, + "keywords": [ + "ai", + "tanstack", + "computer-use", + "e2b", + "sandbox" + ], + "peerDependencies": { + "@tanstack/ai-computer-use": "workspace:^" + }, + "devDependencies": { + "@e2b/desktop": "^2.5.0", + "@tanstack/ai-computer-use": "workspace:*", + "@vitest/coverage-v8": "4.1.10" + } +} diff --git a/packages/ai-computer-e2b/src/driver.ts b/packages/ai-computer-e2b/src/driver.ts new file mode 100644 index 0000000000..0eb9c5c4bd --- /dev/null +++ b/packages/ai-computer-e2b/src/driver.ts @@ -0,0 +1,152 @@ +import type { ComputerDriver, MouseButton } from '@tanstack/ai-computer-use' + +/** The parts of an `@e2b/desktop` `Sandbox` that the driver uses. */ +export interface E2BDesktopLike { + getScreenSize: () => Promise<{ width: number; height: number }> + getCursorPosition: () => Promise<{ x: number; y: number }> + screenshot: (format: 'bytes') => Promise + moveMouse: (x: number, y: number) => Promise + leftClick: () => Promise + doubleClick: () => Promise + rightClick: () => Promise + middleClick: () => Promise + mousePress: (button: 'left' | 'right' | 'middle') => Promise + mouseRelease: (button: 'left' | 'right' | 'middle') => Promise + scroll: (direction: 'up' | 'down', amount: number) => Promise + write: (text: string) => Promise + commands: { run: (cmd: string) => Promise } +} + +/** Options for {@link createE2BComputerDriver}. */ +export interface E2BComputerDriverOptions { + /** The E2B desktop sandbox to control. The driver does not create or kill it. */ + desktop: E2BDesktopLike +} + +/** `KeyboardEvent.key` names that differ from the xdotool names. Other names pass through. */ +const XDOTOOL_KEYS: Record = { + Control: 'ctrl', + Shift: 'shift', + Alt: 'alt', + Meta: 'super', + Enter: 'Return', + Backspace: 'BackSpace', + ArrowLeft: 'Left', + ArrowRight: 'Right', + ArrowUp: 'Up', + ArrowDown: 'Down', + PageUp: 'Page_Up', + PageDown: 'Page_Down', + CapsLock: 'Caps_Lock', + ' ': 'space', + '+': 'plus', +} + +/** xdotool button numbers. */ +const BUTTONS: Record = { + left: 1, + middle: 2, + right: 3, + back: 8, + forward: 9, +} + +// ponytail: one scroll click is 100 px. Fine steps below 100 px round to one click. +const PIXELS_PER_SCROLL_CLICK = 100 + +function toXdotoolKey(key: string) { + return XDOTOOL_KEYS[key] ?? key +} + +/** Quote one value for a POSIX shell. The model picks key names, so they never reach the shell raw. */ +function quote(value: string) { + return `'${value.replaceAll("'", `'"'"'`)}'` +} + +function int(value: number) { + if (!Number.isInteger(value)) { + throw new Error(`Expected an integer, got ${value}`) + } + return value +} + +function scrollClicks(pixels: number) { + return Math.max(1, Math.round(Math.abs(pixels) / PIXELS_PER_SCROLL_CLICK)) +} + +/** + * Create a `ComputerDriver` for an E2B desktop sandbox. + * + * It reads the screen size once, so call it after the desktop is ready. + * + * @example + * const desktop = await Sandbox.create() + * const driver = await createE2BComputerDriver({ desktop }) + */ +export async function createE2BComputerDriver({ + desktop, +}: E2BComputerDriverOptions) { + const display = await desktop.getScreenSize() + const run = (cmd: string) => desktop.commands.run(cmd) + + const sdkClick = async (button: MouseButton, count: number) => { + const isLeft = button === 'left' + if (isLeft && count === 1) return desktop.leftClick() + if (isLeft && count === 2) return desktop.doubleClick() + if (button === 'right' && count === 1) return desktop.rightClick() + if (button === 'middle' && count === 1) return desktop.middleClick() + await run(`xdotool click --repeat ${int(count)} ${BUTTONS[button]}`) + } + + const driver: ComputerDriver = { + display, + environment: 'linux', + screenshot: () => desktop.screenshot('bytes'), + async click(x, y, { button, count }) { + // The SDK click methods skip the move when x or y is 0, so move first. + await desktop.moveMouse(int(x), int(y)) + await sdkClick(button, count) + }, + async move(x, y) { + await desktop.moveMouse(int(x), int(y)) + }, + async mouseDown(button) { + if (button === 'back' || button === 'forward') { + await run(`xdotool mousedown ${BUTTONS[button]}`) + return + } + await desktop.mousePress(button) + }, + async mouseUp(button) { + if (button === 'back' || button === 'forward') { + await run(`xdotool mouseup ${BUTTONS[button]}`) + return + } + await desktop.mouseRelease(button) + }, + async scroll(x, y, deltaX, deltaY) { + await desktop.moveMouse(int(x), int(y)) + if (deltaY !== 0) { + await desktop.scroll(deltaY > 0 ? 'down' : 'up', scrollClicks(deltaY)) + } + if (deltaX !== 0) { + // The SDK has no horizontal scroll. xdotool buttons 6 and 7 scroll left and right. + const button = deltaX > 0 ? 7 : 6 + await run(`xdotool click --repeat ${scrollClicks(deltaX)} ${button}`) + } + }, + async keyDown(key) { + await run(`xdotool keydown -- ${quote(toXdotoolKey(key))}`) + }, + async keyUp(key) { + await run(`xdotool keyup -- ${quote(toXdotoolKey(key))}`) + }, + async press(keys) { + // The SDK press puts the names in the shell raw and lowercases them, so quote them here. + await run(`xdotool key -- ${quote(keys.map(toXdotoolKey).join('+'))}`) + }, + type: (text) => desktop.write(text), + cursorPosition: () => desktop.getCursorPosition(), + } + return driver +} diff --git a/packages/ai-computer-e2b/src/index.ts b/packages/ai-computer-e2b/src/index.ts new file mode 100644 index 0000000000..abd637a598 --- /dev/null +++ b/packages/ai-computer-e2b/src/index.ts @@ -0,0 +1,5 @@ +export { + createE2BComputerDriver, + type E2BComputerDriverOptions, + type E2BDesktopLike, +} from './driver' diff --git a/packages/ai-computer-e2b/tests/driver.test.ts b/packages/ai-computer-e2b/tests/driver.test.ts new file mode 100644 index 0000000000..daa1735d13 --- /dev/null +++ b/packages/ai-computer-e2b/tests/driver.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, it } from 'vitest' +import { createE2BComputerDriver } from '../src/index' +import type { E2BDesktopLike } from '../src/index' + +const PNG = new Uint8Array([137, 80, 78, 71]) + +/** A fake desktop. Each SDK call adds one line to `calls`. */ +function fakeDesktop() { + const calls: Array = [] + const record = + (name: string) => + async (...args: Array) => { + calls.push([name, ...args].join(' ')) + } + const desktop: E2BDesktopLike = { + getScreenSize: async () => ({ width: 1024, height: 768 }), + getCursorPosition: async () => ({ x: 12, y: 34 }), + screenshot: async () => PNG, + moveMouse: record('moveMouse'), + leftClick: record('leftClick'), + doubleClick: record('doubleClick'), + rightClick: record('rightClick'), + middleClick: record('middleClick'), + mousePress: record('mousePress'), + mouseRelease: record('mouseRelease'), + scroll: record('scroll'), + write: record('write'), + commands: { run: record('run') }, + } + return { desktop, calls } +} + +async function setup() { + const { desktop, calls } = fakeDesktop() + const driver = await createE2BComputerDriver({ desktop }) + return { driver, calls } +} + +describe('createE2BComputerDriver', () => { + it('reads the display once and reports linux', async () => { + const { driver } = await setup() + expect(driver.display).toEqual({ width: 1024, height: 768 }) + expect(driver.environment).toBe('linux') + expect(driver.zoom).toBeUndefined() + }) + + it('returns the screenshot bytes and the cursor position', async () => { + const { driver } = await setup() + expect(await driver.screenshot()).toBe(PNG) + expect(await driver.cursorPosition?.()).toEqual({ x: 12, y: 34 }) + }) + + it.each([ + ['left', 1, 'leftClick'], + ['left', 2, 'doubleClick'], + ['right', 1, 'rightClick'], + ['middle', 1, 'middleClick'], + ['left', 3, 'run xdotool click --repeat 3 1'], + ['back', 1, 'run xdotool click --repeat 1 8'], + ['forward', 1, 'run xdotool click --repeat 1 9'], + ] as const)('clicks %s x%i with %s', async (button, count, expected) => { + const { driver, calls } = await setup() + await driver.click(0, 5, { button, count }) + expect(calls).toEqual(['moveMouse 0 5', expected]) + }) + + it('moves the mouse', async () => { + const { driver, calls } = await setup() + await driver.move(10, 20) + expect(calls).toEqual(['moveMouse 10 20']) + }) + + it('rejects a coordinate that is not an integer', async () => { + const { driver, calls } = await setup() + await expect(driver.move(1.5, 2)).rejects.toThrow('Expected an integer') + expect(calls).toEqual([]) + }) + + it('presses and releases buttons', async () => { + const { driver, calls } = await setup() + await driver.mouseDown('left') + await driver.mouseUp('right') + await driver.mouseDown('back') + await driver.mouseUp('forward') + expect(calls).toEqual([ + 'mousePress left', + 'mouseRelease right', + 'run xdotool mousedown 8', + 'run xdotool mouseup 9', + ]) + }) + + it('scrolls down and right in 100 px clicks', async () => { + const { driver, calls } = await setup() + await driver.scroll(5, 6, 200, 300) + expect(calls).toEqual([ + 'moveMouse 5 6', + 'scroll down 3', + 'run xdotool click --repeat 2 7', + ]) + }) + + it('scrolls up and left, at least one click', async () => { + const { driver, calls } = await setup() + await driver.scroll(5, 6, -10, -40) + expect(calls).toEqual([ + 'moveMouse 5 6', + 'scroll up 1', + 'run xdotool click --repeat 1 6', + ]) + }) + + it('maps key names to xdotool names', async () => { + const { driver, calls } = await setup() + await driver.press(['Control', 'Shift', 'ArrowLeft']) + await driver.press([' ']) + await driver.keyDown('Meta') + await driver.keyUp('Enter') + expect(calls).toEqual([ + "run xdotool key -- 'ctrl+shift+Left'", + "run xdotool key -- 'space'", + "run xdotool keydown -- 'super'", + "run xdotool keyup -- 'Return'", + ]) + }) + + it('passes an unknown key name through', async () => { + const { driver, calls } = await setup() + await driver.press(['F1']) + await driver.press(['a']) + expect(calls).toEqual(["run xdotool key -- 'F1'", "run xdotool key -- 'a'"]) + }) + + it('quotes a key name with shell characters', async () => { + const { driver, calls } = await setup() + await driver.keyDown("a'; rm -rf / #") + expect(calls).toEqual([`run xdotool keydown -- 'a'"'"'; rm -rf / #'`]) + }) + + it('types text with the SDK write', async () => { + const { driver, calls } = await setup() + await driver.type('hello') + expect(calls).toEqual(['write hello']) + }) +}) diff --git a/packages/ai-computer-e2b/tests/types.test.ts b/packages/ai-computer-e2b/tests/types.test.ts new file mode 100644 index 0000000000..130a2ad924 --- /dev/null +++ b/packages/ai-computer-e2b/tests/types.test.ts @@ -0,0 +1,9 @@ +import { describe, expectTypeOf, it } from 'vitest' +import type { Sandbox } from '@e2b/desktop' +import type { E2BDesktopLike } from '../src/index' + +describe('E2BDesktopLike', () => { + it('accepts a real @e2b/desktop Sandbox', () => { + expectTypeOf().toExtend() + }) +}) diff --git a/packages/ai-computer-e2b/tsconfig.json b/packages/ai-computer-e2b/tsconfig.json new file mode 100644 index 0000000000..c38689f4ea --- /dev/null +++ b/packages/ai-computer-e2b/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": ["src", "tests"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/ai-computer-e2b/vite.config.ts b/packages/ai-computer-e2b/vite.config.ts new file mode 100644 index 0000000000..77bcc2e60b --- /dev/null +++ b/packages/ai-computer-e2b/vite.config.ts @@ -0,0 +1,36 @@ +import { defineConfig, mergeConfig } from 'vitest/config' +import { tanstackViteConfig } from '@tanstack/vite-config' +import packageJson from './package.json' + +const config = defineConfig({ + test: { + name: packageJson.name, + dir: './', + watch: false, + globals: true, + environment: 'node', + include: ['tests/**/*.test.ts'], + coverage: { + provider: 'v8', + reporter: ['text', 'json', 'html', 'lcov'], + exclude: [ + 'node_modules/', + 'dist/', + 'tests/', + '**/*.test.ts', + '**/*.config.ts', + '**/types.ts', + ], + include: ['src/**/*.ts'], + }, + }, +}) + +export default mergeConfig( + config, + tanstackViteConfig({ + entry: ['./src/index.ts'], + srcDir: './src', + cjs: false, + }), +) diff --git a/packages/ai-computer-playwright/README.md b/packages/ai-computer-playwright/README.md new file mode 100644 index 0000000000..1707ea2d0f --- /dev/null +++ b/packages/ai-computer-playwright/README.md @@ -0,0 +1,72 @@ +
+ + + + TanStack AI + +
+ +
+ +# @tanstack/ai-computer-playwright + +Playwright browser driver for TanStack AI computer use. + +Let the model see and control one browser page. The driver takes screenshots of the page and sends mouse and keyboard input to it. + +## Installation + +```bash +pnpm add @tanstack/ai-computer-playwright @tanstack/ai-computer-use playwright +``` + +## Usage + +1. Open a page that has a viewport size. +2. Pass the page to the driver: + +```typescript +import { chromium } from 'playwright' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { createPlaywrightComputerDriver } from '@tanstack/ai-computer-playwright' + +const browser = await chromium.launch() +const page = await browser.newPage({ viewport: { width: 1280, height: 720 } }) + +const { tool, middleware, interrupt } = createComputerUse({ + driver: createPlaywrightComputerDriver({ page }), +}) +``` + +3. Give `tool`, `middleware`, and `interrupt` to `chat()`. The model can now use the page. + +The driver accepts any object with the shape of a Playwright `Page`. A page from `playwright-core` or `@playwright/test` also works. + +## What your application owns + +- **The browser.** The driver does not launch or close it. Close it when you are done. +- **The viewport.** The driver reads the viewport size once, when you create it. A page with no viewport size throws an error. + +## What the driver does + +- The `back` and `forward` mouse buttons go back and forward in the page history. +- Key names go to Playwright as they are. `['Control', 's']` becomes `Control+s`. +- `zoom` takes a screenshot of one region of the page. + +Read the full guide: [Computer use drivers](https://tanstack.com/ai/latest/docs/computer-use/drivers). + +## Tests + +```bash +pnpm --filter @tanstack/ai-computer-playwright test:lib +``` diff --git a/packages/ai-computer-playwright/package.json b/packages/ai-computer-playwright/package.json new file mode 100644 index 0000000000..a15d4cc40d --- /dev/null +++ b/packages/ai-computer-playwright/package.json @@ -0,0 +1,55 @@ +{ + "name": "@tanstack/ai-computer-playwright", + "version": "0.0.0", + "description": "Playwright browser driver for TanStack AI computer use.", + "author": "", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/TanStack/ai.git", + "directory": "packages/ai-computer-playwright" + }, + "type": "module", + "module": "./dist/esm/index.js", + "types": "./dist/esm/index.d.ts", + "exports": { + ".": { + "types": "./dist/esm/index.d.ts", + "import": "./dist/esm/index.js" + } + }, + "sideEffects": false, + "engines": { + "node": ">=18" + }, + "files": [ + "dist", + "src" + ], + "scripts": { + "build": "vite build", + "clean": "premove ./build ./dist", + "lint:fix": "oxlint src --type-aware --fix", + "test:build": "publint --strict", + "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:lib": "vitest --passWithNoTests", + "test:lib:dev": "pnpm test:lib --watch", + "test:types": "tsc" + }, + "keywords": [ + "ai", + "tanstack", + "computer-use", + "playwright", + "browser" + ], + "peerDependencies": { + "@tanstack/ai-computer-use": "workspace:^" + }, + "devDependencies": { + "@tanstack/ai-computer-use": "workspace:*", + "@vitest/coverage-v8": "4.1.10", + "playwright-core": "^1.57.0" + } +} diff --git a/packages/ai-computer-playwright/src/driver.ts b/packages/ai-computer-playwright/src/driver.ts new file mode 100644 index 0000000000..f08f6aa67f --- /dev/null +++ b/packages/ai-computer-playwright/src/driver.ts @@ -0,0 +1,93 @@ +import type { ComputerDriver, MouseButton } from '@tanstack/ai-computer-use' + +type PageMouseButton = 'left' | 'right' | 'middle' + +/** The slice of a Playwright `Page` the driver uses. */ +export interface PlaywrightPageLike { + viewportSize: () => { width: number; height: number } | null + screenshot: (options: { + type: 'png' + clip?: { x: number; y: number; width: number; height: number } + }) => Promise + mouse: { + move: (x: number, y: number) => Promise + down: (options: { button: PageMouseButton }) => Promise + up: (options: { button: PageMouseButton }) => Promise + click: ( + x: number, + y: number, + options: { button: PageMouseButton; clickCount: number }, + ) => Promise + wheel: (deltaX: number, deltaY: number) => Promise + } + keyboard: { + down: (key: string) => Promise + up: (key: string) => Promise + press: (key: string) => Promise + type: (text: string) => Promise + } + goBack: () => Promise + goForward: () => Promise +} + +export interface PlaywrightComputerDriverOptions { + /** The page to control. It must have a viewport size. */ + page: PlaywrightPageLike +} + +/** + * Create a computer driver that controls one Playwright page. + * + * The driver reads the viewport size once, at create. Key names go to + * Playwright as they are, because Playwright uses `KeyboardEvent.key` names. + * + * @example + * const driver = createPlaywrightComputerDriver({ page }) + * + * @throws Error when the page has no viewport size. + */ +export function createPlaywrightComputerDriver({ + page, +}: PlaywrightComputerDriverOptions) { + const display = page.viewportSize() + if (display === null) { + throw new Error( + 'The page has no viewport size. Set a viewport on the browser context.', + ) + } + + async function navigate(button: 'back' | 'forward') { + if (button === 'back') await page.goBack() + else await page.goForward() + } + + const driver: ComputerDriver = { + display, + environment: 'browser', + screenshot: () => page.screenshot({ type: 'png' }), + async click(x, y, { button, count }) { + if (button === 'back' || button === 'forward') return navigate(button) + await page.mouse.click(x, y, { button, clickCount: count }) + }, + move: (x, y) => page.mouse.move(x, y), + async mouseDown(button: MouseButton) { + // ponytail: back and forward navigate once, on mouseUp. mouseDown does nothing for them. + if (button === 'back' || button === 'forward') return + await page.mouse.down({ button }) + }, + async mouseUp(button: MouseButton) { + if (button === 'back' || button === 'forward') return navigate(button) + await page.mouse.up({ button }) + }, + async scroll(x, y, deltaX, deltaY) { + await page.mouse.move(x, y) + await page.mouse.wheel(deltaX, deltaY) + }, + keyDown: (key) => page.keyboard.down(key), + keyUp: (key) => page.keyboard.up(key), + press: (keys) => page.keyboard.press(keys.join('+')), + type: (text) => page.keyboard.type(text), + zoom: (region) => page.screenshot({ type: 'png', clip: region }), + } + return driver +} diff --git a/packages/ai-computer-playwright/src/index.ts b/packages/ai-computer-playwright/src/index.ts new file mode 100644 index 0000000000..251ffb19a3 --- /dev/null +++ b/packages/ai-computer-playwright/src/index.ts @@ -0,0 +1,5 @@ +export { createPlaywrightComputerDriver } from './driver' +export type { + PlaywrightComputerDriverOptions, + PlaywrightPageLike, +} from './driver' diff --git a/packages/ai-computer-playwright/tests/driver.test.ts b/packages/ai-computer-playwright/tests/driver.test.ts new file mode 100644 index 0000000000..b64e85b1b2 --- /dev/null +++ b/packages/ai-computer-playwright/tests/driver.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from 'vitest' +import { createPlaywrightComputerDriver } from '../src/index' +import type { PlaywrightPageLike } from '../src/index' + +const PNG = new Uint8Array([137, 80, 78, 71]) + +function createFakePage( + viewport: { width: number; height: number } | null = { + width: 1280, + height: 720, + }, +) { + const calls: Array<[string, ...Array]> = [] + const record = + (name: string) => + async (...args: Array) => { + calls.push([name, ...args]) + } + const page: PlaywrightPageLike = { + viewportSize: () => viewport, + screenshot: async (options) => { + calls.push(['screenshot', options]) + return PNG + }, + mouse: { + move: record('mouse.move'), + down: record('mouse.down'), + up: record('mouse.up'), + click: record('mouse.click'), + wheel: record('mouse.wheel'), + }, + keyboard: { + down: record('keyboard.down'), + up: record('keyboard.up'), + press: record('keyboard.press'), + type: record('keyboard.type'), + }, + goBack: record('goBack'), + goForward: record('goForward'), + } + return { page, calls } +} + +function setup() { + const { page, calls } = createFakePage() + return { driver: createPlaywrightComputerDriver({ page }), calls } +} + +describe('createPlaywrightComputerDriver', () => { + it('reads display from the viewport and reports a browser', () => { + const { driver } = setup() + expect(driver.display).toEqual({ width: 1280, height: 720 }) + expect(driver.environment).toBe('browser') + }) + + it('throws at create when the page has no viewport', () => { + const { page } = createFakePage(null) + expect(() => createPlaywrightComputerDriver({ page })).toThrow( + 'The page has no viewport size. Set a viewport on the browser context.', + ) + }) + + it('takes a PNG screenshot', async () => { + const { driver, calls } = setup() + expect(await driver.screenshot()).toEqual(PNG) + expect(calls).toEqual([['screenshot', { type: 'png' }]]) + }) + + it('zooms with a clipped screenshot', async () => { + const { driver, calls } = setup() + const region = { x: 10, y: 20, width: 100, height: 50 } + expect(await driver.zoom?.(region)).toEqual(PNG) + expect(calls).toEqual([['screenshot', { type: 'png', clip: region }]]) + }) + + it('clicks with the button and the click count', async () => { + const { driver, calls } = setup() + await driver.click(5, 6, { button: 'right', count: 2 }) + expect(calls).toEqual([ + ['mouse.click', 5, 6, { button: 'right', clickCount: 2 }], + ]) + }) + + it('navigates back and forward on a click', async () => { + const { driver, calls } = setup() + await driver.click(1, 1, { button: 'back', count: 1 }) + await driver.click(1, 1, { button: 'forward', count: 1 }) + expect(calls).toEqual([['goBack'], ['goForward']]) + }) + + it('moves, presses, and releases the mouse', async () => { + const { driver, calls } = setup() + await driver.move(3, 4) + await driver.mouseDown('left') + await driver.mouseUp('middle') + expect(calls).toEqual([ + ['mouse.move', 3, 4], + ['mouse.down', { button: 'left' }], + ['mouse.up', { button: 'middle' }], + ]) + }) + + it('navigates once for a back press and release', async () => { + const { driver, calls } = setup() + await driver.mouseDown('back') + await driver.mouseUp('back') + expect(calls).toEqual([['goBack']]) + }) + + it('moves to the point and then scrolls', async () => { + const { driver, calls } = setup() + await driver.scroll(50, 60, 0, 300) + expect(calls).toEqual([ + ['mouse.move', 50, 60], + ['mouse.wheel', 0, 300], + ]) + }) + + it('sends keys and text to the keyboard', async () => { + const { driver, calls } = setup() + await driver.keyDown('Shift') + await driver.keyUp('Shift') + await driver.press(['Control', 's']) + await driver.type('hello') + expect(calls).toEqual([ + ['keyboard.down', 'Shift'], + ['keyboard.up', 'Shift'], + ['keyboard.press', 'Control+s'], + ['keyboard.type', 'hello'], + ]) + }) +}) diff --git a/packages/ai-computer-playwright/tests/types.test.ts b/packages/ai-computer-playwright/tests/types.test.ts new file mode 100644 index 0000000000..d7c778e232 --- /dev/null +++ b/packages/ai-computer-playwright/tests/types.test.ts @@ -0,0 +1,9 @@ +import { describe, expectTypeOf, it } from 'vitest' +import type { Page } from 'playwright-core' +import type { PlaywrightPageLike } from '../src/index' + +describe('PlaywrightPageLike', () => { + it('accepts a real playwright-core Page', () => { + expectTypeOf().toExtend() + }) +}) diff --git a/packages/ai-computer-playwright/tsconfig.json b/packages/ai-computer-playwright/tsconfig.json new file mode 100644 index 0000000000..c38689f4ea --- /dev/null +++ b/packages/ai-computer-playwright/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": ["src", "tests"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/ai-computer-playwright/vite.config.ts b/packages/ai-computer-playwright/vite.config.ts new file mode 100644 index 0000000000..77bcc2e60b --- /dev/null +++ b/packages/ai-computer-playwright/vite.config.ts @@ -0,0 +1,36 @@ +import { defineConfig, mergeConfig } from 'vitest/config' +import { tanstackViteConfig } from '@tanstack/vite-config' +import packageJson from './package.json' + +const config = defineConfig({ + test: { + name: packageJson.name, + dir: './', + watch: false, + globals: true, + environment: 'node', + include: ['tests/**/*.test.ts'], + coverage: { + provider: 'v8', + reporter: ['text', 'json', 'html', 'lcov'], + exclude: [ + 'node_modules/', + 'dist/', + 'tests/', + '**/*.test.ts', + '**/*.config.ts', + '**/types.ts', + ], + include: ['src/**/*.ts'], + }, + }, +}) + +export default mergeConfig( + config, + tanstackViteConfig({ + entry: ['./src/index.ts'], + srcDir: './src', + cjs: false, + }), +) diff --git a/packages/ai-computer-use/README.md b/packages/ai-computer-use/README.md new file mode 100644 index 0000000000..3f824b45d4 --- /dev/null +++ b/packages/ai-computer-use/README.md @@ -0,0 +1,91 @@ +
+ + + + TanStack AI + +
+ +
+ +# @tanstack/ai-computer-use + +Computer use for TanStack AI. The model sees a screen and uses a mouse and a keyboard. + +You write the code one time. The same code runs on OpenAI and on Anthropic. A driver controls the machine: a Playwright page, an E2B desktop, or your own. + +## Installation + +```bash +pnpm add @tanstack/ai-computer-use zod +``` + +You also need a driver: + +```bash +# A browser page +pnpm add @tanstack/ai-computer-playwright + +# A Linux desktop in an E2B sandbox +pnpm add @tanstack/ai-computer-e2b +``` + +## Usage + +1. Create the tool, the middleware, and the interrupt from a driver. +2. Give all three to `chat()`: + +```typescript +import { chat } from '@tanstack/ai' +import { openaiText } from '@tanstack/ai-openai' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { createPlaywrightComputerDriver } from '@tanstack/ai-computer-playwright' + +const { tool, middleware, interrupt } = createComputerUse({ + driver: createPlaywrightComputerDriver({ page }), +}) + +const stream = chat({ + adapter: openaiText('gpt-6.1-sol'), + messages, + tools: [tool], + middleware: [middleware], + interrupts: [interrupt], +}) +``` + +3. On the client, register the safety-check interrupt: + +```typescript +import { computerSafetyCheck } from '@tanstack/ai-computer-use' + +useChat({ connection, interrupts: [computerSafetyCheck] }) +``` + +`computerSafetyCheck` is the same object as `interrupt`. The client can import it without the driver. + +## What it does + +- Every action returns a screenshot. The screenshot comes 500 ms after the action. Set `screenshotDelay` to change it. +- Actions run one at a time, in call order. +- When an action fails, the other actions of the same model turn do not run. +- OpenAI can attach safety checks to an action. The action waits until a person approves it. When the person declines, the action does not run. + +## What your application owns + +- **The machine.** The driver does not start or stop the browser or the sandbox. +- **One conversation at a time.** Create one `createComputerUse` for each conversation. + +## Docs + +See [Computer Use](https://tanstack.com/ai/latest/docs/computer-use/computer-use). diff --git a/packages/ai-computer-use/package.json b/packages/ai-computer-use/package.json new file mode 100644 index 0000000000..3f0d0fddba --- /dev/null +++ b/packages/ai-computer-use/package.json @@ -0,0 +1,58 @@ +{ + "name": "@tanstack/ai-computer-use", + "version": "0.0.0", + "description": "Provider-agnostic computer use for TanStack AI.", + "author": "", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/TanStack/ai.git", + "directory": "packages/ai-computer-use" + }, + "type": "module", + "module": "./dist/esm/index.js", + "types": "./dist/esm/index.d.ts", + "exports": { + ".": { + "types": "./dist/esm/index.d.ts", + "import": "./dist/esm/index.js" + } + }, + "sideEffects": false, + "engines": { + "node": ">=18" + }, + "files": [ + "dist", + "src" + ], + "scripts": { + "build": "vite build", + "clean": "premove ./build ./dist", + "lint:fix": "oxlint src --type-aware --fix", + "test:build": "publint --strict", + "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:lib": "vitest --passWithNoTests", + "test:lib:dev": "pnpm test:lib --watch", + "test:types": "tsc" + }, + "keywords": [ + "ai", + "tanstack", + "computer-use", + "agent" + ], + "dependencies": { + "@tanstack/ai-utils": "workspace:^" + }, + "peerDependencies": { + "@tanstack/ai": "workspace:^", + "zod": "^3.0.0 || ^4.0.0" + }, + "devDependencies": { + "@tanstack/ai": "workspace:*", + "@vitest/coverage-v8": "4.1.10", + "zod": "^4.2.0" + } +} diff --git a/packages/ai-computer-use/src/actions.ts b/packages/ai-computer-use/src/actions.ts new file mode 100644 index 0000000000..00822d54d3 --- /dev/null +++ b/packages/ai-computer-use/src/actions.ts @@ -0,0 +1,372 @@ +import { parseKeyCombo, toCombo, toKey } from './keys' +import type { ComputerDriver, MouseButton } from './types' + +/** One Anthropic `scroll_amount` step, in pixels. */ +const SCROLL_STEP = 100 + +// ponytail: an OpenAI `wait` has no duration. 2 s is the wait in the OpenAI samples. +const OPENAI_WAIT_MS = 2000 + +type Point = { x: number; y: number } +type Input = Record + +function isRecord(value: unknown): value is Input { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +function invalid(key: string, expected: string) { + return new Error(`Invalid computer action: "${key}" must be ${expected}.`) +} + +function readNumber(input: Input, key: string) { + const value = input[key] + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw invalid(key, 'a number') + } + return value +} + +function readString(input: Input, key: string) { + const value = input[key] + if (typeof value !== 'string') throw invalid(key, 'a string') + return value +} + +function readOptionalString(input: Input, key: string) { + const value = input[key] + if (value === undefined || value === null) return undefined + if (typeof value !== 'string') throw invalid(key, 'a string') + return value +} + +function readNumbers(input: Input, key: string, length: number) { + const value = input[key] + const isValid = + Array.isArray(value) && + value.length === length && + value.every((item) => typeof item === 'number' && Number.isFinite(item)) + if (!isValid) throw invalid(key, `an array of ${length} numbers`) + return value.filter((item) => typeof item === 'number') +} + +function readCoordinate(input: Input, key: string) { + const [x = 0, y = 0] = readNumbers(input, key, 2) + return { x, y } +} + +/** Presses `keys` in order, runs `action`, then releases them in reverse. */ +async function withKeys( + driver: ComputerDriver, + keys: Array, + action: () => Promise, +) { + for (const key of keys) await driver.keyDown(key) + try { + await action() + } finally { + const releaseOrder = [...keys].reverse() + for (const key of releaseOrder) await driver.keyUp(key) + } +} + +async function drag(driver: ComputerDriver, path: Array) { + const [start, ...rest] = path + if (start === undefined) throw invalid('path', 'a non-empty array') + await driver.move(start.x, start.y) + await driver.mouseDown('left') + for (const point of rest) await driver.move(point.x, point.y) + await driver.mouseUp('left') +} + +// --------------------------------------------------------------------------- +// Anthropic: `{ action: 'left_click', coordinate: [x, y], ... }` +// --------------------------------------------------------------------------- + +const ANTHROPIC_CLICKS: Record = + { + left_click: { button: 'left', count: 1 }, + right_click: { button: 'right', count: 1 }, + middle_click: { button: 'middle', count: 1 }, + double_click: { button: 'left', count: 2 }, + triple_click: { button: 'left', count: 3 }, + } + +/** The `coordinate` of the action, or the mouse position when it has none. */ +async function anthropicPoint(driver: ComputerDriver, input: Input) { + if (input['coordinate'] !== undefined && input['coordinate'] !== null) { + return readCoordinate(input, 'coordinate') + } + if (!driver.cursorPosition) { + throw new Error( + 'Invalid computer action: "coordinate" is missing, and the computer driver has no cursorPosition method.', + ) + } + return driver.cursorPosition() +} + +function anthropicModifiers(input: Input) { + const text = readOptionalString(input, 'text') + return text ? parseKeyCombo(text) : [] +} + +function scrollDelta(direction: string, pixels: number) { + switch (direction) { + case 'up': + return { deltaX: 0, deltaY: -pixels } + case 'down': + return { deltaX: 0, deltaY: pixels } + case 'left': + return { deltaX: -pixels, deltaY: 0 } + case 'right': + return { deltaX: pixels, deltaY: 0 } + default: + throw invalid('scroll_direction', '"up", "down", "left", or "right"') + } +} + +async function runAnthropicAction( + driver: ComputerDriver, + action: string, + input: Input, +) { + const click = ANTHROPIC_CLICKS[action] + if (click) { + const { x, y } = await anthropicPoint(driver, input) + await withKeys(driver, anthropicModifiers(input), () => + driver.click(x, y, click), + ) + return undefined + } + + switch (action) { + case 'screenshot': + return undefined + case 'key': { + const combos = readString(input, 'text').split(/\s+/).filter(Boolean) + const hasRepeat = + input['repeat'] !== undefined && input['repeat'] !== null + const repeat = hasRepeat ? readNumber(input, 'repeat') : 1 + for (let i = 0; i < repeat; i++) { + for (const combo of combos) await driver.press(parseKeyCombo(combo)) + } + return undefined + } + case 'type': + await driver.type(readString(input, 'text')) + return undefined + case 'mouse_move': { + const { x, y } = readCoordinate(input, 'coordinate') + await driver.move(x, y) + return undefined + } + case 'left_click_drag': { + const path = [ + readCoordinate(input, 'start_coordinate'), + readCoordinate(input, 'coordinate'), + ] + await withKeys(driver, anthropicModifiers(input), () => + drag(driver, path), + ) + return undefined + } + case 'left_mouse_down': + await driver.mouseDown('left') + return undefined + case 'left_mouse_up': + await driver.mouseUp('left') + return undefined + case 'scroll': { + const { x, y } = await anthropicPoint(driver, input) + const pixels = readNumber(input, 'scroll_amount') * SCROLL_STEP + const direction = readString(input, 'scroll_direction') + const { deltaX, deltaY } = scrollDelta(direction, pixels) + await withKeys(driver, anthropicModifiers(input), () => + driver.scroll(x, y, deltaX, deltaY), + ) + return undefined + } + case 'hold_key': { + const keys = parseKeyCombo(readString(input, 'text')) + const ms = readNumber(input, 'duration') * 1000 + await withKeys(driver, keys, () => sleep(ms)) + return undefined + } + case 'wait': + await sleep(readNumber(input, 'duration') * 1000) + return undefined + case 'cursor_position': { + if (!driver.cursorPosition) { + throw new Error( + 'The computer driver has no cursorPosition method, so the "cursor_position" action is not available.', + ) + } + const { x, y } = await driver.cursorPosition() + return { type: 'text' as const, text: `X=${x},Y=${y}` } + } + case 'zoom': { + if (!driver.zoom) { + throw new Error( + 'The computer driver has no zoom method, so the "zoom" action is not available.', + ) + } + const [x0 = 0, y0 = 0, x1 = 0, y1 = 0] = readNumbers(input, 'region', 4) + const bytes = await driver.zoom({ + x: x0, + y: y0, + width: x1 - x0, + height: y1 - y0, + }) + return { type: 'image' as const, bytes } + } + default: + throw new Error(`Unknown computer action "${action}".`) + } +} + +// --------------------------------------------------------------------------- +// OpenAI: `{ action: { type: 'click', ... } }` or `{ actions: [...] }` +// --------------------------------------------------------------------------- + +const OPENAI_BUTTONS: Record = { + left: 'left', + right: 'right', + wheel: 'middle', + back: 'back', + forward: 'forward', +} + +function openaiModifiers(input: Input) { + const keys = input['keys'] + if (keys === undefined || keys === null) return [] + const isValid = + Array.isArray(keys) && keys.every((key) => typeof key === 'string') + if (!isValid) throw invalid('keys', 'an array of strings') + return keys.filter((key) => typeof key === 'string').map(toKey) +} + +function readPath(input: Input) { + const path = input['path'] + if (!Array.isArray(path)) throw invalid('path', 'an array of points') + return path.map((point) => { + if (!isRecord(point)) throw invalid('path', 'an array of points') + return { x: readNumber(point, 'x'), y: readNumber(point, 'y') } + }) +} + +async function runOpenAIAction(driver: ComputerDriver, input: unknown) { + if (!isRecord(input)) throw invalid('action', 'an object') + const type = readString(input, 'type') + const modifiers = openaiModifiers(input) + + switch (type) { + case 'screenshot': + return + case 'click': { + const button = OPENAI_BUTTONS[readString(input, 'button')] + if (!button) { + throw invalid( + 'button', + '"left", "right", "wheel", "back", or "forward"', + ) + } + const x = readNumber(input, 'x') + const y = readNumber(input, 'y') + await withKeys(driver, modifiers, () => + driver.click(x, y, { button, count: 1 }), + ) + return + } + case 'double_click': { + const x = readNumber(input, 'x') + const y = readNumber(input, 'y') + await withKeys(driver, modifiers, () => + driver.click(x, y, { button: 'left', count: 2 }), + ) + return + } + case 'drag': { + const path = readPath(input) + await withKeys(driver, modifiers, () => drag(driver, path)) + return + } + case 'keypress': { + const keys = input['keys'] + const isValid = + Array.isArray(keys) && + keys.length > 0 && + keys.every((key) => typeof key === 'string') + if (!isValid) throw invalid('keys', 'a non-empty array of strings') + await driver.press(toCombo(keys.filter((key) => typeof key === 'string'))) + return + } + case 'move': { + const x = readNumber(input, 'x') + const y = readNumber(input, 'y') + await withKeys(driver, modifiers, () => driver.move(x, y)) + return + } + case 'scroll': { + const x = readNumber(input, 'x') + const y = readNumber(input, 'y') + const deltaX = readNumber(input, 'scroll_x') + const deltaY = readNumber(input, 'scroll_y') + await withKeys(driver, modifiers, () => + driver.scroll(x, y, deltaX, deltaY), + ) + return + } + case 'type': + await driver.type(readString(input, 'text')) + return + case 'wait': + await sleep(OPENAI_WAIT_MS) + return + default: + throw new Error(`Unknown computer action "${type}".`) + } +} + +/** + * Runs one computer tool call from the model on a driver. + * + * The input shape picks the provider: + * - A string `action` is an Anthropic action. + * - An object `action` or an `actions` array is an OpenAI call. A batch runs + * in order. + * + * The result is `undefined` when the action has no output of its own. Then + * the caller takes a screenshot. `zoom` gives `{ type: 'image', bytes }` and + * `cursor_position` gives `{ type: 'text', text }`. + * + * Throws an `Error` for an input that is not valid, an unknown action, or a + * `zoom` or `cursor_position` action when the driver has no such method. + * + * @param driver - The driver that controls the computer. + * @param input - The raw tool input from the model. + */ +export async function runComputerAction( + driver: ComputerDriver, + input: unknown, +) { + if (!isRecord(input)) throw invalid('input', 'an object') + const { action, actions } = input + if (typeof action === 'string') { + return runAnthropicAction(driver, action, input) + } + if (Array.isArray(actions)) { + for (const item of actions) await runOpenAIAction(driver, item) + return undefined + } + if (isRecord(action)) { + await runOpenAIAction(driver, action) + return undefined + } + throw invalid('action', 'a string, an object, or an "actions" array') +} + +/** The result of `runComputerAction`. */ +export type ComputerActionResult = Awaited> diff --git a/packages/ai-computer-use/src/create-computer-use.ts b/packages/ai-computer-use/src/create-computer-use.ts new file mode 100644 index 0000000000..a224a7a523 --- /dev/null +++ b/packages/ai-computer-use/src/create-computer-use.ts @@ -0,0 +1,219 @@ +import { brandProviderTool } from '@tanstack/ai' +import { arrayBufferToBase64 } from '@tanstack/ai-utils' +import { z } from 'zod' +import { runComputerAction } from './actions' +import { computerSafetyCheck, safetyCheckSchema } from './safety-check' +import type { + ChatMiddleware, + ChatMiddlewareContext, + ImagePart, + ProviderTool, + ToolExecutionContext, +} from '@tanstack/ai' +import type { ComputerUseToolMetadata } from '@tanstack/ai/adapter-internals' +import type { ComputerDriver } from './types' + +/** Options for `createComputerUse`. */ +export interface CreateComputerUseOptions { + /** The driver that controls the computer. */ + driver: ComputerDriver + /** + * The time in ms between an action and its screenshot. The page or the + * desktop gets this time to settle. The default is 500. + */ + screenshotDelay?: number +} + +const TOOL_NAME = 'computer' + +/** The parts of a computer tool input that the safety check reads. */ +const checkedInputSchema = z.object({ + pending_safety_checks: z.array(safetyCheckSchema).nullish(), + action: z.unknown().optional(), + actions: z.unknown().optional(), +}) + +/** The safety checks and the action of an input, or `null` when they do not parse. */ +function readCheckedInput(input: unknown) { + const parsed = checkedInputSchema.safeParse(input) + if (!parsed.success) return null + const { action, actions } = parsed.data + const checks = parsed.data.pending_safety_checks ?? [] + return { checks, action: action ?? actions ?? null } +} + +function parseArguments(text: string) { + try { + return JSON.parse(text) + } catch { + return undefined + } +} + +function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +function imageResult(bytes: Uint8Array) { + const part: ImagePart = { + type: 'image', + source: { + type: 'data', + value: arrayBufferToBase64(bytes.slice().buffer), + mimeType: 'image/png', + }, + } + return [part] +} + +/** The computer tool calls in the last assistant message. */ +function computerToolCalls(ctx: ChatMiddlewareContext) { + const message = ctx.messages.findLast((item) => item.role === 'assistant') + const toolCalls = message?.toolCalls ?? [] + return toolCalls.filter((call) => call.function.name === TOOL_NAME) +} + +/** + * Create a provider-agnostic computer tool for one machine. + * + * The OpenAI and Anthropic adapters swap `tool` for their native computer + * tool. Every action returns a screenshot. `zoom` returns the zoom image, and + * `cursor_position` returns text. + * + * - `tool`: pass it to `chat({ tools })`. + * - `middleware`: pass it to `chat({ middleware })`. It runs the failed-turn + * rule and the OpenAI safety checks. + * - `interrupt`: pass it to `chat({ interrupts })`. It is the same object as + * `computerSafetyCheck`. + * + * Actions run one at a time, in call order. After an action throws, the + * other calls of the same model turn throw too. A call with OpenAI safety + * checks runs only after a person approves it. + * + * @param options - The driver, and the optional `screenshotDelay` in ms. + * + * @example + * ```ts + * const { tool, middleware, interrupt } = createComputerUse({ + * driver: createPlaywrightComputerDriver({ page }), + * }) + * + * chat({ + * adapter: openaiText('gpt-6.1-sol'), + * messages, + * tools: [tool], + * middleware: [middleware], + * interrupts: [interrupt], + * }) + * ``` + */ +export function createComputerUse({ + driver, + screenshotDelay = 500, +}: CreateComputerUseOptions): { + tool: ProviderTool + // The .d.ts names typeof computerSafetyCheck here, not a copy of its type. + // TypeScript 5 sees two copies as different types, and chat() rejects them. + middleware: ChatMiddleware + interrupt: typeof computerSafetyCheck +} { + // ponytail: one createComputerUse drives one machine for one conversation + // at a time. The queue, the failed-turn flag, and the approvals are shared + // by every chat() call that uses it. Create one per conversation. + let queue: Promise = Promise.resolve() + let turnFailed = false + const approved = new Set() + const declined = new Set() + + async function runAction(input: unknown, toolCallId: string | undefined) { + if (turnFailed) { + throw new Error( + 'Not executed: an earlier computer action in this turn failed.', + ) + } + try { + const checked = readCheckedInput(input) + // Fail closed: checks that do not parse also need an approval. + const hasChecks = checked === null || checked.checks.length > 0 + const isApproved = toolCallId !== undefined && approved.has(toolCallId) + if (hasChecks && !isApproved) { + throw new Error( + 'Refused: this computer action has safety checks that nobody approved. Add the `interrupt` from createComputerUse to chat({ interrupts }) and the `middleware` to chat({ middleware }).', + ) + } + const result = await runComputerAction(driver, input) + if (result === undefined) { + await sleep(screenshotDelay) + return imageResult(await driver.screenshot()) + } + return result.type === 'image' ? imageResult(result.bytes) : result.text + } catch (error) { + turnFailed = true + throw error + } + } + + const metadata = { + __kind: 'computer_use', + display: driver.display, + environment: driver.environment, + capabilities: { + zoom: typeof driver.zoom === 'function', + cursorPosition: typeof driver.cursorPosition === 'function', + }, + } satisfies ComputerUseToolMetadata + + const tool = brandProviderTool>({ + name: TOOL_NAME, + description: 'Use a computer with a mouse and a keyboard.', + metadata, + execute(input: unknown, context?: ToolExecutionContext) { + const run = queue.then(() => runAction(input, context?.toolCallId)) + queue = run.catch(() => undefined) + return run + }, + }) + + const middleware: ChatMiddleware = { + name: 'computer-use', + onConfig(ctx) { + if (ctx.phase === 'beforeModel') turnFailed = false + }, + onInterruptBoundary(ctx) { + if (ctx.phase !== 'beforeTools') return undefined + const interrupts = computerToolCalls(ctx).flatMap((call) => { + const hasDecision = approved.has(call.id) || declined.has(call.id) + if (hasDecision) return [] + const checked = readCheckedInput( + parseArguments(call.function.arguments), + ) + // The tool refuses a call whose checks do not parse. + if (checked === null || checked.checks.length === 0) return [] + const { checks, action } = checked + return [ + computerSafetyCheck.interrupt({ + key: call.id, + reason: 'computer-safety-check', + message: + checks.find((check) => check.message)?.message ?? + 'Approve this computer action.', + payload: { checks, action }, + }), + ] + }) + return interrupts.length > 0 ? { interrupts } : undefined + }, + onInterruptResolution(_ctx, resolutions) { + const results = resolutions.for(computerSafetyCheck) + if (results.length === 0) return undefined + const isAllApproved = results.every( + (result) => result.status === 'resolved' && result.response.approved, + ) + const decided = isAllApproved ? approved : declined + for (const result of results) decided.add(result.request.key) + return { toolResume: isAllApproved ? 'continue' : 'cancel' } + }, + } + + return { tool, middleware, interrupt: computerSafetyCheck } +} diff --git a/packages/ai-computer-use/src/index.ts b/packages/ai-computer-use/src/index.ts new file mode 100644 index 0000000000..d8d17a5515 --- /dev/null +++ b/packages/ai-computer-use/src/index.ts @@ -0,0 +1,4 @@ +export { createComputerUse } from './create-computer-use' +export type { CreateComputerUseOptions } from './create-computer-use' +export { computerSafetyCheck } from './safety-check' +export type { ComputerDriver, MouseButton } from './types' diff --git a/packages/ai-computer-use/src/keys.ts b/packages/ai-computer-use/src/keys.ts new file mode 100644 index 0000000000..bf5a54b73c --- /dev/null +++ b/packages/ai-computer-use/src/keys.ts @@ -0,0 +1,84 @@ +/** + * OpenAI key names and Anthropic xdotool key names, lower case, to + * `KeyboardEvent.key` names. + */ +const KEYS: Record = { + ctrl: 'Control', + control: 'Control', + control_l: 'Control', + control_r: 'Control', + shift: 'Shift', + shift_l: 'Shift', + shift_r: 'Shift', + alt: 'Alt', + alt_l: 'Alt', + alt_r: 'Alt', + option: 'Alt', + meta: 'Meta', + meta_l: 'Meta', + meta_r: 'Meta', + cmd: 'Meta', + command: 'Meta', + super: 'Meta', + super_l: 'Meta', + super_r: 'Meta', + win: 'Meta', + enter: 'Enter', + return: 'Enter', + kp_enter: 'Enter', + esc: 'Escape', + escape: 'Escape', + tab: 'Tab', + space: ' ', + backspace: 'Backspace', + delete: 'Delete', + del: 'Delete', + insert: 'Insert', + home: 'Home', + end: 'End', + pageup: 'PageUp', + page_up: 'PageUp', + prior: 'PageUp', + pagedown: 'PageDown', + page_down: 'PageDown', + next: 'PageDown', + up: 'ArrowUp', + down: 'ArrowDown', + left: 'ArrowLeft', + right: 'ArrowRight', + arrowup: 'ArrowUp', + arrowdown: 'ArrowDown', + arrowleft: 'ArrowLeft', + arrowright: 'ArrowRight', + capslock: 'CapsLock', + caps_lock: 'CapsLock', + plus: '+', + minus: '-', +} + +/** + * Maps one OpenAI or xdotool key name to a `KeyboardEvent.key` name. + * The match ignores case. An unknown name comes back unchanged. + */ +export function toKey(name: string) { + const known = KEYS[name.toLowerCase()] + if (known !== undefined) return known + const isFunctionKey = /^f\d{1,2}$/i.test(name) + return isFunctionKey ? name.toUpperCase() : name +} + +/** + * Maps the keys of one combo, like `['CTRL', 'S']`, to `KeyboardEvent.key` + * names. In a combo with more than one key, a single letter becomes lower + * case. Then `Control` + `A` does not also send a capital `A`. + */ +export function toCombo(names: Array) { + const keys = names.map(toKey) + if (keys.length < 2) return keys + return keys.map((key) => (/^[A-Z]$/.test(key) ? key.toLowerCase() : key)) +} + +/** Splits an xdotool combo, like `ctrl+s`, and maps each key. */ +export function parseKeyCombo(combo: string) { + return toCombo(combo.split('+').map((name) => name.trim())) +} diff --git a/packages/ai-computer-use/src/safety-check.ts b/packages/ai-computer-use/src/safety-check.ts new file mode 100644 index 0000000000..1ffc941ffe --- /dev/null +++ b/packages/ai-computer-use/src/safety-check.ts @@ -0,0 +1,29 @@ +import { defineInterrupt } from '@tanstack/ai' +import { z } from 'zod' + +/** One OpenAI pending safety check on a computer call. */ +export const safetyCheckSchema = z.object({ + id: z.string(), + code: z.string().nullish(), + message: z.string().nullish(), +}) + +/** + * The interrupt that asks a person to approve a computer action with OpenAI + * safety checks. The interrupt key is the tool call id. + * + * - Payload: `{ checks, action }`. `checks` are the pending safety checks. + * `action` is the action (or the list of actions) that waits. + * - Response: `{ approved: boolean }`. + * + * Register it on the client with `useChat({ interrupts: [computerSafetyCheck] })`. + * It is the same object as the `interrupt` from `createComputerUse`. + */ +export const computerSafetyCheck = defineInterrupt({ + id: 'computer-safety-check', + payloadSchema: z.object({ + checks: z.array(safetyCheckSchema), + action: z.unknown(), + }), + responseSchema: z.object({ approved: z.boolean() }), +}) diff --git a/packages/ai-computer-use/src/types.ts b/packages/ai-computer-use/src/types.ts new file mode 100644 index 0000000000..4ea1650238 --- /dev/null +++ b/packages/ai-computer-use/src/types.ts @@ -0,0 +1,53 @@ +/** A mouse button a driver can press. */ +export type MouseButton = 'left' | 'right' | 'middle' | 'back' | 'forward' + +/** + * Controls one computer: a browser page, a desktop VM, or a sandbox. + * + * Coordinates are in pixels in `display` space. Key names are + * `KeyboardEvent.key` names, for example `'Enter'`, `'Control'`, or `'a'`. + */ +export interface ComputerDriver { + /** The screen size in pixels. Screenshots have this size. */ + display: { width: number; height: number } + /** The kind of computer the driver controls. */ + environment: 'browser' | 'linux' | 'mac' | 'windows' + /** Take a screenshot. Returns PNG bytes. */ + screenshot: () => Promise + /** Click `count` times at the point with the given button. */ + click: ( + x: number, + y: number, + options: { button: MouseButton; count: number }, + ) => Promise + /** Move the mouse to the point. */ + move: (x: number, y: number) => Promise + /** Press and hold a mouse button at the current mouse position. */ + mouseDown: (button: MouseButton) => Promise + /** Release a mouse button at the current mouse position. */ + mouseUp: (button: MouseButton) => Promise + /** Scroll at the point. The deltas are in pixels. */ + scroll: ( + x: number, + y: number, + deltaX: number, + deltaY: number, + ) => Promise + /** Press and hold one key. */ + keyDown: (key: string) => Promise + /** Release one key. */ + keyUp: (key: string) => Promise + /** Press one key combo, like `['Control', 's']`. */ + press: (keys: Array) => Promise + /** Type text as a user does. */ + type: (text: string) => Promise + /** Take a screenshot of one region at full size. Returns PNG bytes. */ + zoom?: (region: { + x: number + y: number + width: number + height: number + }) => Promise + /** Get the current mouse position. */ + cursorPosition?: () => Promise<{ x: number; y: number }> +} diff --git a/packages/ai-computer-use/tests/actions.test.ts b/packages/ai-computer-use/tests/actions.test.ts new file mode 100644 index 0000000000..81daa83d3c --- /dev/null +++ b/packages/ai-computer-use/tests/actions.test.ts @@ -0,0 +1,413 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { runComputerAction } from '../src/actions' +import type { ComputerDriver } from '../src/types' + +type Call = [string, ...Array] + +const ZOOM_BYTES = new Uint8Array([1, 2, 3]) + +/** A driver that records every call in order. */ +function createDriver(options: { zoom?: boolean; cursor?: boolean } = {}) { + const calls: Array = [] + const record = + (name: string) => + async (...args: Array) => { + calls.push([name, ...args]) + } + const driver: ComputerDriver = { + display: { width: 1024, height: 768 }, + environment: 'browser', + screenshot: async () => new Uint8Array(), + click: record('click'), + move: record('move'), + mouseDown: record('mouseDown'), + mouseUp: record('mouseUp'), + scroll: record('scroll'), + keyDown: record('keyDown'), + keyUp: record('keyUp'), + press: record('press'), + type: record('type'), + } + if (options.zoom) { + driver.zoom = async (region) => { + calls.push(['zoom', region]) + return ZOOM_BYTES + } + } + if (options.cursor) { + driver.cursorPosition = async () => ({ x: 7, y: 9 }) + } + return { driver, calls } +} + +async function run( + input: unknown, + options?: { zoom?: boolean; cursor?: boolean }, +) { + const { driver, calls } = createDriver(options) + const result = await runComputerAction(driver, input) + return { result, calls } +} + +/** Checks that the action ends after `ms` and not before. */ +async function expectToFinishAfter(input: unknown, ms: number) { + vi.useFakeTimers() + const { driver } = createDriver() + let finished = false + void runComputerAction(driver, input).then(() => { + finished = true + }) + await vi.advanceTimersByTimeAsync(ms - 1) + expect(finished).toBe(false) + await vi.advanceTimersByTimeAsync(1) + expect(finished).toBe(true) +} + +afterEach(() => { + vi.useRealTimers() +}) + +describe('Anthropic actions', () => { + it('needs no driver call for a screenshot', async () => { + expect(await run({ action: 'screenshot' })).toEqual({ + result: undefined, + calls: [], + }) + }) + + it('presses each combo in a key action', async () => { + const { calls } = await run({ action: 'key', text: 'ctrl+s Return' }) + expect(calls).toEqual([ + ['press', ['Control', 's']], + ['press', ['Enter']], + ]) + }) + + it('repeats a key action', async () => { + const { calls } = await run({ action: 'key', text: 'Down', repeat: 2 }) + expect(calls).toEqual([ + ['press', ['ArrowDown']], + ['press', ['ArrowDown']], + ]) + }) + + it('types text', async () => { + const { calls } = await run({ action: 'type', text: 'hello' }) + expect(calls).toEqual([['type', 'hello']]) + }) + + it('moves the mouse', async () => { + const { calls } = await run({ action: 'mouse_move', coordinate: [3, 4] }) + expect(calls).toEqual([['move', 3, 4]]) + }) + + it.each([ + ['left_click', 'left', 1], + ['right_click', 'right', 1], + ['middle_click', 'middle', 1], + ['double_click', 'left', 2], + ['triple_click', 'left', 3], + ])('runs %s as a %s click with count %i', async (action, button, count) => { + const { calls } = await run({ action, coordinate: [10, 20] }) + expect(calls).toEqual([['click', 10, 20, { button, count }]]) + }) + + it('holds the modifiers around a click and releases them in reverse', async () => { + const { calls } = await run({ + action: 'left_click', + coordinate: [10, 20], + text: 'ctrl+shift', + }) + expect(calls).toEqual([ + ['keyDown', 'Control'], + ['keyDown', 'Shift'], + ['click', 10, 20, { button: 'left', count: 1 }], + ['keyUp', 'Shift'], + ['keyUp', 'Control'], + ]) + }) + + it('clicks at the mouse position when the click has no coordinate', async () => { + const { calls } = await run({ action: 'left_click' }, { cursor: true }) + expect(calls).toEqual([['click', 7, 9, { button: 'left', count: 1 }]]) + }) + + it('throws for a click with no coordinate when the driver has no cursorPosition', async () => { + await expect(run({ action: 'left_click' })).rejects.toThrow( + '"coordinate" is missing', + ) + }) + + it('drags from the start to the end coordinate', async () => { + const { calls } = await run({ + action: 'left_click_drag', + start_coordinate: [1, 2], + coordinate: [30, 40], + }) + expect(calls).toEqual([ + ['move', 1, 2], + ['mouseDown', 'left'], + ['move', 30, 40], + ['mouseUp', 'left'], + ]) + }) + + it('presses and releases the left button', async () => { + const down = await run({ action: 'left_mouse_down' }) + const up = await run({ action: 'left_mouse_up' }) + expect([...down.calls, ...up.calls]).toEqual([ + ['mouseDown', 'left'], + ['mouseUp', 'left'], + ]) + }) + + it.each([ + ['up', 0, -300], + ['down', 0, 300], + ['left', -300, 0], + ['right', 300, 0], + ])('scrolls %s in 100 px steps', async (direction, deltaX, deltaY) => { + const { calls } = await run({ + action: 'scroll', + coordinate: [50, 60], + scroll_direction: direction, + scroll_amount: 3, + }) + expect(calls).toEqual([['scroll', 50, 60, deltaX, deltaY]]) + }) + + it('holds a modifier while it scrolls', async () => { + const { calls } = await run({ + action: 'scroll', + coordinate: [50, 60], + scroll_direction: 'down', + scroll_amount: 1, + text: 'shift', + }) + expect(calls).toEqual([ + ['keyDown', 'Shift'], + ['scroll', 50, 60, 0, 100], + ['keyUp', 'Shift'], + ]) + }) + + it('throws for an unknown scroll direction', async () => { + await expect( + run({ + action: 'scroll', + coordinate: [1, 1], + scroll_direction: 'sideways', + scroll_amount: 1, + }), + ).rejects.toThrow('"scroll_direction"') + }) + + it('holds a key for the duration, then releases it', async () => { + vi.useFakeTimers() + const { driver, calls } = createDriver() + const done = runComputerAction(driver, { + action: 'hold_key', + text: 'shift', + duration: 2, + }) + await vi.advanceTimersByTimeAsync(1999) + expect(calls).toEqual([['keyDown', 'Shift']]) + await vi.advanceTimersByTimeAsync(1) + await done + expect(calls).toEqual([ + ['keyDown', 'Shift'], + ['keyUp', 'Shift'], + ]) + }) + + it('waits for the duration in seconds', async () => { + await expectToFinishAfter({ action: 'wait', duration: 1.5 }, 1500) + }) + + it('returns the zoom image for the region', async () => { + const { result, calls } = await run( + { action: 'zoom', region: [10, 20, 110, 70] }, + { zoom: true }, + ) + expect(result).toEqual({ type: 'image', bytes: ZOOM_BYTES }) + expect(calls).toEqual([['zoom', { x: 10, y: 20, width: 100, height: 50 }]]) + }) + + it('throws for zoom when the driver has no zoom method', async () => { + await expect(run({ action: 'zoom', region: [0, 0, 1, 1] })).rejects.toThrow( + 'no zoom method', + ) + }) + + it('returns the cursor position as text', async () => { + const { result } = await run( + { action: 'cursor_position' }, + { cursor: true }, + ) + expect(result).toEqual({ type: 'text', text: 'X=7,Y=9' }) + }) + + it('throws for cursor_position when the driver has no cursorPosition method', async () => { + await expect(run({ action: 'cursor_position' })).rejects.toThrow( + 'no cursorPosition method', + ) + }) + + it('throws for an unknown action', async () => { + await expect(run({ action: 'fly' })).rejects.toThrow( + 'Unknown computer action "fly"', + ) + }) + + it('throws for a missing field', async () => { + await expect(run({ action: 'type' })).rejects.toThrow( + '"text" must be a string', + ) + }) +}) + +describe('OpenAI actions', () => { + it.each([ + ['left', 'left'], + ['right', 'right'], + ['wheel', 'middle'], + ['back', 'back'], + ['forward', 'forward'], + ])('clicks the %s button as %s', async (button, driverButton) => { + const { calls } = await run({ + action: { type: 'click', button, x: 5, y: 6 }, + }) + expect(calls).toEqual([['click', 5, 6, { button: driverButton, count: 1 }]]) + }) + + it('holds the keys around a click and releases them in reverse', async () => { + const { calls } = await run({ + action: { + type: 'click', + button: 'left', + x: 5, + y: 6, + keys: ['CTRL', 'shift'], + }, + }) + expect(calls).toEqual([ + ['keyDown', 'Control'], + ['keyDown', 'Shift'], + ['click', 5, 6, { button: 'left', count: 1 }], + ['keyUp', 'Shift'], + ['keyUp', 'Control'], + ]) + }) + + it('throws for an unknown button', async () => { + await expect( + run({ action: { type: 'click', button: 'thumb', x: 1, y: 1 } }), + ).rejects.toThrow('"button"') + }) + + it('double-clicks', async () => { + const { calls } = await run({ + action: { type: 'double_click', x: 5, y: 6, keys: null }, + }) + expect(calls).toEqual([['click', 5, 6, { button: 'left', count: 2 }]]) + }) + + it('drags along every point of the path', async () => { + const { calls } = await run({ + action: { + type: 'drag', + path: [ + { x: 1, y: 1 }, + { x: 2, y: 2 }, + { x: 3, y: 3 }, + ], + }, + }) + expect(calls).toEqual([ + ['move', 1, 1], + ['mouseDown', 'left'], + ['move', 2, 2], + ['move', 3, 3], + ['mouseUp', 'left'], + ]) + }) + + it('drags with a one-point path', async () => { + const { calls } = await run({ + action: { type: 'drag', path: [{ x: 4, y: 4 }] }, + }) + expect(calls).toEqual([ + ['move', 4, 4], + ['mouseDown', 'left'], + ['mouseUp', 'left'], + ]) + }) + + it('throws for a drag with an empty path', async () => { + await expect(run({ action: { type: 'drag', path: [] } })).rejects.toThrow( + '"path"', + ) + }) + + it('presses a keypress combo', async () => { + const { calls } = await run({ + action: { type: 'keypress', keys: ['CTRL', 'S'] }, + }) + expect(calls).toEqual([['press', ['Control', 's']]]) + }) + + it('moves the mouse', async () => { + const { calls } = await run({ action: { type: 'move', x: 8, y: 9 } }) + expect(calls).toEqual([['move', 8, 9]]) + }) + + it('scrolls by the pixel deltas', async () => { + const { calls } = await run({ + action: { type: 'scroll', x: 10, y: 20, scroll_x: -15, scroll_y: 250 }, + }) + expect(calls).toEqual([['scroll', 10, 20, -15, 250]]) + }) + + it('types text', async () => { + const { calls } = await run({ action: { type: 'type', text: 'hi' } }) + expect(calls).toEqual([['type', 'hi']]) + }) + + it('waits 2 seconds', async () => { + await expectToFinishAfter({ action: { type: 'wait' } }, 2000) + }) + + it('needs no driver call for a screenshot', async () => { + expect(await run({ action: { type: 'screenshot' } })).toEqual({ + result: undefined, + calls: [], + }) + }) + + it('runs an actions batch in order', async () => { + const { result, calls } = await run({ + actions: [ + { type: 'click', button: 'left', x: 1, y: 2 }, + { type: 'type', text: 'abc' }, + { type: 'keypress', keys: ['ENTER'] }, + ], + pending_safety_checks: [], + }) + expect(result).toBeUndefined() + expect(calls).toEqual([ + ['click', 1, 2, { button: 'left', count: 1 }], + ['type', 'abc'], + ['press', ['Enter']], + ]) + }) + + it('throws for an unknown action type', async () => { + await expect(run({ action: { type: 'fly' } })).rejects.toThrow( + 'Unknown computer action "fly"', + ) + }) +}) + +it('throws for an input with no action', async () => { + await expect(run({ foo: 1 })).rejects.toThrow('"action"') +}) diff --git a/packages/ai-computer-use/tests/create-computer-use.test.ts b/packages/ai-computer-use/tests/create-computer-use.test.ts new file mode 100644 index 0000000000..3501694273 --- /dev/null +++ b/packages/ai-computer-use/tests/create-computer-use.test.ts @@ -0,0 +1,429 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { CapabilityRegistry } from '@tanstack/ai' +import { createComputerUse } from '../src/create-computer-use' +import { computerSafetyCheck } from '../src/safety-check' +import type { + ChatMiddlewareContext, + GenericInterruptResolution, + InterruptResolutionCollection, + ModelMessage, +} from '@tanstack/ai' +import type { ComputerDriver } from '../src/types' + +const PNG_BYTES = new Uint8Array([137, 80, 78, 71]) +const PNG_BASE64 = 'iVBORw==' +const ZOOM_BYTES = new Uint8Array([1, 2, 3]) +const ZOOM_BASE64 = 'AQID' + +const CLICK = { action: { type: 'click', x: 10, y: 20, button: 'left' } } +const TYPE = { action: { type: 'type', text: 'hi' } } +const CHECK = { id: 'sc_1', code: 'malicious_instructions', message: 'Look.' } +const CHECKED_CLICK = { ...CLICK, pending_safety_checks: [CHECK] } +const NOT_EXECUTED = + 'Not executed: an earlier computer action in this turn failed.' + +function screenshotOf(base64: string) { + return [ + { + type: 'image', + source: { type: 'data', value: base64, mimeType: 'image/png' }, + }, + ] +} + +interface DriverOptions { + zoom?: boolean + cursor?: boolean + /** `click` throws while this returns true. */ + failClick?: () => boolean + /** `click` waits for this promise. */ + clickGate?: Promise +} + +/** A driver that records its calls in order. */ +function createDriver(options: DriverOptions) { + const calls: Array = [] + const record = (name: string) => async () => { + calls.push(name) + } + const driver: ComputerDriver = { + display: { width: 1024, height: 768 }, + environment: 'browser', + screenshot: async () => { + calls.push('screenshot') + return PNG_BYTES + }, + click: async () => { + calls.push('click:start') + await options.clickGate + if (options.failClick?.()) throw new Error('click failed') + calls.push('click:end') + }, + move: record('move'), + mouseDown: record('mouseDown'), + mouseUp: record('mouseUp'), + scroll: record('scroll'), + keyDown: record('keyDown'), + keyUp: record('keyUp'), + press: record('press'), + type: record('type'), + } + if (options.zoom) driver.zoom = async () => ZOOM_BYTES + if (options.cursor) driver.cursorPosition = async () => ({ x: 7, y: 9 }) + return { driver, calls } +} + +function setup(options: DriverOptions & { screenshotDelay?: number } = {}) { + const { driver, calls } = createDriver(options) + const computer = createComputerUse({ + driver, + screenshotDelay: options.screenshotDelay ?? 0, + }) + const execute = (input: unknown, toolCallId = 'call_1') => { + if (!computer.tool.execute) throw new Error('The tool has no execute.') + return computer.tool.execute(input, { toolCallId, emitCustomEvent() {} }) + } + return { ...computer, calls, execute } +} + +type Computer = ReturnType + +function createContext( + phase: ChatMiddlewareContext['phase'], + messages: Array = [], +): ChatMiddlewareContext { + return { + requestId: 'request', + streamId: 'stream', + runId: 'run', + threadId: 'thread', + phase, + iteration: 0, + chunkIndex: 0, + abort() {}, + retry() {}, + emitCustomEvent() {}, + context: undefined, + defer() {}, + activity: 'chat', + provider: 'openai', + model: 'gpt-6.1-sol', + source: 'server', + streaming: true, + systemPrompts: [], + messageCount: messages.length, + hasTools: true, + currentMessageId: null, + accumulatedContent: '', + messages, + createId: (prefix) => prefix, + capabilities: new CapabilityRegistry(), + get: () => { + throw new Error('unused') + }, + getOptional: () => undefined, + provide() {}, + } +} + +function toolCall(id: string, input: unknown, name = 'computer') { + return { + id, + type: 'function' as const, + function: { name, arguments: JSON.stringify(input) }, + } +} + +/** Run the boundary hook at `beforeTools` with one assistant tool-call message. */ +function boundary( + computer: Computer, + toolCalls: Array>, +) { + const messages: Array = [ + { role: 'user', content: 'Click the button.' }, + { role: 'assistant', content: null, toolCalls }, + ] + return computer.middleware.onInterruptBoundary?.({ + ...createContext('beforeTools', messages), + phase: 'beforeTools', + }) +} + +function beforeModel(computer: Computer) { + return computer.middleware.onConfig?.(createContext('beforeModel'), { + messages: [], + systemPrompts: [], + tools: [computer.tool], + }) +} + +type Resolution = GenericInterruptResolution + +function safetyRequest(key: string) { + return computerSafetyCheck.interrupt({ + key, + reason: 'computer-safety-check', + message: 'Look.', + payload: { checks: [CHECK], action: CLICK.action }, + }) +} + +function answer(key: string, approved: boolean): Resolution { + return { + request: safetyRequest(key), + status: 'resolved', + response: { approved }, + } +} + +function cancelled(key: string): Resolution { + return { request: safetyRequest(key), status: 'cancelled' } +} + +function resolve(computer: Computer, items: Array) { + const collection: InterruptResolutionCollection = + { + // TypeScript cannot tie a generic return to one definition. The chat + // engine uses the same cast. + for: () => items as never, + all: () => items, + } + return computer.middleware.onInterruptResolution?.( + createContext('init'), + collection, + ) +} + +afterEach(() => { + vi.useRealTimers() +}) + +describe('createComputerUse tool', () => { + it('is the computer tool with the driver metadata', () => { + const { tool, interrupt } = setup({ zoom: true }) + + expect(tool.name).toBe('computer') + expect(tool.metadata).toEqual({ + __kind: 'computer_use', + display: { width: 1024, height: 768 }, + environment: 'browser', + capabilities: { zoom: true, cursorPosition: false }, + }) + expect(interrupt).toBe(computerSafetyCheck) + }) + + it('gives the cursorPosition capability from the driver', () => { + const { tool } = setup({ cursor: true }) + + expect(tool.metadata?.capabilities).toEqual({ + zoom: false, + cursorPosition: true, + }) + }) + + it('returns a screenshot after an action', async () => { + const computer = setup() + + await expect(computer.execute(CLICK)).resolves.toEqual( + screenshotOf(PNG_BASE64), + ) + expect(computer.calls).toEqual(['click:start', 'click:end', 'screenshot']) + }) + + it('returns the zoom image and the cursor position as text', async () => { + const computer = setup({ zoom: true, cursor: true }) + + await expect( + computer.execute({ action: 'zoom', region: [0, 0, 10, 10] }), + ).resolves.toEqual(screenshotOf(ZOOM_BASE64)) + await expect(computer.execute({ action: 'cursor_position' })).resolves.toBe( + 'X=7,Y=9', + ) + expect(computer.calls).toEqual([]) + }) + + it('takes the screenshot only after screenshotDelay', async () => { + vi.useFakeTimers() + const computer = setup({ screenshotDelay: 500 }) + + const result = computer.execute(CLICK) + await vi.advanceTimersByTimeAsync(499) + expect(computer.calls).toEqual(['click:start', 'click:end']) + await vi.advanceTimersByTimeAsync(1) + + await expect(result).resolves.toEqual(screenshotOf(PNG_BASE64)) + expect(computer.calls).toEqual(['click:start', 'click:end', 'screenshot']) + }) + + it('runs concurrent calls one at a time, in call order', async () => { + let openGate = () => {} + const clickGate = new Promise((open) => { + openGate = open + }) + const computer = setup({ clickGate }) + + const first = computer.execute(CLICK, 'call_1') + const second = computer.execute(TYPE, 'call_2') + await vi.waitFor(() => expect(computer.calls).toEqual(['click:start'])) + openGate() + await Promise.all([first, second]) + + expect(computer.calls).toEqual([ + 'click:start', + 'click:end', + 'screenshot', + 'type', + 'screenshot', + ]) + }) +}) + +describe('createComputerUse failed turn', () => { + it('stops the turn after a failed action until the next model call', async () => { + let isBroken = true + const computer = setup({ failClick: () => isBroken }) + + await expect(computer.execute(CLICK, 'call_1')).rejects.toThrow( + 'click failed', + ) + await expect(computer.execute(TYPE, 'call_2')).rejects.toThrow(NOT_EXECUTED) + isBroken = false + await beforeModel(computer) + + await expect(computer.execute(CLICK, 'call_3')).resolves.toEqual( + screenshotOf(PNG_BASE64), + ) + expect(computer.calls).toEqual([ + 'click:start', + 'click:start', + 'click:end', + 'screenshot', + ]) + }) +}) + +describe('createComputerUse safety checks', () => { + it('refuses a call with checks that nobody approved', async () => { + const computer = setup() + + await expect(computer.execute(CHECKED_CLICK)).rejects.toThrow('Refused') + expect(computer.calls).toEqual([]) + }) + + it('refuses a call whose checks do not parse', async () => { + const computer = setup() + + await expect( + computer.execute({ ...CLICK, pending_safety_checks: [{ code: 'x' }] }), + ).rejects.toThrow('Refused') + expect(computer.calls).toEqual([]) + }) + + it('emits one interrupt per computer call with checks', async () => { + const computer = setup() + + const result = await boundary(computer, [ + toolCall('call_1', CHECKED_CLICK), + toolCall('call_2', CLICK), + toolCall('call_3', CHECKED_CLICK, 'search'), + ]) + + expect(result?.interrupts).toEqual([ + expect.objectContaining({ + definition: computerSafetyCheck, + key: 'call_1', + payload: { checks: [CHECK], action: CLICK.action }, + }), + ]) + }) + + it('sends the OpenAI actions list as the action', async () => { + const computer = setup() + const actions = [CLICK.action, TYPE.action] + + const result = await boundary(computer, [ + toolCall('call_1', { actions, pending_safety_checks: [CHECK] }), + ]) + + expect(result?.interrupts[0]?.payload).toEqual({ + checks: [CHECK], + action: actions, + }) + }) + + it('emits nothing outside beforeTools', async () => { + const computer = setup() + + const result = await computer.middleware.onInterruptBoundary?.({ + ...createContext('afterTools', [ + { + role: 'assistant', + content: null, + toolCalls: [toolCall('c', CHECKED_CLICK)], + }, + ]), + phase: 'afterTools', + }) + + expect(result).toBeUndefined() + }) + + it('runs an approved call and does not ask again', async () => { + const computer = setup() + + expect(await resolve(computer, [answer('call_1', true)])).toEqual({ + toolResume: 'continue', + }) + expect( + await boundary(computer, [toolCall('call_1', CHECKED_CLICK)]), + ).toBeUndefined() + await expect(computer.execute(CHECKED_CLICK, 'call_1')).resolves.toEqual( + screenshotOf(PNG_BASE64), + ) + }) + + it('cancels a declined call and does not ask again', async () => { + const computer = setup() + + expect(await resolve(computer, [answer('call_1', false)])).toEqual({ + toolResume: 'cancel', + }) + expect( + await boundary(computer, [toolCall('call_1', CHECKED_CLICK)]), + ).toBeUndefined() + await expect(computer.execute(CHECKED_CLICK, 'call_1')).rejects.toThrow( + 'Refused', + ) + }) + + it('cancels the batch when one answer is cancelled', async () => { + const computer = setup() + + const result = await resolve(computer, [ + answer('call_1', true), + cancelled('call_2'), + ]) + + expect(result).toEqual({ toolResume: 'cancel' }) + await expect(computer.execute(CHECKED_CLICK, 'call_1')).rejects.toThrow( + 'Refused', + ) + }) + + it('does not unlock a call with an approval for another call', async () => { + const computer = setup() + + await resolve(computer, [answer('call_2', true)]) + + await expect(computer.execute(CHECKED_CLICK, 'call_1')).rejects.toThrow( + 'Refused', + ) + expect(computer.calls).toEqual([]) + }) + + it('has no opinion without safety-check answers', async () => { + const computer = setup() + + expect(await resolve(computer, [])).toBeUndefined() + }) +}) diff --git a/packages/ai-computer-use/tests/keys.test.ts b/packages/ai-computer-use/tests/keys.test.ts new file mode 100644 index 0000000000..cb87ba5f35 --- /dev/null +++ b/packages/ai-computer-use/tests/keys.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from 'vitest' +import { parseKeyCombo, toCombo, toKey } from '../src/keys' + +describe('toKey', () => { + it.each([ + ['ctrl', 'Control'], + ['CTRL', 'Control'], + ['esc', 'Escape'], + ['cmd', 'Meta'], + ['super', 'Meta'], + ['alt', 'Alt'], + ['shift', 'Shift'], + ['enter', 'Enter'], + ['space', ' '], + ['backspace', 'Backspace'], + ['left', 'ArrowLeft'], + ['arrowdown', 'ArrowDown'], + ['pageup', 'PageUp'], + ['f5', 'F5'], + ])('maps the OpenAI name %s to %s', (name, key) => { + expect(toKey(name)).toBe(key) + }) + + it.each([ + ['Page_Down', 'PageDown'], + ['Return', 'Enter'], + ['BackSpace', 'Backspace'], + ['KP_Enter', 'Enter'], + ['Super_L', 'Meta'], + ['Prior', 'PageUp'], + ])('maps the xdotool name %s to %s', (name, key) => { + expect(toKey(name)).toBe(key) + }) + + it('passes an unknown key name through unchanged', () => { + expect(toKey('super_special')).toBe('super_special') + expect(toKey('a')).toBe('a') + }) +}) + +describe('toCombo', () => { + it('lower-cases a letter in a combo with a modifier', () => { + expect(toCombo(['CTRL', 'S'])).toEqual(['Control', 's']) + }) + + it('keeps a single capital letter as it is', () => { + expect(toCombo(['A'])).toEqual(['A']) + }) +}) + +describe('parseKeyCombo', () => { + it('splits an xdotool combo and maps each key', () => { + expect(parseKeyCombo('ctrl+shift+Page_Down')).toEqual([ + 'Control', + 'Shift', + 'PageDown', + ]) + }) + + it('maps a single key', () => { + expect(parseKeyCombo('Return')).toEqual(['Enter']) + }) +}) diff --git a/packages/ai-computer-use/tests/safety-check.test.ts b/packages/ai-computer-use/tests/safety-check.test.ts new file mode 100644 index 0000000000..cd3515676c --- /dev/null +++ b/packages/ai-computer-use/tests/safety-check.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest' +import { computerSafetyCheck } from '../src/safety-check' + +const payload = { + checks: [ + { id: 'sc_1', code: 'malicious_instructions', message: 'Check the page.' }, + { id: 'sc_2', code: null, message: null }, + ], + action: { type: 'click', x: 10, y: 20, button: 'left' }, +} + +describe('computerSafetyCheck', () => { + it('builds a request with the OpenAI checks and the action', () => { + const request = computerSafetyCheck.interrupt({ + key: 'call_1', + reason: 'computer-safety-check', + message: 'Check the page.', + payload, + }) + + expect(request.key).toBe('call_1') + expect(request.payload).toEqual(payload) + }) + + it('rejects a check without an id', () => { + const validate = computerSafetyCheck.payloadSchema['~standard'].validate + + expect(validate({ checks: [{ code: 'x' }], action: {} })).toHaveProperty( + 'issues', + ) + }) + + it('accepts an approval and rejects a response without one', () => { + const validate = computerSafetyCheck.responseSchema['~standard'].validate + + expect(validate({ approved: true })).toEqual({ value: { approved: true } }) + expect(validate({})).toHaveProperty('issues') + }) + + it('exports both schemas as JSON Schema', () => { + expect( + computerSafetyCheck.responseSchema['~standard'].jsonSchema.input({ + target: 'draft-07', + }), + ).toMatchObject({ + type: 'object', + properties: { approved: { type: 'boolean' } }, + required: ['approved'], + }) + expect( + computerSafetyCheck.payloadSchema['~standard'].jsonSchema.input({ + target: 'draft-07', + }), + ).toMatchObject({ + type: 'object', + properties: { + checks: { + type: 'array', + items: { type: 'object', required: ['id'] }, + }, + }, + required: ['checks', 'action'], + }) + }) +}) diff --git a/packages/ai-computer-use/tsconfig.json b/packages/ai-computer-use/tsconfig.json new file mode 100644 index 0000000000..c38689f4ea --- /dev/null +++ b/packages/ai-computer-use/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": ["src", "tests"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/ai-computer-use/vite.config.ts b/packages/ai-computer-use/vite.config.ts new file mode 100644 index 0000000000..77bcc2e60b --- /dev/null +++ b/packages/ai-computer-use/vite.config.ts @@ -0,0 +1,36 @@ +import { defineConfig, mergeConfig } from 'vitest/config' +import { tanstackViteConfig } from '@tanstack/vite-config' +import packageJson from './package.json' + +const config = defineConfig({ + test: { + name: packageJson.name, + dir: './', + watch: false, + globals: true, + environment: 'node', + include: ['tests/**/*.test.ts'], + coverage: { + provider: 'v8', + reporter: ['text', 'json', 'html', 'lcov'], + exclude: [ + 'node_modules/', + 'dist/', + 'tests/', + '**/*.test.ts', + '**/*.config.ts', + '**/types.ts', + ], + include: ['src/**/*.ts'], + }, + }, +}) + +export default mergeConfig( + config, + tanstackViteConfig({ + entry: ['./src/index.ts'], + srcDir: './src', + cjs: false, + }), +) diff --git a/packages/ai-openai/src/model-meta.ts b/packages/ai-openai/src/model-meta.ts index 600dcaf14c..5f99ff1361 100644 --- a/packages/ai-openai/src/model-meta.ts +++ b/packages/ai-openai/src/model-meta.ts @@ -108,6 +108,7 @@ interface ModelMeta { | 'code_interpreter' | 'mcp' | 'computer_use' + | 'computer' | 'local_shell' | 'shell' | 'apply_patch' @@ -156,6 +157,7 @@ const GPT5_2 = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -196,6 +198,7 @@ const GPT5_2_PRO = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -276,6 +279,7 @@ const GPT5_1 = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -359,6 +363,7 @@ const GPT5 = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -400,6 +405,7 @@ const GPT5_MINI = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -450,6 +456,7 @@ const GPT5_NANO = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -490,6 +497,7 @@ const GPT5_PRO = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -728,6 +736,7 @@ const O3_DEEP_RESEARCH = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -764,6 +773,7 @@ const O4_MINI_DEEP_RESEARCH = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -799,6 +809,7 @@ const O3_PRO = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -952,6 +963,7 @@ const O3 = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -998,6 +1010,7 @@ const O4_MINI = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -1051,6 +1064,7 @@ const GPT4_1 = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -1102,6 +1116,7 @@ const GPT4_1_MINI = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -1154,6 +1169,7 @@ const GPT4_1_NANO = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -1214,7 +1230,7 @@ const COMPUTER_USE_PREVIEW = { output: ['text'], endpoints: ['chat', 'batch'], features: ['function_calling'], - tools: ['computer_use'], + tools: ['computer_use', 'computer'], }, } as const satisfies ModelMeta< OpenAIBaseOptions & @@ -1298,6 +1314,7 @@ const O3_MINI = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -2027,6 +2044,7 @@ const GPT_5_4_MINI = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -2072,6 +2090,7 @@ const GPT_5_4_NANO = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -2117,6 +2136,7 @@ const GPT_5_4_IMAGE_2 = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -2163,6 +2183,7 @@ const GPT_5_6 = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -2210,6 +2231,7 @@ const GPT_5_6_SOL = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -2257,6 +2279,7 @@ const GPT_5_6_TERRA = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -2304,6 +2327,7 @@ const GPT_5_6_LUNA = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -2350,6 +2374,7 @@ const GPT_5_5 = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -2395,6 +2420,7 @@ const GPT_5_5_PRO = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -2439,6 +2465,7 @@ const GPT_CHAT_LATEST = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -2566,6 +2593,7 @@ const GPT_6_ASTRA = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'shell', 'apply_patch', ], @@ -2634,6 +2662,7 @@ const GPT_6_LUNA = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'shell', 'apply_patch', ], @@ -2702,6 +2731,7 @@ const GPT_6_SOL = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'shell', 'apply_patch', ], @@ -2770,6 +2800,7 @@ const GPT_6_1_SOL = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', @@ -2810,6 +2841,7 @@ const GPT_6_1_SOL_PRO = { 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', diff --git a/packages/ai-openai/tests/computer-tool-kind.test.ts b/packages/ai-openai/tests/computer-tool-kind.test.ts new file mode 100644 index 0000000000..7583c4e97b --- /dev/null +++ b/packages/ai-openai/tests/computer-tool-kind.test.ts @@ -0,0 +1,46 @@ +/** + * Type-level checks for the generic `'computer'` tool kind. + * + * A model that lists `computer_use` also accepts the generic computer tool. + * A model without `computer_use` rejects it at compile time. + */ +import { beforeAll, describe, it } from 'vitest' +import { brandProviderTool } from '@tanstack/ai' +import { openaiText } from '../src' +import type { ProviderTool } from '@tanstack/ai' +import type { TextActivityOptions } from '@tanstack/ai/adapters' + +function typedTools>( + adapter: TAdapter, + tools: TextActivityOptions['tools'], +) { + return { adapter, tools } +} + +// The same brand the generic computer tool carries. +const computerTool = brandProviderTool>({ + name: 'computer', + description: 'controls a computer', +}) + +// Adapter construction needs a key. No network calls happen here. +beforeAll(() => { + process.env['OPENAI_API_KEY'] = 'sk-test-dummy' +}) + +describe('OpenAI computer tool kind', () => { + it('gpt-6.1-sol accepts the computer tool', () => { + typedTools(openaiText('gpt-6.1-sol'), [computerTool]) + }) + + it('computer-use-preview accepts the computer tool', () => { + typedTools(openaiText('computer-use-preview'), [computerTool]) + }) + + it('gpt-4o rejects the computer tool', () => { + typedTools(openaiText('gpt-4o'), [ + // @ts-expect-error - gpt-4o does not list computer_use + computerTool, + ]) + }) +}) diff --git a/packages/ai/src/adapter-internals.ts b/packages/ai/src/adapter-internals.ts index e3ebb0ff56..fc0cf8d563 100644 --- a/packages/ai/src/adapter-internals.ts +++ b/packages/ai/src/adapter-internals.ts @@ -80,6 +80,8 @@ export type { BlockOrderEntry, OrderedAssistantBlock, } from './utilities/block-order' +export { getComputerUseToolMetadata } from './tools/computer-use' +export type { ComputerUseToolMetadata } from './tools/computer-use' export { transformMessagesForReplay, hashToolCallId, diff --git a/packages/ai/src/tools/computer-use.ts b/packages/ai/src/tools/computer-use.ts new file mode 100644 index 0000000000..835f23b058 --- /dev/null +++ b/packages/ai/src/tools/computer-use.ts @@ -0,0 +1,59 @@ +import type { Tool } from '../types' + +/** + * Metadata that marks a tool as a portable computer-use tool. Adapters read + * it to map the tool to their native computer-use format. + */ +export interface ComputerUseToolMetadata { + __kind: 'computer_use' + display: { width: number; height: number } + environment: 'browser' | 'linux' | 'mac' | 'windows' + capabilities: { zoom: boolean; cursorPosition: boolean } +} + +const ENVIRONMENTS: ReadonlyArray = [ + 'browser', + 'linux', + 'mac', + 'windows', +] + +function isEnvironment( + value: unknown, +): value is ComputerUseToolMetadata['environment'] { + return ENVIRONMENTS.includes(value) +} + +/** + * Read the portable computer-use metadata from a tool. + * + * Native provider tools use namespaced kinds (for example + * `anthropic.computer_use`), so they return `undefined`. + * + * @param tool - Any tool from the chat tool list. + * @returns The metadata, or `undefined` when the tool is not a portable + * computer-use tool. + */ +export function getComputerUseToolMetadata( + tool: Tool, +): ComputerUseToolMetadata | undefined { + const metadata = tool.metadata + if (metadata?.['__kind'] !== 'computer_use') return undefined + const { display, environment, capabilities } = metadata + const isValid = + typeof display?.width === 'number' && + typeof display?.height === 'number' && + isEnvironment(environment) && + typeof capabilities?.zoom === 'boolean' && + typeof capabilities?.cursorPosition === 'boolean' + if (!isValid) return undefined + return { + __kind: 'computer_use', + display: { width: display.width, height: display.height }, + environment, + capabilities: { + zoom: capabilities.zoom, + cursorPosition: capabilities.cursorPosition, + }, + } +} diff --git a/packages/ai/tests/computer-use-metadata.test.ts b/packages/ai/tests/computer-use-metadata.test.ts new file mode 100644 index 0000000000..17fba3f9e9 --- /dev/null +++ b/packages/ai/tests/computer-use-metadata.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest' +import { toolDefinition } from '../src/activities/chat/tools/tool-definition' +import { getComputerUseToolMetadata } from '../src/tools/computer-use' +import type { Tool } from '../src/types' + +function toolWithMetadata(metadata: Record): Tool { + return { name: 'computer', description: 'Control a computer', metadata } +} + +describe('getComputerUseToolMetadata', () => { + it('returns the metadata of a portable computer-use tool', () => { + const tool = toolWithMetadata({ + __kind: 'computer_use', + display: { width: 1280, height: 720 }, + environment: 'browser', + capabilities: { zoom: true, cursorPosition: false }, + }) + + expect(getComputerUseToolMetadata(tool)).toEqual({ + __kind: 'computer_use', + display: { width: 1280, height: 720 }, + environment: 'browser', + capabilities: { zoom: true, cursorPosition: false }, + }) + }) + + it('returns undefined for a plain tool', () => { + const tool = toolDefinition({ name: 'getWeather', description: 'Weather' }) + + expect(getComputerUseToolMetadata(tool)).toBeUndefined() + }) + + it('returns undefined for a native provider computer-use tool', () => { + const tool = toolWithMetadata({ + __kind: 'anthropic.computer_use', + display_width_px: 1280, + display_height_px: 720, + }) + + expect(getComputerUseToolMetadata(tool)).toBeUndefined() + }) + + it('returns undefined when the computer_use metadata has a bad shape', () => { + const tool = toolWithMetadata({ + __kind: 'computer_use', + display: { width: 1280 }, + environment: 'browser', + capabilities: { zoom: true, cursorPosition: false }, + }) + + expect(getComputerUseToolMetadata(tool)).toBeUndefined() + }) +}) diff --git a/packages/openai-base/src/adapters/chat-completions-text.ts b/packages/openai-base/src/adapters/chat-completions-text.ts index 6c4698e476..f9e8494869 100644 --- a/packages/openai-base/src/adapters/chat-completions-text.ts +++ b/packages/openai-base/src/adapters/chat-completions-text.ts @@ -7,6 +7,7 @@ import { } from '@tanstack/ai' import { BaseTextAdapter } from '@tanstack/ai/adapters' import { + getComputerUseToolMetadata, hashToolCallId, resolveReasoning, toRunErrorPayload, @@ -144,6 +145,14 @@ export abstract class OpenAIBaseChatCompletionsTextAdapter< // other failure mode here — callers iterating chatStream then only need // one error-handling path instead of both a try/catch around iteration // and a RUN_ERROR handler. + const hasComputerTool = options.tools?.some( + (tool) => getComputerUseToolMetadata(tool) !== undefined, + ) + if (hasComputerTool) { + throw new Error( + 'Computer use needs the Responses API. The Chat Completions API has no computer tool.', + ) + } const requestParams = this.mapOptionsToRequest(options) options.logger.request( `activity=chat provider=${this.name} model=${this.model} messages=${options.messages.length} tools=${options.tools?.length ?? 0} stream=true`, diff --git a/packages/openai-base/src/adapters/responses-text.ts b/packages/openai-base/src/adapters/responses-text.ts index e8a34cd1e7..5c4794cd55 100644 --- a/packages/openai-base/src/adapters/responses-text.ts +++ b/packages/openai-base/src/adapters/responses-text.ts @@ -36,6 +36,7 @@ import { } from '../utils/json-object-mode' import { toResponsesToolChoice } from '../tools/tool-choice' import { getOpenAIProviderToolKind } from '../tools/openai-provider-tool' +import { resolveComputerUseTool } from '../tools/computer-use-tool' import { convertToolsToResponsesFormat } from './responses-tool-converter' import { hostedShellCallIds, @@ -518,8 +519,18 @@ export abstract class OpenAIBaseResponsesTextAdapter< } async *chatStream( - options: TextOptions, + chatOptions: TextOptions, ): AsyncIterable { + // A portable computer tool becomes the native computer tool of the model. + // The request and the stream both read the native tool. + const options = chatOptions.tools + ? { + ...chatOptions, + tools: chatOptions.tools.map((tool) => + resolveComputerUseTool(tool, chatOptions.model), + ), + } + : chatOptions // Key streamed state by output item ID because argument deltas reference // `item_id`. The state separately retains `call_id`, which is the public // tool-call ID and the correlation key for function_call_output. diff --git a/packages/openai-base/src/tools/computer-use-tool.ts b/packages/openai-base/src/tools/computer-use-tool.ts index fb5b6a66b0..6fd5ad8745 100644 --- a/packages/openai-base/src/tools/computer-use-tool.ts +++ b/packages/openai-base/src/tools/computer-use-tool.ts @@ -2,6 +2,7 @@ import { getOpenAIProviderToolMetadata, openAIProviderTool, } from './openai-provider-tool' +import { getComputerUseToolMetadata } from '@tanstack/ai/adapter-internals' import type { ComputerTool, ComputerUsePreviewTool, @@ -52,3 +53,31 @@ export function computerUseTool(toolData: ComputerUseToolConfig): Tool { 'computer_use', ) } + +/** + * Swaps a portable computer-use tool for the native OpenAI computer tool of + * the model. The `computer-use-preview` models get `computer_use_preview` + * with the display and environment of the tool. Other models get + * `{ type: 'computer' }`. The result keeps the name `computer` and the + * `execute` of the portable tool. Other tools come back unchanged. + * + * @param tool - A tool from the chat tool list. + * @param model - The model of the request. + */ +export function resolveComputerUseTool(tool: Tool, model: string): Tool { + const metadata = getComputerUseToolMetadata(tool) + if (!metadata) return tool + // Dated ids such as `computer-use-preview-2025-03-11` match too. + const isPreviewModel = model.startsWith('computer-use-preview') + const native = computerUseTool( + isPreviewModel + ? { + type: 'computer_use_preview', + display_width: metadata.display.width, + display_height: metadata.display.height, + environment: metadata.environment, + } + : { type: 'computer' }, + ) + return { ...native, name: 'computer', execute: tool.execute } +} diff --git a/packages/openai-base/tests/computer-use-generic.test.ts b/packages/openai-base/tests/computer-use-generic.test.ts new file mode 100644 index 0000000000..10c16dd1f3 --- /dev/null +++ b/packages/openai-base/tests/computer-use-generic.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, it, vi } from 'vitest' +import { chat } from '@tanstack/ai' +import { OpenAIBaseChatCompletionsTextAdapter } from '../src/adapters/chat-completions-text' +import { OpenAIBaseResponsesTextAdapter } from '../src/adapters/responses-text' +import { convertToolsToProviderFormat } from '../src/tools/tool-converter' +import type OpenAI from 'openai' +import type { StreamChunk, Tool } from '@tanstack/ai' + +const safetyCheck = { + id: 'sc_1', + code: 'malicious_instructions', + message: 'The page asks the model to do something else.', +} + +async function* completed(output: Array>) { + yield* [ + { + type: 'response.created', + response: { id: 'resp_1', model: 'm', status: 'in_progress' }, + }, + { + type: 'response.completed', + response: { + id: 'resp_1', + model: 'm', + status: 'completed', + output, + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + }, + }, + ] +} + +const answer = { + type: 'message', + id: 'msg_1', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: 'Done.', annotations: [] }], +} + +const screenshot = [ + { + type: 'image', + source: { type: 'data', value: 'iVBORw0KGgo=', mimeType: 'image/png' }, + }, +] + +const screenshotOutput = { + type: 'computer_screenshot', + image_url: 'data:image/png;base64,iVBORw0KGgo=', + detail: 'original', +} + +// The stub clients below implement only the one SDK method the adapter calls. +// A full OpenAI client is too large to build, so they are cast. + +// The OpenAI adapter sends provider tools in their native shape. +class TestAdapter extends OpenAIBaseResponsesTextAdapter { + protected override convertTools(tools: Array) { + return convertToolsToProviderFormat(tools) + } +} + +// The portable tool, as `@tanstack/ai-computer-use` builds it. +function genericComputerTool() { + return { + name: 'computer', + description: 'Use the computer.', + metadata: { + __kind: 'computer_use', + display: { width: 1280, height: 800 }, + environment: 'linux', + capabilities: { zoom: false, cursorPosition: true }, + }, + execute: vi.fn(async (_input: unknown) => screenshot), + } satisfies Tool +} + +async function runLoop(model: string, call: Record) { + const create = vi + .fn() + .mockResolvedValueOnce(completed([call])) + .mockResolvedValueOnce(completed([answer])) + const client = { + responses: { create: (params: unknown) => create(params) }, + } as unknown as OpenAI + const tool = genericComputerTool() + for await (const _ of chat({ + adapter: new TestAdapter(model, 'openai', client), + messages: [{ role: 'user', content: 'Click the button.' }], + tools: [tool], + })) { + // Drain the stream. + } + const [first, second] = create.mock.calls.map(([params]) => params) + return { first, second, execute: tool.execute } +} + +describe('portable computer tool on the Responses API', () => { + it('sends computer_use_preview to computer-use-preview and runs the single action', async () => { + const action = { type: 'click', button: 'left', x: 10, y: 20 } + const { first, second, execute } = await runLoop('computer-use-preview', { + type: 'computer_call', + id: 'cu_1', + call_id: 'call_cu', + status: 'completed', + action, + pending_safety_checks: [safetyCheck], + }) + + expect(first.tools).toEqual([ + { + type: 'computer_use_preview', + display_width: 1280, + display_height: 800, + environment: 'linux', + }, + ]) + expect(execute.mock.calls[0]?.[0]).toEqual({ + action, + pending_safety_checks: [safetyCheck], + }) + expect(second.input).toContainEqual({ + type: 'computer_call_output', + call_id: 'call_cu', + output: screenshotOutput, + acknowledged_safety_checks: [safetyCheck], + }) + }) + + it('sends the computer tool to other models and runs the batch', async () => { + const actions = [ + { type: 'click', button: 'left', x: 10, y: 20 }, + { type: 'type', text: 'hello' }, + ] + const { first, second, execute } = await runLoop('gpt-6.1-sol', { + type: 'computer_call', + id: 'cu_2', + call_id: 'call_batch', + status: 'completed', + actions, + pending_safety_checks: [safetyCheck], + }) + + expect(first.tools).toEqual([{ type: 'computer' }]) + expect(execute.mock.calls[0]?.[0]).toEqual({ + actions, + pending_safety_checks: [safetyCheck], + }) + expect(second.input).toContainEqual({ + type: 'computer_call_output', + call_id: 'call_batch', + output: screenshotOutput, + acknowledged_safety_checks: [safetyCheck], + }) + }) +}) + +class TestChatCompletionsAdapter extends OpenAIBaseChatCompletionsTextAdapter {} + +describe('portable computer tool on the Chat Completions API', () => { + it('fails before any request', async () => { + const create = vi.fn() + const client = { + chat: { completions: { create } }, + } as unknown as OpenAI + const chunks: Array = [] + for await (const chunk of chat({ + adapter: new TestChatCompletionsAdapter('gpt-6.1-sol', 'openai', client), + messages: [{ role: 'user', content: 'Click the button.' }], + tools: [genericComputerTool()], + })) { + chunks.push(chunk) + } + + expect(create).not.toHaveBeenCalled() + expect(chunks).toContainEqual( + expect.objectContaining({ + type: 'RUN_ERROR', + message: + 'Computer use needs the Responses API. The Chat Completions API has no computer tool.', + }), + ) + }) +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1e072c4214..4e2a1e6126 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2164,7 +2164,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@vitest/coverage-v8': specifier: 4.1.10 @@ -2183,7 +2183,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@tanstack/ai': specifier: workspace:* @@ -2251,7 +2251,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@tanstack/ai': specifier: workspace:* @@ -2396,6 +2396,46 @@ importers: specifier: 4.1.10 version: 4.1.10(vitest@4.1.11) + packages/ai-computer-e2b: + devDependencies: + '@e2b/desktop': + specifier: ^2.5.0 + version: 2.5.0 + '@tanstack/ai-computer-use': + specifier: workspace:* + version: link:../ai-computer-use + '@vitest/coverage-v8': + specifier: 4.1.10 + version: 4.1.10(vitest@4.1.11) + + packages/ai-computer-playwright: + devDependencies: + '@tanstack/ai-computer-use': + specifier: workspace:* + version: link:../ai-computer-use + '@vitest/coverage-v8': + specifier: 4.1.10 + version: 4.1.10(vitest@4.1.11) + playwright-core: + specifier: ^1.57.0 + version: 1.57.0 + + packages/ai-computer-use: + dependencies: + '@tanstack/ai-utils': + specifier: workspace:^ + version: link:../ai-utils + devDependencies: + '@tanstack/ai': + specifier: workspace:* + version: link:../ai + '@vitest/coverage-v8': + specifier: 4.1.10 + version: 4.1.10(vitest@4.1.11) + zod: + specifier: ^4.2.0 + version: 4.6.5 + packages/ai-devtools: dependencies: '@tanstack/ai': @@ -2531,7 +2571,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@tanstack/ai': specifier: workspace:* @@ -2578,7 +2618,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@vitest/coverage-v8': specifier: 4.1.10 @@ -2688,7 +2728,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@tanstack/ai': specifier: workspace:* @@ -2710,7 +2750,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@tanstack/ai': specifier: workspace:* @@ -2887,7 +2927,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@tanstack/ai': specifier: workspace:* @@ -3436,7 +3476,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@vitest/coverage-v8': specifier: 4.1.10 @@ -3453,7 +3493,7 @@ importers: devDependencies: '@google/genai': specifier: ^2.10.0 - version: 2.10.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6)) + version: 2.10.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.6.5)) '@tanstack/ai': specifier: workspace:* version: link:../ai @@ -3549,7 +3589,7 @@ importers: version: link:../ai-utils openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@tanstack/ai': specifier: workspace:* @@ -3678,6 +3718,12 @@ importers: '@tanstack/ai-compaction': specifier: workspace:* version: link:../../packages/ai-compaction + '@tanstack/ai-computer-playwright': + specifier: workspace:* + version: link:../../packages/ai-computer-playwright + '@tanstack/ai-computer-use': + specifier: workspace:* + version: link:../../packages/ai-computer-use '@tanstack/ai-elevenlabs': specifier: workspace:* version: link:../../packages/ai-elevenlabs @@ -5347,6 +5393,14 @@ packages: '@dimforge/rapier3d-compat@0.12.0': resolution: {integrity: sha512-uekIGetywIgopfD97oDL5PfeezkFpNhwlzlaEYNOA0N6ghdsOvh/HYjSMek5Q2O1PYvRSDFcqFVJl4r4ZBwOow==} + '@e2b/desktop@2.5.0': + resolution: {integrity: sha512-2RgSRF+YQMNdJeIFNmbBG1CV0beEmLfgMR5BdK3y2oP+VyH/8ymNUjXohpufIIhiVRmY3kaWWQ4LhH2b6ih7rA==} + engines: {node: '>=22.19.0'} + + '@e2b/dockerfile-utils@0.1.1': + resolution: {integrity: sha512-eDmlWRMrJNzSUbBOf6Y/qTChd1Ys/o3YY8PK6tRQmwkV2u7z3ZS2cak/YJB07odHMT/Z6HJKoUElR/b0N6bHUw==} + engines: {node: '>=22'} + '@electric-sql/pglite-socket@0.1.3': resolution: {integrity: sha512-LAciWM0M1dCL8hlsxu2venbVZcdxema0BtDfpWYVqr+Y468UADw0pFWidhKw1M8sfJ8rdLT71tjMmnirf/IZRQ==} hasBin: true @@ -13205,6 +13259,10 @@ packages: resolution: {integrity: sha512-P+50lx1nOYpgCFyE1xuz6JVN/TqJivMUiH7v+RW0CUSfb4+NCu8IkC5KruT2sNPgp6aexddQQ77+OZKGpZZJ+Q==} engines: {node: '>=20.18.1 <21 || >=22'} + e2b@2.56.0: + resolution: {integrity: sha512-67vAYTERArdKrPujKnAVZuiIKLFX3vQT2gh1+TFphwEmYcXfU2iGKraRxpwn9qWIdUPFVym+KOo+QFhnySWYNQ==} + engines: {node: '>=22.19.0'} + eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} @@ -15973,9 +16031,15 @@ packages: openapi-fetch@0.14.1: resolution: {integrity: sha512-l7RarRHxlEZYjMLd/PR0slfMVse2/vvIAGm75/F7J6MlQ8/b9uUQmUF2kCPrQhJqMXSxmYWObVgeYXbFYzZR+A==} + openapi-fetch@0.17.0: + resolution: {integrity: sha512-PsbZR1wAPcG91eEthKhN+Zn92FMHxv+/faECIwjXdxfTODGSGegYv0sc1Olz+HYPvKOuoXfp+0pA2XVt2cI0Ig==} + openapi-typescript-helpers@0.0.15: resolution: {integrity: sha512-opyTPaunsklCBpTK8JGef6mfPhLSnyy5a0IN9vKtx3+4aExf+KxEqYwIy3hqkedXIB97u357uLMJsOnm3GVjsw==} + openapi-typescript-helpers@0.1.0: + resolution: {integrity: sha512-OKTGPthhivLw/fHz6c3OPtg72vi86qaMlqbJuVJ23qOvQ+53uw1n7HdmkJFibloF7QEjDrDkzJiOJuockM/ljw==} + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -17952,6 +18016,10 @@ packages: resolution: {integrity: sha512-/y4/bH9YNU5hi9NIrpOuvGXFcxrj3CMrV+/AYpowAYTpHn8gX/XPFjNy766FPoYY0miQhdW977JFWKGNhBdwyQ==} engines: {node: '>=22.19.0'} + undici@8.11.2: + resolution: {integrity: sha512-u4UB2/IrKdU6lFxumHmmo1a3fCQO5tzQllRorfoRS63txhrB7xTpSn1PftwC4qEHkOaqP95fCWW4lJzwErwzhQ==} + engines: {node: '>=22.19.0'} + unenv@2.0.0-rc.24: resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} @@ -20964,7 +21032,7 @@ snapshots: '@blaxel/core@0.3.11(@cfworker/json-schema@4.1.1)(@hey-api/openapi-ts@0.99.0(magicast@0.5.2)(typescript@7.0.2))': dependencies: '@hey-api/client-fetch': 0.10.2(@hey-api/openapi-ts@0.99.0(magicast@0.5.2)(typescript@7.0.2)) - '@modelcontextprotocol/sdk': 1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6) + '@modelcontextprotocol/sdk': 1.29.0(@cfworker/json-schema@4.1.1)(zod@4.6.5) archiver: 7.0.1 axios: 1.18.1(debug@4.4.3(supports-color@7.2.0))(supports-color@7.2.0) dockerfile-ast: 0.7.1 @@ -20975,7 +21043,7 @@ snapshots: uuid: 11.1.1 ws: 8.21.0 yaml: 2.9.0 - zod: 4.3.6 + zod: 4.6.5 transitivePeerDependencies: - '@cfworker/json-schema' - '@hey-api/openapi-ts' @@ -21340,6 +21408,12 @@ snapshots: '@dimforge/rapier3d-compat@0.12.0': {} + '@e2b/desktop@2.5.0': + dependencies: + e2b: 2.56.0 + + '@e2b/dockerfile-utils@0.1.1': {} + '@electric-sql/pglite-socket@0.1.3(@electric-sql/pglite@0.4.3)': dependencies: '@electric-sql/pglite': 0.4.3 @@ -22602,7 +22676,7 @@ snapshots: node-fetch: 2.7.0 nopt: 8.1.0 semver: 7.8.5 - tar: 7.5.16 + tar: 7.5.22 transitivePeerDependencies: - encoding - supports-color @@ -22688,8 +22762,8 @@ snapshots: '@mistralai/mistralai@2.2.0': dependencies: ws: 8.21.0 - zod: 4.3.6 - zod-to-json-schema: 3.25.2(zod@4.3.6) + zod: 4.6.5 + zod-to-json-schema: 3.25.2(zod@4.6.5) transitivePeerDependencies: - bufferutil - utf-8-validate @@ -22702,11 +22776,11 @@ snapshots: eventsource-parser: 3.1.1 jose: 6.2.12 pkce-challenge: 5.0.1 - zod: 4.3.6 + zod: 4.6.5 '@modelcontextprotocol/core@2.1.0': dependencies: - zod: 4.3.6 + zod: 4.6.5 '@modelcontextprotocol/ext-apps@1.7.4(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(supports-color@7.2.0)(zod@4.2.1))(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(zod@3.25.76)': dependencies: @@ -22812,12 +22886,11 @@ snapshots: '@cfworker/json-schema': 4.1.1 transitivePeerDependencies: - supports-color - optional: true '@modelcontextprotocol/server@2.1.0': dependencies: '@modelcontextprotocol/core': 2.1.0 - zod: 4.3.6 + zod: 4.6.5 '@msgpack/msgpack@3.1.3': {} @@ -23077,7 +23150,7 @@ snapshots: '@openrouter/sdk@0.13.20': dependencies: - zod: 4.3.6 + zod: 4.6.5 '@opentelemetry/api-logs@0.217.0': dependencies: @@ -28553,7 +28626,7 @@ snapshots: async-sema: 3.1.1 bindings: 1.5.0 estree-walker: 2.0.2 - glob: 13.0.0 + glob: 13.0.6 graceful-fs: 4.2.11 node-gyp-build: 4.8.4 picomatch: 4.0.7 @@ -28577,7 +28650,7 @@ snapshots: tar-stream: 3.1.7 undici: 7.28.0 xdg-app-paths: 5.1.0 - zod: 4.3.6 + zod: 4.6.5 transitivePeerDependencies: - bare-abort-controller - react-native-b4a @@ -29787,7 +29860,7 @@ snapshots: parse5: 7.3.0 parse5-htmlparser2-tree-adapter: 7.1.0 parse5-parser-stream: 7.1.2 - undici: 7.29.0 + undici: 7.29.1 whatwg-mimetype: 4.0.0 chokidar@4.0.3: @@ -30500,6 +30573,20 @@ snapshots: optionalDependencies: undici8: undici@8.10.2 + e2b@2.56.0: + dependencies: + '@bufbuild/protobuf': 2.15.0 + '@connectrpc/connect': 2.2.0(@bufbuild/protobuf@2.15.0) + '@connectrpc/connect-web': 2.2.0(@bufbuild/protobuf@2.15.0)(@connectrpc/connect@2.2.0(@bufbuild/protobuf@2.15.0)) + '@e2b/dockerfile-utils': 0.1.1 + chalk: 5.6.2 + compare-versions: 6.1.1 + glob: 13.0.6 + openapi-fetch: 0.17.0 + platform: 1.3.6 + tar: 7.5.22 + undici: 8.11.2 + eastasianwidth@0.2.0: {} ecdsa-sig-formatter@1.0.11: @@ -34324,11 +34411,11 @@ snapshots: is-docker: 2.2.1 is-wsl: 2.2.0 - openai@7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6): + openai@7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6): optionalDependencies: '@aws-sdk/credential-provider-node': 3.972.58 '@smithy/signature-v4': 5.5.2 - undici: 8.10.2 + undici: 8.11.2 ws: 8.21.3 zod: 4.3.6 @@ -34336,8 +34423,14 @@ snapshots: dependencies: openapi-typescript-helpers: 0.0.15 + openapi-fetch@0.17.0: + dependencies: + openapi-typescript-helpers: 0.1.0 + openapi-typescript-helpers@0.0.15: {} + openapi-typescript-helpers@0.1.0: {} + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -37277,6 +37370,8 @@ snapshots: undici@8.10.2: optional: true + undici@8.11.2: {} + unenv@2.0.0-rc.24: dependencies: pathe: 2.0.3 @@ -38325,7 +38420,6 @@ snapshots: zod-to-json-schema@3.25.2(zod@4.6.5): dependencies: zod: 4.6.5 - optional: true zod@3.25.76: {} diff --git a/scripts/model-sync/provider-supports.test.ts b/scripts/model-sync/provider-supports.test.ts index c311313371..1c541d409e 100644 --- a/scripts/model-sync/provider-supports.test.ts +++ b/scripts/model-sync/provider-supports.test.ts @@ -15,6 +15,7 @@ describe('buildProviderSupportsBody', () => { expect(body).toContain("'file_search'") expect(body).toContain("'code_interpreter'") expect(body).toContain("'mcp'") + expect(body).toContain("'computer_use', 'computer'") expect(body).toContain("endpoints: ['chat', 'chat-completions']") expect(body).toContain('function_calling') expect(body).toContain('structured_outputs') @@ -27,6 +28,7 @@ describe('buildProviderSupportsBody', () => { supportedParameters: ['temperature'], }) expect(body).toContain('tools: []') + expect(body).not.toContain("'computer'") }) it('lists Anthropic server tools and budget-thinking flags', () => { @@ -38,7 +40,7 @@ describe('buildProviderSupportsBody', () => { expect(body).toContain("'web_search'") expect(body).toContain("'web_fetch'") expect(body).toContain("'code_execution'") - expect(body).toContain("'computer_toolset'") + expect(body).toContain("'computer', 'computer_toolset'") expect(body).not.toContain("'computer_use'") expect(body).toContain('extended_thinking: true') expect(body).toContain('priority_tier: true') diff --git a/scripts/model-sync/provider-supports.ts b/scripts/model-sync/provider-supports.ts index 127b9c2c8d..b025193b09 100644 --- a/scripts/model-sync/provider-supports.ts +++ b/scripts/model-sync/provider-supports.ts @@ -18,16 +18,20 @@ export interface ProviderSupportsInput { } // New Claude models take computer use only through the computer toolset. +// `computer` is the generic tool from `@tanstack/ai-computer-use`. Every +// model with a computer version gets it. const ANTHROPIC_SERVER_TOOLS = [ 'web_search', 'web_fetch', 'code_execution', + 'computer', 'computer_toolset', 'bash', 'text_editor', 'memory', ] as const +// `computer` goes with `computer_use`, as in the Claude list above. const OPENAI_SERVER_TOOLS = [ 'web_search', 'web_search_preview', @@ -36,6 +40,7 @@ const OPENAI_SERVER_TOOLS = [ 'code_interpreter', 'mcp', 'computer_use', + 'computer', 'local_shell', 'shell', 'apply_patch', diff --git a/testing/e2e/package.json b/testing/e2e/package.json index 78f353b467..26a8457dd3 100644 --- a/testing/e2e/package.json +++ b/testing/e2e/package.json @@ -26,6 +26,8 @@ "@tanstack/ai-client": "workspace:*", "@tanstack/ai-cloudflare": "workspace:*", "@tanstack/ai-compaction": "workspace:*", + "@tanstack/ai-computer-playwright": "workspace:*", + "@tanstack/ai-computer-use": "workspace:*", "@tanstack/ai-elevenlabs": "workspace:*", "@tanstack/ai-fal": "workspace:*", "@tanstack/ai-gemini": "workspace:*", diff --git a/testing/e2e/tests/computer-use.spec.ts b/testing/e2e/tests/computer-use.spec.ts new file mode 100644 index 0000000000..ea7a2488be --- /dev/null +++ b/testing/e2e/tests/computer-use.spec.ts @@ -0,0 +1,445 @@ +import { test, expect } from '@playwright/test' +import { + EventType, + INTERRUPT_PAYLOAD_METADATA_KEY, + chat, + genericInterruptContinuationFromDescriptor, + wrapGenericInterruptContinuation, +} from '@tanstack/ai' +import { createAnthropicChat } from '@tanstack/ai-anthropic' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { createPlaywrightComputerDriver } from '@tanstack/ai-computer-playwright' +import { createOpenaiChat } from '@tanstack/ai-openai' +import type { Locator, Page } from '@playwright/test' +import type { ModelMessage, StreamChunk } from '@tanstack/ai' + +/** + * Computer use from end to end. A real Playwright page is the computer. + * `createComputerUse` with the Playwright driver runs the actions on it. + * + * `chat()` runs in the spec process. A mock `fetch` sends the model streams + * and keeps each request body. No app server and no API key are used. + */ + +const PAGE_HTML = ` + +

idle

+ +` + +const USER_MESSAGE: ModelMessage = { role: 'user', content: 'Fill the form.' } + +const SAFETY_CHECK = { + id: 'sc_e2e', + code: 'malicious_instructions', + message: 'The page asks the model to do something else.', +} + +/** A mock `fetch` that sends one stream per request, in order. */ +function mockModel(streams: Array<() => ReadableStream>) { + const bodies: Array = [] + const fetch: typeof globalThis.fetch = async (input, init) => { + const request = input instanceof Request ? input : new Request(input, init) + bodies.push(JSON.parse(await request.text())) + const stream = streams[bodies.length - 1] + if (!stream) + throw new Error(`No model response for request ${bodies.length}`) + return new Response(stream(), { + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + }) + } + return { fetch, bodies } +} + +function sse(lines: Array) { + const encoder = new TextEncoder() + return new ReadableStream({ + start(controller) { + for (const line of lines) controller.enqueue(encoder.encode(line)) + controller.close() + }, + }) +} + +/** An OpenAI Responses stream that completes with `output`. */ +function openaiStream(id: string, output: Array) { + const events = [ + { + type: 'response.created', + response: { id, object: 'response', status: 'in_progress', output: [] }, + }, + { + type: 'response.completed', + response: { + id, + object: 'response', + status: 'completed', + output, + usage: { input_tokens: 5, output_tokens: 2, total_tokens: 7 }, + }, + }, + ] + return () => + sse([ + ...events.map((event) => `data: ${JSON.stringify(event)}\n\n`), + 'data: [DONE]\n\n', + ]) +} + +const OPENAI_ANSWER = { + type: 'message', + id: 'msg_cu_e2e', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: 'Done.', annotations: [] }], +} + +/** An Anthropic Messages stream. */ +function anthropicStream(events: Array<{ type: string }>) { + return () => + sse( + events.map( + (event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`, + ), + ) +} + +const ANTHROPIC_START = { + type: 'message_start', + message: { + id: 'msg_cu_e2e', + type: 'message', + role: 'assistant', + content: [], + model: 'claude-opus-5-5', + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 5, output_tokens: 0 }, + }, +} + +function anthropicEnd(stopReason: string) { + return [ + { + type: 'message_delta', + delta: { stop_reason: stopReason, stop_sequence: null }, + usage: { output_tokens: 1 }, + }, + { type: 'message_stop' }, + ] +} + +/** The blocks of one toolset call of the `computer` toolset. */ +function toolsetCall(index: number, id: string, name: string, input: unknown) { + return [ + { + type: 'content_block_start', + index, + content_block: { + type: 'tool_use', + id, + name, + toolset_name: 'computer', + input: {}, + }, + }, + { + type: 'content_block_delta', + index, + delta: { type: 'input_json_delta', partial_json: JSON.stringify(input) }, + }, + { type: 'content_block_stop', index }, + ] +} + +async function openForm(page: Page) { + await page.setContent(PAGE_HTML) + return { + button: await centerOf(page.locator('#go')), + input: await centerOf(page.locator('#name')), + } +} + +async function centerOf(locator: Locator) { + const box = await locator.boundingBox() + if (!box) throw new Error('The element has no bounding box.') + return { + x: Math.round(box.x + box.width / 2), + y: Math.round(box.y + box.height / 2), + } +} + +function computerUseFor(page: Page) { + return createComputerUse({ + driver: createPlaywrightComputerDriver({ page }), + screenshotDelay: 0, + }) +} + +async function drain(stream: AsyncIterable) { + const chunks: Array = [] + for await (const chunk of stream) chunks.push(chunk) + return chunks +} + +/** The assistant message with the tool calls of a paused run. */ +function assistantFromChunks(chunks: Array): ModelMessage { + const toolCalls = chunks.flatMap((chunk) => { + if (chunk.type !== EventType.TOOL_CALL_START) return [] + const end = chunks.find( + (item) => + item.type === EventType.TOOL_CALL_END && + item.toolCallId === chunk.toolCallId, + ) + const input = end?.type === EventType.TOOL_CALL_END ? end.input : {} + return [ + { + id: chunk.toolCallId, + type: 'function' as const, + function: { + name: chunk.toolCallName, + arguments: JSON.stringify(input), + }, + metadata: chunk.metadata, + }, + ] + }) + return { role: 'assistant', content: null, toolCalls } +} + +function interruptOutcome(chunks: Array) { + const finished = chunks.find((chunk) => chunk.type === EventType.RUN_FINISHED) + const outcome = + finished?.type === EventType.RUN_FINISHED ? finished.outcome : undefined + if (outcome?.type !== 'interrupt') { + throw new Error('The run did not end with an interrupt.') + } + return outcome.interrupts +} + +test.describe('computer use with a Playwright page', () => { + test('openai computer tool runs a batch of actions on the page', async ({ + page, + }) => { + const { button, input } = await openForm(page) + const { tool, middleware, interrupt } = computerUseFor(page) + const model = mockModel([ + openaiStream('resp_cu_1', [ + { + type: 'computer_call', + id: 'cu_batch', + call_id: 'call_cu_batch', + status: 'completed', + actions: [ + { type: 'click', button: 'left', ...button }, + { type: 'click', button: 'left', ...input }, + { type: 'type', text: 'hello' }, + ], + pending_safety_checks: [], + }, + ]), + openaiStream('resp_cu_2', [OPENAI_ANSWER]), + ]) + + await drain( + chat({ + adapter: createOpenaiChat('gpt-6.1-sol', 'sk-e2e-dummy', { + fetch: model.fetch, + }), + messages: [USER_MESSAGE], + tools: [tool], + middleware: [middleware], + interrupts: [interrupt], + }), + ) + + await expect(page.locator('#status')).toHaveText('clicked') + await expect(page.locator('#name')).toHaveValue('hello') + expect(model.bodies).toHaveLength(2) + expect(model.bodies[0]).toMatchObject({ tools: [{ type: 'computer' }] }) + expect(model.bodies[1]).toMatchObject({ + input: expect.arrayContaining([ + expect.objectContaining({ + type: 'computer_call_output', + call_id: 'call_cu_batch', + output: expect.objectContaining({ + type: 'computer_screenshot', + image_url: expect.stringMatching(/^data:image\/png;base64,./), + }), + }), + ]), + }) + }) + + test('anthropic computer toolset runs each call on the page', async ({ + page, + }) => { + const { button, input } = await openForm(page) + const { tool, middleware, interrupt } = computerUseFor(page) + const model = mockModel([ + anthropicStream([ + ANTHROPIC_START, + ...toolsetCall(0, 'toolu_button', 'left_click', { + coordinate: [button.x, button.y], + }), + ...toolsetCall(1, 'toolu_input', 'left_click', { + coordinate: [input.x, input.y], + }), + ...toolsetCall(2, 'toolu_type', 'type', { text: 'hello' }), + ...anthropicEnd('tool_use'), + ]), + anthropicStream([ANTHROPIC_START, ...anthropicEnd('end_turn')]), + ]) + + await drain( + chat({ + adapter: createAnthropicChat('claude-opus-5-5', 'sk-ant-e2e-dummy', { + fetch: model.fetch, + }), + messages: [USER_MESSAGE], + tools: [tool], + middleware: [middleware], + interrupts: [interrupt], + }), + ) + + await expect(page.locator('#status')).toHaveText('clicked') + await expect(page.locator('#name')).toHaveValue('hello') + expect(model.bodies).toHaveLength(2) + // The Playwright driver has zoom and no cursor position. + expect(model.bodies[0]).toMatchObject({ + tools: [ + { + type: 'computer_toolset_20260801', + configs: { + zoom: { enabled: true }, + cursor_position: { enabled: false }, + }, + }, + ], + }) + const screenshotResult = (toolUseId: string) => + expect.objectContaining({ + type: 'tool_result', + tool_use_id: toolUseId, + content: expect.arrayContaining([ + expect.objectContaining({ + type: 'image', + source: expect.objectContaining({ + type: 'base64', + data: expect.stringMatching(/^./), + }), + }), + ]), + }) + expect(model.bodies[1]).toMatchObject({ + messages: expect.arrayContaining([ + expect.objectContaining({ + role: 'user', + content: expect.arrayContaining([ + screenshotResult('toolu_button'), + screenshotResult('toolu_input'), + screenshotResult('toolu_type'), + ]), + }), + ]), + }) + }) + + test('openai safety check pauses the action until a person approves it', async ({ + page, + }) => { + const { button } = await openForm(page) + const { tool, middleware, interrupt } = computerUseFor(page) + const click = { type: 'click', button: 'left', ...button } + const model = mockModel([ + openaiStream('resp_cu_1', [ + { + type: 'computer_call', + id: 'cu_checked', + call_id: 'call_cu_checked', + status: 'completed', + action: click, + pending_safety_checks: [SAFETY_CHECK], + }, + ]), + openaiStream('resp_cu_2', [OPENAI_ANSWER]), + ]) + const adapter = createOpenaiChat('gpt-6.1-sol', 'sk-e2e-dummy', { + fetch: model.fetch, + }) + const threadId = 'thread-cu-safety' + + // Run 1 ends with the safety-check interrupt. The click waits. + const first = await drain( + chat({ + adapter, + messages: [USER_MESSAGE], + tools: [tool], + middleware: [middleware], + interrupts: [interrupt], + threadId, + runId: 'run-cu-safety-1', + }), + ) + const [paused, ...others] = interruptOutcome(first) + expect(others).toEqual([]) + if (!paused) throw new Error('No interrupt in the outcome.') + expect(paused.metadata?.[INTERRUPT_PAYLOAD_METADATA_KEY]).toEqual({ + checks: [SAFETY_CHECK], + action: click, + }) + await expect(page.locator('#status')).toHaveText('idle') + expect(model.bodies).toHaveLength(1) + + const continuation = genericInterruptContinuationFromDescriptor(paused) + if (!continuation) throw new Error('The interrupt has no continuation.') + expect(continuation).toMatchObject({ + definitionId: 'computer-safety-check', + key: 'call_cu_checked', + }) + + // Run 2 approves the check. The click runs, and the model gets the result. + await drain( + chat({ + adapter, + messages: [USER_MESSAGE, assistantFromChunks(first)], + tools: [tool], + middleware: [middleware], + interrupts: [interrupt], + threadId, + runId: 'run-cu-safety-2', + parentRunId: 'run-cu-safety-1', + resume: [ + { + interruptId: paused.id, + status: 'resolved', + payload: { approved: true }, + metadata: wrapGenericInterruptContinuation(continuation), + }, + ], + }), + ) + + await expect(page.locator('#status')).toHaveText('clicked') + expect(model.bodies).toHaveLength(2) + expect(model.bodies[1]).toMatchObject({ + input: expect.arrayContaining([ + expect.objectContaining({ + type: 'computer_call', + call_id: 'call_cu_checked', + action: click, + }), + expect.objectContaining({ + type: 'computer_call_output', + call_id: 'call_cu_checked', + output: expect.objectContaining({ + image_url: expect.stringMatching(/^data:image\/png;base64,./), + }), + acknowledged_safety_checks: [SAFETY_CHECK], + }), + ]), + }) + }) +}) From 9db448f196e2ae37ae2f6d4ae14b692e679bdea1 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:52:45 +0000 Subject: [PATCH 16/32] ci: apply automated fixes --- .agent/domain-map.json | 66 ++++++++++-------------------------------- 1 file changed, 15 insertions(+), 51 deletions(-) diff --git a/.agent/domain-map.json b/.agent/domain-map.json index c64a3c996c..408d5a005c 100644 --- a/.agent/domain-map.json +++ b/.agent/domain-map.json @@ -3,11 +3,7 @@ "domains": [ { "id": "core", - "owns": [ - "packages/ai", - "packages/ai-utils", - "packages/ai-event-client" - ], + "owns": ["packages/ai", "packages/ai-utils", "packages/ai-event-client"], "talksTo": [] }, { @@ -38,9 +34,7 @@ "packages/ai-vertex", "packages/ai-worldlabs" ], - "talksTo": [ - "core" - ] + "talksTo": ["core"] }, { "id": "harness", @@ -51,10 +45,7 @@ "packages/ai-opencode", "packages/ai-grok-build" ], - "talksTo": [ - "core", - "sandbox" - ] + "talksTo": ["core", "sandbox"] }, { "id": "client", @@ -72,9 +63,7 @@ "packages/ai-solid-ui", "packages/ai-vue-ui" ], - "talksTo": [ - "core" - ] + "talksTo": ["core"] }, { "id": "code-mode", @@ -88,9 +77,7 @@ "packages/ai-isolate-quickjs", "packages/ai-isolate-quickjs-bun" ], - "talksTo": [ - "core" - ] + "talksTo": ["core"] }, { "id": "sandbox", @@ -108,11 +95,7 @@ "packages/ai-sandbox-upstash-box", "packages/ai-sandbox-vercel" ], - "talksTo": [ - "core", - "middleware", - "persistence" - ] + "talksTo": ["core", "middleware", "persistence"] }, { "id": "middleware", @@ -122,19 +105,12 @@ "packages/ai-skills", "packages/ai-mcp" ], - "talksTo": [ - "core" - ] + "talksTo": ["core"] }, { "id": "persistence", - "owns": [ - "packages/ai-persistence", - "packages/ai-durable-stream" - ], - "talksTo": [ - "core" - ] + "owns": ["packages/ai-persistence", "packages/ai-durable-stream"], + "talksTo": ["core"] }, { "id": "devtools", @@ -145,9 +121,7 @@ "packages/svelte-ai-devtools", "packages/preact-ai-devtools" ], - "talksTo": [ - "core" - ] + "talksTo": ["core"] }, { "id": "computer-use", @@ -156,15 +130,11 @@ "packages/ai-computer-playwright", "packages/ai-computer-e2b" ], - "talksTo": [ - "core" - ] + "talksTo": ["core"] }, { "id": "e2e", - "owns": [ - "testing/e2e" - ], + "owns": ["testing/e2e"], "talksTo": [ "client", "code-mode", @@ -179,19 +149,13 @@ }, { "id": "docs", - "owns": [ - "docs" - ], + "owns": ["docs"], "talksTo": [] }, { "id": "model-sync", - "owns": [ - "scripts/model-sync" - ], - "talksTo": [ - "provider-adapters" - ] + "owns": ["scripts/model-sync"], + "talksTo": ["provider-adapters"] } ] } From e314c0d81df608947173e129997819281b83e47a Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Tue, 20 Oct 2026 15:36:19 +0200 Subject: [PATCH 17/32] feat(ai-computer-use): give each thread its own machine - createComputerUse returns { tools, middleware, interrupt, close }. Each threadId gets its own machine, queue, failed-turn flag, and approvals. - idleTimeout closes the machine of a thread after 10 minutes without an action. false turns it off. - New navigate tool for drivers that can go to a URL. It accepts only http and https URLs. - playwrightComputer() launches Chromium once and opens one browser context for each thread. playwright-core is an optional peer. - e2bComputer() creates one E2B desktop sandbox for each thread. @e2b/desktop is a peer. - @tanstack/ai: ToolExecutionContext.threadId. chat() sets it. - Docs: the OpenAI and Anthropic pages lead with createComputerUse. - Lockfile: the new packages reuse zod 4.3.6, so ai-mcp sees one zod. --- .changeset/ai-computer-use.md | 30 +- docs/adapters/anthropic.md | 91 ++-- docs/adapters/openai.md | 75 ++- docs/api/ai.md | 3 + docs/computer-use/computer-use.md | 142 +++-- docs/computer-use/custom-driver.md | 145 ++++-- docs/computer-use/drivers.md | 149 ++++-- docs/computer-use/safety-checks.md | 25 +- docs/config.json | 2 +- docs/tools/provider-tools.md | 21 +- packages/ai-computer-e2b/README.md | 38 +- packages/ai-computer-e2b/package.json | 1 + packages/ai-computer-e2b/src/driver.ts | 88 +++- packages/ai-computer-e2b/src/index.ts | 2 + packages/ai-computer-e2b/tests/driver.test.ts | 111 +++- packages/ai-computer-e2b/tests/types.test.ts | 18 +- packages/ai-computer-playwright/README.md | 48 +- packages/ai-computer-playwright/package.json | 8 +- packages/ai-computer-playwright/src/driver.ts | 111 +++- packages/ai-computer-playwright/src/index.ts | 5 +- .../tests/driver.test.ts | 181 ++++++- .../tests/types.test.ts | 20 +- packages/ai-computer-use/README.md | 45 +- packages/ai-computer-use/src/actions.ts | 3 - .../src/create-computer-use.ts | 306 ++++++++--- packages/ai-computer-use/src/index.ts | 7 +- packages/ai-computer-use/src/types.ts | 38 +- .../tests/create-computer-use.test.ts | 490 +++++++++++++++++- packages/ai/src/activities/chat/index.ts | 3 + .../src/activities/chat/tools/tool-calls.ts | 5 + packages/ai/src/types.ts | 6 + packages/ai/tests/tool-calls-context.test.ts | 111 +++- pnpm-lock.yaml | 54 +- testing/e2e/tests/computer-use.spec.ts | 238 +++++++-- 34 files changed, 2145 insertions(+), 475 deletions(-) diff --git a/.changeset/ai-computer-use.md b/.changeset/ai-computer-use.md index b08bc7c173..e650ded209 100644 --- a/.changeset/ai-computer-use.md +++ b/.changeset/ai-computer-use.md @@ -8,24 +8,34 @@ '@tanstack/ai-anthropic': minor --- -Add computer use that runs the same code on OpenAI and on Anthropic. A driver controls the machine. +Add computer use that runs the same code on OpenAI and on Anthropic. A driver controls the machine, and each thread gets its own machine. -- `@tanstack/ai-computer-use` adds `createComputerUse()`. It returns a tool, a middleware, and a safety-check interrupt for `chat()`. -- `@tanstack/ai-computer-playwright` adds `createPlaywrightComputerDriver()`. It controls a Playwright page. -- `@tanstack/ai-computer-e2b` adds `createE2BComputerDriver()`. It controls a Linux desktop in an E2B sandbox. -- `@tanstack/ai-anthropic` and `@tanstack/openai-base` send the generic computer tool to the model as the native computer tool of the provider. `@tanstack/ai` adds the metadata reader for this in `@tanstack/ai/adapter-internals`. +New packages: + +- `@tanstack/ai-computer-use` adds `createComputerUse()`. It returns `tools`, a middleware, a safety-check interrupt, and `close`. Each `threadId` gets its own machine, action queue, and approvals. A thread that stays idle for `idleTimeout` (10 minutes by default) closes. +- `@tanstack/ai-computer-playwright` adds `playwrightComputer()`, which opens one browser page for each thread. `createPlaywrightComputerDriver()` controls one page that you give it. Both add a `navigate` tool that opens `http:` and `https:` URLs. +- `@tanstack/ai-computer-e2b` adds `e2bComputer()`, which starts one E2B desktop sandbox for each thread. `createE2BComputerDriver()` controls one sandbox that you give it. + +Changes in existing packages: + +- `@tanstack/ai` adds `threadId` to `ToolExecutionContext`. It also adds the computer tool metadata reader in `@tanstack/ai/adapter-internals`. +- `@tanstack/ai-anthropic` and `@tanstack/openai-base` send the generic computer tool to the model as the native computer tool of the provider. - `@tanstack/ai-openai` marks the models that support the `computer` tool. ```ts -const { tool, middleware, interrupt } = createComputerUse({ - driver: createPlaywrightComputerDriver({ page }), +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: 'https://tanstack.com' }), }) chat({ adapter, messages, - tools: [tool], - middleware: [middleware], - interrupts: [interrupt], + threadId, + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], }) + +await computerUse.close(threadId) // one thread +await computerUse.close() // every thread, and the browser ``` diff --git a/docs/adapters/anthropic.md b/docs/adapters/anthropic.md index a24edc98e9..f0bfd88b17 100644 --- a/docs/adapters/anthropic.md +++ b/docs/adapters/anthropic.md @@ -618,64 +618,51 @@ equivalent — see [Provider Skills](../tools/provider-skills.md). ### `computerUseTool` -Claude can look at a screen and use the mouse and the keyboard. Your app owns the screen: it runs each action that Claude asks for and sends back a screenshot. - -To run the actions on a browser or a desktop without your own action code, use `createComputerUse` from `@tanstack/ai-computer-use`. It runs each action on a Playwright page or an E2B desktop, and the adapter picks the tool version for your model. See [Computer use](../computer-use/computer-use.md). The rest of this section shows how to run the actions yourself. - -1. Pick the version that your model takes: - - | Version | Models | - | --- | --- | - | `computer_toolset_20260801` | Claude Opus 5.5, Sonnet 5.5, Haiku 5.5, Fable 5.1, Fable 5, Opus 5, Sonnet 5, Opus 4.8 | - | `computer_20251124` | Claude Fable 5.1, Fable 5, Opus 5, Sonnet 5, Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 4.6, Opus 4.5 | - | `computer_20250124` | Claude Sonnet 4.5, Haiku 4.5, Opus 4.1 | - -2. Give the tool an `execute` that runs each action on your screen. -3. Return a screenshot for the `screenshot` action, and `"OK"` for the other actions. +Claude can look at a screen and use the mouse and the keyboard. Your app runs each action and sends back a screenshot. `createComputerUse` from `@tanstack/ai-computer-use` does this on a Playwright browser or an E2B desktop. Each thread gets its own machine: ```typescript import { chat } from "@tanstack/ai"; import { anthropicText } from "@tanstack/ai-anthropic"; -import { computerUseTool } from "@tanstack/ai-anthropic/tools"; +import { createComputerUse } from "@tanstack/ai-computer-use"; +import { playwrightComputer } from "@tanstack/ai-computer-playwright"; -// Your screen, for example a virtual machine. Replace these two functions. -async function runAction(input: { action: string }): Promise { - console.log("run", input); -} -async function takeScreenshot(): Promise { - return ""; // A PNG, as base64. -} +declare const threadId: string; -const computer = { - ...computerUseTool({ - type: "computer_toolset_20260801", - configs: { zoom: { enabled: false } }, - }), - // Every version calls `execute` with the action in `input.action`. - execute: async (input: { action: string }) => { - if (input.action === "screenshot") { - const value = await takeScreenshot(); - return [ - { - type: "image", - source: { type: "data", value, mimeType: "image/png" }, - }, - ]; - } - await runAction(input); - return "OK"; - }, -}; +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: "https://tanstack.com" }), +}); const stream = chat({ adapter: anthropicText("claude-opus-5-5"), - messages: [{ role: "user", content: "Open the display settings." }], - tools: [computer], - // Claude can ask for several actions in one turn. Run them in order. - toolExecution: "sequential", + messages: [{ role: "user", content: "Find the TanStack AI docs" }], + threadId, + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], }); ``` +The adapter sends the `computer` tool to Claude as the newest native computer tool that your model takes. + +- For the setup, the threads, and how to close the machines, see [Computer use](../computer-use/computer-use.md). +- For your own machine, see [Custom driver](../computer-use/custom-driver.md). + +#### The raw tool + +`computerUseTool` from `@tanstack/ai-anthropic/tools` is the native tool with no `execute`. Use it when your app runs the actions itself. Pick the version that your model takes: + +| Version | Models | +| --- | --- | +| `computer_toolset_20260801` | Claude Opus 5.5, Sonnet 5.5, Haiku 5.5, Fable 5.1, Fable 5, Opus 5, Sonnet 5, Opus 4.8 | +| `computer_20251124` | Claude Fable 5.1, Fable 5, Opus 5, Sonnet 5, Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 4.6, Opus 4.5 | +| `computer_20250124` | Claude Sonnet 4.5, Haiku 4.5, Opus 4.1 | + +```typescript +import { computerUseTool } from "@tanstack/ai-anthropic/tools"; + +const computer = computerUseTool({ type: "computer_toolset_20260801" }); +``` + The earlier versions also take a name and the screen size: ```typescript @@ -689,13 +676,15 @@ const computer = computerUseTool({ }); ``` -Good to know: +Add an `execute` that runs the action and returns a result: +- Every version calls `execute` with the action in `input.action`. With the toolset, Claude calls each action as its own tool. The adapter gives you each call as one `computer` call. +- Return a screenshot as an image part for the `screenshot` action, and `"OK"` for the other actions. +- Send screenshots that fit the image limits of the model. The toolset refuses a large screenshot. It does not make it smaller. +- The toolset turns on `zoom` by default. Return an image for the `zoom` action, or turn it off with `configs: { zoom: { enabled: false } }`. - The adapter sends the beta header that an earlier version needs. The toolset needs no beta header. -- With the toolset, Claude calls each action as its own tool. The adapter gives you each call as one `computer` call, and sends it back in the form that Claude sent. So one `execute` works for every version. -- The toolset turns on `zoom` by default. To let Claude zoom, remove `configs` and return an image for the `zoom` action too. -- Send screenshots that fit the image limits of the model. The toolset does not make a large screenshot smaller. It refuses it. Claude gives coordinates in the pixels of your screenshots. -- If an action fails, answer each later action of the same turn with `Not executed: an earlier computer action in this turn failed.` + +Claude gives coordinates in the pixels of your screenshots. A version that your model does not take is a type error. `claude-opus-5-fast` takes no provider tools. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). diff --git a/docs/adapters/openai.md b/docs/adapters/openai.md index 277c49ea69..d6a0985b7c 100644 --- a/docs/adapters/openai.md +++ b/docs/adapters/openai.md @@ -992,65 +992,50 @@ const stream = chat({ ### `computerUseTool` -Lets the model observe a virtual desktop via screenshots and interact with -it using keyboard and mouse events. - -To run the actions on a browser or a desktop without your own action code, use `createComputerUse` from `@tanstack/ai-computer-use`. It runs each action on a Playwright page or an E2B desktop. See [Computer use](../computer-use/computer-use.md). For the approval of safety checks, see [Safety checks](../computer-use/safety-checks.md). The rest of this section shows how to run the actions yourself. - -The model asks for actions, for example a click and then some typing. Your app runs the actions, then returns a screenshot of the screen. Add an `execute` to the tool that does this: +The model can look at a screen and use the mouse and the keyboard. Your app runs each action and sends back a screenshot. `createComputerUse` from `@tanstack/ai-computer-use` does this on a Playwright browser or an E2B desktop. Each thread gets its own machine: ```typescript import { chat } from "@tanstack/ai"; import { openaiText } from "@tanstack/ai-openai"; -import { computerUseTool } from "@tanstack/ai-openai/tools"; +import { createComputerUse } from "@tanstack/ai-computer-use"; +import { playwrightComputer } from "@tanstack/ai-computer-playwright"; -interface SafetyCheck { - id: string; - code?: string; - message?: string; -} +declare const threadId: string; -// Your code: run one action on the desktop. -declare function runAction(action: unknown): Promise; -// Your code: return a PNG of the screen as base64. -declare function takeScreenshot(): Promise; -// Your code: show the checks to a person and return their answer. -declare function askPerson(checks: Array): Promise; +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: "https://tanstack.com" }), +}); const stream = chat({ adapter: openaiText("gpt-6.1-sol"), - messages: [{ role: "user", content: "Open example.com and sign up for the newsletter" }], - tools: [ - { - ...computerUseTool({ type: "computer" }), - execute: async (input: { - actions: Array; - pending_safety_checks: Array; - }) => { - const checks = input.pending_safety_checks; - if (checks.length > 0 && !(await askPerson(checks))) { - throw new Error("A person did not allow this action."); - } - for (const action of input.actions) { - await runAction(action); - } - const screenshot = await takeScreenshot(); - return [ - { - type: "image", - source: { type: "data", value: screenshot, mimeType: "image/png" }, - }, - ]; - }, - }, - ], + messages: [{ role: "user", content: "Find the TanStack AI docs" }], + threadId, + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], }); ``` -The input of each call has two fields: +The adapter sends the `computer` tool to OpenAI as its native computer tool. + +- For the setup, the threads, and how to close the machines, see [Computer use](../computer-use/computer-use.md). +- For the approval of OpenAI safety checks, see [Safety checks](../computer-use/safety-checks.md). +- For your own machine, see [Custom driver](../computer-use/custom-driver.md). + +#### The raw tool + +`computerUseTool` from `@tanstack/ai-openai/tools` is the native tool with no `execute`. Use it when your app runs the actions itself: + +```typescript +import { computerUseTool } from "@tanstack/ai-openai/tools"; + +const computer = computerUseTool({ type: "computer" }); +``` + +Add an `execute` that runs the actions and returns a screenshot as an image part. The input of each call has two fields: - `actions`: the actions to run, in order, as OpenAI sends them. For example `[{ type: "click", button: "left", x: 120, y: 340 }, { type: "type", text: "ada@example.com" }]`. -- `pending_safety_checks`: the safety checks of the actions. If the list is not empty, ask a person before you run the actions. +- `pending_safety_checks`: the safety checks that OpenAI attached to the actions. The adapter sends the first image of the result back as the screenshot. When the call does not fail, the adapter also tells OpenAI that the safety checks are accepted. When the call fails, they stay unaccepted. diff --git a/docs/api/ai.md b/docs/api/ai.md index 4e125f4ee4..6f377a4075 100644 --- a/docs/api/ai.md +++ b/docs/api/ai.md @@ -664,6 +664,7 @@ interface Tool { ```typescript ignore type ToolExecutionContext = { toolCallId?: string; + threadId?: string; emitCustomEvent: ( eventName: string, value: Record, @@ -674,6 +675,8 @@ type ToolExecutionContext = { `context` is the runtime value from `chat({ context })` for server tools, or from `ChatClient` / framework hook options for client tools. It is required when a tool declares a concrete `TContext` and optional for untyped tools where the context type is `unknown`. +`threadId` is the thread id of the chat run. `chat()` sets it to the same value as `ctx.threadId` in middleware. A tool can use it to keep its own state for each conversation. + ### `ChatMiddleware` ```typescript diff --git a/docs/computer-use/computer-use.md b/docs/computer-use/computer-use.md index 5e2e623890..3199d9e168 100644 --- a/docs/computer-use/computer-use.md +++ b/docs/computer-use/computer-use.md @@ -2,71 +2,60 @@ title: Computer Use id: computer-use order: 1 -description: "Let the model use a web page for the user. It sees screenshots and sends clicks and key presses. The same code runs on OpenAI and Anthropic." +description: "Let the model use a web page for the user. It sees screenshots and sends clicks and key presses. Each conversation gets its own browser page, and the same code runs on OpenAI and Anthropic." keywords: - tanstack ai - computer use - createComputerUse + - playwrightComputer - browser agent - playwright - openai - anthropic --- -You have a chat app. You want the model to do a task on a web page for the user, for example fill in a form or find a price. Computer use gives the model a screen, a mouse, and a keyboard. The same code runs on OpenAI and on Anthropic. +You have a chat app. You want the model to do a task on a web page for the user, for example fill in a form or find a price. Computer use gives the model a screen, a mouse, and a keyboard. Each conversation gets its own browser page. The same code runs on OpenAI and on Anthropic. ## Install -react: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod -vue: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod -solid: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod -svelte: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod -preact: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod -angular: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod -vanilla: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod -octane: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod +react: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +vue: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +solid: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +svelte: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +preact: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +angular: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +vanilla: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +octane: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod -## Give the model a browser page +Then download Chromium one time: -1. Open a Playwright page that has a viewport size. -2. Create the computer from a driver for that page. -3. Give `tool`, `middleware`, and `interrupt` to `chat()`. - -```ts -// app/api/chat/computers.ts -import { createComputerUse } from '@tanstack/ai-computer-use' -import { createPlaywrightComputerDriver } from '@tanstack/ai-computer-playwright' -import { chromium } from 'playwright' - -const browser = await chromium.launch() -const computers = new Map>() - -export async function computerFor(threadId: string) { - const existing = computers.get(threadId) - if (existing) return existing - const page = await browser.newPage({ viewport: { width: 1280, height: 720 } }) - await page.goto('https://example.com') - const computer = createComputerUse({ - driver: createPlaywrightComputerDriver({ page }), - }) - computers.set(threadId, computer) - return computer -} +```bash +npx playwright-core install chromium ``` +## Give the model a browser + +1. Create the computer with `playwrightComputer()`, one time, at module scope. +2. Give `tools`, `middleware`, and `interrupt` to `chat()`. +3. Give the `threadId` of the request to `chat()`. + ```ts // app/api/chat/route.ts import { chat, chatParamsFromRequest, toServerSentEventsResponse } from '@tanstack/ai' import { openaiText } from '@tanstack/ai-openai' -import { computerFor } from './computers' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { playwrightComputer } from '@tanstack/ai-computer-playwright' + +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: 'https://tanstack.com' }), +}) export async function POST(request: Request) { const params = await chatParamsFromRequest(request) - const { tool, middleware, interrupt } = await computerFor(params.threadId) const stream = chat({ adapter: openaiText('gpt-6.1-sol'), messages: params.messages, @@ -74,17 +63,19 @@ export async function POST(request: Request) { runId: params.runId, ...(params.parentRunId ? { parentRunId: params.parentRunId } : {}), ...(params.resume ? { resume: params.resume } : {}), - tools: [tool], - middleware: [middleware], - interrupts: [interrupt], + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], }) return toServerSentEventsResponse(stream) } ``` -One `createComputerUse` drives one machine for one conversation at a time. The map keeps one computer for each thread. Your app owns the browser. When the conversation ends, close the page. +- `tools`: the `computer` tool and the `navigate` tool. With `computer`, the model clicks, types, and takes screenshots. With `navigate`, it opens a URL. +- `middleware`: stops the rest of a turn after a failed action. It also runs the OpenAI safety checks. +- `interrupt`: asks a person to approve an action that has a safety check. -The `middleware` stops the rest of a turn after a failed action. It also runs the OpenAI safety checks. The `interrupt` asks a person to approve an action that has a safety check. +Chromium starts on the first action. Each thread gets a new page on its own first action, and the page opens `startUrl`. To use Anthropic, change only the adapter: @@ -96,6 +87,54 @@ const adapter = anthropicText('claude-opus-5-5') Need a full Linux desktop and not a browser page? See [Drivers](./drivers). +## One page for each thread + +Each `threadId` gets its own machine. With `playwrightComputer()`, the machine is a browser context with one page: + +- Each thread has its own cookies and storage. +- Each thread has its own action queue and its own safety-check approvals. +- All threads share one Chromium process. + +Give a stable `threadId` to `chat()`. If `chat()` gets no `threadId`, it makes a new id for each request. Then each request gets a new page. + +`useChat` sends the same thread id with each request of a chat. `chatParamsFromRequest` reads it, as in the route above. + +## Close the pages + +Each open page uses memory on your server. Close the page of a thread when its conversation ends. Close all pages when the server stops: + +```ts +import { createComputerUse } from '@tanstack/ai-computer-use' +import { playwrightComputer } from '@tanstack/ai-computer-playwright' + +declare const threadId: string + +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: 'https://tanstack.com' }), + idleTimeout: 30 * 60_000, +}) + +// The conversation ended. Close the page of this thread. +await computerUse.close(threadId) + +// The server stops. Close every page, then Chromium. +await computerUse.close() +``` + +A thread that stays idle closes by itself. `idleTimeout` sets this time in ms: + +- The default is 10 minutes. +- When the time ends, the page of the thread closes. The next action of that thread opens a new page at `startUrl`. +- `false` keeps every thread open until you call `close`. + +## Keep the browser away from private addresses + +CAUTION: Run the browser in a network where it cannot reach private services. The browser can open every address that your server can reach. + +For example, a page can link to an internal admin page or to a cloud metadata URL. The `navigate` tool accepts only `http:` and `https:` URLs. It does not block private addresses. + +To keep the machine off your network, use an [E2B sandbox](./drivers#e2b). + ## Register the interrupt on the client The client imports `computerSafetyCheck`. It is the same object as `interrupt`, so the client does not load the driver or Playwright. @@ -147,19 +186,23 @@ The model clicks, double-clicks, drags, moves the mouse, scrolls, types text, an - The screenshot comes 500 ms after the action. This time lets the page settle. - `zoom` returns an image of one region at full size. - `cursor_position` returns the mouse position as text. +- `navigate` opens a URL, then returns a screenshot. To change the delay, set `screenshotDelay` in ms: ```ts -const computer = createComputerUse({ - driver: createPlaywrightComputerDriver({ page }), +import { createComputerUse } from '@tanstack/ai-computer-use' +import { playwrightComputer } from '@tanstack/ai-computer-playwright' + +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: 'https://tanstack.com' }), screenshotDelay: 1000, }) ``` ## Order and failures -Actions run one at a time, in the order that the model calls them. +In one thread, actions run one at a time, in the order that the model calls them. Two threads run at the same time, each on its own page. If one action fails, the other actions of that model turn do not run. The model gets this result for each of them: @@ -169,6 +212,11 @@ Not executed: an earlier computer action in this turn failed. The next model turn starts clean. The model sees the error and can try again. +## Limits + +- **One server process.** The pages live in the memory of one process. If the next request of a thread goes to another server, the thread gets a new page there. +- **Time to approve.** If a person answers a safety check after `idleTimeout`, the page is already closed. The approved action then runs on a new page. Set `idleTimeout` above the time that a person takes to answer. + ## Model support - A model with no computer use is a type error at `chat()`. @@ -180,9 +228,9 @@ The next model turn starts clean. The model sees the error and can try again. Start the app and send this message: ```text -Open example.com and tell me the title of the page. +Go to https://tanstack.com/ai and tell me the title of the page. ``` -The model takes a screenshot, reads the page, and answers in the chat. You now have a browser agent. +The model calls `navigate`, reads the screenshot, and answers in the chat. You now have a browser agent. To control your own machine with no driver package, see [Custom driver](./custom-driver). diff --git a/docs/computer-use/custom-driver.md b/docs/computer-use/custom-driver.md index 972aa36eff..5cc3236560 100644 --- a/docs/computer-use/custom-driver.md +++ b/docs/computer-use/custom-driver.md @@ -2,11 +2,12 @@ title: Custom Computer Use Driver id: computer-use-custom-driver order: 4 -description: "Let the model use your own machine. Write a ComputerDriver with a small set of mouse, keyboard, and screenshot methods." +description: "Let the model use your own machine. Write a ComputerDriver with a small set of mouse, keyboard, and screenshot methods, and a ComputerDriverFactory to give each thread its own machine." keywords: - tanstack ai - computer use - ComputerDriver + - ComputerDriverFactory - custom driver - virtual machine --- @@ -21,46 +22,54 @@ A driver is an object with a screen size and a small set of methods. This exampl // app/my-driver.ts import type { ComputerDriver } from '@tanstack/ai-computer-use' -const base = 'http://localhost:8000' - -async function send(path: string, body: object) { - const response = await fetch(`${base}${path}`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - }) - if (!response.ok) throw new Error(`${path} failed: ${response.status}`) -} - -export const myDriver: ComputerDriver = { - display: { width: 1920, height: 1080 }, - environment: 'windows', - async screenshot() { - const response = await fetch(`${base}/screenshot`) - if (!response.ok) throw new Error(`/screenshot failed: ${response.status}`) - return new Uint8Array(await response.arrayBuffer()) - }, - click: (x, y, { button, count }) => send('/click', { x, y, button, count }), - move: (x, y) => send('/move', { x, y }), - mouseDown: (button) => send('/mouse-down', { button }), - mouseUp: (button) => send('/mouse-up', { button }), - scroll: (x, y, deltaX, deltaY) => send('/scroll', { x, y, deltaX, deltaY }), - keyDown: (key) => send('/key-down', { key }), - keyUp: (key) => send('/key-up', { key }), - press: (keys) => send('/press', { keys }), - type: (text) => send('/type', { text }), +export function createMyDriver(base: string) { + async function send(path: string, body: object) { + const response = await fetch(`${base}${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + if (!response.ok) throw new Error(`${path} failed: ${response.status}`) + } + + const driver: ComputerDriver = { + display: { width: 1920, height: 1080 }, + environment: 'windows', + async screenshot() { + const response = await fetch(`${base}/screenshot`) + if (!response.ok) throw new Error(`/screenshot failed: ${response.status}`) + return new Uint8Array(await response.arrayBuffer()) + }, + click: (x, y, { button, count }) => send('/click', { x, y, button, count }), + move: (x, y) => send('/move', { x, y }), + mouseDown: (button) => send('/mouse-down', { button }), + mouseUp: (button) => send('/mouse-up', { button }), + scroll: (x, y, deltaX, deltaY) => send('/scroll', { x, y, deltaX, deltaY }), + keyDown: (key) => send('/key-down', { key }), + keyUp: (key) => send('/key-up', { key }), + press: (keys) => send('/press', { keys }), + type: (text) => send('/type', { text }), + } + return driver } ``` -Then pass it to `createComputerUse`: +Then give it to `createComputerUse`: ```ts import { createComputerUse } from '@tanstack/ai-computer-use' -import { myDriver } from './my-driver' +import type { ComputerDriver } from '@tanstack/ai-computer-use' -const { tool, middleware, interrupt } = createComputerUse({ driver: myDriver }) +// From app/my-driver.ts above. +declare function createMyDriver(base: string): ComputerDriver + +const computerUse = createComputerUse({ + driver: createMyDriver('http://localhost:8000'), +}) ``` +A driver is one machine. Every thread shares it, so it serves one conversation at a time. `createComputerUse` never closes it. To give each thread its own machine, write a [driver factory](#one-machine-for-each-thread). + ## The contract Coordinates are in pixels, in `display` space. Key names are `KeyboardEvent.key` names, for example `'Enter'`, `'Control'`, or `'a'`. @@ -80,15 +89,17 @@ Coordinates are in pixels, in `display` space. Key names are `KeyboardEvent.key` ## Add the optional methods -Two methods are optional. When the driver has one, the Anthropic adapter turns on that action for the model. +Four methods are optional: -- `zoom(region)`: returns PNG bytes of one region at full size. The region has `x`, `y`, `width`, and `height`. -- `cursorPosition()`: returns the mouse position as `{ x, y }`. +- `zoom(region)`: returns PNG bytes of one region at full size. The region has `x`, `y`, `width`, and `height`. The Anthropic adapter then turns on the `zoom` action. +- `cursorPosition()`: returns the mouse position as `{ x, y }`. The Anthropic adapter then turns on the `cursor_position` action. +- `navigate(url)`: opens the URL. `computerUse.tools` then has the `navigate` tool. The tool accepts only `http:` and `https:` URLs. +- `close()`: frees the machine. `createComputerUse` calls it only on a driver that a factory made. Add this function to `app/my-driver.ts`. It narrows the JSON from your agent before it returns it: ```ts -async function cursorPosition() { +export async function readCursorPosition(base: string) { const response = await fetch(`${base}/cursor`) const data: unknown = await response.json() const isPoint = @@ -103,7 +114,67 @@ async function cursorPosition() { } ``` -Then add `cursorPosition` to `myDriver`. +Then add `cursorPosition: () => readCursorPosition(base)` to `driver` in `createMyDriver`. + +## One machine for each thread + +One shared machine serves one conversation at a time. To give each thread its own machine, write a `ComputerDriverFactory`. `createComputerUse` calls `create` on the first action of each thread. + +This example asks a pool service for one VM for each thread, and stops the VM when the thread closes: + +```ts +// app/my-machines.ts +import type { + ComputerDriver, + ComputerDriverFactory, +} from '@tanstack/ai-computer-use' + +// From app/my-driver.ts above. +declare function createMyDriver(base: string): ComputerDriver + +const pool = 'http://localhost:9000' + +export const myMachines: ComputerDriverFactory = { + display: { width: 1920, height: 1080 }, + environment: 'windows', + capabilities: { zoom: false, cursorPosition: false, navigate: false }, + async create({ threadId }) { + const machine = `${pool}/machines/${encodeURIComponent(threadId)}` + const response = await fetch(machine, { method: 'PUT' }) + if (!response.ok) throw new Error(`Start failed: ${response.status}`) + return { + ...createMyDriver(`${machine}/agent`), + close: async () => { + await fetch(machine, { method: 'DELETE' }) + }, + } + }, +} +``` + +Then give the factory to `createComputerUse`: + +```ts +import { createComputerUse } from '@tanstack/ai-computer-use' +import type { ComputerDriverFactory } from '@tanstack/ai-computer-use' + +// From app/my-machines.ts above. +declare const myMachines: ComputerDriverFactory + +const computerUse = createComputerUse({ driver: myMachines }) +``` + +The factory has these members: + +| Member | What it does | +| --- | --- | +| `display` | The screen size of every machine. Each driver from `create` has this `display`. | +| `environment` | The kind of every machine. | +| `capabilities` | `{ zoom, cursorPosition, navigate }`: the optional methods that every driver from `create` has. The tool is built before any machine exists, so the factory tells it. | +| `create({ threadId })` | Makes the driver of one thread. If it throws, the model gets the error. The next model turn calls `create` again. | +| `close()` | Optional. Frees what the threads share. `computerUse.close()` calls it after it closes every thread. | + +When a thread closes or goes idle, `createComputerUse` calls `close()` on the driver of that thread. ## What the core builds for you diff --git a/docs/computer-use/drivers.md b/docs/computer-use/drivers.md index 287d1f51cc..6759f07d31 100644 --- a/docs/computer-use/drivers.md +++ b/docs/computer-use/drivers.md @@ -2,11 +2,13 @@ title: Computer Use Drivers id: computer-use-drivers order: 2 -description: "Pick the machine that the model uses: a Playwright browser page or a full Linux desktop in an E2B sandbox." +description: "Pick the machine that the model uses: a browser page for each thread with Playwright, or a full Linux desktop for each thread in an E2B sandbox." keywords: - tanstack ai - computer use - driver + - playwrightComputer + - e2bComputer - playwright - e2b - linux desktop @@ -17,14 +19,15 @@ A browser page is not always enough. Sometimes the model must use a desktop app, ## Pick a driver -| | Playwright | E2B | -| ------------------- | ----------------------------------- | -------------------------------- | -| **Machine** | One browser page | A full Linux desktop in a sandbox | -| **Environment** | `browser` | `linux` | -| **Runs on** | Your server | E2B cloud | -| **`zoom`** | Yes | No | -| **`cursor_position`** | No | Yes | -| **Setup** | `pnpm add`, then launch a browser | An E2B API key, then create a sandbox | +| | Playwright | E2B | +| --------------------- | ----------------------------------------- | ------------------------------------- | +| **Machine** | A browser page for each thread | A Linux desktop sandbox for each thread | +| **Environment** | `browser` | `linux` | +| **Runs on** | Your server | E2B cloud | +| **`navigate` tool** | Yes | No | +| **`zoom`** | Yes | No | +| **`cursor_position`** | No | Yes | +| **Setup** | `playwright-core`, then download Chromium | An E2B API key | When a driver has `zoom` or `cursorPosition`, the Anthropic adapter turns on that action for the model. @@ -34,40 +37,73 @@ If the task stays inside a web page, use Playwright. -react: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod -vue: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod -solid: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod -svelte: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod -preact: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod -angular: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod -vanilla: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod -octane: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright zod +react: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +vue: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +solid: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +svelte: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +preact: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +angular: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +vanilla: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +octane: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +Then download Chromium one time: + +```bash +npx playwright-core install chromium +``` + +```ts +import { createComputerUse } from '@tanstack/ai-computer-use' +import { playwrightComputer } from '@tanstack/ai-computer-playwright' + +const computerUse = createComputerUse({ + driver: playwrightComputer({ + startUrl: 'https://tanstack.com', + display: { width: 1280, height: 800 }, + }), +}) +``` + +`playwrightComputer()` takes these options. All of them are optional: + +- `startUrl`: the URL that each new page opens. Without it, the page stays blank. +- `display`: the viewport of each page in pixels. The default is 1280 x 800. +- `browser`: a Playwright `Browser` that makes the pages. `createComputerUse` never closes it. +- `launch`: the options for `chromium.launch()`. When you give `browser`, the factory ignores them. + +Without `browser`, Chromium starts on the first action. `computerUse.close()` closes it. + +The `back` and `forward` mouse buttons go back and forward in the page history. + +### Share one page + +You already have a Playwright page, for example in a test. Give it to `createPlaywrightComputerDriver`. Use the Playwright package that your app already has. + ```ts -import { chromium } from 'playwright' import { createComputerUse } from '@tanstack/ai-computer-use' import { createPlaywrightComputerDriver } from '@tanstack/ai-computer-playwright' +import type { PlaywrightPageLike } from '@tanstack/ai-computer-playwright' -const browser = await chromium.launch() -const page = await browser.newPage({ viewport: { width: 1280, height: 720 } }) +// Your Playwright page. It must have a viewport size. +declare const page: PlaywrightPageLike -const { tool, middleware, interrupt } = createComputerUse({ +const computerUse = createComputerUse({ driver: createPlaywrightComputerDriver({ page }), }) ``` +- Every thread shares this page, so it serves one conversation at a time. - The page must have a viewport size. If it has none, the driver throws an error. - The driver reads the viewport size one time, when you create it. -- The `back` and `forward` mouse buttons go back and forward in the page history. -- A page from `playwright-core` or `@playwright/test` also works. +- A page from `playwright`, `playwright-core`, or `@playwright/test` works. -Your app owns the browser. The driver does not launch it or close it. +Your app owns this page. `createComputerUse` never closes it. ## E2B -If the model must use any app on a Linux desktop, use E2B. The desktop runs in a sandbox, away from your server. +If the model must use any app on a Linux desktop, use E2B. Each thread gets its own desktop in a sandbox, away from your server. @@ -83,42 +119,79 @@ octane: @tanstack/ai-computer-use @tanstack/ai-computer-e2b @e2b/desktop zod 1. Set `E2B_API_KEY` in your server environment. -2. Create a desktop sandbox. -3. Pass the sandbox to the driver. +2. Give `e2bComputer()` to `createComputerUse`. + +```ts +import { createComputerUse } from '@tanstack/ai-computer-use' +import { e2bComputer } from '@tanstack/ai-computer-e2b' + +const computerUse = createComputerUse({ + driver: e2bComputer({ timeoutMs: 60 * 60_000 }), +}) +``` + +`e2bComputer()` takes `display` and the options of `Sandbox.create` from `@e2b/desktop`: + +- `display`: the screen size of each sandbox in pixels. The default is 1024 x 768. +- `apiKey`: the E2B API key. Without it, E2B reads `E2B_API_KEY`. +- `timeoutMs`: the time in ms after which E2B kills the sandbox. +- `template`: the sandbox template. + +CAUTION: Set `timeoutMs` above `idleTimeout` plus the longest time that one thread stays in use. E2B kills a sandbox at its `timeoutMs`, also when the thread is busy. + +- There is no `navigate` tool, because a desktop has no address. The model opens the browser on the desktop. +- When a thread closes or goes idle, `createComputerUse` kills its sandbox. +- One scroll step is 100 pixels. +- The driver quotes every key name before it goes to the shell. + +### Share one sandbox + +You already have a desktop sandbox. Give it to `createE2BComputerDriver`: ```ts -import { Sandbox } from '@e2b/desktop' import { createComputerUse } from '@tanstack/ai-computer-use' import { createE2BComputerDriver } from '@tanstack/ai-computer-e2b' +import type { E2BDesktopLike } from '@tanstack/ai-computer-e2b' -const desktop = await Sandbox.create() +// Your sandbox, from `Sandbox.create()` of `@e2b/desktop`. +declare const desktop: E2BDesktopLike -const { tool, middleware, interrupt } = createComputerUse({ +const computerUse = createComputerUse({ driver: await createE2BComputerDriver({ desktop }), }) ``` +- Every thread shares this sandbox, so it serves one conversation at a time. - `createE2BComputerDriver` is async. It reads the screen size one time, so call it after the desktop is ready. -- One scroll step is 100 pixels. -- The driver quotes every key name before it goes to the shell. -Your app owns the sandbox. The driver does not create it or kill it. When the conversation ends, call `await desktop.kill()`. +Your app owns this sandbox. `createComputerUse` never kills it. When you are done, call `await desktop.kill()`. ## Use it in chat -Both drivers give the same three values. Pass them to `chat()`: +With every driver, `createComputerUse` gives `tools`, `middleware`, and `interrupt`. Give them to `chat()` with the thread id: ```ts import { chat } from '@tanstack/ai' +import type { ModelMessage } from '@tanstack/ai' import { anthropicText } from '@tanstack/ai-anthropic' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { e2bComputer } from '@tanstack/ai-computer-e2b' + +declare const messages: Array +declare const threadId: string + +const computerUse = createComputerUse({ + driver: e2bComputer({ timeoutMs: 60 * 60_000 }), +}) const stream = chat({ adapter: anthropicText('claude-opus-5-5'), messages, - tools: [tool], - middleware: [middleware], - interrupts: [interrupt], + threadId, + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], }) ``` -The model now uses a browser page or a full Linux desktop. For a machine that neither driver controls, write a [custom driver](./custom-driver). +The model now uses a browser page or a full Linux desktop, one for each thread. For a machine that no driver controls, write a [custom driver](./custom-driver). diff --git a/docs/computer-use/safety-checks.md b/docs/computer-use/safety-checks.md index bae65c2d03..72b2858fff 100644 --- a/docs/computer-use/safety-checks.md +++ b/docs/computer-use/safety-checks.md @@ -29,17 +29,23 @@ Safety checks are an OpenAI feature. Anthropic models do not send them, so this ## Server -Give `middleware` and `interrupt` to `chat()`. Also pass `parentRunId` and `resume`, so that the approval can continue the run. +Give `tools`, `middleware`, and `interrupt` to `chat()`. Also give `parentRunId` and `resume`, so that the approval can continue the run. ```ts // app/api/chat/route.ts import { chat, chatParamsFromRequest, toServerSentEventsResponse } from '@tanstack/ai' import { openaiText } from '@tanstack/ai-openai' -import { computerFor } from './computers' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { playwrightComputer } from '@tanstack/ai-computer-playwright' + +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: 'https://tanstack.com' }), + // A person has up to one hour to answer before the page closes. + idleTimeout: 60 * 60_000, +}) export async function POST(request: Request) { const params = await chatParamsFromRequest(request) - const { tool, middleware, interrupt } = await computerFor(params.threadId) const stream = chat({ adapter: openaiText('gpt-6.1-sol'), messages: params.messages, @@ -47,15 +53,20 @@ export async function POST(request: Request) { runId: params.runId, ...(params.parentRunId ? { parentRunId: params.parentRunId } : {}), ...(params.resume ? { resume: params.resume } : {}), - tools: [tool], - middleware: [middleware], - interrupts: [interrupt], + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], }) return toServerSentEventsResponse(stream) } ``` -`computerFor` returns the same `createComputerUse` result for each thread. The approval is stored on that result, so the continuation must use the same one. [Computer Use](./computer-use) shows `computerFor`. +`computerUse` keeps the approval for the thread until the resume request comes. To keep it there: + +- Create `computerUse` one time, at module scope. Every request uses the same one. +- Give `params.threadId` to `chat()`. The resume request has the same thread id as the request that paused. +- Set `idleTimeout` above the time that a person takes to answer. If the time ends first, the page closes, and the approved action runs on a new page. +- Send the resume request to the same server process. The approvals live in the memory of that process. ## Client diff --git a/docs/config.json b/docs/config.json index ded4edfa89..1e77cc7fc8 100644 --- a/docs/config.json +++ b/docs/config.json @@ -1505,7 +1505,7 @@ "label": "@tanstack/ai", "to": "api/ai", "addedAt": "2026-04-15", - "updatedAt": "2026-10-13" + "updatedAt": "2026-10-20" }, { "label": "@tanstack/ai-client", diff --git a/docs/tools/provider-tools.md b/docs/tools/provider-tools.md index 7be0949fec..929e438b87 100644 --- a/docs/tools/provider-tools.md +++ b/docs/tools/provider-tools.md @@ -92,28 +92,27 @@ assistant message; the agent loop never tries to run it client-side. OpenAI `applyPatchTool`, `localShellTool`, `computerUseTool`, and `shellTool({ environment: { type: "local" } })` are different. The model returns the call, and your app runs it. See [OpenAI adapter](../adapters/openai.md#applypatchtool) and [`computerUseTool`](../adapters/openai.md#computerusetool). -To run computer use actions on a browser or a desktop without your own action code, use `createComputerUse` from `@tanstack/ai-computer-use`. The same `tool` works with OpenAI and Anthropic. See [Computer use](../computer-use/computer-use.md). +To run computer use actions on a browser or a desktop without your own action code, use `createComputerUse` from `@tanstack/ai-computer-use`. The same `tools` work with OpenAI and Anthropic, and each thread gets its own machine. See [Computer use](../computer-use/computer-use.md). ```typescript import { chat } from '@tanstack/ai' import { openaiText } from '@tanstack/ai-openai' import { createComputerUse } from '@tanstack/ai-computer-use' -import { createPlaywrightComputerDriver } from '@tanstack/ai-computer-playwright' -import { chromium } from 'playwright' +import { playwrightComputer } from '@tanstack/ai-computer-playwright' -const browser = await chromium.launch() -const page = await browser.newPage({ viewport: { width: 1280, height: 800 } }) +declare const threadId: string -const { tool, middleware, interrupt } = createComputerUse({ - driver: createPlaywrightComputerDriver({ page }), +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: 'https://tanstack.com' }), }) const stream = chat({ adapter: openaiText('gpt-6.1-sol'), - messages: [{ role: 'user', content: 'Open example.com' }], - tools: [tool], - middleware: [middleware], - interrupts: [interrupt], + messages: [{ role: 'user', content: 'Find the TanStack AI docs' }], + threadId, + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], }) ``` diff --git a/packages/ai-computer-e2b/README.md b/packages/ai-computer-e2b/README.md index 3de53c87e9..f25554a595 100644 --- a/packages/ai-computer-e2b/README.md +++ b/packages/ai-computer-e2b/README.md @@ -22,7 +22,7 @@ E2B desktop driver for TanStack AI computer use. -Let the model use a Linux desktop in an E2B sandbox. The model sees screenshots and sends clicks, scrolls, and key presses. The driver runs them on the desktop. +Let the model use a Linux desktop in an E2B sandbox. Each thread gets its own sandbox. The model sees screenshots and sends clicks, scrolls, and key presses. The driver runs them on the desktop. ## Installation @@ -33,7 +33,26 @@ pnpm add @tanstack/ai-computer-e2b @tanstack/ai-computer-use @e2b/desktop ## Usage 1. Set `E2B_API_KEY` in your server environment. -2. Create a desktop sandbox and pass it to the driver: +2. Create the computer with `e2bComputer()`: + +```typescript +import { createComputerUse } from '@tanstack/ai-computer-use' +import { e2bComputer } from '@tanstack/ai-computer-e2b' + +const computerUse = createComputerUse({ + driver: e2bComputer({ timeoutMs: 60 * 60_000 }), +}) +``` + +3. Give `computerUse.tools`, `middleware`, `interrupt`, and the thread id to `chat()`. The model can now use a desktop. + +`e2bComputer()` takes `display` and the options of `Sandbox.create`, like `apiKey`, `timeoutMs`, and `template`. The default `display` is 1024 x 768. When a thread closes or goes idle, `createComputerUse` kills its sandbox. + +Set `timeoutMs` above `idleTimeout` plus the longest time that one thread stays in use. E2B kills a sandbox at its `timeoutMs`, also when the thread is busy. + +## Share one sandbox + +To drive a sandbox that you already have, use `createE2BComputerDriver`. Every thread shares this sandbox, so it serves one conversation at a time. ```typescript import { Sandbox } from '@e2b/desktop' @@ -42,19 +61,14 @@ import { createE2BComputerDriver } from '@tanstack/ai-computer-e2b' const desktop = await Sandbox.create() -const { tool, middleware, interrupt } = createComputerUse({ +const computerUse = createComputerUse({ driver: await createE2BComputerDriver({ desktop }), }) ``` -3. Give `tool`, `middleware`, and `interrupt` to `chat()`. The model can now use the desktop. - -The driver accepts any object with the shape of an `@e2b/desktop` `Sandbox`. - -## What your application owns - -- **The sandbox.** The driver does not create, pause, or kill it. Kill it when you are done. -- **The screen size.** The driver reads it once, when you create the driver. +- The driver accepts any object with the shape of an `@e2b/desktop` `Sandbox`. +- The driver reads the screen size once, when you create it. +- Your app owns this sandbox. `createComputerUse` never kills it. Kill it when you are done. ## What the driver does @@ -62,7 +76,7 @@ The driver accepts any object with the shape of an `@e2b/desktop` `Sandbox`. - It maps `KeyboardEvent.key` names, like `Control` and `Enter`, to `xdotool` names. - It quotes every key name before it goes to the shell. - One scroll step is 100 pixels. -- It supports `cursorPosition`. It does not support `zoom`. +- It supports `cursorPosition`. It has no `zoom` and no `navigate`. The model opens the browser on the desktop. See the [drivers guide](https://tanstack.com/ai/latest/docs/computer-use/drivers) for more. diff --git a/packages/ai-computer-e2b/package.json b/packages/ai-computer-e2b/package.json index 9f6cb3d457..ca5b84a21a 100644 --- a/packages/ai-computer-e2b/package.json +++ b/packages/ai-computer-e2b/package.json @@ -45,6 +45,7 @@ "sandbox" ], "peerDependencies": { + "@e2b/desktop": "^2.0.0", "@tanstack/ai-computer-use": "workspace:^" }, "devDependencies": { diff --git a/packages/ai-computer-e2b/src/driver.ts b/packages/ai-computer-e2b/src/driver.ts index 0eb9c5c4bd..c538326a34 100644 --- a/packages/ai-computer-e2b/src/driver.ts +++ b/packages/ai-computer-e2b/src/driver.ts @@ -1,4 +1,9 @@ -import type { ComputerDriver, MouseButton } from '@tanstack/ai-computer-use' +import { Sandbox } from '@e2b/desktop' +import type { + ComputerDriver, + ComputerDriverFactory, + MouseButton, +} from '@tanstack/ai-computer-use' /** The parts of an `@e2b/desktop` `Sandbox` that the driver uses. */ export interface E2BDesktopLike { @@ -150,3 +155,84 @@ export async function createE2BComputerDriver({ } return driver } + +/** + * The options of `Sandbox.create` from `@e2b/desktop`. The package does not + * export this type. Its `SandboxOpts` export is the base `e2b` type, without + * `resolution`. Both `create` signatures take these options last. + */ +type DesktopSandboxOpts = NonNullable[1]> + +/** + * Options for {@link e2bComputer}. Every option except `display` goes to + * `Sandbox.create` from `@e2b/desktop`, like `apiKey`, `timeoutMs`, and + * `template`. + */ +export type E2BComputerOptions = Omit< + DesktopSandboxOpts, + 'resolution' | 'display' +> & { + /** + * The screen size in pixels. Each sandbox starts at this resolution. + * The default is `{ width: 1024, height: 768 }`. + */ + display?: { width: number; height: number } +} + +/** + * Create a driver factory that starts one E2B desktop sandbox for each thread. + * + * When `createComputerUse` closes a thread, or the thread goes idle, the + * sandbox of that thread is killed. The drivers have `cursorPosition`. They + * have no `zoom` and no `navigate`, because a desktop has no address. The + * model opens the browser on the desktop. + * + * ponytail: E2B kills a sandbox after its own `timeoutMs`, even when the + * thread is busy. Pass a `timeoutMs` above `idleTimeout` plus the longest + * thread. + * + * @param options - The `display` size and the `Sandbox.create` options. + * + * @example + * const computer = createComputerUse({ + * driver: e2bComputer({ timeoutMs: 30 * 60_000 }), + * }) + */ +export function e2bComputer(options: E2BComputerOptions = {}) { + const { + display = { width: 1024, height: 768 }, + template, + ...sandboxOptions + } = options + + const factory: ComputerDriverFactory = { + display, + environment: 'linux', + capabilities: { zoom: false, cursorPosition: true, navigate: false }, + async create() { + const createOptions: DesktopSandboxOpts = { + ...sandboxOptions, + resolution: [display.width, display.height], + } + // The desktop SDK reads the template only from the first argument. + const desktop = + template === undefined + ? await Sandbox.create(createOptions) + : await Sandbox.create(template, createOptions) + const driver = await createE2BComputerDriver({ desktop }).catch( + async (error: unknown) => { + // Keep the setup error. A failed kill does not replace it. + await desktop.kill().catch(() => undefined) + throw error + }, + ) + return { + ...driver, + close: async () => { + await desktop.kill() + }, + } + }, + } + return factory +} diff --git a/packages/ai-computer-e2b/src/index.ts b/packages/ai-computer-e2b/src/index.ts index abd637a598..cd9116570f 100644 --- a/packages/ai-computer-e2b/src/index.ts +++ b/packages/ai-computer-e2b/src/index.ts @@ -1,5 +1,7 @@ export { createE2BComputerDriver, + e2bComputer, type E2BComputerDriverOptions, + type E2BComputerOptions, type E2BDesktopLike, } from './driver' diff --git a/packages/ai-computer-e2b/tests/driver.test.ts b/packages/ai-computer-e2b/tests/driver.test.ts index daa1735d13..8d8a42a7cc 100644 --- a/packages/ai-computer-e2b/tests/driver.test.ts +++ b/packages/ai-computer-e2b/tests/driver.test.ts @@ -1,18 +1,27 @@ -import { describe, expect, it } from 'vitest' -import { createE2BComputerDriver } from '../src/index' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { createE2BComputerDriver, e2bComputer } from '../src/index' import type { E2BDesktopLike } from '../src/index' +/** A desktop as `Sandbox.create` returns it, with `kill`. */ +type FakeDesktop = E2BDesktopLike & { kill: () => Promise } + +const { create } = vi.hoisted(() => ({ + create: vi.fn<(...args: Array) => Promise>(), +})) + +vi.mock('@e2b/desktop', () => ({ Sandbox: { create } })) + const PNG = new Uint8Array([137, 80, 78, 71]) /** A fake desktop. Each SDK call adds one line to `calls`. */ -function fakeDesktop() { +function fakeDesktop(overrides: Partial = {}) { const calls: Array = [] const record = (name: string) => async (...args: Array) => { calls.push([name, ...args].join(' ')) } - const desktop: E2BDesktopLike = { + const desktop: FakeDesktop = { getScreenSize: async () => ({ width: 1024, height: 768 }), getCursorPosition: async () => ({ x: 12, y: 34 }), screenshot: async () => PNG, @@ -26,6 +35,8 @@ function fakeDesktop() { scroll: record('scroll'), write: record('write'), commands: { run: record('run') }, + kill: record('kill'), + ...overrides, } return { desktop, calls } } @@ -36,6 +47,17 @@ async function setup() { return { driver, calls } } +/** Make each `Sandbox.create` call return a new fake desktop. Returns the fakes in create order. */ +function mockSandboxes() { + const sandboxes: Array> = [] + create.mockImplementation(async () => { + const sandbox = fakeDesktop() + sandboxes.push(sandbox) + return sandbox.desktop + }) + return sandboxes +} + describe('createE2BComputerDriver', () => { it('reads the display once and reports linux', async () => { const { driver } = await setup() @@ -143,3 +165,84 @@ describe('createE2BComputerDriver', () => { expect(calls).toEqual(['write hello']) }) }) + +describe('e2bComputer', () => { + beforeEach(() => { + create.mockReset() + }) + + it('reports a linux desktop with cursor position only', () => { + const factory = e2bComputer() + expect(factory.display).toEqual({ width: 1024, height: 768 }) + expect(factory.environment).toBe('linux') + expect(factory.capabilities).toEqual({ + zoom: false, + cursorPosition: true, + navigate: false, + }) + expect(factory.close).toBeUndefined() + }) + + it('creates the sandbox at the default resolution', async () => { + mockSandboxes() + await e2bComputer().create({ threadId: 'a' }) + expect(create).toHaveBeenCalledWith({ resolution: [1024, 768] }) + }) + + it('passes the sandbox options through and takes the resolution from display', async () => { + mockSandboxes() + const factory = e2bComputer({ + apiKey: 'key', + timeoutMs: 60_000, + display: { width: 800, height: 600 }, + }) + await factory.create({ threadId: 'a' }) + expect(factory.display).toEqual({ width: 800, height: 600 }) + expect(create).toHaveBeenCalledWith({ + apiKey: 'key', + timeoutMs: 60_000, + resolution: [800, 600], + }) + }) + + it('gives the template to Sandbox.create as the first argument', async () => { + mockSandboxes() + await e2bComputer({ template: 'my-desktop' }).create({ threadId: 'a' }) + expect(create).toHaveBeenCalledWith('my-desktop', { + resolution: [1024, 768], + }) + }) + + it('creates one sandbox for each thread', async () => { + const sandboxes = mockSandboxes() + const factory = e2bComputer() + const a = await factory.create({ threadId: 'a' }) + const b = await factory.create({ threadId: 'b' }) + await a.move(1, 2) + await b.move(3, 4) + expect(sandboxes.map(({ calls }) => calls)).toEqual([ + ['moveMouse 1 2'], + ['moveMouse 3 4'], + ]) + }) + + it('kills only the sandbox of the closed driver', async () => { + const sandboxes = mockSandboxes() + const factory = e2bComputer() + const a = await factory.create({ threadId: 'a' }) + await factory.create({ threadId: 'b' }) + await a.close?.() + expect(sandboxes.map(({ calls }) => calls)).toEqual([['kill'], []]) + }) + + it('kills the sandbox and rethrows when the driver setup fails', async () => { + const { desktop, calls } = fakeDesktop({ + getScreenSize: () => Promise.reject(new Error('no screen')), + }) + create.mockResolvedValue(desktop) + await expect(e2bComputer().create({ threadId: 'a' })).rejects.toThrow( + 'no screen', + ) + expect(calls).toEqual(['kill']) + }) +}) diff --git a/packages/ai-computer-e2b/tests/types.test.ts b/packages/ai-computer-e2b/tests/types.test.ts index 130a2ad924..d87e1dde0a 100644 --- a/packages/ai-computer-e2b/tests/types.test.ts +++ b/packages/ai-computer-e2b/tests/types.test.ts @@ -1,9 +1,25 @@ import { describe, expectTypeOf, it } from 'vitest' import type { Sandbox } from '@e2b/desktop' -import type { E2BDesktopLike } from '../src/index' +import type { E2BComputerOptions, E2BDesktopLike } from '../src/index' describe('E2BDesktopLike', () => { it('accepts a real @e2b/desktop Sandbox', () => { expectTypeOf().toExtend() }) }) + +describe('E2BComputerOptions', () => { + it('accepts the sandbox options and a display', () => { + expectTypeOf< + Pick + >().toEqualTypeOf<{ + apiKey?: string + timeoutMs?: number + display?: { width: number; height: number } + }>() + }) + + it('has no resolution, because display sets it', () => { + expectTypeOf().not.toHaveProperty('resolution') + }) +}) diff --git a/packages/ai-computer-playwright/README.md b/packages/ai-computer-playwright/README.md index 1707ea2d0f..a4e7146916 100644 --- a/packages/ai-computer-playwright/README.md +++ b/packages/ai-computer-playwright/README.md @@ -22,43 +22,59 @@ Playwright browser driver for TanStack AI computer use. -Let the model see and control one browser page. The driver takes screenshots of the page and sends mouse and keyboard input to it. +Let the model see and control a browser page. Each thread gets its own page. The driver takes screenshots of the page and sends mouse and keyboard input to it. ## Installation ```bash -pnpm add @tanstack/ai-computer-playwright @tanstack/ai-computer-use playwright +pnpm add @tanstack/ai-computer-playwright @tanstack/ai-computer-use playwright-core +npx playwright-core install chromium ``` ## Usage -1. Open a page that has a viewport size. -2. Pass the page to the driver: +1. Create the computer with `playwrightComputer()`: ```typescript -import { chromium } from 'playwright' import { createComputerUse } from '@tanstack/ai-computer-use' -import { createPlaywrightComputerDriver } from '@tanstack/ai-computer-playwright' - -const browser = await chromium.launch() -const page = await browser.newPage({ viewport: { width: 1280, height: 720 } }) +import { playwrightComputer } from '@tanstack/ai-computer-playwright' -const { tool, middleware, interrupt } = createComputerUse({ - driver: createPlaywrightComputerDriver({ page }), +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: 'https://tanstack.com' }), }) ``` -3. Give `tool`, `middleware`, and `interrupt` to `chat()`. The model can now use the page. +2. Give `computerUse.tools`, `middleware`, `interrupt`, and the thread id to `chat()`. The model can now use a page. + +Each thread gets its own browser context, with its own cookies and storage, and one page. Chromium starts on the first action. `computerUse.close()` closes it. + +`playwrightComputer()` takes these options. All of them are optional: + +- `startUrl`: the URL that each new page opens. +- `display`: the viewport of each page in pixels. The default is 1280 x 800. +- `browser`: a Playwright `Browser` that makes the pages. `createComputerUse` never closes it. +- `launch`: the options for `chromium.launch()`. + +## Share one page -The driver accepts any object with the shape of a Playwright `Page`. A page from `playwright-core` or `@playwright/test` also works. +To drive a page that you already have, use `createPlaywrightComputerDriver`. Every thread shares this page, so it serves one conversation at a time. -## What your application owns +```typescript +import { createComputerUse } from '@tanstack/ai-computer-use' +import { createPlaywrightComputerDriver } from '@tanstack/ai-computer-playwright' + +const computerUse = createComputerUse({ + driver: createPlaywrightComputerDriver({ page }), +}) +``` -- **The browser.** The driver does not launch or close it. Close it when you are done. -- **The viewport.** The driver reads the viewport size once, when you create it. A page with no viewport size throws an error. +- The driver accepts any object with the shape of a Playwright `Page`. A page from `playwright`, `playwright-core`, or `@playwright/test` works. +- The driver reads the viewport size once, when you create it. A page with no viewport size throws an error. +- Your app owns this page. `createComputerUse` never closes it. ## What the driver does +- `navigate` opens a URL in the page. Then `computerUse.tools` has the `navigate` tool. - The `back` and `forward` mouse buttons go back and forward in the page history. - Key names go to Playwright as they are. `['Control', 's']` becomes `Control+s`. - `zoom` takes a screenshot of one region of the page. diff --git a/packages/ai-computer-playwright/package.json b/packages/ai-computer-playwright/package.json index a15d4cc40d..f642c94699 100644 --- a/packages/ai-computer-playwright/package.json +++ b/packages/ai-computer-playwright/package.json @@ -45,7 +45,13 @@ "browser" ], "peerDependencies": { - "@tanstack/ai-computer-use": "workspace:^" + "@tanstack/ai-computer-use": "workspace:^", + "playwright-core": "^1.40.0" + }, + "peerDependenciesMeta": { + "playwright-core": { + "optional": true + } }, "devDependencies": { "@tanstack/ai-computer-use": "workspace:*", diff --git a/packages/ai-computer-playwright/src/driver.ts b/packages/ai-computer-playwright/src/driver.ts index f08f6aa67f..52cd5442a0 100644 --- a/packages/ai-computer-playwright/src/driver.ts +++ b/packages/ai-computer-playwright/src/driver.ts @@ -1,4 +1,9 @@ -import type { ComputerDriver, MouseButton } from '@tanstack/ai-computer-use' +import type { + ComputerDriver, + ComputerDriverFactory, + MouseButton, +} from '@tanstack/ai-computer-use' +import type { LaunchOptions } from 'playwright-core' type PageMouseButton = 'left' | 'right' | 'middle' @@ -28,6 +33,21 @@ export interface PlaywrightPageLike { } goBack: () => Promise goForward: () => Promise + goto: (url: string) => Promise +} + +/** The slice of a Playwright `BrowserContext` the factory uses. */ +export interface PlaywrightContextLike { + newPage: () => Promise + close: () => Promise +} + +/** The slice of a Playwright `Browser` the factory uses. */ +export interface PlaywrightBrowserLike { + newContext: (options: { + viewport: { width: number; height: number } + }) => Promise + close: () => Promise } export interface PlaywrightComputerDriverOptions { @@ -35,6 +55,20 @@ export interface PlaywrightComputerDriverOptions { page: PlaywrightPageLike } +export interface PlaywrightComputerOptions { + /** The URL that each new page opens. Without it, the page stays blank. */ + startUrl?: string + /** The viewport of each page in pixels. The default is 1280 x 800. */ + display?: { width: number; height: number } + /** + * The browser that makes the pages. Without it, the factory launches + * Chromium on the first thread. A browser you pass in stays open. + */ + browser?: PlaywrightBrowserLike + /** The options for `chromium.launch`. With `browser`, the factory ignores them. */ + launch?: LaunchOptions +} + /** * Create a computer driver that controls one Playwright page. * @@ -56,7 +90,7 @@ export function createPlaywrightComputerDriver({ ) } - async function navigate(button: 'back' | 'forward') { + async function history(button: 'back' | 'forward') { if (button === 'back') await page.goBack() else await page.goForward() } @@ -66,7 +100,7 @@ export function createPlaywrightComputerDriver({ environment: 'browser', screenshot: () => page.screenshot({ type: 'png' }), async click(x, y, { button, count }) { - if (button === 'back' || button === 'forward') return navigate(button) + if (button === 'back' || button === 'forward') return history(button) await page.mouse.click(x, y, { button, clickCount: count }) }, move: (x, y) => page.mouse.move(x, y), @@ -76,7 +110,7 @@ export function createPlaywrightComputerDriver({ await page.mouse.down({ button }) }, async mouseUp(button: MouseButton) { - if (button === 'back' || button === 'forward') return navigate(button) + if (button === 'back' || button === 'forward') return history(button) await page.mouse.up({ button }) }, async scroll(x, y, deltaX, deltaY) { @@ -88,6 +122,75 @@ export function createPlaywrightComputerDriver({ press: (keys) => page.keyboard.press(keys.join('+')), type: (text) => page.keyboard.type(text), zoom: (region) => page.screenshot({ type: 'png', clip: region }), + navigate: async (url) => { + await page.goto(url) + }, } return driver } + +/** + * Create a driver factory that gives each thread its own browser page. + * + * Each thread gets its own browser context, with its own cookies and + * storage. When the thread closes, its context closes. When you pass no + * `browser`, the factory launches Chromium once, on the first thread, and + * `close()` on the factory closes it. + * + * Without `browser`, the app installs `playwright-core` and runs + * `npx playwright-core install chromium` once. + * + * @param options - The start URL, the display, and the browser or the launch options. + * + * @example + * const computer = createComputerUse({ + * driver: playwrightComputer({ startUrl: 'https://tanstack.com' }), + * }) + */ +export function playwrightComputer(options: PlaywrightComputerOptions = {}) { + const { startUrl, browser, launch } = options + const display = options.display ?? { width: 1280, height: 800 } + let launched: Promise | null = null + + async function launchChromium() { + const { chromium } = await import('playwright-core') + return chromium.launch(launch) + } + + function getBrowser() { + if (browser !== undefined) return Promise.resolve(browser) + launched ??= launchChromium().catch((error: unknown) => { + launched = null + throw error + }) + return launched + } + + const factory: ComputerDriverFactory = { + display, + environment: 'browser', + capabilities: { zoom: true, cursorPosition: false, navigate: true }, + async create() { + const shared = await getBrowser() + const context = await shared.newContext({ viewport: display }) + try { + const page = await context.newPage() + if (startUrl !== undefined) await page.goto(startUrl) + return { + ...createPlaywrightComputerDriver({ page }), + close: () => context.close(), + } + } catch (error) { + await context.close() + throw error + } + }, + async close() { + const current = launched + launched = null + if (current === null) return + await (await current).close() + }, + } + return factory +} diff --git a/packages/ai-computer-playwright/src/index.ts b/packages/ai-computer-playwright/src/index.ts index 251ffb19a3..343871da0f 100644 --- a/packages/ai-computer-playwright/src/index.ts +++ b/packages/ai-computer-playwright/src/index.ts @@ -1,5 +1,8 @@ -export { createPlaywrightComputerDriver } from './driver' +export { createPlaywrightComputerDriver, playwrightComputer } from './driver' export type { + PlaywrightBrowserLike, PlaywrightComputerDriverOptions, + PlaywrightComputerOptions, + PlaywrightContextLike, PlaywrightPageLike, } from './driver' diff --git a/packages/ai-computer-playwright/tests/driver.test.ts b/packages/ai-computer-playwright/tests/driver.test.ts index b64e85b1b2..0b5dbd5733 100644 --- a/packages/ai-computer-playwright/tests/driver.test.ts +++ b/packages/ai-computer-playwright/tests/driver.test.ts @@ -1,6 +1,16 @@ -import { describe, expect, it } from 'vitest' -import { createPlaywrightComputerDriver } from '../src/index' -import type { PlaywrightPageLike } from '../src/index' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + createPlaywrightComputerDriver, + playwrightComputer, +} from '../src/index' +import type { LaunchOptions } from 'playwright-core' +import type { PlaywrightBrowserLike, PlaywrightPageLike } from '../src/index' + +const launch = vi.hoisted(() => + vi.fn<(options?: LaunchOptions) => Promise>(), +) + +vi.mock('playwright-core', () => ({ chromium: { launch } })) const PNG = new Uint8Array([137, 80, 78, 71]) @@ -37,6 +47,7 @@ function createFakePage( }, goBack: record('goBack'), goForward: record('goForward'), + goto: record('goto'), } return { page, calls } } @@ -46,6 +57,37 @@ function setup() { return { driver: createPlaywrightComputerDriver({ page }), calls } } +function createFakeBrowser({ gotoError }: { gotoError?: Error } = {}) { + const contexts: Array<{ + viewport: { width: number; height: number } + calls: Array<[string, ...Array]> + closed: boolean + }> = [] + const state = { closed: false } + const browser: PlaywrightBrowserLike = { + newContext: async ({ viewport }) => { + const { page, calls } = createFakePage(viewport) + if (gotoError) { + page.goto = async () => { + throw gotoError + } + } + const context = { viewport, calls, closed: false } + contexts.push(context) + return { + newPage: async () => page, + close: async () => { + context.closed = true + }, + } + }, + close: async () => { + state.closed = true + }, + } + return { browser, contexts, state } +} + describe('createPlaywrightComputerDriver', () => { it('reads display from the viewport and reports a browser', () => { const { driver } = setup() @@ -129,4 +171,137 @@ describe('createPlaywrightComputerDriver', () => { ['keyboard.type', 'hello'], ]) }) + + it('goes to a URL', async () => { + const { driver, calls } = setup() + await driver.navigate?.('https://tanstack.com') + expect(calls).toEqual([['goto', 'https://tanstack.com']]) + }) +}) + +describe('playwrightComputer', () => { + beforeEach(() => { + launch.mockReset() + }) + + it('reports a browser with zoom and navigate', async () => { + const { browser } = createFakeBrowser() + const factory = playwrightComputer({ browser }) + const driver = await factory.create({ threadId: 'a' }) + expect(factory.environment).toBe('browser') + expect(factory.capabilities).toEqual({ + zoom: true, + cursorPosition: false, + navigate: true, + }) + expect(driver.zoom).toBeTypeOf('function') + expect(driver.navigate).toBeTypeOf('function') + expect(driver.cursorPosition).toBeUndefined() + }) + + it('makes one 1280x800 context for each thread by default', async () => { + const { browser, contexts } = createFakeBrowser() + const factory = playwrightComputer({ browser }) + const driver = await factory.create({ threadId: 'a' }) + await factory.create({ threadId: 'b' }) + expect(factory.display).toEqual({ width: 1280, height: 800 }) + expect(driver.display).toEqual({ width: 1280, height: 800 }) + expect(contexts.map((context) => context.viewport)).toEqual([ + { width: 1280, height: 800 }, + { width: 1280, height: 800 }, + ]) + }) + + it('uses the display option as the viewport', async () => { + const { browser, contexts } = createFakeBrowser() + const display = { width: 1024, height: 768 } + const factory = playwrightComputer({ browser, display }) + const driver = await factory.create({ threadId: 'a' }) + expect(factory.display).toEqual({ width: 1024, height: 768 }) + expect(driver.display).toEqual({ width: 1024, height: 768 }) + expect(contexts[0]?.viewport).toEqual({ width: 1024, height: 768 }) + }) + + it('opens startUrl on the new page', async () => { + const { browser, contexts } = createFakeBrowser() + const factory = playwrightComputer({ + browser, + startUrl: 'https://tanstack.com', + }) + await factory.create({ threadId: 'a' }) + expect(contexts[0]?.calls).toEqual([['goto', 'https://tanstack.com']]) + }) + + it('does not navigate without startUrl', async () => { + const { browser, contexts } = createFakeBrowser() + await playwrightComputer({ browser }).create({ threadId: 'a' }) + expect(contexts[0]?.calls).toEqual([]) + }) + + it('closes the context and rethrows when startUrl fails to load', async () => { + const { browser, contexts } = createFakeBrowser({ + gotoError: new Error('net::ERR_NAME_NOT_RESOLVED'), + }) + const factory = playwrightComputer({ + browser, + startUrl: 'https://tanstack.invalid', + }) + await expect(factory.create({ threadId: 'a' })).rejects.toThrow( + 'net::ERR_NAME_NOT_RESOLVED', + ) + expect(contexts[0]?.closed).toBe(true) + }) + + it('closes only its own context when a driver closes', async () => { + const { browser, contexts, state } = createFakeBrowser() + const factory = playwrightComputer({ browser }) + const first = await factory.create({ threadId: 'a' }) + await factory.create({ threadId: 'b' }) + await first.close?.() + expect(contexts.map((context) => context.closed)).toEqual([true, false]) + expect(state.closed).toBe(false) + }) + + it('leaves a passed browser open on factory close', async () => { + const { browser, state } = createFakeBrowser() + const factory = playwrightComputer({ browser }) + await factory.create({ threadId: 'a' }) + await factory.close?.() + expect(state.closed).toBe(false) + }) + + it('launches Chromium once for two threads at once', async () => { + const { browser, contexts } = createFakeBrowser() + launch.mockResolvedValue(browser) + const factory = playwrightComputer({ launch: { headless: false } }) + await Promise.all([ + factory.create({ threadId: 'a' }), + factory.create({ threadId: 'b' }), + ]) + expect(launch.mock.calls).toEqual([[{ headless: false }]]) + expect(contexts).toHaveLength(2) + }) + + it('closes the launched browser on factory close', async () => { + const { browser, state } = createFakeBrowser() + launch.mockResolvedValue(browser) + const factory = playwrightComputer() + await factory.create({ threadId: 'a' }) + await factory.close?.() + expect(state.closed).toBe(true) + }) + + it('tries the launch again after a failed launch', async () => { + const { browser, contexts } = createFakeBrowser() + launch + .mockRejectedValueOnce(new Error('Chromium is not installed')) + .mockResolvedValueOnce(browser) + const factory = playwrightComputer() + await expect(factory.create({ threadId: 'a' })).rejects.toThrow( + 'Chromium is not installed', + ) + await factory.create({ threadId: 'a' }) + expect(launch).toHaveBeenCalledTimes(2) + expect(contexts).toHaveLength(1) + }) }) diff --git a/packages/ai-computer-playwright/tests/types.test.ts b/packages/ai-computer-playwright/tests/types.test.ts index d7c778e232..43d31f2ddb 100644 --- a/packages/ai-computer-playwright/tests/types.test.ts +++ b/packages/ai-computer-playwright/tests/types.test.ts @@ -1,9 +1,25 @@ import { describe, expectTypeOf, it } from 'vitest' -import type { Page } from 'playwright-core' -import type { PlaywrightPageLike } from '../src/index' +import type { Browser, BrowserContext, Page } from 'playwright-core' +import type { + PlaywrightBrowserLike, + PlaywrightContextLike, + PlaywrightPageLike, +} from '../src/index' describe('PlaywrightPageLike', () => { it('accepts a real playwright-core Page', () => { expectTypeOf().toExtend() }) }) + +describe('PlaywrightContextLike', () => { + it('accepts a real playwright-core BrowserContext', () => { + expectTypeOf().toExtend() + }) +}) + +describe('PlaywrightBrowserLike', () => { + it('accepts a real playwright-core Browser', () => { + expectTypeOf().toExtend() + }) +}) diff --git a/packages/ai-computer-use/README.md b/packages/ai-computer-use/README.md index 3f824b45d4..3fa376294b 100644 --- a/packages/ai-computer-use/README.md +++ b/packages/ai-computer-use/README.md @@ -22,7 +22,7 @@ Computer use for TanStack AI. The model sees a screen and uses a mouse and a keyboard. -You write the code one time. The same code runs on OpenAI and on Anthropic. A driver controls the machine: a Playwright page, an E2B desktop, or your own. +You write the code one time. The same code runs on OpenAI and on Anthropic. A driver controls the machine: a Playwright browser, an E2B desktop, or your own. Each thread gets its own machine. ## Installation @@ -33,34 +33,36 @@ pnpm add @tanstack/ai-computer-use zod You also need a driver: ```bash -# A browser page -pnpm add @tanstack/ai-computer-playwright +# A browser page for each thread +pnpm add @tanstack/ai-computer-playwright playwright-core +npx playwright-core install chromium -# A Linux desktop in an E2B sandbox -pnpm add @tanstack/ai-computer-e2b +# A Linux desktop in an E2B sandbox for each thread +pnpm add @tanstack/ai-computer-e2b @e2b/desktop ``` ## Usage -1. Create the tool, the middleware, and the interrupt from a driver. -2. Give all three to `chat()`: +1. Create the computer from a driver, one time, at module scope. +2. Give `tools`, `middleware`, `interrupt`, and the thread id to `chat()`: ```typescript import { chat } from '@tanstack/ai' import { openaiText } from '@tanstack/ai-openai' import { createComputerUse } from '@tanstack/ai-computer-use' -import { createPlaywrightComputerDriver } from '@tanstack/ai-computer-playwright' +import { playwrightComputer } from '@tanstack/ai-computer-playwright' -const { tool, middleware, interrupt } = createComputerUse({ - driver: createPlaywrightComputerDriver({ page }), +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: 'https://tanstack.com' }), }) const stream = chat({ adapter: openaiText('gpt-6.1-sol'), messages, - tools: [tool], - middleware: [middleware], - interrupts: [interrupt], + threadId, // the thread id of the client + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], }) ``` @@ -74,17 +76,28 @@ useChat({ connection, interrupts: [computerSafetyCheck] }) `computerSafetyCheck` is the same object as `interrupt`. The client can import it without the driver. +4. When a conversation ends, close its machine: + +```typescript +await computerUse.close(threadId) // the machine of one thread +await computerUse.close() // every thread, then the browser +``` + ## What it does +- Each `threadId` gets its own machine, action queue, and safety-check approvals. In one thread, actions run one at a time, in call order. +- `tools` is `[computer]`. When the driver can navigate, it is `[computer, navigate]`. `navigate` opens only `http:` and `https:` URLs. - Every action returns a screenshot. The screenshot comes 500 ms after the action. Set `screenshotDelay` to change it. -- Actions run one at a time, in call order. - When an action fails, the other actions of the same model turn do not run. - OpenAI can attach safety checks to an action. The action waits until a person approves it. When the person declines, the action does not run. ## What your application owns -- **The machine.** The driver does not start or stop the browser or the sandbox. -- **One conversation at a time.** Create one `createComputerUse` for each conversation. +- **A stable thread id.** If `chat()` gets no `threadId`, each request gets a new machine. +- **The idle time.** A thread that stays idle for `idleTimeout` closes. The default is 10 minutes. `false` keeps every thread until `close`. +- **One server process.** The machines live in the memory of one process. +- **The network.** The browser can open every address that your server can reach. Run it where that is safe. +- **A driver you pass in.** `createComputerUse` never closes it. ## Docs diff --git a/packages/ai-computer-use/src/actions.ts b/packages/ai-computer-use/src/actions.ts index 00822d54d3..ffa5266733 100644 --- a/packages/ai-computer-use/src/actions.ts +++ b/packages/ai-computer-use/src/actions.ts @@ -367,6 +367,3 @@ export async function runComputerAction( } throw invalid('action', 'a string, an object, or an "actions" array') } - -/** The result of `runComputerAction`. */ -export type ComputerActionResult = Awaited> diff --git a/packages/ai-computer-use/src/create-computer-use.ts b/packages/ai-computer-use/src/create-computer-use.ts index a224a7a523..7600bb0193 100644 --- a/packages/ai-computer-use/src/create-computer-use.ts +++ b/packages/ai-computer-use/src/create-computer-use.ts @@ -1,4 +1,4 @@ -import { brandProviderTool } from '@tanstack/ai' +import { brandProviderTool, toolDefinition } from '@tanstack/ai' import { arrayBufferToBase64 } from '@tanstack/ai-utils' import { z } from 'zod' import { runComputerAction } from './actions' @@ -11,21 +11,69 @@ import type { ToolExecutionContext, } from '@tanstack/ai' import type { ComputerUseToolMetadata } from '@tanstack/ai/adapter-internals' -import type { ComputerDriver } from './types' +import type { ComputerDriver, ComputerDriverFactory } from './types' /** Options for `createComputerUse`. */ export interface CreateComputerUseOptions { - /** The driver that controls the computer. */ - driver: ComputerDriver + /** + * The machine to control. + * + * - A driver is one machine. Every thread shares it. `createComputerUse` + * never closes it. + * - A driver factory makes one machine per thread, on the first call of the + * thread. `createComputerUse` closes that machine when the thread closes + * or goes idle. + */ + driver: ComputerDriver | ComputerDriverFactory /** * The time in ms between an action and its screenshot. The page or the * desktop gets this time to settle. The default is 500. */ screenshotDelay?: number + /** + * The time in ms that a thread can stay idle. After this time, the state of + * the thread is dropped, and its machine from the factory is closed. The + * next call of the thread gets a new machine. `false` keeps every thread + * until `close`. The default is 600000 (10 minutes). + */ + idleTimeout?: number | false } const TOOL_NAME = 'computer' +/** The thread of a call with no `threadId`, like a direct `execute()` call. */ +const SHARED_THREAD = 'computer-use:shared' + +/** The state of one thread. */ +interface ThreadState { + id: string + /** The machine from the factory. It stays unset for a driver that the app passed in. */ + driver: Promise | undefined + /** The actions of the thread run one at a time on this queue. */ + queue: Promise + /** The number of actions in the queue. */ + pending: number + turnFailed: boolean + approved: Set + declined: Set + timer: ReturnType | undefined +} + +const navigateDefinition = toolDefinition({ + name: 'navigate', + description: 'Go to a URL in the browser. Returns a screenshot.', + inputSchema: z.object({ + url: z + .string() + .describe( + 'The full URL that starts with https:// or http://, like https://tanstack.com.', + ), + }), +}) + +type ComputerTool = ProviderTool +type NavigateTool = ReturnType + /** The parts of a computer tool input that the safety check reads. */ const checkedInputSchema = z.object({ pending_safety_checks: z.array(safetyCheckSchema).nullish(), @@ -50,6 +98,16 @@ function parseArguments(text: string) { } } +/** True for an `http:` or an `https:` URL. */ +function isWebUrl(url: string) { + try { + const { protocol } = new URL(url) + return protocol === 'https:' || protocol === 'http:' + } catch { + return false + } +} + function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)) } @@ -74,115 +132,224 @@ function computerToolCalls(ctx: ChatMiddlewareContext) { } /** - * Create a provider-agnostic computer tool for one machine. + * Create a provider-agnostic computer tool. Each thread gets its own state. + * With a driver factory, each thread also gets its own machine. * - * The OpenAI and Anthropic adapters swap `tool` for their native computer - * tool. Every action returns a screenshot. `zoom` returns the zoom image, and - * `cursor_position` returns text. + * The OpenAI and Anthropic adapters swap the computer tool for their native + * computer tool. Every action returns a screenshot. `zoom` returns the zoom + * image, and `cursor_position` returns text. * - * - `tool`: pass it to `chat({ tools })`. + * - `tools`: pass it to `chat({ tools })`. It is `[computer]`. When the + * driver can navigate, it is `[computer, navigate]`. * - `middleware`: pass it to `chat({ middleware })`. It runs the failed-turn * rule and the OpenAI safety checks. * - `interrupt`: pass it to `chat({ interrupts })`. It is the same object as * `computerSafetyCheck`. + * - `close(threadId)`: close the machine of one thread. `close()` closes + * every thread, then the driver factory. * - * Actions run one at a time, in call order. After an action throws, the - * other calls of the same model turn throw too. A call with OpenAI safety - * checks runs only after a person approves it. + * The thread is the `threadId` of `chat()`. Pass the thread id of the client + * to `chat()`. In one thread, actions run one at a time, in call order. After + * an action throws, the other calls of the same model turn throw too. A call + * with OpenAI safety checks runs only after a person approves it. When a + * thread stays idle for `idleTimeout`, its state is dropped and its machine + * from the factory is closed. * - * @param options - The driver, and the optional `screenshotDelay` in ms. + * @param options - The driver or the driver factory, the optional + * `screenshotDelay` in ms, and the optional `idleTimeout` in ms. * * @example * ```ts - * const { tool, middleware, interrupt } = createComputerUse({ - * driver: createPlaywrightComputerDriver({ page }), + * const computerUse = createComputerUse({ + * driver: playwrightComputer({ startUrl: 'https://tanstack.com' }), * }) * * chat({ * adapter: openaiText('gpt-6.1-sol'), * messages, - * tools: [tool], - * middleware: [middleware], - * interrupts: [interrupt], + * threadId, + * tools: computerUse.tools, + * middleware: [computerUse.middleware], + * interrupts: [computerUse.interrupt], * }) * ``` */ export function createComputerUse({ driver, screenshotDelay = 500, + idleTimeout = 600_000, }: CreateComputerUseOptions): { - tool: ProviderTool + tools: [ComputerTool] | [ComputerTool, NavigateTool] // The .d.ts names typeof computerSafetyCheck here, not a copy of its type. // TypeScript 5 sees two copies as different types, and chat() rejects them. middleware: ChatMiddleware interrupt: typeof computerSafetyCheck + close: (threadId?: string) => Promise } { - // ponytail: one createComputerUse drives one machine for one conversation - // at a time. The queue, the failed-turn flag, and the approvals are shared - // by every chat() call that uses it. Create one per conversation. - let queue: Promise = Promise.resolve() - let turnFailed = false - const approved = new Set() - const declined = new Set() - - async function runAction(input: unknown, toolCallId: string | undefined) { - if (turnFailed) { - throw new Error( - 'Not executed: an earlier computer action in this turn failed.', - ) + // ponytail: the machines live in this process only. When the next request + // of a thread reaches another server, the thread gets a new machine there. + // ponytail: without a stable threadId, chat() makes a new id for each + // request, so each request gets a new machine. Pass threadId to chat(). + // ponytail: when a person answers a safety check after idleTimeout, the + // approved action runs on a new machine. Set idleTimeout above that wait. + const threads = new Map() + + function startTimer(thread: ThreadState) { + clearTimeout(thread.timer) + const isOpen = threads.get(thread.id) === thread + if (idleTimeout === false || !isOpen) return + thread.timer = setTimeout(() => { + // Nobody waits for an idle close, so a close error has no caller. + closeThread(thread).catch(() => undefined) + }, idleTimeout) + thread.timer.unref?.() + } + + function threadOf(threadId: string) { + const known = threads.get(threadId) + if (known) return known + const thread: ThreadState = { + id: threadId, + driver: undefined, + queue: Promise.resolve(), + pending: 0, + turnFailed: false, + approved: new Set(), + declined: new Set(), + timer: undefined, } - try { - const checked = readCheckedInput(input) - // Fail closed: checks that do not parse also need an approval. - const hasChecks = checked === null || checked.checks.length > 0 - const isApproved = toolCallId !== undefined && approved.has(toolCallId) - if (hasChecks && !isApproved) { + threads.set(threadId, thread) + startTimer(thread) + return thread + } + + /** Drop the thread, wait for its queue, then close its machine from the factory. */ + async function closeThread(thread: ThreadState) { + if (threads.get(thread.id) === thread) threads.delete(thread.id) + clearTimeout(thread.timer) + await thread.queue + const machine = await thread.driver + await machine?.close?.() + } + + function machineOf(thread: ThreadState) { + if (!('create' in driver)) return driver + thread.driver ??= driver + .create({ threadId: thread.id }) + .catch((error: unknown) => { + // The next call of the thread calls create again. + thread.driver = undefined + throw error + }) + return thread.driver + } + + /** Run `work` in the queue of the thread, with the failed-turn rule. */ + function runInThread( + threadId: string | undefined, + work: (thread: ThreadState) => Promise, + ) { + const thread = threadOf(threadId ?? SHARED_THREAD) + clearTimeout(thread.timer) + thread.pending += 1 + const run = thread.queue.then(async () => { + if (thread.turnFailed) { throw new Error( - 'Refused: this computer action has safety checks that nobody approved. Add the `interrupt` from createComputerUse to chat({ interrupts }) and the `middleware` to chat({ middleware }).', + 'Not executed: an earlier computer action in this turn failed.', ) } - const result = await runComputerAction(driver, input) - if (result === undefined) { - await sleep(screenshotDelay) - return imageResult(await driver.screenshot()) + try { + return await work(thread) + } catch (error) { + thread.turnFailed = true + throw error } - return result.type === 'image' ? imageResult(result.bytes) : result.text - } catch (error) { - turnFailed = true - throw error - } + }) + thread.queue = run + .catch(() => undefined) + .then(() => { + thread.pending -= 1 + if (thread.pending === 0) startTimer(thread) + }) + return run } + const capabilities = + 'create' in driver + ? driver.capabilities + : { + zoom: typeof driver.zoom === 'function', + cursorPosition: typeof driver.cursorPosition === 'function', + navigate: typeof driver.navigate === 'function', + } + const metadata = { __kind: 'computer_use', display: driver.display, environment: driver.environment, capabilities: { - zoom: typeof driver.zoom === 'function', - cursorPosition: typeof driver.cursorPosition === 'function', + zoom: capabilities.zoom, + cursorPosition: capabilities.cursorPosition, }, } satisfies ComputerUseToolMetadata - const tool = brandProviderTool>({ + const computer = brandProviderTool({ name: TOOL_NAME, description: 'Use a computer with a mouse and a keyboard.', metadata, execute(input: unknown, context?: ToolExecutionContext) { - const run = queue.then(() => runAction(input, context?.toolCallId)) - queue = run.catch(() => undefined) - return run + return runInThread(context?.threadId, async (thread) => { + const checked = readCheckedInput(input) + // Fail closed: checks that do not parse also need an approval. + const hasChecks = checked === null || checked.checks.length > 0 + const toolCallId = context?.toolCallId + const isApproved = + toolCallId !== undefined && thread.approved.has(toolCallId) + if (hasChecks && !isApproved) { + throw new Error( + 'Refused: this computer action has safety checks that nobody approved. Add the `interrupt` from createComputerUse to chat({ interrupts }) and the `middleware` to chat({ middleware }).', + ) + } + const machine = await machineOf(thread) + const result = await runComputerAction(machine, input) + if (result === undefined) { + await sleep(screenshotDelay) + return imageResult(await machine.screenshot()) + } + return result.type === 'image' ? imageResult(result.bytes) : result.text + }) }, }) + const navigate = navigateDefinition.server(({ url }, context) => + runInThread(context?.threadId, async (thread) => { + if (!isWebUrl(url)) { + throw new Error( + 'Refused: navigate needs a full URL that starts with https:// or http://, like https://tanstack.com.', + ) + } + const machine = await machineOf(thread) + if (machine.navigate === undefined) { + throw new Error('The computer driver has no navigate method.') + } + await machine.navigate(url) + await sleep(screenshotDelay) + return imageResult(await machine.screenshot()) + }), + ) + const middleware: ChatMiddleware = { name: 'computer-use', onConfig(ctx) { - if (ctx.phase === 'beforeModel') turnFailed = false + const thread = threads.get(ctx.threadId) + if (ctx.phase === 'beforeModel' && thread) thread.turnFailed = false }, onInterruptBoundary(ctx) { if (ctx.phase !== 'beforeTools') return undefined + const thread = threads.get(ctx.threadId) const interrupts = computerToolCalls(ctx).flatMap((call) => { - const hasDecision = approved.has(call.id) || declined.has(call.id) + const hasDecision = + thread?.approved.has(call.id) || thread?.declined.has(call.id) if (hasDecision) return [] const checked = readCheckedInput( parseArguments(call.function.arguments), @@ -203,17 +370,38 @@ export function createComputerUse({ }) return interrupts.length > 0 ? { interrupts } : undefined }, - onInterruptResolution(_ctx, resolutions) { + onInterruptResolution(ctx, resolutions) { const results = resolutions.for(computerSafetyCheck) if (results.length === 0) return undefined + const thread = threadOf(ctx.threadId) const isAllApproved = results.every( (result) => result.status === 'resolved' && result.response.approved, ) - const decided = isAllApproved ? approved : declined + const decided = isAllApproved ? thread.approved : thread.declined for (const result of results) decided.add(result.request.key) return { toolResume: isAllApproved ? 'continue' : 'cancel' } }, } - return { tool, middleware, interrupt: computerSafetyCheck } + async function close(threadId?: string) { + if (threadId !== undefined) { + const thread = threads.get(threadId) + if (thread) await closeThread(thread) + return + } + const all = [...threads.values()] + // A failed thread close does not keep the browser open. Every close runs, + // then the first error is thrown. + const results = await Promise.allSettled( + all.map((thread) => closeThread(thread)), + ) + if ('create' in driver) await driver.close?.() + const failed = results.find((result) => result.status === 'rejected') + if (failed) throw failed.reason + } + + const tools: [ComputerTool] | [ComputerTool, NavigateTool] = + capabilities.navigate ? [computer, navigate] : [computer] + + return { tools, middleware, interrupt: computerSafetyCheck, close } } diff --git a/packages/ai-computer-use/src/index.ts b/packages/ai-computer-use/src/index.ts index d8d17a5515..969615718d 100644 --- a/packages/ai-computer-use/src/index.ts +++ b/packages/ai-computer-use/src/index.ts @@ -1,4 +1,9 @@ export { createComputerUse } from './create-computer-use' export type { CreateComputerUseOptions } from './create-computer-use' export { computerSafetyCheck } from './safety-check' -export type { ComputerDriver, MouseButton } from './types' +export type { + ComputerDriver, + ComputerDriverFactory, + ComputerEnvironment, + MouseButton, +} from './types' diff --git a/packages/ai-computer-use/src/types.ts b/packages/ai-computer-use/src/types.ts index 4ea1650238..0aecbc2e3c 100644 --- a/packages/ai-computer-use/src/types.ts +++ b/packages/ai-computer-use/src/types.ts @@ -1,6 +1,9 @@ /** A mouse button a driver can press. */ export type MouseButton = 'left' | 'right' | 'middle' | 'back' | 'forward' +/** The kind of computer a driver controls. */ +export type ComputerEnvironment = 'browser' | 'linux' | 'mac' | 'windows' + /** * Controls one computer: a browser page, a desktop VM, or a sandbox. * @@ -11,7 +14,7 @@ export interface ComputerDriver { /** The screen size in pixels. Screenshots have this size. */ display: { width: number; height: number } /** The kind of computer the driver controls. */ - environment: 'browser' | 'linux' | 'mac' | 'windows' + environment: ComputerEnvironment /** Take a screenshot. Returns PNG bytes. */ screenshot: () => Promise /** Click `count` times at the point with the given button. */ @@ -50,4 +53,37 @@ export interface ComputerDriver { }) => Promise /** Get the current mouse position. */ cursorPosition?: () => Promise<{ x: number; y: number }> + /** + * Go to a URL. When the driver has this method, `createComputerUse` adds + * the `navigate` tool. + */ + navigate?: (url: string) => Promise + /** + * Free the machine. When a factory created the driver, `createComputerUse` + * calls this when the thread closes or goes idle. A driver that the app + * passes in stays open. + */ + close?: () => Promise +} + +/** + * Makes one machine per thread. + * + * `createComputerUse` builds the tool before any machine exists. The factory + * tells it the display, the environment, and the optional methods. + */ +export interface ComputerDriverFactory { + /** The screen size in pixels. Every created driver has this `display`. */ + display: { width: number; height: number } + /** The kind of computer. Every created driver has this `environment`. */ + environment: ComputerEnvironment + /** The optional methods that every created driver has. */ + capabilities: { zoom: boolean; cursorPosition: boolean; navigate: boolean } + /** Make the machine for one thread. */ + create: (context: { threadId: string }) => Promise + /** + * Free what the threads share, like the browser. + * `createComputerUse(...).close()` with no thread id calls this. + */ + close?: () => Promise } diff --git a/packages/ai-computer-use/tests/create-computer-use.test.ts b/packages/ai-computer-use/tests/create-computer-use.test.ts index 3501694273..0d204f9c6d 100644 --- a/packages/ai-computer-use/tests/create-computer-use.test.ts +++ b/packages/ai-computer-use/tests/create-computer-use.test.ts @@ -8,19 +8,21 @@ import type { InterruptResolutionCollection, ModelMessage, } from '@tanstack/ai' -import type { ComputerDriver } from '../src/types' +import type { ComputerDriver, ComputerDriverFactory } from '../src/types' const PNG_BYTES = new Uint8Array([137, 80, 78, 71]) const PNG_BASE64 = 'iVBORw==' const ZOOM_BYTES = new Uint8Array([1, 2, 3]) const ZOOM_BASE64 = 'AQID' +const THREAD = 'thread' const CLICK = { action: { type: 'click', x: 10, y: 20, button: 'left' } } const TYPE = { action: { type: 'type', text: 'hi' } } const CHECK = { id: 'sc_1', code: 'malicious_instructions', message: 'Look.' } const CHECKED_CLICK = { ...CLICK, pending_safety_checks: [CHECK] } const NOT_EXECUTED = 'Not executed: an earlier computer action in this turn failed.' +const FULL_URL = 'a full URL that starts with https:// or http://' function screenshotOf(base64: string) { return [ @@ -31,9 +33,19 @@ function screenshotOf(base64: string) { ] } +/** A gate that holds an action until the test opens it. */ +function createGate() { + let open = () => {} + const gate = new Promise((resolve) => { + open = resolve + }) + return { gate, open } +} + interface DriverOptions { zoom?: boolean cursor?: boolean + navigate?: boolean /** `click` throws while this returns true. */ failClick?: () => boolean /** `click` waits for this promise. */ @@ -41,7 +53,7 @@ interface DriverOptions { } /** A driver that records its calls in order. */ -function createDriver(options: DriverOptions) { +function createDriver(options: DriverOptions = {}) { const calls: Array = [] const record = (name: string) => async () => { calls.push(name) @@ -67,36 +79,108 @@ function createDriver(options: DriverOptions) { keyUp: record('keyUp'), press: record('press'), type: record('type'), + close: record('close'), } if (options.zoom) driver.zoom = async () => ZOOM_BYTES if (options.cursor) driver.cursorPosition = async () => ({ x: 7, y: 9 }) + if (options.navigate) { + driver.navigate = async (url) => { + calls.push(`navigate:${url}`) + } + } return { driver, calls } } -function setup(options: DriverOptions & { screenshotDelay?: number } = {}) { - const { driver, calls } = createDriver(options) +interface FactoryOptions extends DriverOptions { + /** `create` throws while this returns true. */ + failCreate?: () => boolean +} + +/** A factory that records each created machine and each close. */ +function createFactory(options: FactoryOptions = {}) { + const created: Array = [] + const closed: Array = [] + const machines = new Map>() + const factory: ComputerDriverFactory = { + display: { width: 1280, height: 800 }, + environment: 'linux', + capabilities: { + zoom: options.zoom ?? false, + cursorPosition: options.cursor ?? false, + navigate: options.navigate ?? false, + }, + async create({ threadId }) { + created.push(threadId) + if (options.failCreate?.()) throw new Error('create failed') + const { driver, calls } = createDriver(options) + driver.close = async () => { + closed.push(threadId) + } + machines.set(threadId, calls) + return driver + }, + async close() { + closed.push('factory') + }, + } + return { factory, created, closed, machines } +} + +interface SetupOptions { + screenshotDelay?: number + idleTimeout?: number | false +} + +/** `createComputerUse`, plus helpers that call its tools as `chat()` does. */ +function setupWith( + driver: ComputerDriver | ComputerDriverFactory, + options: SetupOptions = {}, +) { const computer = createComputerUse({ driver, screenshotDelay: options.screenshotDelay ?? 0, + idleTimeout: options.idleTimeout, }) - const execute = (input: unknown, toolCallId = 'call_1') => { - if (!computer.tool.execute) throw new Error('The tool has no execute.') - return computer.tool.execute(input, { toolCallId, emitCustomEvent() {} }) + const [computerTool, navigateTool] = computer.tools + const execute = ( + input: unknown, + toolCallId = 'call_1', + threadId = THREAD, + ) => { + if (!computerTool.execute) throw new Error('The tool has no execute.') + return computerTool.execute(input, { + toolCallId, + threadId, + emitCustomEvent() {}, + }) + } + const navigate = (url: string, threadId = THREAD) => { + if (!navigateTool?.execute) throw new Error('There is no navigate tool.') + return navigateTool.execute( + { url }, + { toolCallId: 'call_nav', threadId, emitCustomEvent() {} }, + ) } - return { ...computer, calls, execute } + return { ...computer, execute, navigate } } -type Computer = ReturnType +function setup(options: DriverOptions & SetupOptions = {}) { + const { driver, calls } = createDriver(options) + return { ...setupWith(driver, options), calls } +} + +type Computer = ReturnType function createContext( phase: ChatMiddlewareContext['phase'], messages: Array = [], + threadId = THREAD, ): ChatMiddlewareContext { return { requestId: 'request', streamId: 'stream', runId: 'run', - threadId: 'thread', + threadId, phase, iteration: 0, chunkIndex: 0, @@ -138,23 +222,23 @@ function toolCall(id: string, input: unknown, name = 'computer') { function boundary( computer: Computer, toolCalls: Array>, + threadId = THREAD, ) { const messages: Array = [ { role: 'user', content: 'Click the button.' }, { role: 'assistant', content: null, toolCalls }, ] return computer.middleware.onInterruptBoundary?.({ - ...createContext('beforeTools', messages), + ...createContext('beforeTools', messages, threadId), phase: 'beforeTools', }) } -function beforeModel(computer: Computer) { - return computer.middleware.onConfig?.(createContext('beforeModel'), { - messages: [], - systemPrompts: [], - tools: [computer.tool], - }) +function beforeModel(computer: Computer, threadId = THREAD) { + return computer.middleware.onConfig?.( + createContext('beforeModel', [], threadId), + { messages: [], systemPrompts: [], tools: [] }, + ) } type Resolution = GenericInterruptResolution @@ -180,7 +264,11 @@ function cancelled(key: string): Resolution { return { request: safetyRequest(key), status: 'cancelled' } } -function resolve(computer: Computer, items: Array) { +function resolve( + computer: Computer, + items: Array, + threadId = THREAD, +) { const collection: InterruptResolutionCollection = { // TypeScript cannot tie a generic return to one definition. The chat @@ -189,18 +277,26 @@ function resolve(computer: Computer, items: Array) { all: () => items, } return computer.middleware.onInterruptResolution?.( - createContext('init'), + createContext('init', [], threadId), collection, ) } +/** Wait for a macrotask, so that every pending promise step runs. */ +function settle() { + return new Promise((done) => setTimeout(done, 0)) +} + afterEach(() => { vi.useRealTimers() }) describe('createComputerUse tool', () => { it('is the computer tool with the driver metadata', () => { - const { tool, interrupt } = setup({ zoom: true }) + const { + tools: [tool], + interrupt, + } = setup({ zoom: true }) expect(tool.name).toBe('computer') expect(tool.metadata).toEqual({ @@ -213,7 +309,9 @@ describe('createComputerUse tool', () => { }) it('gives the cursorPosition capability from the driver', () => { - const { tool } = setup({ cursor: true }) + const { + tools: [tool], + } = setup({ cursor: true }) expect(tool.metadata?.capabilities).toEqual({ zoom: false, @@ -221,6 +319,21 @@ describe('createComputerUse tool', () => { }) }) + it('takes the metadata from the factory before any machine exists', () => { + const { factory, created } = createFactory({ cursor: true }) + const { + tools: [tool], + } = setupWith(factory) + + expect(tool.metadata).toEqual({ + __kind: 'computer_use', + display: { width: 1280, height: 800 }, + environment: 'linux', + capabilities: { zoom: false, cursorPosition: true }, + }) + expect(created).toEqual([]) + }) + it('returns a screenshot after an action', async () => { const computer = setup() @@ -256,16 +369,13 @@ describe('createComputerUse tool', () => { }) it('runs concurrent calls one at a time, in call order', async () => { - let openGate = () => {} - const clickGate = new Promise((open) => { - openGate = open - }) - const computer = setup({ clickGate }) + const { gate, open } = createGate() + const computer = setup({ clickGate: gate }) const first = computer.execute(CLICK, 'call_1') const second = computer.execute(TYPE, 'call_2') await vi.waitFor(() => expect(computer.calls).toEqual(['click:start'])) - openGate() + open() await Promise.all([first, second]) expect(computer.calls).toEqual([ @@ -427,3 +537,329 @@ describe('createComputerUse safety checks', () => { expect(await resolve(computer, [])).toBeUndefined() }) }) + +describe('createComputerUse threads', () => { + it('gives each thread its own machine from the factory', async () => { + const { factory, created, machines } = createFactory() + const computer = setupWith(factory) + + await computer.execute(CLICK, 'call_1', 'a') + await computer.execute(TYPE, 'call_2', 'b') + + expect(created).toEqual(['a', 'b']) + expect(machines.get('a')).toEqual([ + 'click:start', + 'click:end', + 'screenshot', + ]) + expect(machines.get('b')).toEqual(['type', 'screenshot']) + }) + + it('creates one machine for two calls at once on a new thread', async () => { + const { factory, created, machines } = createFactory() + const computer = setupWith(factory) + + await Promise.all([ + computer.execute(CLICK, 'call_1', 'a'), + computer.execute(TYPE, 'call_2', 'a'), + ]) + + expect(created).toEqual(['a']) + expect(machines.get('a')).toEqual([ + 'click:start', + 'click:end', + 'screenshot', + 'type', + 'screenshot', + ]) + }) + + it('uses one shared thread for calls with no threadId', async () => { + const { factory, created } = createFactory() + const { + tools: [tool], + } = setupWith(factory) + const context = { emitCustomEvent() {} } + + await tool.execute?.(CLICK, context) + await tool.execute?.(TYPE, context) + + expect(created).toHaveLength(1) + }) + + it('keeps the approvals per thread', async () => { + const computer = setup() + + await resolve(computer, [answer('call_1', true)], 'a') + const otherThread = await boundary( + computer, + [toolCall('call_1', CHECKED_CLICK)], + 'b', + ) + + expect(otherThread?.interrupts).toHaveLength(1) + await expect( + computer.execute(CHECKED_CLICK, 'call_1', 'b'), + ).rejects.toThrow('Refused') + await expect( + computer.execute(CHECKED_CLICK, 'call_1', 'a'), + ).resolves.toEqual(screenshotOf(PNG_BASE64)) + }) + + it('keeps the failed-turn flag per thread', async () => { + const computer = setup({ failClick: () => true }) + + await expect(computer.execute(CLICK, 'call_1', 'a')).rejects.toThrow( + 'click failed', + ) + await expect(computer.execute(TYPE, 'call_2', 'b')).resolves.toEqual( + screenshotOf(PNG_BASE64), + ) + await beforeModel(computer, 'b') + + await expect(computer.execute(TYPE, 'call_3', 'a')).rejects.toThrow( + NOT_EXECUTED, + ) + }) + + it('calls create again after it rejects', async () => { + let isBroken = true + const { factory, created } = createFactory({ failCreate: () => isBroken }) + const computer = setupWith(factory) + + await expect(computer.execute(CLICK, 'call_1', 'a')).rejects.toThrow( + 'create failed', + ) + isBroken = false + await beforeModel(computer, 'a') + + await expect(computer.execute(CLICK, 'call_2', 'a')).resolves.toEqual( + screenshotOf(PNG_BASE64), + ) + expect(created).toEqual(['a', 'a']) + }) +}) + +describe('createComputerUse close', () => { + it('closes the machine of one thread with close(threadId)', async () => { + const { factory, created, closed } = createFactory() + const computer = setupWith(factory) + await computer.execute(CLICK, 'call_1', 'a') + await computer.execute(CLICK, 'call_2', 'b') + + await computer.close('a') + await computer.execute(CLICK, 'call_3', 'a') + + expect(closed).toEqual(['a']) + expect(created).toEqual(['a', 'b', 'a']) + }) + + it('closes every thread, then the factory, with close()', async () => { + const { factory, closed } = createFactory() + const computer = setupWith(factory) + await computer.execute(CLICK, 'call_1', 'a') + await computer.execute(CLICK, 'call_2', 'b') + + await computer.close() + + expect(closed).toEqual(['a', 'b', 'factory']) + }) + + it('closes the factory when a thread close fails, then throws', async () => { + const { factory, closed } = createFactory() + const create = factory.create + factory.create = async (context) => { + const driver = await create(context) + if (context.threadId === 'a') { + driver.close = async () => { + throw new Error('close failed') + } + } + return driver + } + const computer = setupWith(factory) + await computer.execute(CLICK, 'call_1', 'a') + await computer.execute(CLICK, 'call_2', 'b') + + await expect(computer.close()).rejects.toThrow('close failed') + expect(closed).toEqual(['b', 'factory']) + }) + + it('does not throw for a closed or an unknown thread', async () => { + const { factory, closed } = createFactory() + const computer = setupWith(factory) + await computer.execute(CLICK, 'call_1', 'a') + + await computer.close('a') + await computer.close('a') + await computer.close('unknown') + + expect(closed).toEqual(['a']) + }) + + it('waits for the running action before it closes the thread', async () => { + const { gate, open } = createGate() + const { factory, closed, machines } = createFactory({ clickGate: gate }) + const computer = setupWith(factory) + const result = computer.execute(CLICK, 'call_1', 'a') + await vi.waitFor(() => expect(machines.get('a')).toEqual(['click:start'])) + + const closing = computer.close('a') + await settle() + expect(closed).toEqual([]) + open() + await closing + + expect(closed).toEqual(['a']) + await expect(result).resolves.toEqual(screenshotOf(PNG_BASE64)) + expect(machines.get('a')).toEqual([ + 'click:start', + 'click:end', + 'screenshot', + ]) + }) + + it('never closes a driver that the app passed in', async () => { + vi.useFakeTimers() + let isBroken = true + const computer = setup({ idleTimeout: 1000, failClick: () => isBroken }) + await expect(computer.execute(CLICK, 'call_1')).rejects.toThrow( + 'click failed', + ) + isBroken = false + + await vi.advanceTimersByTimeAsync(1000) + const next = computer.execute(TYPE, 'call_2') + await vi.advanceTimersByTimeAsync(0) + await computer.close(THREAD) + await computer.close() + + // The idle timeout dropped the failed turn of the thread. + await expect(next).resolves.toEqual(screenshotOf(PNG_BASE64)) + expect(computer.calls).not.toContain('close') + }) +}) + +describe('createComputerUse idle timeout', () => { + it('closes and drops a thread after idleTimeout', async () => { + vi.useFakeTimers() + const { factory, created, closed } = createFactory() + const computer = setupWith(factory, { idleTimeout: 1000 }) + const first = computer.execute(CLICK, 'call_1', 'a') + await vi.advanceTimersByTimeAsync(0) + await first + + await vi.advanceTimersByTimeAsync(999) + expect(closed).toEqual([]) + await vi.advanceTimersByTimeAsync(1) + expect(closed).toEqual(['a']) + + const second = computer.execute(CLICK, 'call_2', 'a') + await vi.advanceTimersByTimeAsync(0) + await second + expect(created).toEqual(['a', 'a']) + }) + + it('starts the idle time only after a long action ends', async () => { + vi.useFakeTimers() + const { gate, open } = createGate() + const { factory, closed } = createFactory({ clickGate: gate }) + const computer = setupWith(factory, { idleTimeout: 1000 }) + const result = computer.execute(CLICK, 'call_1', 'a') + + await vi.advanceTimersByTimeAsync(5000) + expect(closed).toEqual([]) + open() + await vi.advanceTimersByTimeAsync(0) + await result + await vi.advanceTimersByTimeAsync(999) + expect(closed).toEqual([]) + await vi.advanceTimersByTimeAsync(1) + + expect(closed).toEqual(['a']) + }) + + it('never closes a thread with idleTimeout false', async () => { + vi.useFakeTimers() + const { factory, created, closed } = createFactory() + const computer = setupWith(factory, { idleTimeout: false }) + const first = computer.execute(CLICK, 'call_1', 'a') + await vi.advanceTimersByTimeAsync(0) + await first + + await vi.advanceTimersByTimeAsync(24 * 60 * 60 * 1000) + const second = computer.execute(CLICK, 'call_2', 'a') + await vi.advanceTimersByTimeAsync(0) + await second + + expect(closed).toEqual([]) + expect(created).toEqual(['a']) + }) +}) + +describe('createComputerUse navigate', () => { + it('has no navigate tool for a driver without navigate', () => { + const { tools } = setup() + + expect(tools.map((tool) => tool.name)).toEqual(['computer']) + }) + + it('has a navigate tool for a factory that can navigate', () => { + const { factory } = createFactory({ navigate: true }) + const { tools } = setupWith(factory) + + expect(tools.map((tool) => tool.name)).toEqual(['computer', 'navigate']) + }) + + it('goes to the URL on the machine of the thread and returns a screenshot', async () => { + const { factory, machines } = createFactory({ navigate: true }) + const computer = setupWith(factory) + + await expect( + computer.navigate('https://tanstack.com', 'a'), + ).resolves.toEqual(screenshotOf(PNG_BASE64)) + expect(machines.get('a')).toEqual([ + 'navigate:https://tanstack.com', + 'screenshot', + ]) + }) + + it('asks for a full URL when the URL has no scheme', async () => { + const computer = setup({ navigate: true }) + + await expect(computer.navigate('tanstack.com')).rejects.toThrow(FULL_URL) + expect(computer.calls).toEqual([]) + }) + + it.each(['file:///etc/passwd', 'javascript:alert(1)'])( + 'refuses %s', + async (url) => { + const computer = setup({ navigate: true }) + + await expect(computer.navigate(url)).rejects.toThrow(FULL_URL) + expect(computer.calls).toEqual([]) + }, + ) + + it('stops the turn after a refused navigate', async () => { + const computer = setup({ navigate: true }) + + await expect(computer.navigate('file:///etc/passwd')).rejects.toThrow( + FULL_URL, + ) + + await expect(computer.execute(CLICK)).rejects.toThrow(NOT_EXECUTED) + }) + + it('fails when the factory says navigate but its driver has none', async () => { + const { factory } = createFactory() + const computer = setupWith({ + ...factory, + capabilities: { ...factory.capabilities, navigate: true }, + }) + + await expect(computer.navigate('https://tanstack.com')).rejects.toThrow( + 'no navigate method', + ) + }) +}) diff --git a/packages/ai/src/activities/chat/index.ts b/packages/ai/src/activities/chat/index.ts index 5fead436a2..6f19634622 100644 --- a/packages/ai/src/activities/chat/index.ts +++ b/packages/ai/src/activities/chat/index.ts @@ -2633,6 +2633,7 @@ class TextEngine< this.toolResumeState(), this.params.toolExecution, this.params.repairToolInput, + this.threadId, ) // Consume the async generator, yielding custom events and collecting the return value @@ -2867,6 +2868,7 @@ class TextEngine< this.toolResumeState(), this.params.toolExecution, this.params.repairToolInput, + this.threadId, ) // Consume the async generator, yielding custom events and collecting the return value @@ -3958,6 +3960,7 @@ class TextEngine< this.toolResumeState(), undefined, this.params.repairToolInput, + this.threadId, ) } diff --git a/packages/ai/src/activities/chat/tools/tool-calls.ts b/packages/ai/src/activities/chat/tools/tool-calls.ts index bd697e8edf..bd03775986 100644 --- a/packages/ai/src/activities/chat/tools/tool-calls.ts +++ b/packages/ai/src/activities/chat/tools/tool-calls.ts @@ -981,6 +981,8 @@ async function buildClientToolResult( * call at a time. Results come back in call order either way. * @param repairToolInput - How an input that fails the final check is * repaired: the built-in repair (default), none (`false`), or a function. + * @param threadId - The thread id of the run. Each server tool gets it on + * its context. */ export async function* executeToolCalls( toolCalls: Array, @@ -998,6 +1000,7 @@ export async function* executeToolCalls( resumeState?: ToolResumeExecutionState, toolExecution: NonNullable = 'parallel', repairToolInput?: TextOptions['repairToolInput'], + threadId?: string, ): AsyncGenerator { const results: Array = [] const needsApproval: Array = [] @@ -1292,6 +1295,7 @@ export async function* executeToolCalls( toolCallId: toolCall.id, context: userContext, abortSignal, + ...(threadId !== undefined ? { threadId } : {}), ...(inputResponse !== undefined ? { inputResponse } : {}), emitCustomEvent: ( eventName: string, @@ -1355,6 +1359,7 @@ export async function* executeToolCalls( resumeState, toolExecution, repairToolInput, + threadId, ) results.push(...rest.results) needsClientExecution.push(...rest.needsClientExecution) diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 336fcbe2a1..552366403e 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -873,6 +873,12 @@ export type ToolExecutionContext = RuntimeContextField & { /** The ID of the tool call being executed */ toolCallId?: string + /** + * The thread id of the chat run. `chat()` sets it to the same value as + * `ctx.threadId` in middleware. A tool can use it to keep its own state + * for each conversation. + */ + threadId?: string /** * Abort signal for the current chat run. Aborts when the run's * `abortController` fires (or middleware aborts). Long-running tools — diff --git a/packages/ai/tests/tool-calls-context.test.ts b/packages/ai/tests/tool-calls-context.test.ts index 7ac9842ba2..8f2c33480c 100644 --- a/packages/ai/tests/tool-calls-context.test.ts +++ b/packages/ai/tests/tool-calls-context.test.ts @@ -1,9 +1,29 @@ import { describe, expect, it } from 'vitest' import { z } from 'zod' -import { toolDefinition } from '../src' +import { chat, toolDefinition } from '../src' import { executeToolCalls } from '../src/activities/chat/tools/tool-calls' +import { collectChunks, createMockAdapter, ev } from './test-utils' import type { ToolCall } from '../src/types' +async function drain(generator: ReturnType) { + let result = await generator.next() + while (!result.done) { + result = await generator.next() + } + return result.value +} + +function threadIdTool(seen: Array) { + return toolDefinition({ + name: 'read_thread', + description: 'Read the thread id', + inputSchema: z.object({}), + }).server((_input, ctx) => { + seen.push(ctx?.threadId) + return 'ok' + }) +} + describe('executeToolCalls runtime context', () => { it('passes runtime context to server tools', async () => { type AppContext = { userId: string; requestId: string } @@ -33,25 +53,86 @@ describe('executeToolCalls runtime context', () => { }, ] - const generator = executeToolCalls( - toolCalls, - [tool], - new Map(), - new Map(), - undefined, - undefined, - { userId: 'u-1', requestId: 'req-1' }, + const result = await drain( + executeToolCalls( + toolCalls, + [tool], + new Map(), + new Map(), + undefined, + undefined, + { userId: 'u-1', requestId: 'req-1' }, + ), ) - let result = await generator.next() - while (!result.done) { - result = await generator.next() - } - - expect(result.value.results[0]?.result).toEqual({ + expect(result.results[0]?.result).toEqual({ userId: 'u-1', requestId: 'req-1', toolCallId: 'tc-context', }) }) + + it('passes the thread id to server tools', async () => { + const seen: Array = [] + const tool = threadIdTool(seen) + const toolCalls: Array = [ + { + id: 'tc-thread', + type: 'function', + function: { name: tool.name, arguments: '{}' }, + }, + ] + + await drain( + executeToolCalls( + toolCalls, + [tool], + new Map(), + new Map(), + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + 'thread-a', + ), + ) + + expect(seen).toEqual(['thread-a']) + }) + + it('chat() gives tools its threadId', async () => { + const seen: Array = [] + const tool = threadIdTool(seen) + const { adapter } = createMockAdapter({ + iterations: [ + [ + ev.runStarted(), + ev.toolStart('tc-thread', tool.name), + ev.toolArgs('tc-thread', '{}'), + ev.runFinished('tool_calls'), + ], + [ + ev.runStarted(), + ev.textStart(), + ev.textContent('done'), + ev.textEnd(), + ev.runFinished('stop'), + ], + ], + }) + + await collectChunks( + chat({ + adapter, + messages: [{ role: 'user', content: 'read the thread' }], + tools: [tool], + threadId: 'thread-a', + }), + ) + + expect(seen).toEqual(['thread-a']) + }) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4e2a1e6126..abdd684063 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2164,7 +2164,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@vitest/coverage-v8': specifier: 4.1.10 @@ -2183,7 +2183,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@tanstack/ai': specifier: workspace:* @@ -2251,7 +2251,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@tanstack/ai': specifier: workspace:* @@ -2434,7 +2434,7 @@ importers: version: 4.1.10(vitest@4.1.11) zod: specifier: ^4.2.0 - version: 4.6.5 + version: 4.3.6 packages/ai-devtools: dependencies: @@ -2571,7 +2571,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@tanstack/ai': specifier: workspace:* @@ -2618,7 +2618,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@vitest/coverage-v8': specifier: 4.1.10 @@ -2728,7 +2728,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@tanstack/ai': specifier: workspace:* @@ -2750,7 +2750,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@tanstack/ai': specifier: workspace:* @@ -2927,7 +2927,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@tanstack/ai': specifier: workspace:* @@ -3476,7 +3476,7 @@ importers: version: link:../openai-base openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@vitest/coverage-v8': specifier: 4.1.10 @@ -3493,7 +3493,7 @@ importers: devDependencies: '@google/genai': specifier: ^2.10.0 - version: 2.10.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.6.5)) + version: 2.10.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6)) '@tanstack/ai': specifier: workspace:* version: link:../ai @@ -3589,7 +3589,7 @@ importers: version: link:../ai-utils openai: specifier: ^7.30.0 - version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6) + version: 7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6) devDependencies: '@tanstack/ai': specifier: workspace:* @@ -21032,7 +21032,7 @@ snapshots: '@blaxel/core@0.3.11(@cfworker/json-schema@4.1.1)(@hey-api/openapi-ts@0.99.0(magicast@0.5.2)(typescript@7.0.2))': dependencies: '@hey-api/client-fetch': 0.10.2(@hey-api/openapi-ts@0.99.0(magicast@0.5.2)(typescript@7.0.2)) - '@modelcontextprotocol/sdk': 1.29.0(@cfworker/json-schema@4.1.1)(zod@4.6.5) + '@modelcontextprotocol/sdk': 1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6) archiver: 7.0.1 axios: 1.18.1(debug@4.4.3(supports-color@7.2.0))(supports-color@7.2.0) dockerfile-ast: 0.7.1 @@ -21043,7 +21043,7 @@ snapshots: uuid: 11.1.1 ws: 8.21.0 yaml: 2.9.0 - zod: 4.6.5 + zod: 4.3.6 transitivePeerDependencies: - '@cfworker/json-schema' - '@hey-api/openapi-ts' @@ -22676,7 +22676,7 @@ snapshots: node-fetch: 2.7.0 nopt: 8.1.0 semver: 7.8.5 - tar: 7.5.22 + tar: 7.5.16 transitivePeerDependencies: - encoding - supports-color @@ -22762,8 +22762,8 @@ snapshots: '@mistralai/mistralai@2.2.0': dependencies: ws: 8.21.0 - zod: 4.6.5 - zod-to-json-schema: 3.25.2(zod@4.6.5) + zod: 4.3.6 + zod-to-json-schema: 3.25.2(zod@4.3.6) transitivePeerDependencies: - bufferutil - utf-8-validate @@ -22776,11 +22776,11 @@ snapshots: eventsource-parser: 3.1.1 jose: 6.2.12 pkce-challenge: 5.0.1 - zod: 4.6.5 + zod: 4.3.6 '@modelcontextprotocol/core@2.1.0': dependencies: - zod: 4.6.5 + zod: 4.3.6 '@modelcontextprotocol/ext-apps@1.7.4(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(supports-color@7.2.0)(zod@4.2.1))(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(zod@3.25.76)': dependencies: @@ -22886,11 +22886,12 @@ snapshots: '@cfworker/json-schema': 4.1.1 transitivePeerDependencies: - supports-color + optional: true '@modelcontextprotocol/server@2.1.0': dependencies: '@modelcontextprotocol/core': 2.1.0 - zod: 4.6.5 + zod: 4.3.6 '@msgpack/msgpack@3.1.3': {} @@ -23150,7 +23151,7 @@ snapshots: '@openrouter/sdk@0.13.20': dependencies: - zod: 4.6.5 + zod: 4.3.6 '@opentelemetry/api-logs@0.217.0': dependencies: @@ -28626,7 +28627,7 @@ snapshots: async-sema: 3.1.1 bindings: 1.5.0 estree-walker: 2.0.2 - glob: 13.0.6 + glob: 13.0.0 graceful-fs: 4.2.11 node-gyp-build: 4.8.4 picomatch: 4.0.7 @@ -28650,7 +28651,7 @@ snapshots: tar-stream: 3.1.7 undici: 7.28.0 xdg-app-paths: 5.1.0 - zod: 4.6.5 + zod: 4.3.6 transitivePeerDependencies: - bare-abort-controller - react-native-b4a @@ -29860,7 +29861,7 @@ snapshots: parse5: 7.3.0 parse5-htmlparser2-tree-adapter: 7.1.0 parse5-parser-stream: 7.1.2 - undici: 7.29.1 + undici: 7.29.0 whatwg-mimetype: 4.0.0 chokidar@4.0.3: @@ -34411,11 +34412,11 @@ snapshots: is-docker: 2.2.1 is-wsl: 2.2.0 - openai@7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.11.2)(ws@8.21.3)(zod@4.3.6): + openai@7.30.0(@aws-sdk/credential-provider-node@3.972.58)(@smithy/signature-v4@5.5.2)(undici@8.10.2)(ws@8.21.3)(zod@4.3.6): optionalDependencies: '@aws-sdk/credential-provider-node': 3.972.58 '@smithy/signature-v4': 5.5.2 - undici: 8.11.2 + undici: 8.10.2 ws: 8.21.3 zod: 4.3.6 @@ -38420,6 +38421,7 @@ snapshots: zod-to-json-schema@3.25.2(zod@4.6.5): dependencies: zod: 4.6.5 + optional: true zod@3.25.76: {} diff --git a/testing/e2e/tests/computer-use.spec.ts b/testing/e2e/tests/computer-use.spec.ts index ea7a2488be..b12818e01c 100644 --- a/testing/e2e/tests/computer-use.spec.ts +++ b/testing/e2e/tests/computer-use.spec.ts @@ -8,14 +8,19 @@ import { } from '@tanstack/ai' import { createAnthropicChat } from '@tanstack/ai-anthropic' import { createComputerUse } from '@tanstack/ai-computer-use' -import { createPlaywrightComputerDriver } from '@tanstack/ai-computer-playwright' +import { + createPlaywrightComputerDriver, + playwrightComputer, +} from '@tanstack/ai-computer-playwright' import { createOpenaiChat } from '@tanstack/ai-openai' -import type { Locator, Page } from '@playwright/test' +import type { Browser, BrowserContext, Locator, Page } from '@playwright/test' import type { ModelMessage, StreamChunk } from '@tanstack/ai' /** * Computer use from end to end. A real Playwright page is the computer. * `createComputerUse` with the Playwright driver runs the actions on it. + * `playwrightComputer` makes one page for each thread from the `browser` + * fixture. * * `chat()` runs in the spec process. A mock `fetch` sends the model streams * and keeps each request body. No app server and no API key are used. @@ -27,6 +32,9 @@ const PAGE_HTML = ` ` +const START_URL = 'https://computer.test/start' +const FORM_URL = 'https://computer.test/form' + const USER_MESSAGE: ModelMessage = { role: 'user', content: 'Fill the form.' } const SAFETY_CHECK = { @@ -35,6 +43,12 @@ const SAFETY_CHECK = { message: 'The page asks the model to do something else.', } +/** The `navigate` function tool in an OpenAI request body. */ +const OPENAI_NAVIGATE_TOOL = { type: 'function', name: 'navigate' } + +type MockModel = ReturnType +type ComputerUse = ReturnType + /** A mock `fetch` that sends one stream per request, in order. */ function mockModel(streams: Array<() => ReadableStream>) { const bodies: Array = [] @@ -87,6 +101,21 @@ function openaiStream(id: string, output: Array) { ]) } +/** An OpenAI `computer_call` item with one action and no safety checks. */ +function computerCall( + name: string, + action: { type: string; [field: string]: unknown }, +) { + return { + type: 'computer_call', + id: `cu_${name}`, + call_id: `call_cu_${name}`, + status: 'completed', + action, + pending_safety_checks: [], + } +} + const OPENAI_ANSWER = { type: 'message', id: 'msg_cu_e2e', @@ -177,6 +206,64 @@ function computerUseFor(page: Page) { }) } +/** + * Computer use with `playwrightComputer` on the `browser` fixture. Each new + * context serves `PAGE_HTML` for every `https://computer.test` URL. The + * contexts stay in `contexts` in the order the threads open them. + */ +function routedComputerUse(browser: Browser) { + const contexts: Array = [] + const computerUse = createComputerUse({ + driver: playwrightComputer({ + startUrl: START_URL, + browser: { + async newContext(options) { + const context = await browser.newContext(options) + await context.route('https://computer.test/**', (route) => + route.fulfill({ contentType: 'text/html', body: PAGE_HTML }), + ) + contexts.push(context) + return context + }, + close: () => browser.close(), + }, + }), + screenshotDelay: 0, + }) + return { computerUse, contexts } +} + +/** The one page that the factory opens in a context. */ +function pageOf(context: BrowserContext) { + const [page] = context.pages() + if (!page) throw new Error('The context has no page.') + return page +} + +function openaiAdapter(model: MockModel) { + return createOpenaiChat('gpt-6.1-sol', 'sk-e2e-dummy', { + fetch: model.fetch, + }) +} + +/** Run one OpenAI chat with computer use in a thread, to the end. */ +function runOpenai( + computerUse: ComputerUse, + model: MockModel, + threadId: string, +) { + return drain( + chat({ + adapter: openaiAdapter(model), + messages: [USER_MESSAGE], + threadId, + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], + }), + ) +} + async function drain(stream: AsyncIterable) { const chunks: Array = [] for await (const chunk of stream) chunks.push(chunk) @@ -223,7 +310,7 @@ test.describe('computer use with a Playwright page', () => { page, }) => { const { button, input } = await openForm(page) - const { tool, middleware, interrupt } = computerUseFor(page) + const computerUse = computerUseFor(page) const model = mockModel([ openaiStream('resp_cu_1', [ { @@ -242,22 +329,15 @@ test.describe('computer use with a Playwright page', () => { openaiStream('resp_cu_2', [OPENAI_ANSWER]), ]) - await drain( - chat({ - adapter: createOpenaiChat('gpt-6.1-sol', 'sk-e2e-dummy', { - fetch: model.fetch, - }), - messages: [USER_MESSAGE], - tools: [tool], - middleware: [middleware], - interrupts: [interrupt], - }), - ) + await runOpenai(computerUse, model, 'thread-cu-batch') await expect(page.locator('#status')).toHaveText('clicked') await expect(page.locator('#name')).toHaveValue('hello') expect(model.bodies).toHaveLength(2) - expect(model.bodies[0]).toMatchObject({ tools: [{ type: 'computer' }] }) + // The Playwright driver can navigate, so the `navigate` tool goes too. + expect(model.bodies[0]).toMatchObject({ + tools: [{ type: 'computer' }, OPENAI_NAVIGATE_TOOL], + }) expect(model.bodies[1]).toMatchObject({ input: expect.arrayContaining([ expect.objectContaining({ @@ -276,7 +356,7 @@ test.describe('computer use with a Playwright page', () => { page, }) => { const { button, input } = await openForm(page) - const { tool, middleware, interrupt } = computerUseFor(page) + const computerUse = computerUseFor(page) const model = mockModel([ anthropicStream([ ANTHROPIC_START, @@ -298,16 +378,18 @@ test.describe('computer use with a Playwright page', () => { fetch: model.fetch, }), messages: [USER_MESSAGE], - tools: [tool], - middleware: [middleware], - interrupts: [interrupt], + threadId: 'thread-cu-toolset', + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], }), ) await expect(page.locator('#status')).toHaveText('clicked') await expect(page.locator('#name')).toHaveValue('hello') expect(model.bodies).toHaveLength(2) - // The Playwright driver has zoom and no cursor position. + // The Playwright driver has zoom and no cursor position. It can + // navigate, so the `navigate` tool goes too, as a custom tool. expect(model.bodies[0]).toMatchObject({ tools: [ { @@ -317,6 +399,7 @@ test.describe('computer use with a Playwright page', () => { cursor_position: { enabled: false }, }, }, + { type: 'custom', name: 'navigate' }, ], }) const screenshotResult = (toolUseId: string) => @@ -351,24 +434,20 @@ test.describe('computer use with a Playwright page', () => { page, }) => { const { button } = await openForm(page) - const { tool, middleware, interrupt } = computerUseFor(page) + const computerUse = computerUseFor(page) const click = { type: 'click', button: 'left', ...button } const model = mockModel([ openaiStream('resp_cu_1', [ { - type: 'computer_call', - id: 'cu_checked', - call_id: 'call_cu_checked', - status: 'completed', - action: click, + ...computerCall('checked', click), pending_safety_checks: [SAFETY_CHECK], }, ]), openaiStream('resp_cu_2', [OPENAI_ANSWER]), ]) - const adapter = createOpenaiChat('gpt-6.1-sol', 'sk-e2e-dummy', { - fetch: model.fetch, - }) + const adapter = openaiAdapter(model) + // `createComputerUse` keeps the approvals per thread. So both runs use + // the same thread. const threadId = 'thread-cu-safety' // Run 1 ends with the safety-check interrupt. The click waits. @@ -376,9 +455,9 @@ test.describe('computer use with a Playwright page', () => { chat({ adapter, messages: [USER_MESSAGE], - tools: [tool], - middleware: [middleware], - interrupts: [interrupt], + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], threadId, runId: 'run-cu-safety-1', }), @@ -405,9 +484,9 @@ test.describe('computer use with a Playwright page', () => { chat({ adapter, messages: [USER_MESSAGE, assistantFromChunks(first)], - tools: [tool], - middleware: [middleware], - interrupts: [interrupt], + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], threadId, runId: 'run-cu-safety-2', parentRunId: 'run-cu-safety-1', @@ -443,3 +522,92 @@ test.describe('computer use with a Playwright page', () => { }) }) }) + +test.describe('computer use with playwrightComputer', () => { + test('playwrightComputer navigates, then clicks', async ({ + browser, + page, + }) => { + // The fixture page has the same HTML and width as the factory pages. + // So the button has the same center there. + const { button } = await openForm(page) + const { computerUse, contexts } = routedComputerUse(browser) + const model = mockModel([ + openaiStream('resp_cu_1', [ + { + type: 'function_call', + id: 'fc_cu_navigate', + call_id: 'call_cu_navigate', + name: 'navigate', + arguments: JSON.stringify({ url: FORM_URL }), + status: 'completed', + }, + ]), + openaiStream('resp_cu_2', [ + computerCall('click', { type: 'click', button: 'left', ...button }), + ]), + openaiStream('resp_cu_3', [OPENAI_ANSWER]), + ]) + + await runOpenai(computerUse, model, 'thread-cu-navigate') + + expect(contexts).toHaveLength(1) + const [formPage] = contexts.map(pageOf) + expect(formPage.url()).toBe(FORM_URL) + await expect(formPage.locator('#status')).toHaveText('clicked') + expect(model.bodies).toHaveLength(3) + expect(model.bodies[0]).toMatchObject({ + tools: [{ type: 'computer' }, OPENAI_NAVIGATE_TOOL], + }) + expect(model.bodies[1]).toMatchObject({ + input: expect.arrayContaining([ + expect.objectContaining({ + type: 'function_call_output', + call_id: 'call_cu_navigate', + output: [ + expect.objectContaining({ + type: 'input_image', + image_url: expect.stringMatching(/^data:image\/png;base64,./), + }), + ], + }), + ]), + }) + + await computerUse.close() + }) + + test('two threads get two pages', async ({ browser, page }) => { + const { button } = await openForm(page) + const { computerUse, contexts } = routedComputerUse(browser) + // Thread A clicks Go. Thread B only takes a screenshot. + const clickModel = mockModel([ + openaiStream('resp_cu_a1', [ + computerCall('click', { type: 'click', button: 'left', ...button }), + ]), + openaiStream('resp_cu_a2', [OPENAI_ANSWER]), + ]) + const lookModel = mockModel([ + openaiStream('resp_cu_b1', [ + computerCall('look', { type: 'screenshot' }), + ]), + openaiStream('resp_cu_b2', [OPENAI_ANSWER]), + ]) + + await runOpenai(computerUse, clickModel, 'a') + await runOpenai(computerUse, lookModel, 'b') + + expect(contexts).toHaveLength(2) + const [pageA, pageB] = contexts.map(pageOf) + await expect(pageA.locator('#status')).toHaveText('clicked') + await expect(pageB.locator('#status')).toHaveText('idle') + + // `close('a')` closes the context of thread A. Thread B keeps its page. + await computerUse.close('a') + expect(pageA.isClosed()).toBe(true) + expect(pageB.isClosed()).toBe(false) + + await computerUse.close() + expect(pageB.isClosed()).toBe(true) + }) +}) From 16ff74f816ae9837e30d66a4a2e936098e88cff2 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Tue, 20 Oct 2026 16:27:39 +0200 Subject: [PATCH 18/32] feat(ai-computer-local): let the model use the local computer Add @tanstack/ai-computer-local with localComputer(). The model controls the mouse, keyboard, and main screen of the machine that runs chat(), on Windows, macOS, and Linux, through @jitsi/robotjs. Screenshots scale down to the display (1280 x 800 at most by default), and clicks scale back up to the screen. The PNG encoder uses node:zlib. --- .changeset/ai-computer-use.md | 4 +- docs/computer-use/computer-use.md | 4 +- docs/computer-use/drivers.md | 71 ++++- packages/ai-computer-local/README.md | 72 +++++ packages/ai-computer-local/package.json | 56 ++++ packages/ai-computer-local/src/driver.ts | 177 +++++++++++ packages/ai-computer-local/src/index.ts | 5 + packages/ai-computer-local/src/png.ts | 85 ++++++ .../ai-computer-local/tests/driver.test.ts | 275 ++++++++++++++++++ .../ai-computer-local/tests/types.test.ts | 9 + packages/ai-computer-local/tsconfig.json | 8 + packages/ai-computer-local/vite.config.ts | 36 +++ pnpm-lock.yaml | 29 ++ pnpm-workspace.yaml | 3 + testing/e2e/package.json | 1 + testing/e2e/tests/computer-use.spec.ts | 86 +++++- 16 files changed, 906 insertions(+), 15 deletions(-) create mode 100644 packages/ai-computer-local/README.md create mode 100644 packages/ai-computer-local/package.json create mode 100644 packages/ai-computer-local/src/driver.ts create mode 100644 packages/ai-computer-local/src/index.ts create mode 100644 packages/ai-computer-local/src/png.ts create mode 100644 packages/ai-computer-local/tests/driver.test.ts create mode 100644 packages/ai-computer-local/tests/types.test.ts create mode 100644 packages/ai-computer-local/tsconfig.json create mode 100644 packages/ai-computer-local/vite.config.ts diff --git a/.changeset/ai-computer-use.md b/.changeset/ai-computer-use.md index e650ded209..be6eb6fbf5 100644 --- a/.changeset/ai-computer-use.md +++ b/.changeset/ai-computer-use.md @@ -2,19 +2,21 @@ '@tanstack/ai-computer-use': minor '@tanstack/ai-computer-playwright': minor '@tanstack/ai-computer-e2b': minor +'@tanstack/ai-computer-local': minor '@tanstack/ai': minor '@tanstack/openai-base': minor '@tanstack/ai-openai': minor '@tanstack/ai-anthropic': minor --- -Add computer use that runs the same code on OpenAI and on Anthropic. A driver controls the machine, and each thread gets its own machine. +Add computer use that runs the same code on OpenAI and on Anthropic. A driver controls the machine. The Playwright and E2B drivers give each thread its own machine. New packages: - `@tanstack/ai-computer-use` adds `createComputerUse()`. It returns `tools`, a middleware, a safety-check interrupt, and `close`. Each `threadId` gets its own machine, action queue, and approvals. A thread that stays idle for `idleTimeout` (10 minutes by default) closes. - `@tanstack/ai-computer-playwright` adds `playwrightComputer()`, which opens one browser page for each thread. `createPlaywrightComputerDriver()` controls one page that you give it. Both add a `navigate` tool that opens `http:` and `https:` URLs. - `@tanstack/ai-computer-e2b` adds `e2bComputer()`, which starts one E2B desktop sandbox for each thread. `createE2BComputerDriver()` controls one sandbox that you give it. +- `@tanstack/ai-computer-local` adds `localComputer()`, which lets the model use the mouse, keyboard, and main screen of the machine that runs `chat()`. It works on Windows, macOS, and Linux with `@jitsi/robotjs`. Changes in existing packages: diff --git a/docs/computer-use/computer-use.md b/docs/computer-use/computer-use.md index 3199d9e168..9f2699daad 100644 --- a/docs/computer-use/computer-use.md +++ b/docs/computer-use/computer-use.md @@ -85,7 +85,7 @@ import { anthropicText } from '@tanstack/ai-anthropic' const adapter = anthropicText('claude-opus-5-5') ``` -Need a full Linux desktop and not a browser page? See [Drivers](./drivers). +Need a full Linux desktop or your own computer, and not a browser page? See [Drivers](./drivers). ## One page for each thread @@ -233,4 +233,4 @@ Go to https://tanstack.com/ai and tell me the title of the page. The model calls `navigate`, reads the screenshot, and answers in the chat. You now have a browser agent. -To control your own machine with no driver package, see [Custom driver](./custom-driver). +To let the model use your own computer, see [Your own computer](./drivers#your-own-computer). For a machine that no driver controls, see [Custom driver](./custom-driver). diff --git a/docs/computer-use/drivers.md b/docs/computer-use/drivers.md index 6759f07d31..0b5e069c5d 100644 --- a/docs/computer-use/drivers.md +++ b/docs/computer-use/drivers.md @@ -2,32 +2,35 @@ title: Computer Use Drivers id: computer-use-drivers order: 2 -description: "Pick the machine that the model uses: a browser page for each thread with Playwright, or a full Linux desktop for each thread in an E2B sandbox." +description: "Pick the machine that the model uses: a browser page for each thread with Playwright, a full Linux desktop for each thread in an E2B sandbox, or your own computer." keywords: - tanstack ai - computer use - driver - playwrightComputer - e2bComputer + - localComputer - playwright - e2b - linux desktop - sandbox + - robotjs + - local desktop --- A browser page is not always enough. Sometimes the model must use a desktop app, a file manager, or a terminal. A driver decides which machine the model uses. Change the driver, and the rest of your [computer use](./computer-use) code stays the same. ## Pick a driver -| | Playwright | E2B | -| --------------------- | ----------------------------------------- | ------------------------------------- | -| **Machine** | A browser page for each thread | A Linux desktop sandbox for each thread | -| **Environment** | `browser` | `linux` | -| **Runs on** | Your server | E2B cloud | -| **`navigate` tool** | Yes | No | -| **`zoom`** | Yes | No | -| **`cursor_position`** | No | Yes | -| **Setup** | `playwright-core`, then download Chromium | An E2B API key | +| | Playwright | E2B | Your own computer | +| --------------------- | ----------------------------------------- | --------------------------------------- | ------------------------------------------ | +| **Machine** | A browser page for each thread | A Linux desktop sandbox for each thread | Your mouse, keyboard, and main screen | +| **Environment** | `browser` | `linux` | `windows`, `mac`, or `linux` | +| **Runs on** | Your server | E2B cloud | The machine that runs `chat()` | +| **`navigate` tool** | Yes | No | No | +| **`zoom`** | Yes | No | Yes | +| **`cursor_position`** | No | Yes | Yes | +| **Setup** | `playwright-core`, then download Chromium | An E2B API key | `@jitsi/robotjs`, and permissions on macOS | When a driver has `zoom` or `cursorPosition`, the Anthropic adapter turns on that action for the model. @@ -166,6 +169,52 @@ const computerUse = createComputerUse({ Your app owns this sandbox. `createComputerUse` never kills it. When you are done, call `await desktop.kill()`. +## Your own computer + +If the model must use the apps on the machine that runs `chat()`, use `localComputer()`. The model moves your real mouse, presses your real keys, and reads your main screen. + +CAUTION: Watch the model while it uses your computer. It can do what you can do there: read files, send messages, and delete data. If you cannot watch it, use [E2B](#e2b) or a VM. + + + +react: @tanstack/ai-computer-use @tanstack/ai-computer-local @jitsi/robotjs zod +vue: @tanstack/ai-computer-use @tanstack/ai-computer-local @jitsi/robotjs zod +solid: @tanstack/ai-computer-use @tanstack/ai-computer-local @jitsi/robotjs zod +svelte: @tanstack/ai-computer-use @tanstack/ai-computer-local @jitsi/robotjs zod +preact: @tanstack/ai-computer-use @tanstack/ai-computer-local @jitsi/robotjs zod +angular: @tanstack/ai-computer-use @tanstack/ai-computer-local @jitsi/robotjs zod +vanilla: @tanstack/ai-computer-use @tanstack/ai-computer-local @jitsi/robotjs zod +octane: @tanstack/ai-computer-use @tanstack/ai-computer-local @jitsi/robotjs zod + + + +`@jitsi/robotjs` has prebuilt binaries for Windows, macOS, and Linux. If pnpm asks to allow its install script, you can deny it. + +1. On macOS, open System Settings > Privacy & Security. Give the Accessibility and Screen Recording permissions to the app that runs Node, for example your terminal. +2. On Linux, use an X11 session. Wayland is not supported. +3. Give `localComputer()` to `createComputerUse`. + +```ts +import { createComputerUse } from '@tanstack/ai-computer-use' +import { localComputer } from '@tanstack/ai-computer-local' + +const computerUse = createComputerUse({ + driver: localComputer(), +}) +``` + +`localComputer()` takes these options. Both are optional: + +- `display`: the size of the screenshots that the model sees. The driver scales each screenshot down to this size, and scales each click up to the screen. The default is your screen size, scaled down to fit in 1280 x 800. Keep the aspect ratio of your screen. +- `robot`: the object that moves the mouse and presses the keys. The default is `@jitsi/robotjs`. A test can give a fake with the same methods. + +Your computer is one machine: + +- Every thread shares it, so it serves one conversation at a time. +- The model sees only the main screen. +- There is no `navigate` tool. The model opens the browser on your desktop. +- There are no `back` and `forward` mouse buttons. + ## Use it in chat With every driver, `createComputerUse` gives `tools`, `middleware`, and `interrupt`. Give them to `chat()` with the thread id: @@ -194,4 +243,4 @@ const stream = chat({ }) ``` -The model now uses a browser page or a full Linux desktop, one for each thread. For a machine that no driver controls, write a [custom driver](./custom-driver). +The model now uses a browser page or a full Linux desktop for each thread, or your own computer. For a machine that no driver controls, write a [custom driver](./custom-driver). diff --git a/packages/ai-computer-local/README.md b/packages/ai-computer-local/README.md new file mode 100644 index 0000000000..8e0b3679b9 --- /dev/null +++ b/packages/ai-computer-local/README.md @@ -0,0 +1,72 @@ +
+ + + + TanStack AI + +
+ +
+ +# @tanstack/ai-computer-local + +Local computer driver for TanStack AI computer use. + +Let the model use the machine that runs `chat()`, on Windows, macOS, or Linux. The model sees screenshots of your main screen. Then it moves your real mouse and presses your real keys. + +CAUTION: Watch the model while it uses your computer. It can do what you can do there. If you cannot watch it, use `@tanstack/ai-computer-e2b` or a VM. + +## Installation + +```bash +pnpm add @tanstack/ai-computer-local @tanstack/ai-computer-use @jitsi/robotjs +``` + +`@jitsi/robotjs` has prebuilt binaries for Windows, macOS, and Linux. If pnpm asks to allow its install script, you can deny it. + +## Usage + +1. On macOS, give the Accessibility and Screen Recording permissions to the app that runs Node. +2. On Linux, use an X11 session. Wayland is not supported. +3. Create the computer with `localComputer()`: + +```typescript +import { createComputerUse } from '@tanstack/ai-computer-use' +import { localComputer } from '@tanstack/ai-computer-local' + +const computerUse = createComputerUse({ + driver: localComputer(), +}) +``` + +4. Give `computerUse.tools`, `middleware`, `interrupt`, and the thread id to `chat()`. The model can now use your computer. + +`localComputer()` takes two options: + +- `display`: the size of the screenshots that the model sees. The default is your screen size, scaled down to fit in 1280 x 800. +- `robot`: the object that moves the mouse and presses the keys. The default is `@jitsi/robotjs`. + +## What the driver does + +- It scales each screenshot down to `display`, and each click up to the screen. +- It maps `KeyboardEvent.key` names, like `Control` and `ArrowLeft`, to robotjs names. +- It supports `zoom` and `cursorPosition`. It has no `navigate` and no `back` or `forward` mouse buttons. +- Every thread shares your one computer, so it serves one conversation at a time. + +See the [drivers guide](https://tanstack.com/ai/latest/docs/computer-use/drivers#your-own-computer) for more. + +## Tests + +```bash +pnpm --filter @tanstack/ai-computer-local test:lib +``` diff --git a/packages/ai-computer-local/package.json b/packages/ai-computer-local/package.json new file mode 100644 index 0000000000..a747398766 --- /dev/null +++ b/packages/ai-computer-local/package.json @@ -0,0 +1,56 @@ +{ + "name": "@tanstack/ai-computer-local", + "version": "0.0.0", + "description": "Local computer driver for TanStack AI computer use. The model controls the mouse, keyboard, and screen of the machine that runs the server.", + "author": "", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/TanStack/ai.git", + "directory": "packages/ai-computer-local" + }, + "type": "module", + "module": "./dist/esm/index.js", + "types": "./dist/esm/index.d.ts", + "exports": { + ".": { + "types": "./dist/esm/index.d.ts", + "import": "./dist/esm/index.js" + } + }, + "sideEffects": false, + "engines": { + "node": ">=18" + }, + "files": [ + "dist", + "src" + ], + "scripts": { + "build": "vite build", + "clean": "premove ./build ./dist", + "lint:fix": "oxlint src --type-aware --fix", + "test:build": "publint --strict", + "test:oxlint": "oxlint src --type-aware", + "test:coverage": "vitest run --passWithNoTests --coverage --coverage.include='src/**' --coverage.reporter=text-summary --coverage.reporter=json-summary", + "test:lib": "vitest --passWithNoTests", + "test:lib:dev": "pnpm test:lib --watch", + "test:types": "tsc" + }, + "keywords": [ + "ai", + "tanstack", + "computer-use", + "desktop", + "robotjs" + ], + "peerDependencies": { + "@jitsi/robotjs": "^0.6.0", + "@tanstack/ai-computer-use": "workspace:^" + }, + "devDependencies": { + "@jitsi/robotjs": "^0.6.24", + "@tanstack/ai-computer-use": "workspace:*", + "@vitest/coverage-v8": "4.1.10" + } +} diff --git a/packages/ai-computer-local/src/driver.ts b/packages/ai-computer-local/src/driver.ts new file mode 100644 index 0000000000..c73266d652 --- /dev/null +++ b/packages/ai-computer-local/src/driver.ts @@ -0,0 +1,177 @@ +import { createRequire } from 'node:module' +import { toPng } from './png' +import type { ComputerDriver, MouseButton } from '@tanstack/ai-computer-use' +import type { Bitmap } from './png' + +/** The parts of `@jitsi/robotjs` that the driver uses. */ +export interface RobotLike { + getScreenSize: () => { width: number; height: number } + getMousePos: () => { x: number; y: number } + moveMouse: (x: number, y: number) => void + mouseClick: (button?: string, double?: boolean) => void + mouseToggle: (down?: string, button?: string) => void + scrollMouse: (x: number, y: number) => void + keyToggle: (key: string, down: string) => void + typeString: (text: string) => void + screen: { + capture: (x?: number, y?: number, width?: number, height?: number) => Bitmap + } +} + +/** Options for {@link localComputer}. */ +export interface LocalComputerOptions { + /** + * The size of the screenshots that the model sees. The driver scales each + * screenshot down to this size, and scales each point from the model up to + * the screen. The default is the screen size, scaled down to fit in + * 1280 x 800. + */ + display?: { width: number; height: number } + /** The robot that moves the mouse and presses the keys. The default is `@jitsi/robotjs`. */ + robot?: RobotLike +} + +/** The largest default display. Bigger screenshots cost more tokens, and the providers scale them down anyway. */ +const MAX_DISPLAY = { width: 1280, height: 800 } + +/** `KeyboardEvent.key` names that differ from the robotjs names. Other names are lowercased. */ +const ROBOT_KEYS: Record = { + ArrowUp: 'up', + ArrowDown: 'down', + ArrowLeft: 'left', + ArrowRight: 'right', + Meta: 'command', + ContextMenu: 'menu', + ' ': 'space', +} + +function toRobotKey(key: string) { + return ROBOT_KEYS[key] ?? (key.length === 1 ? key : key.toLowerCase()) +} + +function toRobotButton(button: MouseButton) { + if (button === 'back' || button === 'forward') { + throw new Error(`The local computer has no "${button}" mouse button.`) + } + return button +} + +function currentEnvironment() { + if (process.platform === 'win32') return 'windows' + if (process.platform === 'darwin') return 'mac' + return 'linux' +} + +/** robotjs scrolls in pixels on macOS, in wheel clicks on Linux, and in 120 units for each wheel notch on Windows. */ +function scrollUnits(pixels: number) { + if (pixels === 0 || process.platform === 'darwin') return pixels + // ponytail: one wheel notch is about 100 px. + const notches = pixels / 100 + if (process.platform === 'win32') return Math.round(notches * 120) + return Math.sign(pixels) * Math.max(1, Math.round(Math.abs(notches))) +} + +function fitDisplay(screen: { width: number; height: number }) { + const scale = Math.min( + 1, + MAX_DISPLAY.width / screen.width, + MAX_DISPLAY.height / screen.height, + ) + return { + width: Math.round(screen.width * scale), + height: Math.round(screen.height * scale), + } +} + +/** Load robotjs on the first call, so an import of this package does not load the native addon. */ +function loadRobot() { + return createRequire(import.meta.url)('@jitsi/robotjs') +} + +/** + * Create a `ComputerDriver` for the machine that runs this code. The model + * controls the real mouse, keyboard, and main screen. + * + * All threads share this one machine. `createComputerUse` never closes it. + * + * CAUTION: the model can do on this machine what you can do. Use a VM or + * `e2bComputer` from `@tanstack/ai-computer-e2b` when you cannot watch it. + * + * Needs `@jitsi/robotjs`. On macOS, give the app that runs Node the + * Accessibility and Screen Recording permissions. On Linux, it needs X11. + * + * @example + * const computerUse = createComputerUse({ driver: localComputer() }) + */ +export function localComputer(options: LocalComputerOptions = {}) { + const robot: RobotLike = options.robot ?? loadRobot() + const screen = robot.getScreenSize() + const display = options.display ?? fitDisplay(screen) + + /** Scale a point or a size from the display to the screen. */ + const toScreen = (x: number, y: number) => ({ + x: Math.round((x * screen.width) / display.width), + y: Math.round((y * screen.height) / display.height), + }) + const moveTo = (x: number, y: number) => { + const point = toScreen(x, y) + robot.moveMouse(point.x, point.y) + } + + return { + display, + environment: currentEnvironment(), + screenshot: async () => + toPng(robot.screen.capture(), display.width, display.height), + async click(x, y, { button, count }) { + const name = toRobotButton(button) + moveTo(x, y) + // robotjs sends a real double click, which macOS needs. + if (count === 2) return robot.mouseClick(name, true) + // ponytail: macOS sees a triple click as three single clicks. + for (let i = 0; i < count; i++) robot.mouseClick(name) + }, + async move(x, y) { + moveTo(x, y) + }, + async mouseDown(button) { + robot.mouseToggle('down', toRobotButton(button)) + }, + async mouseUp(button) { + robot.mouseToggle('up', toRobotButton(button)) + }, + async scroll(x, y, deltaX, deltaY) { + moveTo(x, y) + const delta = toScreen(deltaX, deltaY) + // robotjs scrolls right and down for negative values. + robot.scrollMouse(scrollUnits(-delta.x), scrollUnits(-delta.y)) + }, + async keyDown(key) { + robot.keyToggle(toRobotKey(key), 'down') + }, + async keyUp(key) { + robot.keyToggle(toRobotKey(key), 'up') + }, + async press(keys) { + const names = keys.map(toRobotKey) + for (const name of names) robot.keyToggle(name, 'down') + for (const name of [...names].reverse()) robot.keyToggle(name, 'up') + }, + async type(text) { + robot.typeString(text) + }, + async cursorPosition() { + const { x, y } = robot.getMousePos() + return { + x: Math.round((x * display.width) / screen.width), + y: Math.round((y * display.height) / screen.height), + } + }, + async zoom(region) { + const start = toScreen(region.x, region.y) + const size = toScreen(region.width, region.height) + const bitmap = robot.screen.capture(start.x, start.y, size.x, size.y) + return toPng(bitmap, bitmap.width, bitmap.height) + }, + } satisfies ComputerDriver +} diff --git a/packages/ai-computer-local/src/index.ts b/packages/ai-computer-local/src/index.ts new file mode 100644 index 0000000000..15237d941d --- /dev/null +++ b/packages/ai-computer-local/src/index.ts @@ -0,0 +1,5 @@ +export { + localComputer, + type LocalComputerOptions, + type RobotLike, +} from './driver' diff --git a/packages/ai-computer-local/src/png.ts b/packages/ai-computer-local/src/png.ts new file mode 100644 index 0000000000..f61588dba2 --- /dev/null +++ b/packages/ai-computer-local/src/png.ts @@ -0,0 +1,85 @@ +import { deflateSync } from 'node:zlib' + +/** Raw screen pixels, as robotjs captures them: blue, green, red, then one unused byte. */ +export interface Bitmap { + width: number + height: number + /** Bytes in one row. */ + byteWidth: number + bytesPerPixel: number + image: Uint8Array +} + +const SIGNATURE = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]) + +const CRC_TABLE = Array.from({ length: 256 }, (_, n) => { + let c = n + for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1 + return c >>> 0 +}) + +// ponytail: zlib.crc32 needs Node 20.15. The package supports Node 18. +function crc32(bytes: Uint8Array) { + let c = 0xffffffff + for (const byte of bytes) c = (CRC_TABLE[(c ^ byte) & 0xff] ?? 0) ^ (c >>> 8) + return (c ^ 0xffffffff) >>> 0 +} + +function chunk(type: string, data: Uint8Array) { + const out = Buffer.alloc(12 + data.length) + out.writeUInt32BE(data.length, 0) + out.write(type, 4, 'latin1') + out.set(data, 8) + out.writeUInt32BE(crc32(out.subarray(4, 8 + data.length)), 8 + data.length) + return out +} + +/** + * Scale a bitmap to `width` x `height`, then encode it as an RGB PNG. + * + * Each output pixel is the average of the source pixels that it covers, so + * small text stays readable. + */ +export function toPng(bitmap: Bitmap, width: number, height: number) { + const { image, byteWidth, bytesPerPixel } = bitmap + const rowLength = 1 + width * 3 + // Each row starts with filter byte 0, which Buffer.alloc already set. + const rows = Buffer.alloc(rowLength * height) + + for (let ty = 0; ty < height; ty++) { + const y0 = Math.floor((ty * bitmap.height) / height) + const y1 = Math.max(y0 + 1, Math.floor(((ty + 1) * bitmap.height) / height)) + for (let tx = 0; tx < width; tx++) { + const x0 = Math.floor((tx * bitmap.width) / width) + const x1 = Math.max(x0 + 1, Math.floor(((tx + 1) * bitmap.width) / width)) + let red = 0 + let green = 0 + let blue = 0 + for (let y = y0; y < y1; y++) { + for (let x = x0; x < x1; x++) { + const i = y * byteWidth + x * bytesPerPixel + blue += image[i] ?? 0 + green += image[i + 1] ?? 0 + red += image[i + 2] ?? 0 + } + } + const count = (y1 - y0) * (x1 - x0) + const o = ty * rowLength + 1 + tx * 3 + rows[o] = Math.round(red / count) + rows[o + 1] = Math.round(green / count) + rows[o + 2] = Math.round(blue / count) + } + } + + const header = Buffer.alloc(13) + header.writeUInt32BE(width, 0) + header.writeUInt32BE(height, 4) + header[8] = 8 // 8 bits for each color + header[9] = 2 // RGB + return Buffer.concat([ + SIGNATURE, + chunk('IHDR', header), + chunk('IDAT', deflateSync(rows)), + chunk('IEND', new Uint8Array()), + ]) +} diff --git a/packages/ai-computer-local/tests/driver.test.ts b/packages/ai-computer-local/tests/driver.test.ts new file mode 100644 index 0000000000..335e6b7582 --- /dev/null +++ b/packages/ai-computer-local/tests/driver.test.ts @@ -0,0 +1,275 @@ +import { crc32, inflateSync } from 'node:zlib' +import { afterEach, describe, expect, it } from 'vitest' +import { localComputer } from '../src/index' +import type { RobotLike } from '../src/index' + +type Size = { width: number; height: number } +type Rgb = readonly [number, number, number] + +const RED: Rgb = [255, 0, 0] +const BLACK: Rgb = [0, 0, 0] +const BLUE: Rgb = [0, 0, 255] + +/** Extra bytes at the end of each bitmap row, as macOS adds. */ +const ROW_PADDING = 4 + +/** A 4 x 2 screen: red over black on the left half, blue on the right half. */ +function testColor(x: number, y: number) { + if (x >= 2) return BLUE + return y === 0 ? RED : BLACK +} + +/** A robotjs bitmap of one screen region: blue, green, red, unused, with padded rows. */ +function capture( + color: (x: number, y: number) => Rgb, + region: { x: number; y: number } & Size, +) { + const byteWidth = region.width * 4 + ROW_PADDING + const image = new Uint8Array(byteWidth * region.height).fill(99) + for (let y = 0; y < region.height; y++) { + for (let x = 0; x < region.width; x++) { + const [red, green, blue] = color(region.x + x, region.y + y) + image.set([blue, green, red, 255], y * byteWidth + x * 4) + } + } + return { ...region, byteWidth, bytesPerPixel: 4, image } +} + +/** A fake robot on a screen of the given size. Each call adds one line to `calls`. */ +function fakeRobot(screen: Size = { width: 2560, height: 1440 }) { + const calls: Array = [] + const record = + (name: string) => + (...args: Array) => { + calls.push([name, ...args].join(' ').trim()) + } + const robot: RobotLike = { + getScreenSize: () => screen, + getMousePos: () => ({ x: 640, y: 360 }), + moveMouse: record('moveMouse'), + mouseClick: record('mouseClick'), + mouseToggle: record('mouseToggle'), + scrollMouse: record('scrollMouse'), + keyToggle: record('keyToggle'), + typeString: record('typeString'), + screen: { + capture: (x = 0, y = 0, width = screen.width, height = screen.height) => { + record('capture')(x, y, width, height) + return capture(testColor, { x, y, width, height }) + }, + }, + } + return { robot, calls } +} + +/** Decode a PNG. Checks the signature and the CRC of each chunk. */ +function readPng(png: Uint8Array) { + const bytes = Buffer.from(png) + expect([...bytes.subarray(0, 8)]).toEqual([137, 80, 78, 71, 13, 10, 26, 10]) + const chunks = new Map() + let offset = 8 + while (offset < bytes.length) { + const length = bytes.readUInt32BE(offset) + const end = offset + 8 + length + expect(bytes.readUInt32BE(end)).toBe(crc32(bytes.subarray(offset + 4, end))) + chunks.set( + bytes.toString('latin1', offset + 4, offset + 8), + bytes.subarray(offset + 8, end), + ) + offset = end + 4 + } + const header = chunks.get('IHDR') ?? Buffer.alloc(13) + return { + width: header.readUInt32BE(0), + height: header.readUInt32BE(4), + colorType: header[9], + rows: [...inflateSync(chunks.get('IDAT') ?? Buffer.alloc(0))], + } +} + +const realPlatform = Object.getOwnPropertyDescriptor(process, 'platform') + +function setPlatform(platform: NodeJS.Platform) { + Object.defineProperty(process, 'platform', { value: platform }) +} + +afterEach(() => { + if (realPlatform) Object.defineProperty(process, 'platform', realPlatform) +}) + +describe('localComputer display', () => { + it('scales a big screen down to fit in 1280 x 800', () => { + const { robot } = fakeRobot({ width: 2560, height: 1440 }) + expect(localComputer({ robot }).display).toEqual({ + width: 1280, + height: 720, + }) + }) + + it('keeps a small screen at its own size', () => { + const { robot } = fakeRobot({ width: 1024, height: 768 }) + expect(localComputer({ robot }).display).toEqual({ + width: 1024, + height: 768, + }) + }) + + it.each([ + ['win32', 'windows'], + ['darwin', 'mac'], + ['linux', 'linux'], + ] as const)('reports %s as %s', (platform, environment) => { + setPlatform(platform) + expect(localComputer({ robot: fakeRobot().robot }).environment).toBe( + environment, + ) + }) +}) + +describe('localComputer mouse', () => { + it('clicks at the screen point, scaled up from the display', async () => { + const { robot, calls } = fakeRobot() + await localComputer({ robot }).click(100, 50, { button: 'left', count: 1 }) + expect(calls).toEqual(['moveMouse 200 100', 'mouseClick left']) + }) + + it('sends a double click as one robotjs double click', async () => { + const { robot, calls } = fakeRobot() + await localComputer({ robot }).click(100, 50, { button: 'right', count: 2 }) + expect(calls).toEqual(['moveMouse 200 100', 'mouseClick right true']) + }) + + it('sends a triple click as three clicks', async () => { + const { robot, calls } = fakeRobot() + await localComputer({ robot }).click(100, 50, { button: 'left', count: 3 }) + expect(calls).toEqual([ + 'moveMouse 200 100', + 'mouseClick left', + 'mouseClick left', + 'mouseClick left', + ]) + }) + + it('rejects the back button before the mouse moves', async () => { + const { robot, calls } = fakeRobot() + const click = localComputer({ robot }).click(1, 1, { + button: 'back', + count: 1, + }) + await expect(click).rejects.toThrow( + 'The local computer has no "back" mouse button.', + ) + expect(calls).toEqual([]) + }) + + it('holds and releases a mouse button', async () => { + const { robot, calls } = fakeRobot() + const driver = localComputer({ robot }) + await driver.mouseDown('middle') + await driver.mouseUp('middle') + expect(calls).toEqual(['mouseToggle down middle', 'mouseToggle up middle']) + }) + + it('reports the mouse position in display space', async () => { + const { robot } = fakeRobot() + expect(await localComputer({ robot }).cursorPosition()).toEqual({ + x: 320, + y: 180, + }) + }) + + it.each([ + ['darwin', 0, 120, 'scrollMouse 0 -120'], + ['darwin', 40, 0, 'scrollMouse -40 0'], + ['win32', 0, 100, 'scrollMouse 0 -120'], + ['win32', -50, 0, 'scrollMouse 60 0'], + ['linux', 0, 250, 'scrollMouse 0 -3'], + ['linux', 0, 30, 'scrollMouse 0 -1'], + ] as const)( + 'on %s, scrolls %i, %i px as "%s"', + async (platform, deltaX, deltaY, expected) => { + setPlatform(platform) + const { robot, calls } = fakeRobot({ width: 1280, height: 800 }) + await localComputer({ robot }).scroll(10, 20, deltaX, deltaY) + expect(calls).toEqual(['moveMouse 10 20', expected]) + }, + ) +}) + +describe('localComputer keyboard', () => { + it('presses a combo in order, then releases it in reverse', async () => { + const { robot, calls } = fakeRobot() + await localComputer({ robot }).press(['Control', 'Shift', 'ArrowLeft']) + expect(calls).toEqual([ + 'keyToggle control down', + 'keyToggle shift down', + 'keyToggle left down', + 'keyToggle left up', + 'keyToggle shift up', + 'keyToggle control up', + ]) + }) + + it('sends robotjs key names', async () => { + const { robot, calls } = fakeRobot() + const driver = localComputer({ robot }) + const keys = [ + 'Meta', + ' ', + 'Enter', + 'F5', + 'PageDown', + 'ContextMenu', + 'a', + 'A', + ] + for (const key of keys) await driver.keyDown(key) + expect(calls).toEqual([ + 'keyToggle command down', + 'keyToggle space down', + 'keyToggle enter down', + 'keyToggle f5 down', + 'keyToggle pagedown down', + 'keyToggle menu down', + 'keyToggle a down', + 'keyToggle A down', + ]) + }) + + it('types the text', async () => { + const { robot, calls } = fakeRobot() + await localComputer({ robot }).type('hello world') + expect(calls).toEqual(['typeString hello world']) + }) +}) + +describe('localComputer screen', () => { + it('takes a screenshot at the display size, with each pixel the average of the pixels it covers', async () => { + const { robot } = fakeRobot({ width: 4, height: 2 }) + const png = readPng( + await localComputer({ + robot, + display: { width: 2, height: 1 }, + }).screenshot(), + ) + expect(png).toEqual({ + width: 2, + height: 1, + colorType: 2, + rows: [0, 128, 0, 0, 0, 0, 255], + }) + }) + + it('zooms into a region at the full screen size', async () => { + const { robot, calls } = fakeRobot({ width: 4, height: 2 }) + const driver = localComputer({ robot, display: { width: 2, height: 1 } }) + const png = readPng(await driver.zoom({ x: 1, y: 0, width: 1, height: 1 })) + expect(calls).toEqual(['capture 2 0 2 2']) + expect(png).toEqual({ + width: 2, + height: 2, + colorType: 2, + rows: [0, 0, 0, 255, 0, 0, 255, 0, 0, 0, 255, 0, 0, 255], + }) + }) +}) diff --git a/packages/ai-computer-local/tests/types.test.ts b/packages/ai-computer-local/tests/types.test.ts new file mode 100644 index 0000000000..f2d8339ce5 --- /dev/null +++ b/packages/ai-computer-local/tests/types.test.ts @@ -0,0 +1,9 @@ +import { describe, expectTypeOf, it } from 'vitest' +import type * as robotjs from '@jitsi/robotjs' +import type { RobotLike } from '../src/index' + +describe('RobotLike', () => { + it('accepts the real @jitsi/robotjs module', () => { + expectTypeOf().toExtend() + }) +}) diff --git a/packages/ai-computer-local/tsconfig.json b/packages/ai-computer-local/tsconfig.json new file mode 100644 index 0000000000..c38689f4ea --- /dev/null +++ b/packages/ai-computer-local/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": ["src", "tests"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/ai-computer-local/vite.config.ts b/packages/ai-computer-local/vite.config.ts new file mode 100644 index 0000000000..77bcc2e60b --- /dev/null +++ b/packages/ai-computer-local/vite.config.ts @@ -0,0 +1,36 @@ +import { defineConfig, mergeConfig } from 'vitest/config' +import { tanstackViteConfig } from '@tanstack/vite-config' +import packageJson from './package.json' + +const config = defineConfig({ + test: { + name: packageJson.name, + dir: './', + watch: false, + globals: true, + environment: 'node', + include: ['tests/**/*.test.ts'], + coverage: { + provider: 'v8', + reporter: ['text', 'json', 'html', 'lcov'], + exclude: [ + 'node_modules/', + 'dist/', + 'tests/', + '**/*.test.ts', + '**/*.config.ts', + '**/types.ts', + ], + include: ['src/**/*.ts'], + }, + }, +}) + +export default mergeConfig( + config, + tanstackViteConfig({ + entry: ['./src/index.ts'], + srcDir: './src', + cjs: false, + }), +) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index abdd684063..58044759de 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2408,6 +2408,18 @@ importers: specifier: 4.1.10 version: 4.1.10(vitest@4.1.11) + packages/ai-computer-local: + devDependencies: + '@jitsi/robotjs': + specifier: ^0.6.24 + version: 0.6.24 + '@tanstack/ai-computer-use': + specifier: workspace:* + version: link:../ai-computer-use + '@vitest/coverage-v8': + specifier: 4.1.10 + version: 4.1.10(vitest@4.1.11) + packages/ai-computer-playwright: devDependencies: '@tanstack/ai-computer-use': @@ -3718,6 +3730,9 @@ importers: '@tanstack/ai-compaction': specifier: workspace:* version: link:../../packages/ai-compaction + '@tanstack/ai-computer-local': + specifier: workspace:* + version: link:../../packages/ai-computer-local '@tanstack/ai-computer-playwright': specifier: workspace:* version: link:../../packages/ai-computer-playwright @@ -6502,6 +6517,9 @@ packages: '@jitl/quickjs-wasmfile-release-sync@0.31.0': resolution: {integrity: sha512-hYduecOByj9AsAfsJhZh5nA6exokmuFC8cls39+lYmTCGY51bgjJJJwReEu7Ff7vBWaQCL6TeDdVlnp2WYz0jw==} + '@jitsi/robotjs@0.6.24': + resolution: {integrity: sha512-mJrOMfvrI4aqG8jmGruXC3jtIccCptVM1JIcGXgKRyHqXVJIzz8Eb/L9dsJCIDJ2UFvd7yPcgNX819ZlsrJahg==} + '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -15791,6 +15809,10 @@ packages: node-addon-api@7.1.1: resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==} + node-addon-api@8.9.2: + resolution: {integrity: sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg==} + engines: {node: ^18 || ^20 || >= 21} + node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -22588,6 +22610,11 @@ snapshots: dependencies: '@jitl/quickjs-ffi-types': 0.31.0 + '@jitsi/robotjs@0.6.24': + dependencies: + node-addon-api: 8.9.2 + node-gyp-build: 4.8.4 + '@jridgewell/gen-mapping@0.3.13': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -34062,6 +34089,8 @@ snapshots: node-addon-api@7.1.1: {} + node-addon-api@8.9.2: {} + node-domexception@1.0.0: {} node-fetch-native@1.6.7: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 96e73a81da..abccc945d8 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -193,3 +193,6 @@ allowBuilds: '@prisma/engines': false prisma: false better-sqlite3: false + # @tanstack/ai-computer-local needs only the types. Its tests use a fake + # robot, so nothing loads the native addon that this script downloads. + '@jitsi/robotjs': false diff --git a/testing/e2e/package.json b/testing/e2e/package.json index 26a8457dd3..108d79526a 100644 --- a/testing/e2e/package.json +++ b/testing/e2e/package.json @@ -26,6 +26,7 @@ "@tanstack/ai-client": "workspace:*", "@tanstack/ai-cloudflare": "workspace:*", "@tanstack/ai-compaction": "workspace:*", + "@tanstack/ai-computer-local": "workspace:*", "@tanstack/ai-computer-playwright": "workspace:*", "@tanstack/ai-computer-use": "workspace:*", "@tanstack/ai-elevenlabs": "workspace:*", diff --git a/testing/e2e/tests/computer-use.spec.ts b/testing/e2e/tests/computer-use.spec.ts index b12818e01c..37a9d70d18 100644 --- a/testing/e2e/tests/computer-use.spec.ts +++ b/testing/e2e/tests/computer-use.spec.ts @@ -12,15 +12,17 @@ import { createPlaywrightComputerDriver, playwrightComputer, } from '@tanstack/ai-computer-playwright' +import { localComputer } from '@tanstack/ai-computer-local' import { createOpenaiChat } from '@tanstack/ai-openai' import type { Browser, BrowserContext, Locator, Page } from '@playwright/test' import type { ModelMessage, StreamChunk } from '@tanstack/ai' +import type { RobotLike } from '@tanstack/ai-computer-local' /** * Computer use from end to end. A real Playwright page is the computer. * `createComputerUse` with the Playwright driver runs the actions on it. * `playwrightComputer` makes one page for each thread from the `browser` - * fixture. + * fixture. `localComputer` runs on a fake robot, because CI has no desktop. * * `chat()` runs in the spec process. A mock `fetch` sends the model streams * and keeps each request body. No app server and no API key are used. @@ -611,3 +613,85 @@ test.describe('computer use with playwrightComputer', () => { expect(pageB.isClosed()).toBe(true) }) }) + +/** A fake robot on a 2560 x 1600 screen. Each call adds one line to `calls`. */ +function fakeRobot() { + const screen = { width: 2560, height: 1600 } + const calls: Array = [] + const record = + (name: string) => + (...args: Array) => { + calls.push([name, ...args].join(' ').trim()) + } + const robot: RobotLike = { + getScreenSize: () => screen, + getMousePos: () => ({ x: 0, y: 0 }), + moveMouse: record('moveMouse'), + mouseClick: record('mouseClick'), + mouseToggle: record('mouseToggle'), + scrollMouse: record('scrollMouse'), + keyToggle: record('keyToggle'), + typeString: record('typeString'), + screen: { + capture: () => ({ + ...screen, + byteWidth: screen.width * 4, + bytesPerPixel: 4, + image: new Uint8Array(screen.width * screen.height * 4), + }), + }, + } + return { robot, calls } +} + +/** The size of the first PNG screenshot in a request body. */ +function screenshotSize(body: unknown) { + const match = /data:image\/png;base64,([A-Za-z0-9+/=]+)/.exec( + JSON.stringify(body), + ) + if (!match?.[1]) throw new Error('The request has no PNG screenshot.') + const png = Buffer.from(match[1], 'base64') + return { width: png.readUInt32BE(16), height: png.readUInt32BE(20) } +} + +test.describe('computer use with localComputer', () => { + test('localComputer clicks on the screen and sends a display-size screenshot', async () => { + const { robot, calls } = fakeRobot() + const computerUse = createComputerUse({ + driver: localComputer({ robot }), + screenshotDelay: 0, + }) + const model = mockModel([ + openaiStream('resp_cu_1', [ + { + type: 'computer_call', + id: 'cu_local', + call_id: 'call_cu_local', + status: 'completed', + actions: [ + { type: 'click', button: 'left', x: 640, y: 400 }, + { type: 'type', text: 'hello' }, + ], + pending_safety_checks: [], + }, + ]), + openaiStream('resp_cu_2', [OPENAI_ANSWER]), + ]) + + await runOpenai(computerUse, model, 'thread-cu-local') + + // The 2560 x 1600 screen fits in 1280 x 800, so each point doubles. + expect(calls).toEqual([ + 'moveMouse 1280 800', + 'mouseClick left', + 'typeString hello', + ]) + expect(model.bodies).toHaveLength(2) + // A desktop has no address, so the `navigate` tool does not go. + expect(model.bodies[0]).toMatchObject({ tools: [{ type: 'computer' }] }) + expect(screenshotSize(model.bodies[1])).toEqual({ + width: 1280, + height: 800, + }) + }) +}) From 2b6923f7896fac08cbf972079ddfaaa2b3c196ea Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Tue, 20 Oct 2026 16:46:16 +0200 Subject: [PATCH 19/32] fix(ai-computer-use): send only the screenshot bytes for a small Node Buffer A small Buffer lives in the shared Node pool, so its ArrayBuffer holds other data too. Buffer.slice() returns a view, so imageResult encoded the whole pool. Copy the bytes into a new Uint8Array before the base64. The localComputer E2E test caught it with a small PNG. --- packages/ai-computer-use/src/create-computer-use.ts | 3 ++- .../tests/create-computer-use.test.ts | 13 ++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/ai-computer-use/src/create-computer-use.ts b/packages/ai-computer-use/src/create-computer-use.ts index 7600bb0193..5f149fbd75 100644 --- a/packages/ai-computer-use/src/create-computer-use.ts +++ b/packages/ai-computer-use/src/create-computer-use.ts @@ -117,7 +117,8 @@ function imageResult(bytes: Uint8Array) { type: 'image', source: { type: 'data', - value: arrayBufferToBase64(bytes.slice().buffer), + // Copy the bytes. The ArrayBuffer of a small Node Buffer is a shared pool. + value: arrayBufferToBase64(new Uint8Array(bytes).buffer), mimeType: 'image/png', }, } diff --git a/packages/ai-computer-use/tests/create-computer-use.test.ts b/packages/ai-computer-use/tests/create-computer-use.test.ts index 0d204f9c6d..ee2d9f95cd 100644 --- a/packages/ai-computer-use/tests/create-computer-use.test.ts +++ b/packages/ai-computer-use/tests/create-computer-use.test.ts @@ -50,6 +50,8 @@ interface DriverOptions { failClick?: () => boolean /** `click` waits for this promise. */ clickGate?: Promise + /** The bytes that `screenshot` returns. The default is `PNG_BYTES`. */ + screenshot?: Uint8Array } /** A driver that records its calls in order. */ @@ -63,7 +65,7 @@ function createDriver(options: DriverOptions = {}) { environment: 'browser', screenshot: async () => { calls.push('screenshot') - return PNG_BYTES + return options.screenshot ?? PNG_BYTES }, click: async () => { calls.push('click:start') @@ -343,6 +345,15 @@ describe('createComputerUse tool', () => { expect(computer.calls).toEqual(['click:start', 'click:end', 'screenshot']) }) + it('sends only the screenshot bytes when the driver returns a small Node Buffer', async () => { + // Node puts a small Buffer in a shared pool, so its ArrayBuffer holds other data too. + const computer = setup({ screenshot: Buffer.from(PNG_BYTES) }) + + await expect(computer.execute(CLICK)).resolves.toEqual( + screenshotOf(PNG_BASE64), + ) + }) + it('returns the zoom image and the cursor position as text', async () => { const computer = setup({ zoom: true, cursor: true }) From df61a96718c3fa6ca84850500f2399992365d66e Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Tue, 20 Oct 2026 17:34:01 +0200 Subject: [PATCH 20/32] fix(ai-computer-local, ai-openai): release keys, clamp zoom, and drop computer from o3-mini - localComputer().press releases the keys that went down when a later key is not valid. Before, Control stayed down on the host. - localComputer().zoom keeps the region on the screen. robotjs does not check the region, and a region off the screen can crash Node. - o3-mini has no image input, so it no longer accepts the computer tool. It cannot read the screenshots. --- packages/ai-computer-local/src/driver.ts | 25 +++++++++--- .../ai-computer-local/tests/driver.test.ts | 40 +++++++++++++++++++ packages/ai-openai/src/model-meta.ts | 1 - .../tests/computer-tool-kind.test.ts | 11 ++++- 4 files changed, 69 insertions(+), 8 deletions(-) diff --git a/packages/ai-computer-local/src/driver.ts b/packages/ai-computer-local/src/driver.ts index c73266d652..280e508208 100644 --- a/packages/ai-computer-local/src/driver.ts +++ b/packages/ai-computer-local/src/driver.ts @@ -153,9 +153,16 @@ export function localComputer(options: LocalComputerOptions = {}) { robot.keyToggle(toRobotKey(key), 'up') }, async press(keys) { - const names = keys.map(toRobotKey) - for (const name of names) robot.keyToggle(name, 'down') - for (const name of [...names].reverse()) robot.keyToggle(name, 'up') + // Release the pressed keys also when a later key is not valid. + const pressed: Array = [] + try { + for (const name of keys.map(toRobotKey)) { + robot.keyToggle(name, 'down') + pressed.push(name) + } + } finally { + for (const name of pressed.reverse()) robot.keyToggle(name, 'up') + } }, async type(text) { robot.typeString(text) @@ -168,9 +175,17 @@ export function localComputer(options: LocalComputerOptions = {}) { } }, async zoom(region) { + // robotjs does not check the region. A region off the screen can crash Node. const start = toScreen(region.x, region.y) - const size = toScreen(region.width, region.height) - const bitmap = robot.screen.capture(start.x, start.y, size.x, size.y) + const end = toScreen(region.x + region.width, region.y + region.height) + const x = Math.min(Math.max(start.x, 0), screen.width - 1) + const y = Math.min(Math.max(start.y, 0), screen.height - 1) + const bitmap = robot.screen.capture( + x, + y, + Math.max(1, Math.min(end.x, screen.width) - x), + Math.max(1, Math.min(end.y, screen.height) - y), + ) return toPng(bitmap, bitmap.width, bitmap.height) }, } satisfies ComputerDriver diff --git a/packages/ai-computer-local/tests/driver.test.ts b/packages/ai-computer-local/tests/driver.test.ts index 335e6b7582..8140382385 100644 --- a/packages/ai-computer-local/tests/driver.test.ts +++ b/packages/ai-computer-local/tests/driver.test.ts @@ -210,6 +210,27 @@ describe('localComputer keyboard', () => { ]) }) + it('releases the pressed keys when a later key is not valid', async () => { + const { robot, calls } = fakeRobot() + const keyToggle = robot.keyToggle + robot.keyToggle = (key, down) => { + if (key === 'not_a_key') throw new Error('Invalid key code specified.') + keyToggle(key, down) + } + const press = localComputer({ robot }).press([ + 'Control', + 'Shift', + 'NOT_A_KEY', + ]) + await expect(press).rejects.toThrow('Invalid key code specified.') + expect(calls).toEqual([ + 'keyToggle control down', + 'keyToggle shift down', + 'keyToggle shift up', + 'keyToggle control up', + ]) + }) + it('sends robotjs key names', async () => { const { robot, calls } = fakeRobot() const driver = localComputer({ robot }) @@ -272,4 +293,23 @@ describe('localComputer screen', () => { rows: [0, 0, 0, 255, 0, 0, 255, 0, 0, 0, 255, 0, 0, 255], }) }) + + it.each([ + [ + 'past the bottom right', + { x: 1, y: 0, width: 5, height: 3 }, + 'capture 2 0 2 2', + ], + [ + 'before the top left', + { x: -3, y: -1, width: 1, height: 1 }, + 'capture 0 0 1 1', + ], + ])('keeps a zoom region %s on the screen', async (_, region, expected) => { + const { robot, calls } = fakeRobot({ width: 4, height: 2 }) + await localComputer({ robot, display: { width: 2, height: 1 } }).zoom( + region, + ) + expect(calls).toEqual([expected]) + }) }) diff --git a/packages/ai-openai/src/model-meta.ts b/packages/ai-openai/src/model-meta.ts index 5f99ff1361..c05abdd6c2 100644 --- a/packages/ai-openai/src/model-meta.ts +++ b/packages/ai-openai/src/model-meta.ts @@ -1314,7 +1314,6 @@ const O3_MINI = { 'code_interpreter', 'mcp', 'computer_use', - 'computer', 'local_shell', 'shell', 'apply_patch', diff --git a/packages/ai-openai/tests/computer-tool-kind.test.ts b/packages/ai-openai/tests/computer-tool-kind.test.ts index 7583c4e97b..3d67c60635 100644 --- a/packages/ai-openai/tests/computer-tool-kind.test.ts +++ b/packages/ai-openai/tests/computer-tool-kind.test.ts @@ -1,8 +1,8 @@ /** * Type-level checks for the generic `'computer'` tool kind. * - * A model that lists `computer_use` also accepts the generic computer tool. - * A model without `computer_use` rejects it at compile time. + * A model that lists `computer` accepts the generic computer tool. A model + * without it rejects it at compile time. */ import { beforeAll, describe, it } from 'vitest' import { brandProviderTool } from '@tanstack/ai' @@ -43,4 +43,11 @@ describe('OpenAI computer tool kind', () => { computerTool, ]) }) + + it('o3-mini rejects the computer tool, because it has no image input', () => { + typedTools(openaiText('o3-mini'), [ + // @ts-expect-error - o3-mini cannot read the screenshots + computerTool, + ]) + }) }) From f9c2d8c8a03047a878c65804f1e929ea229b0354 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Tue, 20 Oct 2026 17:44:15 +0200 Subject: [PATCH 21/32] fix(openai-base): give the model the screenshot and the reason for a failed computer call A failed computer call has no image in its result. Before, the adapter sent a computer_call_output with no image, and the failure reason did not reach the model. - The failed call sends the newest screenshot of the conversation. - The failure reasons go in one user message after the call outputs, because a computer call output has no text. --- .changeset/openai-computer-call-loop.md | 2 + .../src/adapters/responses-text.ts | 28 +++++- .../tests/computer-use-loop.test.ts | 85 ++++++++++++++----- 3 files changed, 94 insertions(+), 21 deletions(-) diff --git a/.changeset/openai-computer-call-loop.md b/.changeset/openai-computer-call-loop.md index 9232a504f1..6cb00c774b 100644 --- a/.changeset/openai-computer-call-loop.md +++ b/.changeset/openai-computer-call-loop.md @@ -3,3 +3,5 @@ --- The computer use loop of `computerUseTool()` runs. A `computer_call` from the model is now a tool call to `computer_use_preview`, with the `action` and the `pending_safety_checks` as its input. The tool result goes back as `computer_call_output`, with the first image of the result as the screenshot. When the call does not fail, the output acknowledges the safety checks of the call. + +A failed computer call sends the newest screenshot of the conversation, because the failed call has no screenshot of its own. A computer call output has no text, so the reason for the failure goes to the model in a user message after the call outputs. diff --git a/packages/openai-base/src/adapters/responses-text.ts b/packages/openai-base/src/adapters/responses-text.ts index 5c4794cd55..2ee4c69657 100644 --- a/packages/openai-base/src/adapters/responses-text.ts +++ b/packages/openai-base/src/adapters/responses-text.ts @@ -2927,18 +2927,41 @@ export abstract class OpenAIBaseResponsesTextAdapter< metadata?: unknown } >() + // A failed computer call has no screenshot of its own, so it sends the + // newest one. A computer call output has no text, so the failure reasons + // go in one user message after the call outputs. + let lastScreenshot: ResponseComputerToolCallOutputScreenshot | undefined + const failures: Array = [] + const pushFailures = () => { + if (failures.length === 0) return + result.push({ + type: 'message', + role: 'user', + content: [{ type: 'input_text', text: failures.splice(0).join('\n') }], + }) + } for (const [index, message] of messages.entries()) { + if (message.role !== 'tool') pushFailures() result.push(...(insertBefore?.get(index) ?? [])) // Handle tool messages - convert to FunctionToolCallOutput if (message.role === 'tool') { const owner = message.toolCallId ? userToolCalls.get(message.toolCallId) : undefined + const error = message.error const userOutput = owner ? userToolResultItem(owner, message.content, { - screenshot: () => this.screenshotOf(message.content), - failed: message.error !== undefined, + screenshot: () => { + if (error !== undefined) { + failures.push(`Computer call ${owner.id} failed: ${error}`) + return lastScreenshot ?? { type: 'computer_screenshot' } + } + const shot = this.screenshotOf(message.content) + if (shot.image_url || shot.file_id) lastScreenshot = shot + return shot + }, + failed: error !== undefined, }) : null if (userOutput) { @@ -3171,6 +3194,7 @@ export abstract class OpenAIBaseResponsesTextAdapter< }) } + pushFailures() result.push(...(insertBefore?.get(messages.length) ?? [])) return result } diff --git a/packages/openai-base/tests/computer-use-loop.test.ts b/packages/openai-base/tests/computer-use-loop.test.ts index f8a5d0b430..36235a5251 100644 --- a/packages/openai-base/tests/computer-use-loop.test.ts +++ b/packages/openai-base/tests/computer-use-loop.test.ts @@ -77,17 +77,25 @@ const screenshot = [ }, ] +/** The user message that gives the model the reason for a failed call. */ +function failureMessage(text: string) { + return { + type: 'message', + role: 'user', + content: [{ type: 'input_text', text }], + } +} + async function runLoop( execute: (input: unknown) => Promise, { computer = previewTool, - call = clickCall, - }: { computer?: Tool; call?: Record } = {}, + calls = [clickCall], + }: { computer?: Tool; calls?: Array> } = {}, ) { - const create = vi - .fn() - .mockResolvedValueOnce(completed([call])) - .mockResolvedValueOnce(completed([answer])) + const create = vi.fn() + for (const call of calls) create.mockResolvedValueOnce(completed([call])) + create.mockResolvedValueOnce(completed([answer])) const client = { responses: { create: (params: unknown) => create(params) }, } as unknown as OpenAI @@ -145,14 +153,16 @@ describe('computer use loop', () => { ] const { create, execute } = await runLoop(async () => screenshot, { computer: computerUseTool({ type: 'computer' }), - call: { - type: 'computer_call', - id: 'cu_2', - call_id: 'call_batch', - status: 'completed', - actions, - pending_safety_checks: [], - }, + calls: [ + { + type: 'computer_call', + id: 'cu_2', + call_id: 'call_batch', + status: 'completed', + actions, + pending_safety_checks: [], + }, + ], }) expect(execute.mock.calls[0]?.[0]).toEqual({ @@ -237,11 +247,48 @@ describe('computer use loop', () => { const input: Array> = create.mock.calls[1]?.[0].input - const output = input.find((item) => item.type === 'computer_call_output') - expect(output).toEqual({ - type: 'computer_call_output', - call_id: 'call_cu', - output: { type: 'computer_screenshot' }, + expect(input.slice(-2)).toEqual([ + { + type: 'computer_call_output', + call_id: 'call_cu', + output: { type: 'computer_screenshot' }, + }, + failureMessage( + 'Computer call call_cu failed: A person did not allow the click.', + ), + ]) + }) + + it('sends the newest screenshot and the reason for a failed call', async () => { + const secondCall = { + ...clickCall, + id: 'cu_2', + call_id: 'call_cu_2', + pending_safety_checks: [], + } + const execute = vi + .fn() + .mockResolvedValueOnce(screenshot) + .mockRejectedValueOnce(new Error('The page has no button there.')) + const { create } = await runLoop(execute, { + calls: [clickCall, secondCall], }) + + const input: Array> = + create.mock.calls[2]?.[0].input + expect(input.slice(-2)).toEqual([ + { + type: 'computer_call_output', + call_id: 'call_cu_2', + output: { + type: 'computer_screenshot', + image_url: 'data:image/png;base64,iVBORw0KGgo=', + detail: 'original', + }, + }, + failureMessage( + 'Computer call call_cu_2 failed: The page has no button there.', + ), + ]) }) }) From 8763d90d43c97f4565125ede1e4e04825e9e213e Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:10:44 +1100 Subject: [PATCH 22/32] fix(ai-computer-use): run a shared driver's threads one at a time and release keys on failure - A driver that the app passes in is one machine. Its actions now run one at a time across all threads, so two threads cannot mix clicks on one page. - Release only the modifier keys that went down, keep releasing after one keyUp fails, and release the drag button when a move fails. - Reject a key repeat outside 1 to 100 and a hold or wait over 300 seconds, the limits in the Anthropic computer tool docs. - openai-base: keep safety checks that do not parse, so the computer tool refuses the action instead of running it with no approval. - E2B: round coordinates to the nearest pixel. E2B and Playwright: log a failed cleanup and keep the setup error. Log a failed idle close. - Docs: screenshots after batches, zoom and cursor_position, and the computer versions that turn them on. --- docs/computer-use/computer-use.md | 8 +- docs/computer-use/custom-driver.md | 6 +- docs/computer-use/drivers.md | 6 +- packages/ai-computer-e2b/src/driver.ts | 13 ++- packages/ai-computer-e2b/tests/driver.test.ts | 27 +++++- packages/ai-computer-playwright/src/driver.ts | 12 ++- .../tests/driver.test.ts | 27 +++++- packages/ai-computer-use/README.md | 13 ++- packages/ai-computer-use/src/actions.ts | 56 +++++++++--- .../src/create-computer-use.ts | 35 +++++-- .../ai-computer-use/tests/actions.test.ts | 91 +++++++++++++++++++ .../tests/create-computer-use.test.ts | 57 ++++++++++++ .../src/adapters/responses-user-tools.ts | 46 ++++++---- .../tests/computer-use-generic.test.ts | 26 ++++++ 14 files changed, 360 insertions(+), 63 deletions(-) diff --git a/docs/computer-use/computer-use.md b/docs/computer-use/computer-use.md index 9f2699daad..0d9289019b 100644 --- a/docs/computer-use/computer-use.md +++ b/docs/computer-use/computer-use.md @@ -182,10 +182,8 @@ To show the approval card for a safety check, see [Safety checks](./safety-check The model clicks, double-clicks, drags, moves the mouse, scrolls, types text, and presses key combos. It can also wait. -- Every action returns a screenshot of the screen. -- The screenshot comes 500 ms after the action. This time lets the page settle. -- `zoom` returns an image of one region at full size. -- `cursor_position` returns the mouse position as text. +- A normal action returns a screenshot 500 ms later, so the page can settle. An OpenAI batch returns one screenshot after the last action. +- `zoom` returns an image of one region. `cursor_position` returns the mouse position as text. Neither takes another screenshot. - `navigate` opens a URL, then returns a screenshot. To change the delay, set `screenshotDelay` in ms: @@ -202,7 +200,7 @@ const computerUse = createComputerUse({ ## Order and failures -In one thread, actions run one at a time, in the order that the model calls them. Two threads run at the same time, each on its own page. +In one thread, actions run one at a time, in the order that the model calls them. A driver factory gives each thread its own machine, and those threads can run at the same time. A driver that you pass in is one machine. Actions on that machine run one at a time, even from two threads. If one action fails, the other actions of that model turn do not run. The model gets this result for each of them: diff --git a/docs/computer-use/custom-driver.md b/docs/computer-use/custom-driver.md index 5cc3236560..3ba89194a6 100644 --- a/docs/computer-use/custom-driver.md +++ b/docs/computer-use/custom-driver.md @@ -91,8 +91,8 @@ Coordinates are in pixels, in `display` space. Key names are `KeyboardEvent.key` Four methods are optional: -- `zoom(region)`: returns PNG bytes of one region at full size. The region has `x`, `y`, `width`, and `height`. The Anthropic adapter then turns on the `zoom` action. -- `cursorPosition()`: returns the mouse position as `{ x, y }`. The Anthropic adapter then turns on the `cursor_position` action. +- `zoom(region)`: returns PNG bytes of one region at full size. The region has `x`, `y`, `width`, and `height`. `computer_toolset_20260801` and `computer_20251124` turn on `zoom` when the driver has this method. `computer_20250124` does not. +- `cursorPosition()`: returns the mouse position as `{ x, y }`. Only `computer_toolset_20260801` turns on `cursor_position`, and only when the driver has this method. - `navigate(url)`: opens the URL. `computerUse.tools` then has the `navigate` tool. The tool accepts only `http:` and `https:` URLs. - `close()`: frees the machine. `createComputerUse` calls it only on a driver that a factory made. @@ -184,7 +184,7 @@ Your driver does only the small steps. The core builds the rest from them: - **Modifier clicks and scrolls**: `keyDown` for each modifier, the action, then `keyUp` in reverse order. - **`hold_key` and `wait`**: the core holds the keys or waits, then continues. - **Key names**: the core changes OpenAI and Anthropic key names to `KeyboardEvent.key` names. -- **Screenshots**: after each action, the core waits `screenshotDelay` ms and calls `screenshot()`. +- **Screenshots**: after a normal action, the core waits `screenshotDelay` ms and calls `screenshot()`. An OpenAI batch waits once, after the last action, then takes one screenshot. `zoom` and `cursor_position` return their own result. The core does not call `screenshot()` for those two. ## Try it diff --git a/docs/computer-use/drivers.md b/docs/computer-use/drivers.md index 0b5e069c5d..84ae7d1d5d 100644 --- a/docs/computer-use/drivers.md +++ b/docs/computer-use/drivers.md @@ -32,7 +32,11 @@ A browser page is not always enough. Sometimes the model must use a desktop app, | **`cursor_position`** | No | Yes | Yes | | **Setup** | `playwright-core`, then download Chromium | An E2B API key | `@jitsi/robotjs`, and permissions on macOS | -When a driver has `zoom` or `cursorPosition`, the Anthropic adapter turns on that action for the model. +The Anthropic adapter turns on `zoom` and `cursor_position` only on the computer versions that have those flags. + +- `computer_toolset_20260801` turns on both when the driver has the method. +- `computer_20251124` turns on `zoom` only, and only when the driver has `zoom`. +- `computer_20250124` turns on neither. ## Playwright diff --git a/packages/ai-computer-e2b/src/driver.ts b/packages/ai-computer-e2b/src/driver.ts index c538326a34..5de30a41ad 100644 --- a/packages/ai-computer-e2b/src/driver.ts +++ b/packages/ai-computer-e2b/src/driver.ts @@ -109,11 +109,11 @@ export async function createE2BComputerDriver({ screenshot: () => desktop.screenshot('bytes'), async click(x, y, { button, count }) { // The SDK click methods skip the move when x or y is 0, so move first. - await desktop.moveMouse(int(x), int(y)) + await desktop.moveMouse(Math.round(x), Math.round(y)) await sdkClick(button, count) }, async move(x, y) { - await desktop.moveMouse(int(x), int(y)) + await desktop.moveMouse(Math.round(x), Math.round(y)) }, async mouseDown(button) { if (button === 'back' || button === 'forward') { @@ -130,7 +130,7 @@ export async function createE2BComputerDriver({ await desktop.mouseRelease(button) }, async scroll(x, y, deltaX, deltaY) { - await desktop.moveMouse(int(x), int(y)) + await desktop.moveMouse(Math.round(x), Math.round(y)) if (deltaY !== 0) { await desktop.scroll(deltaY > 0 ? 'down' : 'up', scrollClicks(deltaY)) } @@ -222,7 +222,12 @@ export function e2bComputer(options: E2BComputerOptions = {}) { const driver = await createE2BComputerDriver({ desktop }).catch( async (error: unknown) => { // Keep the setup error. A failed kill does not replace it. - await desktop.kill().catch(() => undefined) + await desktop.kill().catch((killError: unknown) => { + console.error( + 'Failed to kill the E2B desktop after setup failed.', + killError, + ) + }) throw error }, ) diff --git a/packages/ai-computer-e2b/tests/driver.test.ts b/packages/ai-computer-e2b/tests/driver.test.ts index 8d8a42a7cc..70b2eb2914 100644 --- a/packages/ai-computer-e2b/tests/driver.test.ts +++ b/packages/ai-computer-e2b/tests/driver.test.ts @@ -92,10 +92,10 @@ describe('createE2BComputerDriver', () => { expect(calls).toEqual(['moveMouse 10 20']) }) - it('rejects a coordinate that is not an integer', async () => { + it('rounds a coordinate to the nearest pixel', async () => { const { driver, calls } = await setup() - await expect(driver.move(1.5, 2)).rejects.toThrow('Expected an integer') - expect(calls).toEqual([]) + await driver.move(1.5, 2.2) + expect(calls).toEqual(['moveMouse 2 2']) }) it('presses and releases buttons', async () => { @@ -245,4 +245,25 @@ describe('e2bComputer', () => { ) expect(calls).toEqual(['kill']) }) + + it('keeps the setup error when the sandbox kill throws', async () => { + const error = vi.spyOn(console, 'error').mockImplementation(() => {}) + const { desktop, calls } = fakeDesktop({ + getScreenSize: () => Promise.reject(new Error('no screen')), + kill: async () => { + calls.push('kill') + throw new Error('kill failed') + }, + }) + create.mockResolvedValue(desktop) + await expect(e2bComputer().create({ threadId: 'a' })).rejects.toThrow( + 'no screen', + ) + expect(calls).toEqual(['kill']) + expect(error).toHaveBeenCalledWith( + 'Failed to kill the E2B desktop after setup failed.', + expect.any(Error), + ) + error.mockRestore() + }) }) diff --git a/packages/ai-computer-playwright/src/driver.ts b/packages/ai-computer-playwright/src/driver.ts index 52cd5442a0..305bcfaea8 100644 --- a/packages/ai-computer-playwright/src/driver.ts +++ b/packages/ai-computer-playwright/src/driver.ts @@ -137,8 +137,8 @@ export function createPlaywrightComputerDriver({ * `browser`, the factory launches Chromium once, on the first thread, and * `close()` on the factory closes it. * - * Without `browser`, the app installs `playwright-core` and runs - * `npx playwright-core install chromium` once. + * Without `browser`, the factory launches Chromium on the first thread. + * Install `playwright-core` and Chromium before that. * * @param options - The start URL, the display, and the browser or the launch options. * @@ -181,7 +181,13 @@ export function playwrightComputer(options: PlaywrightComputerOptions = {}) { close: () => context.close(), } } catch (error) { - await context.close() + // Keep the page error. A failed close does not replace it. + await context.close().catch((closeError: unknown) => { + console.error( + 'Failed to close the Playwright context after setup failed.', + closeError, + ) + }) throw error } }, diff --git a/packages/ai-computer-playwright/tests/driver.test.ts b/packages/ai-computer-playwright/tests/driver.test.ts index 0b5dbd5733..c31f72c99d 100644 --- a/packages/ai-computer-playwright/tests/driver.test.ts +++ b/packages/ai-computer-playwright/tests/driver.test.ts @@ -57,7 +57,10 @@ function setup() { return { driver: createPlaywrightComputerDriver({ page }), calls } } -function createFakeBrowser({ gotoError }: { gotoError?: Error } = {}) { +function createFakeBrowser({ + gotoError, + closeError, +}: { gotoError?: Error; closeError?: Error } = {}) { const contexts: Array<{ viewport: { width: number; height: number } calls: Array<[string, ...Array]> @@ -78,6 +81,7 @@ function createFakeBrowser({ gotoError }: { gotoError?: Error } = {}) { newPage: async () => page, close: async () => { context.closed = true + if (closeError) throw closeError }, } }, @@ -238,6 +242,27 @@ describe('playwrightComputer', () => { expect(contexts[0]?.calls).toEqual([]) }) + it('keeps the startUrl error when closing the context throws', async () => { + const error = vi.spyOn(console, 'error').mockImplementation(() => {}) + const { browser, contexts } = createFakeBrowser({ + gotoError: new Error('net::ERR_NAME_NOT_RESOLVED'), + closeError: new Error('close failed'), + }) + const factory = playwrightComputer({ + browser, + startUrl: 'https://tanstack.invalid', + }) + await expect(factory.create({ threadId: 'a' })).rejects.toThrow( + 'net::ERR_NAME_NOT_RESOLVED', + ) + expect(contexts[0]?.closed).toBe(true) + expect(error).toHaveBeenCalledWith( + 'Failed to close the Playwright context after setup failed.', + expect.any(Error), + ) + error.mockRestore() + }) + it('closes the context and rethrows when startUrl fails to load', async () => { const { browser, contexts } = createFakeBrowser({ gotoError: new Error('net::ERR_NAME_NOT_RESOLVED'), diff --git a/packages/ai-computer-use/README.md b/packages/ai-computer-use/README.md index 3fa376294b..e15b80e605 100644 --- a/packages/ai-computer-use/README.md +++ b/packages/ai-computer-use/README.md @@ -22,7 +22,7 @@ Computer use for TanStack AI. The model sees a screen and uses a mouse and a keyboard. -You write the code one time. The same code runs on OpenAI and on Anthropic. A driver controls the machine: a Playwright browser, an E2B desktop, or your own. Each thread gets its own machine. +You write the code one time. The same code runs on OpenAI and on Anthropic. A driver controls the machine: a Playwright browser, an E2B desktop, or your own. A driver factory gives each thread its own machine. A driver that you pass in is one machine. ## Installation @@ -85,15 +85,20 @@ await computerUse.close() // every thread, then the browser ## What it does -- Each `threadId` gets its own machine, action queue, and safety-check approvals. In one thread, actions run one at a time, in call order. +- In one thread, actions run one at a time, in call order. +- A driver factory gives each `threadId` its own machine and action queue. +- A driver that you pass in is one machine. Its actions run one at a time, across all threads. +- Safety-check approvals belong to one thread. - `tools` is `[computer]`. When the driver can navigate, it is `[computer, navigate]`. `navigate` opens only `http:` and `https:` URLs. -- Every action returns a screenshot. The screenshot comes 500 ms after the action. Set `screenshotDelay` to change it. +- A normal action returns a screenshot 500 ms later. Set `screenshotDelay` to change that wait. +- An OpenAI batch returns one screenshot, after the last action. +- `zoom` returns a region image, and `cursor_position` returns text. Neither takes another screenshot. - When an action fails, the other actions of the same model turn do not run. - OpenAI can attach safety checks to an action. The action waits until a person approves it. When the person declines, the action does not run. ## What your application owns -- **A stable thread id.** If `chat()` gets no `threadId`, each request gets a new machine. +- **A stable thread id.** If `chat()` gets no `threadId`, each request gets a new thread. A driver factory then makes a new machine for that thread. - **The idle time.** A thread that stays idle for `idleTimeout` closes. The default is 10 minutes. `false` keeps every thread until `close`. - **One server process.** The machines live in the memory of one process. - **The network.** The browser can open every address that your server can reach. Run it where that is safe. diff --git a/packages/ai-computer-use/src/actions.ts b/packages/ai-computer-use/src/actions.ts index ffa5266733..a96b690195 100644 --- a/packages/ai-computer-use/src/actions.ts +++ b/packages/ai-computer-use/src/actions.ts @@ -7,6 +7,11 @@ const SCROLL_STEP = 100 // ponytail: an OpenAI `wait` has no duration. 2 s is the wait in the OpenAI samples. const OPENAI_WAIT_MS = 2000 +// The Anthropic computer tool docs give these limits: `repeat` is 1 to 100, and a +// `hold_key` or `wait` duration is up to 300 seconds. Reject here, before any key goes down. +const MAX_KEY_REPEAT = 100 +const MAX_WAIT_SECONDS = 300 + type Point = { x: number; y: number } type Input = Record @@ -58,19 +63,47 @@ function readCoordinate(input: Input, key: string) { return { x, y } } -/** Presses `keys` in order, runs `action`, then releases them in reverse. */ +function readRepeat(input: Input) { + if (input['repeat'] === undefined || input['repeat'] === null) return 1 + const repeat = readNumber(input, 'repeat') + if (!Number.isInteger(repeat) || repeat < 1 || repeat > MAX_KEY_REPEAT) { + throw invalid('repeat', `an integer from 1 to ${MAX_KEY_REPEAT}`) + } + return repeat +} + +function readSeconds(input: Input, key: string) { + const value = readNumber(input, key) + if (value < 0 || value > MAX_WAIT_SECONDS) { + throw invalid(key, `a number from 0 to ${MAX_WAIT_SECONDS}`) + } + return value +} + +/** Presses `keys` in order, runs `action`, then releases the keys that went down. */ async function withKeys( driver: ComputerDriver, keys: Array, action: () => Promise, ) { - for (const key of keys) await driver.keyDown(key) + const pressed: Array = [] + let releaseError: unknown try { + for (const key of keys) { + await driver.keyDown(key) + pressed.push(key) + } await action() } finally { - const releaseOrder = [...keys].reverse() - for (const key of releaseOrder) await driver.keyUp(key) + // One failed keyUp does not stop the rest. + for (const key of pressed.reverse()) { + await driver.keyUp(key).catch((error: unknown) => { + releaseError ??= error + }) + } } + // Reached only when the action succeeded, so an action error wins. + if (releaseError !== undefined) throw releaseError } async function drag(driver: ComputerDriver, path: Array) { @@ -78,8 +111,11 @@ async function drag(driver: ComputerDriver, path: Array) { if (start === undefined) throw invalid('path', 'a non-empty array') await driver.move(start.x, start.y) await driver.mouseDown('left') - for (const point of rest) await driver.move(point.x, point.y) - await driver.mouseUp('left') + try { + for (const point of rest) await driver.move(point.x, point.y) + } finally { + await driver.mouseUp('left') + } } // --------------------------------------------------------------------------- @@ -147,9 +183,7 @@ async function runAnthropicAction( return undefined case 'key': { const combos = readString(input, 'text').split(/\s+/).filter(Boolean) - const hasRepeat = - input['repeat'] !== undefined && input['repeat'] !== null - const repeat = hasRepeat ? readNumber(input, 'repeat') : 1 + const repeat = readRepeat(input) for (let i = 0; i < repeat; i++) { for (const combo of combos) await driver.press(parseKeyCombo(combo)) } @@ -191,12 +225,12 @@ async function runAnthropicAction( } case 'hold_key': { const keys = parseKeyCombo(readString(input, 'text')) - const ms = readNumber(input, 'duration') * 1000 + const ms = readSeconds(input, 'duration') * 1000 await withKeys(driver, keys, () => sleep(ms)) return undefined } case 'wait': - await sleep(readNumber(input, 'duration') * 1000) + await sleep(readSeconds(input, 'duration') * 1000) return undefined case 'cursor_position': { if (!driver.cursorPosition) { diff --git a/packages/ai-computer-use/src/create-computer-use.ts b/packages/ai-computer-use/src/create-computer-use.ts index 5f149fbd75..8d2aa0ea54 100644 --- a/packages/ai-computer-use/src/create-computer-use.ts +++ b/packages/ai-computer-use/src/create-computer-use.ts @@ -49,7 +49,7 @@ interface ThreadState { id: string /** The machine from the factory. It stays unset for a driver that the app passed in. */ driver: Promise | undefined - /** The actions of the thread run one at a time on this queue. */ + /** This thread's last queued action. A shared driver uses one queue for every thread. */ queue: Promise /** The number of actions in the queue. */ pending: number @@ -134,11 +134,14 @@ function computerToolCalls(ctx: ChatMiddlewareContext) { /** * Create a provider-agnostic computer tool. Each thread gets its own state. - * With a driver factory, each thread also gets its own machine. + * A driver factory gives each thread its own machine. A driver that you pass + * in is one machine, and its actions run one at a time across threads. * * The OpenAI and Anthropic adapters swap the computer tool for their native - * computer tool. Every action returns a screenshot. `zoom` returns the zoom - * image, and `cursor_position` returns text. + * computer tool. A normal action returns a screenshot. An OpenAI batch + * returns one screenshot after the last action. `zoom` returns the zoom + * image, and `cursor_position` returns text. Those two do not take another + * screenshot. * * - `tools`: pass it to `chat({ tools })`. It is `[computer]`. When the * driver can navigate, it is `[computer, navigate]`. @@ -193,15 +196,24 @@ export function createComputerUse({ // request, so each request gets a new machine. Pass threadId to chat(). // ponytail: when a person answers a safety check after idleTimeout, the // approved action runs on a new machine. Set idleTimeout above that wait. + // ponytail: one queue for a shared driver. Per-thread queues if two + // threads on one machine must overlap. const threads = new Map() + const sharesOneMachine = !('create' in driver) + let sharedQueue: Promise = Promise.resolve() function startTimer(thread: ThreadState) { clearTimeout(thread.timer) const isOpen = threads.get(thread.id) === thread if (idleTimeout === false || !isOpen) return thread.timer = setTimeout(() => { - // Nobody waits for an idle close, so a close error has no caller. - closeThread(thread).catch(() => undefined) + // Nobody waits for an idle close. Log the error so a stuck machine is visible. + closeThread(thread).catch((error: unknown) => { + console.error( + `Failed to close the idle computer thread ${thread.id}.`, + error, + ) + }) }, idleTimeout) thread.timer.unref?.() } @@ -253,7 +265,8 @@ export function createComputerUse({ const thread = threadOf(threadId ?? SHARED_THREAD) clearTimeout(thread.timer) thread.pending += 1 - const run = thread.queue.then(async () => { + const previous = sharesOneMachine ? sharedQueue : thread.queue + const run = previous.then(async () => { if (thread.turnFailed) { throw new Error( 'Not executed: an earlier computer action in this turn failed.', @@ -266,12 +279,18 @@ export function createComputerUse({ throw error } }) - thread.queue = run + // Swallow here so the next action runs. `run` still rejects for the caller. + // On a shared driver, `settled` follows the actions of other threads, so + // closing a thread waits for them too. Closing a thread never closes a + // shared driver, so this wait does no harm. + const settled = run .catch(() => undefined) .then(() => { thread.pending -= 1 if (thread.pending === 0) startTimer(thread) }) + if (sharesOneMachine) sharedQueue = settled + thread.queue = settled return run } diff --git a/packages/ai-computer-use/tests/actions.test.ts b/packages/ai-computer-use/tests/actions.test.ts index 81daa83d3c..b2717b09a8 100644 --- a/packages/ai-computer-use/tests/actions.test.ts +++ b/packages/ai-computer-use/tests/actions.test.ts @@ -127,6 +127,48 @@ describe('Anthropic actions', () => { ]) }) + it('releases the keys that went down when a later keyDown throws', async () => { + const { driver, calls } = createDriver() + driver.keyDown = async (key) => { + calls.push(['keyDown', key]) + if (key === 'Shift') throw new Error('key down failed') + } + await expect( + runComputerAction(driver, { + action: 'left_click', + coordinate: [1, 2], + text: 'ctrl+shift', + }), + ).rejects.toThrow('key down failed') + expect(calls).toEqual([ + ['keyDown', 'Control'], + ['keyDown', 'Shift'], + ['keyUp', 'Control'], + ]) + }) + + it('releases the remaining keys when one keyUp throws', async () => { + const { driver, calls } = createDriver() + driver.keyUp = async (key) => { + calls.push(['keyUp', key]) + if (key === 'Shift') throw new Error('key up failed') + } + await expect( + runComputerAction(driver, { + action: 'left_click', + coordinate: [1, 2], + text: 'ctrl+shift', + }), + ).rejects.toThrow('key up failed') + expect(calls).toEqual([ + ['keyDown', 'Control'], + ['keyDown', 'Shift'], + ['click', 1, 2, { button: 'left', count: 1 }], + ['keyUp', 'Shift'], + ['keyUp', 'Control'], + ]) + }) + it('clicks at the mouse position when the click has no coordinate', async () => { const { calls } = await run({ action: 'left_click' }, { cursor: true }) expect(calls).toEqual([['click', 7, 9, { button: 'left', count: 1 }]]) @@ -138,6 +180,55 @@ describe('Anthropic actions', () => { ) }) + it('releases the drag button when a later move throws', async () => { + const { driver, calls } = createDriver() + let moves = 0 + driver.move = async (x, y) => { + calls.push(['move', x, y]) + moves += 1 + if (moves === 2) throw new Error('move failed') + } + await expect( + runComputerAction(driver, { + action: 'left_click_drag', + start_coordinate: [1, 2], + coordinate: [3, 4], + }), + ).rejects.toThrow('move failed') + expect(calls).toEqual([ + ['move', 1, 2], + ['mouseDown', 'left'], + ['move', 3, 4], + ['mouseUp', 'left'], + ]) + }) + + it('rejects a key repeat outside 1 to 100 before it presses', async () => { + const { driver, calls } = createDriver() + await expect( + runComputerAction(driver, { action: 'key', text: 'a', repeat: 101 }), + ).rejects.toThrow('"repeat"') + await expect( + runComputerAction(driver, { action: 'key', text: 'a', repeat: 1.5 }), + ).rejects.toThrow('"repeat"') + expect(calls).toEqual([]) + }) + + it('rejects a hold or a wait longer than 300 seconds', async () => { + const { driver, calls } = createDriver() + await expect( + runComputerAction(driver, { + action: 'hold_key', + text: 'shift', + duration: 301, + }), + ).rejects.toThrow('"duration"') + await expect( + runComputerAction(driver, { action: 'wait', duration: 301 }), + ).rejects.toThrow('"duration"') + expect(calls).toEqual([]) + }) + it('drags from the start to the end coordinate', async () => { const { calls } = await run({ action: 'left_click_drag', diff --git a/packages/ai-computer-use/tests/create-computer-use.test.ts b/packages/ai-computer-use/tests/create-computer-use.test.ts index ee2d9f95cd..9c32065553 100644 --- a/packages/ai-computer-use/tests/create-computer-use.test.ts +++ b/packages/ai-computer-use/tests/create-computer-use.test.ts @@ -96,6 +96,8 @@ function createDriver(options: DriverOptions = {}) { interface FactoryOptions extends DriverOptions { /** `create` throws while this returns true. */ failCreate?: () => boolean + /** `close` throws while this returns true. */ + failClose?: () => boolean } /** A factory that records each created machine and each close. */ @@ -117,6 +119,7 @@ function createFactory(options: FactoryOptions = {}) { const { driver, calls } = createDriver(options) driver.close = async () => { closed.push(threadId) + if (options.failClose?.()) throw new Error('close failed') } machines.set(threadId, calls) return driver @@ -566,6 +569,41 @@ describe('createComputerUse threads', () => { expect(machines.get('b')).toEqual(['type', 'screenshot']) }) + it('runs two threads one at a time on a shared driver', async () => { + const { gate, open } = createGate() + const { driver, calls } = createDriver({ clickGate: gate }) + const computer = setupWith(driver) + const first = computer.execute(CLICK, 'call_1', 'a') + const second = computer.execute(CLICK, 'call_2', 'b') + await new Promise((done) => setTimeout(done, 0)) + + expect(calls).toEqual(['click:start']) + open() + await Promise.all([first, second]) + expect(calls).toEqual([ + 'click:start', + 'click:end', + 'screenshot', + 'click:start', + 'click:end', + 'screenshot', + ]) + }) + + it('runs two factory threads at the same time', async () => { + const { gate, open } = createGate() + const { factory, machines } = createFactory({ clickGate: gate }) + const computer = setupWith(factory) + const first = computer.execute(CLICK, 'call_1', 'a') + const second = computer.execute(CLICK, 'call_2', 'b') + await new Promise((done) => setTimeout(done, 0)) + + expect(machines.get('a')).toEqual(['click:start']) + expect(machines.get('b')).toEqual(['click:start']) + open() + await Promise.all([first, second]) + }) + it('creates one machine for two calls at once on a new thread', async () => { const { factory, created, machines } = createFactory() const computer = setupWith(factory) @@ -790,6 +828,25 @@ describe('createComputerUse idle timeout', () => { expect(closed).toEqual(['a']) }) + it('logs a close error when the idle timeout closes a thread', async () => { + vi.useFakeTimers() + const error = vi.spyOn(console, 'error').mockImplementation(() => {}) + const { factory, closed } = createFactory({ failClose: () => true }) + const computer = setupWith(factory, { idleTimeout: 1000 }) + const first = computer.execute(CLICK, 'call_1', 'a') + await vi.advanceTimersByTimeAsync(0) + await first + + await vi.advanceTimersByTimeAsync(1000) + + expect(closed).toEqual(['a']) + expect(error).toHaveBeenCalledWith( + 'Failed to close the idle computer thread a.', + expect.any(Error), + ) + error.mockRestore() + }) + it('never closes a thread with idleTimeout false', async () => { vi.useFakeTimers() const { factory, created, closed } = createFactory() diff --git a/packages/openai-base/src/adapters/responses-user-tools.ts b/packages/openai-base/src/adapters/responses-user-tools.ts index d42ae57276..318bf35ad4 100644 --- a/packages/openai-base/src/adapters/responses-user-tools.ts +++ b/packages/openai-base/src/adapters/responses-user-tools.ts @@ -114,24 +114,28 @@ function isComputerActionList(value: unknown): value is ComputerActionList { ) } -/** The safety checks of a computer call, with only their known fields. */ +/** + * The safety checks of a computer call, with only their known fields. + * + * - No list (`undefined` or `null` input): returns `[]`. There are no checks. + * - A list that does not parse: returns `null`. Do not turn that into `[]`. + * An unreadable check must stay visible so the tool can refuse it. + */ function safetyChecks( value: unknown, -): Array { - if (!Array.isArray(value)) return [] - return value.flatMap((check) => - isRecord(check) && typeof check.id === 'string' - ? [ - { - id: check.id, - ...(typeof check.code === 'string' ? { code: check.code } : {}), - ...(typeof check.message === 'string' - ? { message: check.message } - : {}), - }, - ] - : [], - ) +): Array | null { + if (value == null) return [] + if (!Array.isArray(value)) return null + const checks: Array = [] + for (const check of value) { + if (!isRecord(check) || typeof check.id !== 'string') return null + checks.push({ + id: check.id, + ...(typeof check.code === 'string' ? { code: check.code } : {}), + ...(typeof check.message === 'string' ? { message: check.message } : {}), + }) + } + return checks } function readItemId(metadata: unknown): string | undefined { @@ -199,6 +203,7 @@ export function readUserExecutedCall( const hasAction = isComputerAction(action) const hasActions = isComputerActionList(actions) if (!callId || (!hasAction && !hasActions)) return null + const checks = safetyChecks(item.pending_safety_checks) return { name: options.computerTool ?? 'computer_use_preview', callId, @@ -206,7 +211,8 @@ export function readUserExecutedCall( input: { ...(hasAction ? { action } : {}), ...(hasActions ? { actions } : {}), - pending_safety_checks: safetyChecks(item.pending_safety_checks), + // Keep a list that did not parse. The computer tool refuses it. + pending_safety_checks: checks ?? item.pending_safety_checks, }, } } @@ -322,7 +328,7 @@ export function userToolRequestItem( status: 'completed', ...(hasAction ? { action } : {}), ...(hasActions ? { actions } : {}), - pending_safety_checks: safetyChecks(args.pending_safety_checks), + pending_safety_checks: safetyChecks(args.pending_safety_checks) ?? [], } } @@ -407,9 +413,9 @@ export function userToolResultItem( if (name === 'computer' || name === 'computer_use_preview') { const checks = computer?.failed ? [] - : safetyChecks( + : (safetyChecks( parseArguments(toolCall.function.arguments).pending_safety_checks, - ) + ) ?? []) return { type: 'computer_call_output', call_id: toolCall.id, diff --git a/packages/openai-base/tests/computer-use-generic.test.ts b/packages/openai-base/tests/computer-use-generic.test.ts index 10c16dd1f3..fbdd63b4f8 100644 --- a/packages/openai-base/tests/computer-use-generic.test.ts +++ b/packages/openai-base/tests/computer-use-generic.test.ts @@ -155,6 +155,32 @@ describe('portable computer tool on the Responses API', () => { acknowledged_safety_checks: [safetyCheck], }) }) + + it('keeps safety checks that do not parse on the tool input', async () => { + const action = { type: 'click', button: 'left', x: 10, y: 20 } + const badChecks = [{ code: 'x' }] + const { second, execute } = await runLoop('gpt-6.1-sol', { + type: 'computer_call', + id: 'cu_bad', + call_id: 'call_bad', + status: 'completed', + action, + pending_safety_checks: badChecks, + }) + + expect(execute.mock.calls[0]?.[0]).toEqual({ + action, + pending_safety_checks: badChecks, + }) + expect(second.input).toContainEqual({ + type: 'computer_call_output', + call_id: 'call_bad', + output: screenshotOutput, + }) + expect(JSON.stringify(second.input)).not.toContain( + 'acknowledged_safety_checks', + ) + }) }) class TestChatCompletionsAdapter extends OpenAIBaseChatCompletionsTextAdapter {} From bca32339abb92e7a110cd62285cba0634fa98b10 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:32:53 +1100 Subject: [PATCH 23/32] docs(examples): add a computer use example and tutorial The model opens tanstack.com in Chromium and finds the computer use docs. Each screenshot shows in the chat, and OpenAI safety checks show an approval card. --- docs/computer-use/computer-use.md | 2 + docs/config.json | 17 +- docs/getting-started/overview.md | 2 + docs/getting-started/quick-start.md | 2 + docs/tutorials/computer-use.md | 229 ++++++++++++++++++ examples/README.md | 20 ++ examples/react/computer-use/.env.example | 3 + examples/react/computer-use/.gitignore | 7 + examples/react/computer-use/README.md | 19 ++ examples/react/computer-use/package.json | 34 +++ .../react/computer-use/src/routeTree.gen.ts | 86 +++++++ examples/react/computer-use/src/router.tsx | 10 + .../react/computer-use/src/routes/__root.tsx | 41 ++++ .../react/computer-use/src/routes/api.chat.ts | 47 ++++ .../react/computer-use/src/routes/index.tsx | 200 +++++++++++++++ examples/react/computer-use/src/styles.css | 10 + examples/react/computer-use/tsconfig.json | 25 ++ examples/react/computer-use/vite.config.ts | 19 ++ pnpm-lock.yaml | 110 +++++++-- 19 files changed, 856 insertions(+), 27 deletions(-) create mode 100644 docs/tutorials/computer-use.md create mode 100644 examples/react/computer-use/.env.example create mode 100644 examples/react/computer-use/.gitignore create mode 100644 examples/react/computer-use/README.md create mode 100644 examples/react/computer-use/package.json create mode 100644 examples/react/computer-use/src/routeTree.gen.ts create mode 100644 examples/react/computer-use/src/router.tsx create mode 100644 examples/react/computer-use/src/routes/__root.tsx create mode 100644 examples/react/computer-use/src/routes/api.chat.ts create mode 100644 examples/react/computer-use/src/routes/index.tsx create mode 100644 examples/react/computer-use/src/styles.css create mode 100644 examples/react/computer-use/tsconfig.json create mode 100644 examples/react/computer-use/vite.config.ts diff --git a/docs/computer-use/computer-use.md b/docs/computer-use/computer-use.md index 0d9289019b..8ae53df9d4 100644 --- a/docs/computer-use/computer-use.md +++ b/docs/computer-use/computer-use.md @@ -16,6 +16,8 @@ keywords: You have a chat app. You want the model to do a task on a web page for the user, for example fill in a form or find a price. Computer use gives the model a screen, a mouse, and a keyboard. Each conversation gets its own browser page. The same code runs on OpenAI and on Anthropic. +To build a full app step by step, with the screenshots in the chat, open the [Computer Use tutorial](../tutorials/computer-use). + ## Install diff --git a/docs/config.json b/docs/config.json index 1e77cc7fc8..72265a35c0 100644 --- a/docs/config.json +++ b/docs/config.json @@ -13,13 +13,13 @@ "label": "Overview", "to": "getting-started/overview", "addedAt": "2026-04-15", - "updatedAt": "2026-09-29" + "updatedAt": "2026-10-21" }, { "label": "Quick Start", "to": "getting-started/quick-start", "addedAt": "2026-04-15", - "updatedAt": "2026-09-17" + "updatedAt": "2026-10-21" }, { "label": "Quick Start: React Native", @@ -85,6 +85,11 @@ "label": "Build an MCP Server", "to": "tutorials/mcp-server", "addedAt": "2026-09-22" + }, + { + "label": "Computer Use", + "to": "tutorials/computer-use", + "addedAt": "2026-10-21" } ] }, @@ -126,6 +131,11 @@ "label": "MCP Server", "to": "framework/react/examples/mcp-server", "addedAt": "2026-09-22" + }, + { + "label": "Computer Use", + "to": "framework/react/examples/computer-use", + "addedAt": "2026-10-21" } ] }, @@ -925,7 +935,8 @@ { "label": "Computer Use", "to": "computer-use/computer-use", - "addedAt": "2026-10-20" + "addedAt": "2026-10-20", + "updatedAt": "2026-10-21" }, { "label": "Drivers", diff --git a/docs/getting-started/overview.md b/docs/getting-started/overview.md index 7368445499..4807766aaf 100644 --- a/docs/getting-started/overview.md +++ b/docs/getting-started/overview.md @@ -26,6 +26,8 @@ Stream a typed table from `chat({ outputSchema })` in [Streaming Structured Tabl Add three named agents (researcher, SEO, and writer) to a blog-writing chat in [Subagents](../tutorials/subagents). +Let the model use a web page from screenshots in [Computer Use](../tutorials/computer-use). + ## Key Features - ✅ **Type-Safe** - Full TypeScript support with Zod schema inference diff --git a/docs/getting-started/quick-start.md b/docs/getting-started/quick-start.md index dc9f284006..9413b696f3 100644 --- a/docs/getting-started/quick-start.md +++ b/docs/getting-started/quick-start.md @@ -38,6 +38,8 @@ For a table that fills as JSON streams, open [Streaming Structured Table](../tut For a blog desk where a researcher and an SEO agent run together and a writer uses their notes, open [Subagents](../tutorials/subagents). +For a model that clicks through a web page from screenshots, open [Computer Use](../tutorials/computer-use). + > [!TIP] > If you do not want a key per provider, [OpenRouter](../adapters/openrouter) gives you 300+ models with one API key. diff --git a/docs/tutorials/computer-use.md b/docs/tutorials/computer-use.md new file mode 100644 index 0000000000..8d88156d80 --- /dev/null +++ b/docs/tutorials/computer-use.md @@ -0,0 +1,229 @@ +--- +title: Computer Use +id: computer-use +order: 8 +description: "Add computer use to a React chat. The model opens tanstack.com in Chromium, reads each screenshot, and clicks until it finds the computer use docs. The chat shows each screenshot." +keywords: + - tanstack ai + - tutorial + - computer use + - createComputerUse + - playwrightComputer + - useChat + - openai + - tanstack start +--- + +You have a chat app. You want the model to do a task on a web page for your user. In this tutorial, the model opens tanstack.com and finds the TanStack AI computer use docs. The model sees only screenshots. It clicks, types, and scrolls on what it sees. + +This tutorial is React + Start, with OpenAI. + +## 1. Create a Start app + +```bash +npx @tanstack/cli@latest create +``` + +Pick React. Then install the packages: + + + +react: @tanstack/ai @tanstack/ai-react @tanstack/ai-openai @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod + + + +Download Chromium one time: + +```bash +npx playwright-core install chromium +``` + +Put your OpenAI key in `.env.local`: + +```bash +OPENAI_API_KEY=sk-... +``` + +## Client and server + +The **server** owns the browser. It runs each action that the model asks for, then sends a screenshot back to the model. + +The **client** sends the task. It shows each screenshot, so your user can see what the model sees. + +The next steps build the server. Then they build the client. + +## 2. Give the model a browser + +Create `src/routes/api.chat.ts`, next to `index.tsx`. Start maps that file name to the `/api/chat` path. + +```typescript ignore +import { createFileRoute } from '@tanstack/react-router' +import { + chat, + chatParamsFromRequest, + maxIterations, + toServerSentEventsResponse, +} from '@tanstack/ai' +import { openaiText } from '@tanstack/ai-openai' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { playwrightComputer } from '@tanstack/ai-computer-playwright' + +// One computer for the whole server. Each chat thread gets its own page. +const computerUse = createComputerUse({ + driver: playwrightComputer({ + startUrl: 'https://tanstack.com', + // Show the Chromium window, so you can watch the model work. + launch: { headless: false }, + }), + // tanstack.com renders in the browser. Give each page time to draw. + screenshotDelay: 1000, +}) + +export async function POST({ request }: { request: Request }) { + const params = await chatParamsFromRequest(request) + const stream = chat({ + adapter: openaiText('gpt-6.1-sol'), + messages: params.messages, + threadId: params.threadId, + runId: params.runId, + ...(params.parentRunId ? { parentRunId: params.parentRunId } : {}), + ...(params.resume ? { resume: params.resume } : {}), + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], + // Each action is one step. A search takes more than the default 5. + agentLoopStrategy: maxIterations(20), + }) + return toServerSentEventsResponse(stream) +} + +export const Route = createFileRoute('/api/chat')({ + server: { + handlers: { + POST, + }, + }, +}) +``` + +- `createComputerUse` is at module scope. Every request uses the same browser. +- `threadId` gives each chat its own page. The page stays open between requests of the same chat. +- `tools` gives the model the `computer` tool and the `navigate` tool. +- `middleware` and `interrupts` pause a risky action until a person approves it. +- `maxIterations(20)` lets the model take up to 20 actions in one reply. + +CAUTION: The browser can open every address that your server can reach. Run this app only on your own machine. + +## 3. Show the screenshots + +Each computer action returns a screenshot. On the client, the screenshot is an `image` part inside a `tool-result` part. Open `src/routes/index.tsx` and turn those parts into `` sources: + +```tsx ignore +import type { UIMessage } from '@tanstack/ai-react' + +type Part = UIMessage['parts'][number] + +/** The screenshots in one tool result, as `` sources. */ +function screenshots(part: Part) { + if (part.type !== 'tool-result' || typeof part.content === 'string') return [] + return part.content.flatMap((content) => { + if (content.type !== 'image') return [] + const { source } = content + if (source.type === 'data') { + return [`data:${source.mimeType};base64,${source.value}`] + } + return source.type === 'url' ? [source.value] : [] + }) +} +``` + +Then render text parts and screenshots for each message: + +```tsx ignore +{message.parts.map((part, index) => { + if (part.type === 'text' && part.content) { + return
{part.content}
+ } + return screenshots(part).map((src) => ( + What the model sees + )) +})} +``` + +## 4. Approve risky actions + +OpenAI can mark an action as risky, for example a form submit. The run then pauses. Register `computerSafetyCheck` with `useChat` and show a card for each pause: + +```tsx ignore +import { + fetchServerSentEvents, + useChat, + type GenericInterrupt, +} from '@tanstack/ai-react' +import { computerSafetyCheck } from '@tanstack/ai-computer-use' + +function SafetyCheckCard({ + interrupt, +}: { + interrupt: GenericInterrupt +}) { + const checks = interrupt.payload?.checks ?? [] + return ( +
+

Approve this computer action?

+
    + {checks.map((check) => ( +
  • {check.message ?? check.code ?? check.id}
    • + ))} +
+ + +
+ ) +} + +function ChatPage() { + const { interrupts } = useChat({ + connection: fetchServerSentEvents('/api/chat'), + interrupts: [computerSafetyCheck], + }) + + return ( + <> + {interrupts.map((interrupt) => { + if (interrupt.kind !== 'generic') return null + if (!('definitionId' in interrupt)) return null + if (interrupt.definitionId !== computerSafetyCheck.id) return null + return + })} + + ) +} +``` + +Put the cards above the input, next to the messages from step 3. + +- If you click Approve, the action runs. +- If you click Decline, no action of that turn runs. + +For more, see [Safety checks](../computer-use/safety-checks). + +## 5. Send the task + +Run the app and send this task: + +```text +You are on the TanStack home page. Find the TanStack AI documentation page about computer use. +Use the screenshots. Click, type, and scroll only on what you can see. Do not type a URL from memory. +When the page is the computer use docs, say the page title and stop. +``` + +A Chromium window opens on tanstack.com. The model clicks through the site, and each screenshot shows in the chat. At the end, the model says the title of the computer use page. + +The full example is on GitHub: [TanStack/ai `examples/react/computer-use`](https://github.com/TanStack/ai/tree/main/examples/react/computer-use). It has the same task in the input box. + +To use Anthropic, change the adapter only. To run the browser in a sandbox, see [Drivers](../computer-use/drivers). diff --git a/examples/README.md b/examples/README.md index 34a6460347..2e0660075f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -12,6 +12,7 @@ Choose an example based on your use case: - **Want typed ticket routing?** → [Evaluate](#evaluate) - **Want a blog desk with three agents?** → [Subagents](#subagents) - **Want your own MCP server?** → [MCP server](#mcp-server) +- **Want the model to use a web page?** → [Computer Use](#computer-use) - **Want a full-stack TypeScript app?** → [TanStack Chat (ts-react-chat)](#tanstack-chat-ts-react-chat) - **Need a live world stream?** → [World generation (ts-react-media)](#world-generation-ts-react-media) - **Need a vanilla JS frontend?** → [Vanilla Chat](#vanilla-chat) @@ -146,6 +147,25 @@ Open http://localhost:3100. Click Call the server. --- +### Computer Use + +A slim TanStack Start app. The model opens tanstack.com in Chromium and finds the computer use docs. One provider (OpenAI). Each screenshot shows in the chat. + +**Path:** `examples/react/computer-use` + +**Getting Started:** + +```bash +pnpm --filter computer-use exec playwright-core install chromium +pnpm --filter computer-use dev +``` + +Set `OPENAI_API_KEY` in `.env.local`. Open http://localhost:3100. Click Send. + +📖 [Full Documentation](react/computer-use/README.md) + +--- + ### TanStack Chat (ts-react-chat) A full-featured chat application built with the TanStack ecosystem. diff --git a/examples/react/computer-use/.env.example b/examples/react/computer-use/.env.example new file mode 100644 index 0000000000..00b69dfa39 --- /dev/null +++ b/examples/react/computer-use/.env.example @@ -0,0 +1,3 @@ +# The model reads the screenshots and picks each click. Get a key at +# https://platform.openai.com/api-keys +OPENAI_API_KEY= diff --git a/examples/react/computer-use/.gitignore b/examples/react/computer-use/.gitignore new file mode 100644 index 0000000000..b6ceee9913 --- /dev/null +++ b/examples/react/computer-use/.gitignore @@ -0,0 +1,7 @@ +node_modules +dist +.env +.env.local +.nitro +.tanstack +.output diff --git a/examples/react/computer-use/README.md b/examples/react/computer-use/README.md new file mode 100644 index 0000000000..1a8a09e0ae --- /dev/null +++ b/examples/react/computer-use/README.md @@ -0,0 +1,19 @@ +# Computer Use + +You want the model to do a task on a web page for you. This app opens tanstack.com in Chromium, and the model finds the TanStack AI computer use docs. It reads each screenshot, then clicks, types, and scrolls. Each screenshot shows in the chat. + +## Run it + +1. From the repo root, run `pnpm install`. +2. Download Chromium one time: `pnpm --filter computer-use exec playwright-core install chromium`. +3. Copy `.env.example` to `.env.local`, then set `OPENAI_API_KEY`. Get a key at https://platform.openai.com/api-keys. +4. Run `pnpm --filter computer-use dev`. +5. Open http://localhost:3100 and click Send. + +A Chromium window opens on your machine. Watch the model use it. + +If OpenAI marks an action as risky, an approval card shows. Click Approve to run the action, or Decline to skip it. + +The server route is `src/routes/api.chat.ts`. The chat UI is `src/routes/index.tsx`. + +CAUTION: The browser can open every address that your server can reach. Run this app only on your own machine. diff --git a/examples/react/computer-use/package.json b/examples/react/computer-use/package.json new file mode 100644 index 0000000000..52bbba0b84 --- /dev/null +++ b/examples/react/computer-use/package.json @@ -0,0 +1,34 @@ +{ + "name": "computer-use", + "private": true, + "type": "module", + "scripts": { + "dev": "vite dev --port 3100", + "build": "vite build", + "test:types": "tsc" + }, + "dependencies": { + "@tailwindcss/vite": "^4.1.18", + "@tanstack/ai": "workspace:*", + "@tanstack/ai-computer-playwright": "workspace:*", + "@tanstack/ai-computer-use": "workspace:*", + "@tanstack/ai-openai": "workspace:*", + "@tanstack/ai-react": "workspace:*", + "@tanstack/react-router": "^1.170.41", + "@tanstack/react-start": "^1.168.60", + "@tanstack/router-plugin": "^1.168.42", + "playwright-core": "^1.57.0", + "react": "^19.2.3", + "react-dom": "^19.2.3", + "tailwindcss": "^4.1.18", + "zod": "^4.2.0" + }, + "devDependencies": { + "@types/node": "^24.10.1", + "@types/react": "^19.2.7", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react": "^5.2.0", + "typescript": "5.9.3", + "vite": "^8.2.1" + } +} diff --git a/examples/react/computer-use/src/routeTree.gen.ts b/examples/react/computer-use/src/routeTree.gen.ts new file mode 100644 index 0000000000..861dc17e2b --- /dev/null +++ b/examples/react/computer-use/src/routeTree.gen.ts @@ -0,0 +1,86 @@ +/* eslint-disable */ + +// @ts-nocheck + +// noinspection JSUnusedGlobalSymbols + +// This file was automatically generated by TanStack Router. +// You should NOT make any changes in this file as it will be overwritten. +// Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. + +import { Route as rootRouteImport } from './routes/__root' +import { Route as IndexRouteImport } from './routes/index' +import { Route as ApiChatRouteImport } from './routes/api.chat' + +const IndexRoute = IndexRouteImport.update({ + id: '/', + path: '/', + getParentRoute: () => rootRouteImport, +} as any) +const ApiChatRoute = ApiChatRouteImport.update({ + id: '/api/chat', + path: '/api/chat', + getParentRoute: () => rootRouteImport, +} as any) + +export interface FileRoutesByFullPath { + '/': typeof IndexRoute + '/api/chat': typeof ApiChatRoute +} +export interface FileRoutesByTo { + '/': typeof IndexRoute + '/api/chat': typeof ApiChatRoute +} +export interface FileRoutesById { + __root__: typeof rootRouteImport + '/': typeof IndexRoute + '/api/chat': typeof ApiChatRoute +} +export interface FileRouteTypes { + fileRoutesByFullPath: FileRoutesByFullPath + fullPaths: '/' | '/api/chat' + fileRoutesByTo: FileRoutesByTo + to: '/' | '/api/chat' + id: '__root__' | '/' | '/api/chat' + fileRoutesById: FileRoutesById +} +export interface RootRouteChildren { + IndexRoute: typeof IndexRoute + ApiChatRoute: typeof ApiChatRoute +} + +declare module '@tanstack/react-router' { + interface FileRoutesByPath { + '/': { + id: '/' + path: '/' + fullPath: '/' + preLoaderRoute: typeof IndexRouteImport + parentRoute: typeof rootRouteImport + } + '/api/chat': { + id: '/api/chat' + path: '/api/chat' + fullPath: '/api/chat' + preLoaderRoute: typeof ApiChatRouteImport + parentRoute: typeof rootRouteImport + } + } +} + +const rootRouteChildren: RootRouteChildren = { + IndexRoute: IndexRoute, + ApiChatRoute: ApiChatRoute, +} +export const routeTree = rootRouteImport + ._addFileChildren(rootRouteChildren) + ._addFileTypes() + +import type { getRouter } from './router.tsx' +import type { createStart } from '@tanstack/react-start' +declare module '@tanstack/react-start' { + interface Register { + ssr: true + router: Awaited> + } +} diff --git a/examples/react/computer-use/src/router.tsx b/examples/react/computer-use/src/router.tsx new file mode 100644 index 0000000000..a595444648 --- /dev/null +++ b/examples/react/computer-use/src/router.tsx @@ -0,0 +1,10 @@ +import { createRouter } from '@tanstack/react-router' +import { routeTree } from './routeTree.gen' + +export const getRouter = () => { + return createRouter({ + routeTree, + scrollRestoration: true, + defaultPreloadStaleTime: 0, + }) +} diff --git a/examples/react/computer-use/src/routes/__root.tsx b/examples/react/computer-use/src/routes/__root.tsx new file mode 100644 index 0000000000..7486c94f8f --- /dev/null +++ b/examples/react/computer-use/src/routes/__root.tsx @@ -0,0 +1,41 @@ +import { HeadContent, Scripts, createRootRoute } from '@tanstack/react-router' +import appCss from '../styles.css?url' + +export const Route = createRootRoute({ + head: () => ({ + meta: [ + { + charSet: 'utf-8', + }, + { + name: 'viewport', + content: 'width=device-width, initial-scale=1', + }, + { + title: 'Computer Use', + }, + ], + links: [ + { + rel: 'stylesheet', + href: appCss, + }, + ], + }), + + shellComponent: RootDocument, +}) + +function RootDocument({ children }: { children: React.ReactNode }) { + return ( + + + + + + {children} + + + + ) +} diff --git a/examples/react/computer-use/src/routes/api.chat.ts b/examples/react/computer-use/src/routes/api.chat.ts new file mode 100644 index 0000000000..07c1aafa64 --- /dev/null +++ b/examples/react/computer-use/src/routes/api.chat.ts @@ -0,0 +1,47 @@ +import { createFileRoute } from '@tanstack/react-router' +import { + chat, + chatParamsFromRequest, + maxIterations, + toServerSentEventsResponse, +} from '@tanstack/ai' +import { openaiText } from '@tanstack/ai-openai' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { playwrightComputer } from '@tanstack/ai-computer-playwright' + +// One computer for the whole server. Each chat thread gets its own page. +const computerUse = createComputerUse({ + driver: playwrightComputer({ + startUrl: 'https://tanstack.com', + // Show the Chromium window, so you can watch the model work. + launch: { headless: false }, + }), + // tanstack.com renders in the browser. Give each page time to draw. + screenshotDelay: 1000, +}) + +export async function POST({ request }: { request: Request }) { + const params = await chatParamsFromRequest(request) + const stream = chat({ + adapter: openaiText('gpt-6.1-sol'), + messages: params.messages, + threadId: params.threadId, + runId: params.runId, + ...(params.parentRunId ? { parentRunId: params.parentRunId } : {}), + ...(params.resume ? { resume: params.resume } : {}), + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], + // Each action is one step. A search takes more than the default 5. + agentLoopStrategy: maxIterations(20), + }) + return toServerSentEventsResponse(stream) +} + +export const Route = createFileRoute('/api/chat')({ + server: { + handlers: { + POST, + }, + }, +}) diff --git a/examples/react/computer-use/src/routes/index.tsx b/examples/react/computer-use/src/routes/index.tsx new file mode 100644 index 0000000000..49c5e52ffe --- /dev/null +++ b/examples/react/computer-use/src/routes/index.tsx @@ -0,0 +1,200 @@ +import { useEffect, useRef, useState } from 'react' +import { createFileRoute } from '@tanstack/react-router' +import { + fetchServerSentEvents, + useChat, + type GenericInterrupt, + type UIMessage, +} from '@tanstack/ai-react' +import { computerSafetyCheck } from '@tanstack/ai-computer-use' + +const TASK = `You are on the TanStack home page. Find the TanStack AI documentation page about computer use. +Use the screenshots. Click, type, and scroll only on what you can see. Do not type a URL from memory. +When the page is the computer use docs, say the page title and stop.` + +type Part = UIMessage['parts'][number] + +/** The screenshots in one tool result, as `` sources. */ +function screenshots(part: Part) { + if (part.type !== 'tool-result' || typeof part.content === 'string') return [] + return part.content.flatMap((content) => { + if (content.type !== 'image') return [] + const { source } = content + if (source.type === 'data') { + return [`data:${source.mimeType};base64,${source.value}`] + } + return source.type === 'url' ? [source.value] : [] + }) +} + +function Messages({ messages }: { messages: ReadonlyArray }) { + const containerRef = useRef(null) + + useEffect(() => { + if (containerRef.current) { + containerRef.current.scrollTop = containerRef.current.scrollHeight + } + }, [messages]) + + if (!messages.length) { + return ( +
+
+

+ Computer Use +

+

+ Click Send. A Chromium window opens on tanstack.com, and the model + looks for the computer use docs. Each screenshot shows here. +

+
+
+ ) + } + + return ( +
+ {messages.map((message) => ( +
+ {message.parts.map((part, index) => { + if (part.type === 'text' && part.content) { + return ( +
+ {part.content} +
+ ) + } + return screenshots(part).map((src) => ( + What the model sees + )) + })} +
+ ))} +
+ ) +} + +/** OpenAI can mark an action as risky. A person approves it before it runs. */ +function SafetyCheckCard({ + interrupt, +}: { + interrupt: GenericInterrupt +}) { + const checks = interrupt.payload?.checks ?? [] + return ( +
+

Approve this computer action?

+
    + {checks.map((check) => ( +
  • {check.message ?? check.code ?? check.id}
    • + ))} +
+ 
+        {JSON.stringify(interrupt.payload?.action, null, 2)}
+
+
+ + +
+
+ ) +} + +function ChatPage() { + const [input, setInput] = useState(TASK) + const { messages, sendMessage, isLoading, error, stop, interrupts } = useChat( + { + connection: fetchServerSentEvents('/api/chat'), + interrupts: [computerSafetyCheck], + }, + ) + + const handleSendMessage = () => { + if (!input.trim()) return + sendMessage(input.trim()) + setInput('') + } + + return ( +
+
+ + + {interrupts.map((interrupt) => { + if (interrupt.kind !== 'generic') return null + if (!('definitionId' in interrupt)) return null + if (interrupt.definitionId !== computerSafetyCheck.id) return null + return + })} + + {error ? ( +
+ {error.message} +
+ ) : null} + +
+
+ {isLoading ? ( +
+ +
+ ) : null} +
+