diff --git a/.changeset/anthropic-computer-toolset.md b/.changeset/anthropic-computer-toolset.md new file mode 100644 index 0000000000..94212459f0 --- /dev/null +++ b/.changeset/anthropic-computer-toolset.md @@ -0,0 +1,10 @@ +--- +'@tanstack/ai-anthropic': minor +--- + +Run computer use on every Claude model that supports it. + +- `computerUseTool()` takes `computer_toolset_20260801`, the only computer use version of Claude Opus 5.5, Sonnet 5.5, and Haiku 5.5. The model calls each action as its own tool. The adapter runs each call as one `computer` call with the action in `input.action`, and sends it back to Claude in the form that Claude sent. So one `execute` works for every version. +- `computerUseTool()` takes `computer_20251124`, with its `computer-use-2025-11-24` beta header. +- Each model now takes only the computer use versions that it supports. For example, `computer_20250124` on `claude-opus-4-6` is a type error, because Anthropic rejects that pair. +- `@anthropic-ai/sdk` is now `^0.135.0`. diff --git a/docs/adapters/anthropic.md b/docs/adapters/anthropic.md index 8b8b2b73a8..f603d0f027 100644 --- a/docs/adapters/anthropic.md +++ b/docs/adapters/anthropic.md @@ -618,30 +618,84 @@ equivalent — see [Provider Skills](../tools/provider-skills.md). ### `computerUseTool` -Allows Claude to observe a virtual desktop (screenshots) and interact with it -via keyboard and mouse events. Provide the screen resolution so Claude can -calculate accurate coordinates. +Claude can look at a screen and use the mouse and the keyboard. Your app owns the screen: it runs each action that Claude asks for and sends back a screenshot. + +1. Pick the version that your model takes: + + | Version | Models | + | --- | --- | + | `computer_toolset_20260801` | Claude Opus 5.5, Sonnet 5.5, Haiku 5.5, Fable 5.1, Fable 5, Opus 5, Sonnet 5, Opus 4.8 | + | `computer_20251124` | Claude Fable 5.1, Fable 5, Opus 5, Sonnet 5, Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 4.6, Opus 4.5 | + | `computer_20250124` | Claude Sonnet 4.5, Haiku 4.5, Opus 4.1 | + +2. Give the tool an `execute` that runs each action on your screen. +3. Return a screenshot for the `screenshot` action, and `"OK"` for the other actions. ```typescript import { chat } from "@tanstack/ai"; import { anthropicText } from "@tanstack/ai-anthropic"; import { computerUseTool } from "@tanstack/ai-anthropic/tools"; +// Your screen, for example a virtual machine. Replace these two functions. +async function runAction(input: { action: string }): Promise { + console.log("run", input); +} +async function takeScreenshot(): Promise { + return ""; // A PNG, as base64. +} + +const computer = { + ...computerUseTool({ + type: "computer_toolset_20260801", + configs: { zoom: { enabled: false } }, + }), + // Every version calls `execute` with the action in `input.action`. + execute: async (input: { action: string }) => { + if (input.action === "screenshot") { + const value = await takeScreenshot(); + return [ + { + type: "image", + source: { type: "data", value, mimeType: "image/png" }, + }, + ]; + } + await runAction(input); + return "OK"; + }, +}; + const stream = chat({ - adapter: anthropicText("claude-sonnet-4-6"), - messages: [{ role: "user", content: "Open the browser and go to example.com" }], - tools: [ - computerUseTool({ - type: "computer_20250124", - name: "computer", - display_width_px: 1024, - display_height_px: 768, - }), - ], + adapter: anthropicText("claude-opus-5-5"), + messages: [{ role: "user", content: "Open the display settings." }], + tools: [computer], + // Claude can ask for several actions in one turn. Run them in order. + toolExecution: "sequential", }); ``` -**Supported models:** Claude Sonnet 3.5 and above, with two exceptions. `claude-opus-5-fast` takes no provider tools. Claude Opus 5.5, Claude Sonnet 5.5, and Claude Haiku 5.5 accept only the `computer_toolset_20260801` toolset, which the adapter does not offer yet. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). +The earlier versions also take a name and the screen size: + +```typescript +import { computerUseTool } from "@tanstack/ai-anthropic/tools"; + +const computer = computerUseTool({ + type: "computer_20251124", + name: "computer", + display_width_px: 1280, + display_height_px: 800, +}); +``` + +Good to know: + +- The adapter sends the beta header that an earlier version needs. The toolset needs no beta header. +- With the toolset, Claude calls each action as its own tool. The adapter gives you each call as one `computer` call, and sends it back in the form that Claude sent. So one `execute` works for every version. +- The toolset turns on `zoom` by default. To let Claude zoom, remove `configs` and return an image for the `zoom` action too. +- Send screenshots that fit the image limits of the model. The toolset does not make a large screenshot smaller. It refuses it. Claude gives coordinates in the pixels of your screenshots. +- If an action fails, answer each later action of the same turn with `Not executed: an earlier computer action in this turn failed.` + +A version that your model does not take is a type error. `claude-opus-5-fast` takes no provider tools. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). ### `bashTool` diff --git a/docs/config.json b/docs/config.json index 5fed936ceb..c235f5f524 100644 --- a/docs/config.json +++ b/docs/config.json @@ -323,7 +323,7 @@ "label": "Provider Tools", "to": "tools/provider-tools", "addedAt": "2026-04-21", - "updatedAt": "2026-09-24" + "updatedAt": "2026-10-19" }, { "label": "Provider Skills", @@ -1215,7 +1215,7 @@ "label": "Anthropic", "to": "adapters/anthropic", "addedAt": "2026-04-15", - "updatedAt": "2026-10-09" + "updatedAt": "2026-10-19" }, { "label": "Google Gemini", diff --git a/docs/tools/provider-tools.md b/docs/tools/provider-tools.md index 25da0275c4..64504bf5ac 100644 --- a/docs/tools/provider-tools.md +++ b/docs/tools/provider-tools.md @@ -149,10 +149,10 @@ Each adapter's `supports.tools` array is the source of truth. The comparison matrix is maintained alongside `model-meta.ts` and reflected here: - **Anthropic**: every registered model supports the full tool superset, - except four. `claude-opus-5-5`, `claude-sonnet-5-5`, and - `claude-haiku-5-5` support every tool but `computerUseTool`, and - `claude-opus-5-fast` supports none. The - retired Claude 3.x models with narrower support were removed. + except `claude-opus-5-fast`, which supports none. For `computerUseTool`, + each model takes only some versions. The Claude 5.5 models take only + `computer_toolset_20260801`. See the version table in + [`computerUseTool`](../adapters/anthropic.md#computerusetool). - **OpenAI**: GPT-5 family and reasoning models (O-series) support the full superset. GPT-6 Astra, Sol, and Luna support every tool except `webSearchPreviewTool` and `localShellTool`. GPT-4-series supports diff --git a/examples/ts-react-chat/package.json b/examples/ts-react-chat/package.json index dace1d669a..13ddbcc526 100644 --- a/examples/ts-react-chat/package.json +++ b/examples/ts-react-chat/package.json @@ -12,7 +12,7 @@ "test:types": "tsc --noEmit" }, "dependencies": { - "@anthropic-ai/sdk": "^0.97.1", + "@anthropic-ai/sdk": "^0.135.0", "@elevenlabs/elevenlabs-js": "^2.68.0", "@mcp-ui/client": "^7.1.1", "@modelcontextprotocol/sdk": "^1.29.0", diff --git a/packages/ai-anthropic/package.json b/packages/ai-anthropic/package.json index 1afd093b13..ff946b18c2 100644 --- a/packages/ai-anthropic/package.json +++ b/packages/ai-anthropic/package.json @@ -67,7 +67,7 @@ "test:types": "tsc" }, "dependencies": { - "@anthropic-ai/sdk": "^0.97.1", + "@anthropic-ai/sdk": "^0.135.0", "@tanstack/ai-utils": "workspace:^" }, "peerDependencies": { @@ -80,7 +80,6 @@ } }, "devDependencies": { - "@anthropic-ai/sdk-v112": "npm:@anthropic-ai/sdk@0.112.4", "@anthropic-ai/vertex-sdk": "^0.19.0", "@tanstack/ai": "workspace:*", "@vitest/coverage-v8": "4.1.10", diff --git a/packages/ai-anthropic/src/adapters/files.ts b/packages/ai-anthropic/src/adapters/files.ts index 37394d5fb2..133859ddad 100644 --- a/packages/ai-anthropic/src/adapters/files.ts +++ b/packages/ai-anthropic/src/adapters/files.ts @@ -8,7 +8,7 @@ import { getAnthropicCredentialFromEnv, } from '../utils/client' import type Anthropic_SDK from '@anthropic-ai/sdk' -import type { FileMetadata } from '@anthropic-ai/sdk/resources/beta/files' +import type { BetaFileMetadata } from '@anthropic-ai/sdk/resources/beta/files' import type { FileHandle, FileUploadInput } from '@tanstack/ai/adapters' import type { AnthropicClientConfig } from '../utils/client' @@ -55,7 +55,7 @@ export class AnthropicFilesAdapter extends BaseFilesAdapter<'anthropic'> { } } -function toFileHandle(file: FileMetadata): FileHandle<'anthropic'> { +function toFileHandle(file: BetaFileMetadata): FileHandle<'anthropic'> { return { id: file.id, provider: 'anthropic', diff --git a/packages/ai-anthropic/src/adapters/text.ts b/packages/ai-anthropic/src/adapters/text.ts index dd3dec4dbc..4c906aedef 100644 --- a/packages/ai-anthropic/src/adapters/text.ts +++ b/packages/ai-anthropic/src/adapters/text.ts @@ -16,7 +16,13 @@ import { import { BaseTextAdapter } from '@tanstack/ai/adapters' import { convertToolsToProviderFormat } from '../tools/tool-converter' import { getAnthropicProviderToolKind } from '../tools/anthropic-provider-tool' -import { convertComputerUseToolToAdapterFormat } from '../tools/computer-use-tool' +import { + COMPUTER_TOOL_NAME, + convertComputerUseToolToAdapterFormat, + isComputerToolset, + joinComputerToolsetCall, + splitComputerToolsetCall, +} from '../tools/computer-use-tool' import { readCodeExecutionConfig, readCodeExecutionSkills, @@ -106,6 +112,7 @@ import type { AnthropicClientConfig, AnthropicMessagesClient, } from '../utils/client' +import type { ComputerUseSingleToolConfig } from '../tools/computer-use-tool' /** * The block type carried by an Anthropic provider-executed (server) tool's @@ -212,13 +219,20 @@ function toAnthropicToolChoice( return { type: 'tool' as const, name: choice.name } } +/** The beta header of each single `computer` tool version. */ +const COMPUTER_USE_BETAS = { + computer_20241022: 'computer-use-2024-10-22', + computer_20250124: 'computer-use-2025-01-24', + computer_20251124: 'computer-use-2025-11-24', +} as const satisfies Record + /** * Computes the `betas` array for a Messages request. Unions: * - `interleaved-thinking-2025-05-14` when interleaved thinking is enabled, * - `code-execution-2025-08-25` when a `code_execution` tool is present, * - `skills-2025-10-02` when that tool carries skills, - * - `computer-use-2024-10-22` or `computer-use-2025-01-24` for the version - * of a `computer` tool, + * - the `computer-use-*` beta of a single `computer` tool version (the + * computer toolset needs none), * - `context-management-2025-06-27` when `context_management` is set, * - `files-api-2025-04-14` when a message references an uploaded file handle, * - `mid-conversation-tool-changes-2026-07-01` in mid-conversation tool mode. @@ -299,12 +313,10 @@ export function computeAnthropicBetas( (tool) => getAnthropicProviderToolKind(tool) === 'computer_use', ) if (computerTool) { - betas.add( - convertComputerUseToolToAdapterFormat(computerTool).type === - 'computer_20241022' - ? 'computer-use-2024-10-22' - : 'computer-use-2025-01-24', - ) + const { type } = convertComputerUseToolToAdapterFormat(computerTool) + if (type !== 'computer_toolset_20260801') { + betas.add(COMPUTER_USE_BETAS[type]) + } } return betas.size > 0 ? Array.from(betas) : undefined @@ -323,18 +335,6 @@ const DEFERRED_TOOL_PLACEHOLDER: BetaTool = { defer_loading: true, } -/** - * A mid-conversation `system` message. SDK 0.97.1 types only `user` and - * `assistant` messages and has no `tool_addition` block, so this type is local. - */ -interface AnthropicMidConversationSystemMessage { - role: 'system' - content: Array< - | TextBlockParam - | { type: 'tool_addition'; tool: { type: 'tool_reference'; name: string } } - > -} - /** * Configuration for Anthropic text adapter */ @@ -607,7 +607,8 @@ export class AnthropicTextAdapter< messagesHaveFileSource(options.messages), // Mid-conversation tool mode puts the deferred placeholder in `tools`. requestParams.tools?.some( - (tool) => tool.name === DEFERRED_TOOL_PLACEHOLDER.name, + (tool) => + 'name' in tool && tool.name === DEFERRED_TOOL_PLACEHOLDER.name, ), ), ) @@ -821,6 +822,7 @@ export class AnthropicTextAdapter< const formattedMessages = this.formatMessages( replay.messages, mid?.systemMessages, + options.tools?.some(isComputerToolset) ?? false, ) const tools = options.tools ? (mid?.tools ?? convertToolsToProviderFormat(options.tools)) @@ -1064,7 +1066,7 @@ export class AnthropicTextAdapter< ) const systemMessages = new Map() for (const [before, change] of split.at) { - const content: AnthropicMidConversationSystemMessage['content'] = [ + const content: Array = [ ...(channels.systemPrompts ? change.systemPrompts.map( (p): TextBlockParam => ({ type: 'text', text: p.content }), @@ -1078,12 +1080,7 @@ export class AnthropicTextAdapter< : []), ] if (content.length === 0) continue - const message: AnthropicMidConversationSystemMessage = { - role: 'system', - content, - } - // oxlint-disable-next-line eslint-js/no-restricted-syntax -- SDK 0.97.1 types no `system` message in `messages`; the models in the channel map accept one (pi 0.87.1). - systemMessages.set(before, message as unknown as BetaMessageParam) + systemMessages.set(before, { role: 'system', content }) } return { ...(channels.systemPrompts && { @@ -1218,8 +1215,12 @@ export class AnthropicTextAdapter< messages: Array, /** Mid-conversation `system` messages by message index (`before`). */ systemMessages?: ReadonlyMap, + /** The tools have the computer toolset, so `computer` calls are its calls. */ + computerToolset = false, ): InternalTextProviderOptions['messages'] { const formattedMessages: InternalTextProviderOptions['messages'] = [] + // The ids of the computer toolset calls. Their results name the toolset. + const toolsetCalls = computerToolset ? new Set() : undefined // A system message waits for the next assistant message (or the end), // so a `tool_result` still follows its `tool_use` directly. const pendingSystemMessages: InternalTextProviderOptions['messages'] = [] @@ -1240,6 +1241,9 @@ export class AnthropicTextAdapter< { type: 'tool_result', tool_use_id: message.toolCallId, + ...(toolsetCalls?.has(message.toolCallId) && { + toolset_name: COMPUTER_TOOL_NAME, + }), content: Array.isArray(toolContent) ? toolContent.map((part) => this.convertContentPartToAnthropic(part), @@ -1264,7 +1268,11 @@ export class AnthropicTextAdapter< if (block.type === 'thinking') { this.appendThinkingBlocks(contentBlocks, [block.thinking]) } else if (block.type === 'tool-call') { - this.appendToolCallBlocks(contentBlocks, block.toolCall) + this.appendToolCallBlocks( + contentBlocks, + block.toolCall, + toolsetCalls, + ) } else { contentBlocks.push({ type: 'text', text: block.text }) } @@ -1292,7 +1300,7 @@ export class AnthropicTextAdapter< } for (const toolCall of message.toolCalls) { - this.appendToolCallBlocks(contentBlocks, toolCall) + this.appendToolCallBlocks(contentBlocks, toolCall, toolsetCalls) } formattedMessages.push({ @@ -1389,10 +1397,15 @@ export class AnthropicTextAdapter< } } - /** A tool call as a `tool_use` block, or a server tool as its two blocks. */ + /** + * A tool call as a `tool_use` block, or a server tool as its two blocks. + * With `toolsetCalls`, a `computer` call goes back as the computer toolset + * call that the model sent, and its id goes in `toolsetCalls`. + */ private appendToolCallBlocks( contentBlocks: Array, toolCall: ToolCall, + toolsetCalls?: Set, ): void { let parsedInput: unknown = {} try { @@ -1421,6 +1434,25 @@ export class AnthropicTextAdapter< return } + const member = + toolsetCalls && + toolCall.function.name === COMPUTER_TOOL_NAME && + typeof parsedInput === 'object' && + parsedInput !== null + ? splitComputerToolsetCall(parsedInput) + : undefined + if (member) { + toolsetCalls?.add(toolCall.id) + contentBlocks.push({ + type: 'tool_use', + id: toolCall.id, + name: member.name, + toolset_name: COMPUTER_TOOL_NAME, + input: member.input, + }) + return + } + const toolUseBlock: ToolUseBlockParam = { type: 'tool_use', id: toolCall.id, @@ -1534,6 +1566,8 @@ export class AnthropicTextAdapter< input: string started: boolean hasInputDelta: boolean + /** The action of a computer toolset call (`left_click`, …). */ + member?: string } >() let currentToolIndex = -1 @@ -1596,9 +1630,16 @@ export class AnthropicTextAdapter< currentBlockType = event.content_block.type if (event.content_block.type === 'tool_use') { currentToolIndex++ + // A computer toolset call names its action. It runs as one + // `computer` call with the action in `input.action`. + const isComputerCall = + event.content_block.toolset_name === COMPUTER_TOOL_NAME toolCallsMap.set(currentToolIndex, { id: event.content_block.id, - name: event.content_block.name, + name: isComputerCall + ? COMPUTER_TOOL_NAME + : event.content_block.name, + ...(isComputerCall && { member: event.content_block.name }), // The initial input snapshot. The first `input_json_delta` // replaces it. input: JSON.stringify(event.content_block.input) ?? '', @@ -1860,7 +1901,14 @@ export class AnthropicTextAdapter< // never appends onto the prior client tool's buffer. if (currentBlockType === 'tool_use') { const existing = toolCallsMap.get(currentToolIndex) - if (existing) { + // A computer toolset call sends its joined arguments at the end. + if (existing?.member !== undefined) { + if (!existing.hasInputDelta) { + existing.input = '' + existing.hasInputDelta = true + } + existing.input += event.delta.partial_json + } else if (existing) { // Emit TOOL_CALL_START on first args delta if (!existing.started) { existing.started = true @@ -1971,6 +2019,26 @@ export class AnthropicTextAdapter< // Keep `parsedInput` undefined for the tool error. } + if ( + existing.member !== undefined && + typeof parsedInput === 'object' && + parsedInput !== null + ) { + parsedInput = joinComputerToolsetCall( + existing.member, + parsedInput, + ) + existing.input = JSON.stringify(parsedInput) + yield { + type: EventType.TOOL_CALL_ARGS, + toolCallId: existing.id, + model, + timestamp: Date.now(), + delta: existing.input, + args: existing.input, + } + } + yield { type: EventType.TOOL_CALL_END, toolCallId: existing.id, diff --git a/packages/ai-anthropic/src/model-meta.ts b/packages/ai-anthropic/src/model-meta.ts index 4534d13c97..80dc5e69ce 100644 --- a/packages/ai-anthropic/src/model-meta.ts +++ b/packages/ai-anthropic/src/model-meta.ts @@ -92,11 +92,19 @@ interface ModelMeta< * every request, with or without thinking. Absent: it accepts one. */ forced_tool_choice?: false + /** + * Computer use has three kinds, one for each version that a model + * accepts: `computer_use` (`computer_20241022` and `computer_20250124`), + * `computer_use_20251124`, and `computer_toolset` + * (`computer_toolset_20260801`). + */ tools?: Array< | 'web_search' | 'web_fetch' | 'code_execution' | 'computer_use' + | 'computer_use_20251124' + | 'computer_toolset' | 'bash' | 'text_editor' | 'memory' @@ -155,7 +163,7 @@ const CLAUDE_OPUS_4_6 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', 'bash', 'text_editor', 'memory', @@ -195,7 +203,7 @@ const CLAUDE_OPUS_4_5 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', 'bash', 'text_editor', 'memory', @@ -238,7 +246,7 @@ const CLAUDE_SONNET_4_6 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', 'bash', 'text_editor', 'memory', @@ -393,7 +401,7 @@ const CLAUDE_OPUS_4_7 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', 'bash', 'text_editor', 'memory', @@ -437,7 +445,8 @@ const CLAUDE_OPUS_4_8 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -484,7 +493,8 @@ const CLAUDE_FABLE_5 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -529,7 +539,8 @@ const CLAUDE_SONNET_5 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -577,7 +588,8 @@ const CLAUDE_OPUS_5 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -645,7 +657,8 @@ const CLAUDE_FABLE_5_1 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -681,12 +694,12 @@ const CLAUDE_OPUS_5_5 = { input: ['text', 'image', 'document'], forced_tool_choice: false, mid_conversation_channels: { tools: true, systemPrompts: true }, - // No 'computer_use': this model accepts only `computer_toolset_20260801`, - // and `computerUseTool()` sends the older versions, which return a 400. + // Computer use only through `computer_toolset_20260801`. tools: [ 'web_search', 'web_fetch', 'code_execution', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -728,12 +741,12 @@ const CLAUDE_SONNET_5_5 = { forced_tool_choice: false, extended_thinking: false, adaptive_thinking: true, - // No 'computer_use': this model accepts only `computer_toolset_20260801`, - // and `computerUseTool()` sends the older versions, which return a 400. + // Computer use only through `computer_toolset_20260801`. tools: [ 'web_search', 'web_fetch', 'code_execution', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -771,12 +784,12 @@ const CLAUDE_HAIKU_5_5 = { input: ['text', 'image', 'document'], extended_thinking: false, adaptive_thinking: true, - // No 'computer_use': this model accepts only `computer_toolset_20260801`, - // and `computerUseTool()` sends the older versions, which return a 400. + // Computer use only through `computer_toolset_20260801`. tools: [ 'web_search', 'web_fetch', 'code_execution', + 'computer_toolset', 'bash', 'text_editor', 'memory', diff --git a/packages/ai-anthropic/src/prompt-cache.ts b/packages/ai-anthropic/src/prompt-cache.ts index 60f9fde70c..6874d4cb3a 100644 --- a/packages/ai-anthropic/src/prompt-cache.ts +++ b/packages/ai-anthropic/src/prompt-cache.ts @@ -153,7 +153,10 @@ export function applyAnthropicPromptCache( // last tool, as before. const placeholder = request.tools?.findIndex( - (tool) => tool.name === ANTHROPIC_DEFERRED_TOOL_PLACEHOLDER_NAME, + // The computer toolset entry has no name. + (tool) => + 'name' in tool && + tool.name === ANTHROPIC_DEFERRED_TOOL_PLACEHOLDER_NAME, ) ?? -1 const tools = request.tools?.map((tool, index, all) => index === (placeholder > 0 ? placeholder - 1 : all.length - 1) diff --git a/packages/ai-anthropic/src/tools/computer-use-tool.ts b/packages/ai-anthropic/src/tools/computer-use-tool.ts index d37f5ce07b..8959ca6546 100644 --- a/packages/ai-anthropic/src/tools/computer-use-tool.ts +++ b/packages/ai-anthropic/src/tools/computer-use-tool.ts @@ -1,22 +1,50 @@ -import { - brandAnthropicProviderTool, - getAnthropicProviderToolMetadata, -} from './anthropic-provider-tool' +import { brandProviderTool } from '@tanstack/ai' +import { getAnthropicProviderToolMetadata } from './anthropic-provider-tool' import type { + BetaComputerToolset20260801, BetaToolComputerUse20241022, BetaToolComputerUse20250124, + BetaToolComputerUse20251124, } from '@anthropic-ai/sdk/resources/beta' import type { ProviderTool, Tool } from '@tanstack/ai' -export type ComputerUseToolConfig = +/** The single `computer` tool that sends its action in `input.action`. */ +export type ComputerUseSingleToolConfig = | BetaToolComputerUse20241022 | BetaToolComputerUse20250124 + | BetaToolComputerUse20251124 + +/** + * The computer toolset: one entry with no name. The model calls each action + * as its own tool (`left_click`, `screenshot`, …). + */ +export type ComputerToolsetConfig = BetaComputerToolset20260801 + +export type ComputerUseToolConfig = + | ComputerUseSingleToolConfig + | ComputerToolsetConfig /** @deprecated Renamed to `ComputerUseToolConfig`. Will be removed in a future release. */ export type ComputerUseTool = ComputerUseToolConfig +/** `computer_20241022` or `computer_20250124`. */ export type AnthropicComputerUseTool = ProviderTool<'anthropic', 'computer_use'> +/** `computer_20251124`. */ +export type AnthropicComputerUse20251124Tool = ProviderTool< + 'anthropic', + 'computer_use_20251124' +> + +/** `computer_toolset_20260801`. */ +export type AnthropicComputerToolset = ProviderTool< + 'anthropic', + 'computer_toolset' +> + +/** The name of the tool that every computer use version runs as. */ +export const COMPUTER_TOOL_NAME = 'computer' + export function convertComputerUseToolToAdapterFormat( tool: Tool, ): ComputerUseToolConfig { @@ -26,15 +54,58 @@ export function convertComputerUseToolToAdapterFormat( return metadata } +/** True for the `computer_toolset_20260801` entry. */ +export function isComputerToolset(tool: Tool) { + return tool.metadata?.['type'] === 'computer_toolset_20260801' +} + +/** + * One toolset call (`left_click` with `{ coordinate }`) as one `computer` + * call: `{ action: 'left_click', coordinate }`. So one `execute` works for + * every version. + */ +export function joinComputerToolsetCall(member: string, input: object) { + return { action: member, ...input } +} + +/** A `computer` call as the toolset call that the model sent. */ +export function splitComputerToolsetCall(input: object) { + if (!('action' in input) || typeof input.action !== 'string') { + return undefined + } + const { action, ...memberInput } = input + return { name: action, input: memberInput } +} + +/** + * Computer use. Every version runs as one tool named `computer`, with the + * action in `input.action`: + * + * - `computer_toolset_20260801`: Claude Opus 5.5, Sonnet 5.5, Haiku 5.5, and + * the other models that support the toolset. No beta header. + * - `computer_20251124`: the earlier tool of Claude Opus 4.5 to Claude Opus 5, + * Sonnet 4.6, and Sonnet 5. + * - `computer_20250124`: Claude Sonnet 4.5, Haiku 4.5, and Opus 4.1. + */ +export function computerUseTool( + config: ComputerToolsetConfig, +): AnthropicComputerToolset +export function computerUseTool( + config: BetaToolComputerUse20251124, +): AnthropicComputerUse20251124Tool export function computerUseTool( - config: ComputerUseToolConfig, -): AnthropicComputerUseTool { - return brandAnthropicProviderTool( - { - name: 'computer', - description: '', - metadata: config, - }, - 'computer_use', - ) + config: BetaToolComputerUse20241022 | BetaToolComputerUse20250124, +): AnthropicComputerUseTool +export function computerUseTool(config: ComputerUseToolConfig) { + return brandProviderTool< + | AnthropicComputerToolset + | AnthropicComputerUse20251124Tool + | AnthropicComputerUseTool + >({ + name: COMPUTER_TOOL_NAME, + description: '', + // Every version has the same runtime kind. The type kind picks the + // models that accept it. + metadata: { ...config, __kind: 'anthropic.computer_use' }, + }) } diff --git a/packages/ai-anthropic/src/tools/index.ts b/packages/ai-anthropic/src/tools/index.ts index 9ad77c0431..bc6d3d9354 100644 --- a/packages/ai-anthropic/src/tools/index.ts +++ b/packages/ai-anthropic/src/tools/index.ts @@ -26,7 +26,11 @@ export { } from './code-execution-tool' export { computerUseTool, + type AnthropicComputerToolset, + type AnthropicComputerUse20251124Tool, type AnthropicComputerUseTool, + type ComputerToolsetConfig, + type ComputerUseSingleToolConfig, type ComputerUseToolConfig, type ComputerUseTool, } from './computer-use-tool' diff --git a/packages/ai-anthropic/tests/auth-parity.test.ts b/packages/ai-anthropic/tests/auth-parity.test.ts index da1f951822..25af71ff17 100644 --- a/packages/ai-anthropic/tests/auth-parity.test.ts +++ b/packages/ai-anthropic/tests/auth-parity.test.ts @@ -473,7 +473,7 @@ describe('files factories', () => { authorization: null, xApiKey: 'files-key', xApp: null, - anthropicBeta: 'files-api-2025-04-14,files-api-2025-04-14', + anthropicBeta: 'files-api-2025-04-14', }) expect(file).toEqual(FILE) }) diff --git a/packages/ai-anthropic/tests/client-injection-type-safety.test.ts b/packages/ai-anthropic/tests/client-injection-type-safety.test.ts index 57392a90c4..18d3f73fcf 100644 --- a/packages/ai-anthropic/tests/client-injection-type-safety.test.ts +++ b/packages/ai-anthropic/tests/client-injection-type-safety.test.ts @@ -1,18 +1,7 @@ import { expectTypeOf, it } from 'vitest' -import type AnthropicSdkV112 from '@anthropic-ai/sdk-v112' import type { AnthropicVertex } from '@anthropic-ai/vertex-sdk' import type { AnthropicMessagesClient } from '../src' -type V112MessagesClient = { - readonly beta: { - readonly messages: Pick - } -} - it('accepts the official Vertex client', () => { expectTypeOf().toExtend() }) - -it('accepts clients backed by a newer Anthropic SDK version', () => { - expectTypeOf().toExtend() -}) diff --git a/packages/ai-anthropic/tests/computer-toolset.test.ts b/packages/ai-anthropic/tests/computer-toolset.test.ts new file mode 100644 index 0000000000..1f952f1507 --- /dev/null +++ b/packages/ai-anthropic/tests/computer-toolset.test.ts @@ -0,0 +1,225 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { chat } from '@tanstack/ai' +import { AnthropicTextAdapter } from '../src/adapters/text' +import { computerUseTool } from '../src/tools' + +const mocks = vi.hoisted(() => { + const betaMessagesCreate = vi.fn() + const client = { beta: { messages: { create: betaMessagesCreate } } } + return { betaMessagesCreate, client } +}) + +vi.mock('@anthropic-ai/sdk', () => { + const { client } = mocks + class MockAnthropic { + beta = client.beta + constructor(_: { apiKey: string }) {} + } + return { default: MockAnthropic } +}) + +function stream(events: Array) { + mocks.betaMessagesCreate.mockResolvedValueOnce( + (async function* () { + yield* events + })(), + ) +} + +/** A computer toolset call, as the model streams it. */ +function memberCall(index: number, id: string, name: string, json?: string) { + return [ + { + type: 'content_block_start', + index, + content_block: { + type: 'tool_use', + id, + name, + toolset_name: 'computer', + input: {}, + }, + }, + ...(json === undefined + ? [] + : [ + { + type: 'content_block_delta', + index, + delta: { type: 'input_json_delta', partial_json: json }, + }, + ]), + { type: 'content_block_stop', index }, + ] +} + +const end = (stopReason: string) => [ + { + type: 'message_delta', + delta: { stop_reason: stopReason }, + usage: { output_tokens: 1 }, + }, + { type: 'message_stop' }, +] + +const screenshot = [ + { + type: 'image' as const, + source: { type: 'data' as const, value: 'AAAA', mimeType: 'image/png' }, + }, +] + +beforeEach(() => { + mocks.betaMessagesCreate.mockReset() +}) + +describe('anthropic computer toolset', () => { + it('sends the toolset entry with no name and no beta', async () => { + stream(end('end_turn')) + const adapter = new AnthropicTextAdapter({ apiKey: 'k' }, 'claude-opus-5-5') + + for await (const _ of chat({ + adapter, + messages: [{ role: 'user', content: 'Open the settings.' }], + tools: [ + computerUseTool({ + type: 'computer_toolset_20260801', + configs: { zoom: { enabled: false } }, + }), + ], + })) { + // consume + } + + const [payload] = mocks.betaMessagesCreate.mock.calls[0]! + // The prompt cache marker can also sit on the entry. + expect(payload.tools).toMatchObject([ + { + type: 'computer_toolset_20260801', + configs: { zoom: { enabled: false } }, + }, + ]) + expect(payload.tools[0]).not.toHaveProperty('name') + expect(payload.tools[0]).not.toHaveProperty('__kind') + expect(payload.betas).toBeUndefined() + }) + + it('runs each toolset call as one computer call and sends it back', async () => { + stream([ + ...memberCall(0, 'toolu_click', 'left_click', '{"coordinate":[500,300]}'), + ...memberCall(1, 'toolu_shot', 'screenshot'), + ...end('tool_use'), + ]) + stream(end('end_turn')) + const calls: Array = [] + const tool = { + ...computerUseTool({ type: 'computer_toolset_20260801' }), + execute: (input: unknown) => { + calls.push(input) + return calls.length === 1 ? 'OK' : screenshot + }, + } + const adapter = new AnthropicTextAdapter({ apiKey: 'k' }, 'claude-opus-5-5') + + for await (const _ of chat({ + adapter, + messages: [{ role: 'user', content: 'Click the button.' }], + tools: [tool], + toolExecution: 'sequential', + })) { + // consume + } + + expect(calls).toEqual([ + { action: 'left_click', coordinate: [500, 300] }, + { action: 'screenshot' }, + ]) + const [payload] = mocks.betaMessagesCreate.mock.calls[1]! + const [, assistant, results] = payload.messages + expect(assistant.content).toEqual([ + { + type: 'tool_use', + id: 'toolu_click', + name: 'left_click', + toolset_name: 'computer', + input: { coordinate: [500, 300] }, + }, + { + type: 'tool_use', + id: 'toolu_shot', + name: 'screenshot', + toolset_name: 'computer', + input: {}, + }, + ]) + expect(results.content).toMatchObject([ + { + type: 'tool_result', + tool_use_id: 'toolu_click', + toolset_name: 'computer', + content: 'OK', + }, + { + type: 'tool_result', + tool_use_id: 'toolu_shot', + toolset_name: 'computer', + content: [ + { + type: 'image', + source: { type: 'base64', data: 'AAAA', media_type: 'image/png' }, + }, + ], + }, + ]) + }) + + it('sends a computer call back as the single tool when the toolset is not in tools', async () => { + stream(end('end_turn')) + const adapter = new AnthropicTextAdapter({ apiKey: 'k' }, 'claude-opus-4-6') + + for await (const _ of chat({ + adapter, + messages: [ + { role: 'user', content: 'Click the button.' }, + { + role: 'assistant', + content: '', + toolCalls: [ + { + id: 'toolu_click', + type: 'function', + function: { + name: 'computer', + arguments: '{"action":"left_click","coordinate":[1,2]}', + }, + }, + ], + }, + { role: 'tool', toolCallId: 'toolu_click', content: 'OK' }, + ], + tools: [ + computerUseTool({ + type: 'computer_20251124', + name: 'computer', + display_width_px: 1024, + display_height_px: 768, + }), + ], + })) { + // consume + } + + const [payload] = mocks.betaMessagesCreate.mock.calls[0]! + expect(payload.betas).toContain('computer-use-2025-11-24') + const [, assistant, results] = payload.messages + expect(assistant.content).toEqual([ + { + type: 'tool_use', + id: 'toolu_click', + name: 'computer', + input: { action: 'left_click', coordinate: [1, 2] }, + }, + ]) + expect(results.content[0]).not.toHaveProperty('toolset_name') + }) +}) diff --git a/packages/ai-anthropic/tests/tools-per-model-type-safety.test.ts b/packages/ai-anthropic/tests/tools-per-model-type-safety.test.ts index d46b14bf07..abfca22c0f 100644 --- a/packages/ai-anthropic/tests/tools-per-model-type-safety.test.ts +++ b/packages/ai-anthropic/tests/tools-per-model-type-safety.test.ts @@ -58,7 +58,7 @@ describe('Anthropic per-model tool gating', () => { type: 'code_execution_20250825', }), computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -83,7 +83,7 @@ describe('Anthropic per-model tool gating', () => { type: 'code_execution_20250825', }), computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -108,7 +108,7 @@ describe('Anthropic per-model tool gating', () => { type: 'code_execution_20250825', }), computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -133,7 +133,7 @@ describe('Anthropic per-model tool gating', () => { type: 'code_execution_20250825', }), computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -158,7 +158,7 @@ describe('Anthropic per-model tool gating', () => { type: 'code_execution_20250825', }), computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -172,7 +172,7 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-opus-5-5 accepts every provider tool except computer use', () => { + it('claude-opus-5-5 accepts the full tool superset with the computer toolset', () => { const adapter = anthropicText('claude-opus-5-5') typedTools(adapter, [ userTool, @@ -182,6 +182,7 @@ describe('Anthropic per-model tool gating', () => { name: 'code_execution', type: 'code_execution_20250825', }), + computerUseTool({ type: 'computer_toolset_20260801' }), bashTool({ name: 'bash', type: 'bash_20250124' }), textEditorTool({ type: 'text_editor_20250124', @@ -191,12 +192,12 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-opus-5-5 rejects computerUseTool (only computer_toolset_20260801 works)', () => { + it('claude-opus-5-5 rejects computer_20251124 (only the toolset works)', () => { const adapter = anthropicText('claude-opus-5-5') typedTools(adapter, [ - // @ts-expect-error - computer_20250124 returns a 400 on claude-opus-5-5 + // @ts-expect-error - computer_20251124 returns a 400 on claude-opus-5-5 computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -204,7 +205,7 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-sonnet-5-5 accepts every provider tool except computer use', () => { + it('claude-sonnet-5-5 accepts the full tool superset with the computer toolset', () => { const adapter = anthropicText('claude-sonnet-5-5') typedTools(adapter, [ userTool, @@ -214,6 +215,7 @@ describe('Anthropic per-model tool gating', () => { name: 'code_execution', type: 'code_execution_20250825', }), + computerUseTool({ type: 'computer_toolset_20260801' }), bashTool({ name: 'bash', type: 'bash_20250124' }), textEditorTool({ type: 'text_editor_20250124', @@ -223,12 +225,12 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-sonnet-5-5 rejects computerUseTool (only computer_toolset_20260801 works)', () => { + it('claude-sonnet-5-5 rejects computer_20251124 (only the toolset works)', () => { const adapter = anthropicText('claude-sonnet-5-5') typedTools(adapter, [ - // @ts-expect-error - computer_20250124 returns a 400 on claude-sonnet-5-5 + // @ts-expect-error - computer_20251124 returns a 400 on claude-sonnet-5-5 computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -236,7 +238,7 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-haiku-5-5 accepts every provider tool except computer use', () => { + it('claude-haiku-5-5 accepts the full tool superset with the computer toolset', () => { const adapter = anthropicText('claude-haiku-5-5') typedTools(adapter, [ userTool, @@ -246,6 +248,7 @@ describe('Anthropic per-model tool gating', () => { name: 'code_execution', type: 'code_execution_20250825', }), + computerUseTool({ type: 'computer_toolset_20260801' }), bashTool({ name: 'bash', type: 'bash_20250124' }), textEditorTool({ type: 'text_editor_20250124', @@ -255,16 +258,57 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-haiku-5-5 rejects computerUseTool (only computer_toolset_20260801 works)', () => { + it('claude-haiku-5-5 rejects computer_20251124 (only the toolset works)', () => { const adapter = anthropicText('claude-haiku-5-5') typedTools(adapter, [ - // @ts-expect-error - computer_20250124 returns a 400 on claude-haiku-5-5 + // @ts-expect-error - computer_20251124 returns a 400 on claude-haiku-5-5 + computerUseTool({ + type: 'computer_20251124', + name: 'computer', + display_width_px: 1024, + display_height_px: 768, + }), + ]) + }) + + it('claude-opus-4-6 rejects computer_20250124 and the computer toolset', () => { + const adapter = anthropicText('claude-opus-4-6') + typedTools(adapter, [ + // @ts-expect-error - claude-opus-4-6 takes only computer_20251124 computerUseTool({ type: 'computer_20250124', name: 'computer', display_width_px: 1024, display_height_px: 768, }), + // @ts-expect-error - claude-opus-4-6 takes only computer_20251124 + computerUseTool({ type: 'computer_toolset_20260801' }), + ]) + }) + + it('claude-haiku-4-5 accepts computer_20250124 and rejects computer_20251124', () => { + const adapter = anthropicText('claude-haiku-4-5') + typedTools(adapter, [ + computerUseTool({ + type: 'computer_20250124', + name: 'computer', + display_width_px: 1024, + display_height_px: 768, + }), + // @ts-expect-error - claude-haiku-4-5 takes only computer_20250124 + computerUseTool({ + type: 'computer_20251124', + name: 'computer', + display_width_px: 1024, + display_height_px: 768, + }), + ]) + }) + + it('claude-opus-5 accepts the computer toolset', () => { + const adapter = anthropicText('claude-opus-5') + typedTools(adapter, [ + computerUseTool({ type: 'computer_toolset_20260801' }), ]) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bb0a8207db..1e072c4214 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1065,8 +1065,8 @@ importers: examples/ts-react-chat: dependencies: '@anthropic-ai/sdk': - specifier: ^0.97.1 - version: 0.97.1(zod@4.2.1) + specifier: ^0.135.0 + version: 0.135.0(zod@4.2.1) '@elevenlabs/elevenlabs-js': specifier: ^2.68.0 version: 2.68.0 @@ -2120,15 +2120,12 @@ importers: packages/ai-anthropic: dependencies: '@anthropic-ai/sdk': - specifier: ^0.97.1 - version: 0.97.1(zod@4.2.1) + specifier: ^0.135.0 + version: 0.135.0(zod@4.2.1) '@tanstack/ai-utils': specifier: workspace:^ version: link:../ai-utils devDependencies: - '@anthropic-ai/sdk-v112': - specifier: npm:@anthropic-ai/sdk@0.112.4 - version: '@anthropic-ai/sdk@0.112.4(zod@4.2.1)' '@anthropic-ai/vertex-sdk': specifier: ^0.19.0 version: 0.19.0(zod@4.2.1) @@ -3640,8 +3637,8 @@ importers: testing/e2e: dependencies: '@anthropic-ai/sdk': - specifier: ^0.97.1 - version: 0.97.1(zod@4.3.6) + specifier: ^0.135.0 + version: 0.135.0(zod@4.3.6) '@copilotkit/aimock': specifier: ^1.34.0 version: 1.34.0(vitest@4.1.11(@opentelemetry/api@1.9.1)(@types/node@24.10.3)(@vitest/coverage-v8@4.1.10(vitest@4.1.10))(happy-dom@20.11.2)(jsdom@27.4.0(@noble/hashes@2.3.0))(vite@8.2.1(@types/node@24.10.3)(esbuild@0.28.1)(jiti@2.7.0)(less@4.6.6)(sass@1.101.0)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0))) @@ -4155,17 +4152,8 @@ packages: '@angular/animations': optional: true - '@anthropic-ai/sdk@0.112.4': - resolution: {integrity: sha512-7eXJJnrmBI5GMC6drrCiSkycVsT7crRZX3qv5HusLSm+qiILjmtqP7gf+UiT7ASu/7Gdj+Zfl4f2haV8wATKUg==} - hasBin: true - peerDependencies: - zod: ^3.25.0 || ^4.0.0 - peerDependenciesMeta: - zod: - optional: true - - '@anthropic-ai/sdk@0.97.1': - resolution: {integrity: sha512-wOf7AUeJPitcVpvKO4UMu63mWH5SaVipkGd7OOQJt/G6VYGlV8D2Gp9dLxOrttDJh/9gqPqdaBwDGcBevumeAg==} + '@anthropic-ai/sdk@0.135.0': + resolution: {integrity: sha512-WWrdL57JhU5KoynpTjt9GXK6k7ud9nPw2uw/Trn+oBXh6s0KpjhAccWZm2Qxw08ARYG1UDQj5iC+JO42j1rt/g==} hasBin: true peerDependencies: zod: ^3.25.0 || ^4.0.0 @@ -19221,22 +19209,17 @@ snapshots: '@angular/core': 21.2.20(@angular/compiler@21.2.20)(rxjs@7.8.2)(zone.js@0.15.1) tslib: 2.8.1 - '@anthropic-ai/sdk@0.112.4(zod@4.2.1)': - dependencies: - json-schema-to-ts: 3.1.1 - standardwebhooks: 1.0.0 - optionalDependencies: - zod: 4.2.1 - - '@anthropic-ai/sdk@0.97.1(zod@4.2.1)': + '@anthropic-ai/sdk@0.135.0(zod@4.2.1)': dependencies: + '@opentelemetry/api': 1.9.1 json-schema-to-ts: 3.1.1 standardwebhooks: 1.0.0 optionalDependencies: zod: 4.2.1 - '@anthropic-ai/sdk@0.97.1(zod@4.3.6)': + '@anthropic-ai/sdk@0.135.0(zod@4.3.6)': dependencies: + '@opentelemetry/api': 1.9.1 json-schema-to-ts: 3.1.1 standardwebhooks: 1.0.0 optionalDependencies: @@ -19244,7 +19227,7 @@ snapshots: '@anthropic-ai/vertex-sdk@0.19.0(zod@4.2.1)': dependencies: - '@anthropic-ai/sdk': 0.112.4(zod@4.2.1) + '@anthropic-ai/sdk': 0.135.0(zod@4.2.1) google-auth-library: 10.5.0 transitivePeerDependencies: - supports-color diff --git a/scripts/model-sync/provider-supports.test.ts b/scripts/model-sync/provider-supports.test.ts index 355c75eae4..c311313371 100644 --- a/scripts/model-sync/provider-supports.test.ts +++ b/scripts/model-sync/provider-supports.test.ts @@ -38,7 +38,8 @@ describe('buildProviderSupportsBody', () => { expect(body).toContain("'web_search'") expect(body).toContain("'web_fetch'") expect(body).toContain("'code_execution'") - expect(body).toContain("'computer_use'") + expect(body).toContain("'computer_toolset'") + expect(body).not.toContain("'computer_use'") expect(body).toContain('extended_thinking: true') expect(body).toContain('priority_tier: true') expect(body).not.toContain('adaptive_thinking') diff --git a/scripts/model-sync/provider-supports.ts b/scripts/model-sync/provider-supports.ts index d27c86b0af..127b9c2c8d 100644 --- a/scripts/model-sync/provider-supports.ts +++ b/scripts/model-sync/provider-supports.ts @@ -17,11 +17,12 @@ export interface ProviderSupportsInput { reasoningMandatory?: boolean } +// New Claude models take computer use only through the computer toolset. const ANTHROPIC_SERVER_TOOLS = [ 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_toolset', 'bash', 'text_editor', 'memory', diff --git a/testing/e2e/package.json b/testing/e2e/package.json index 7001fbcc32..78f353b467 100644 --- a/testing/e2e/package.json +++ b/testing/e2e/package.json @@ -12,7 +12,7 @@ "postinstall": "playwright install chromium" }, "dependencies": { - "@anthropic-ai/sdk": "^0.97.1", + "@anthropic-ai/sdk": "^0.135.0", "@copilotkit/aimock": "^1.34.0", "@modelcontextprotocol/sdk": "^1.29.0", "@openrouter/sdk": "0.13.20", diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index 70bc3a477a..3d2109a65d 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -54,6 +54,7 @@ import { Route as ApiActivityTestRouteImport } from './routes/api.activity-test' import { Route as ApiAdapterInputModalitiesRouteImport } from './routes/api.adapter-input-modalities' import { Route as ApiAnthropicAuthWireRouteImport } from './routes/api.anthropic-auth-wire' import { Route as ApiAnthropicBugTestRouteImport } from './routes/api.anthropic-bug-test' +import { Route as ApiAnthropicComputerToolsetWireRouteImport } from './routes/api.anthropic-computer-toolset-wire' import { Route as ApiAnthropicComputerUseWireRouteImport } from './routes/api.anthropic-computer-use-wire' import { Route as ApiAnthropicHaiku55WireRouteImport } from './routes/api.anthropic-haiku-5-5-wire' import { Route as ApiAnthropicMaxTokensUsageRouteImport } from './routes/api.anthropic-max-tokens-usage' @@ -414,6 +415,12 @@ const ApiAnthropicBugTestRoute = ApiAnthropicBugTestRouteImport.update({ path: '/api/anthropic-bug-test', getParentRoute: () => rootRouteImport, } as any) +const ApiAnthropicComputerToolsetWireRoute = + ApiAnthropicComputerToolsetWireRouteImport.update({ + id: '/api/anthropic-computer-toolset-wire', + path: '/api/anthropic-computer-toolset-wire', + getParentRoute: () => rootRouteImport, + } as any) const ApiAnthropicComputerUseWireRoute = ApiAnthropicComputerUseWireRouteImport.update({ id: '/api/anthropic-computer-use-wire', @@ -1150,6 +1157,7 @@ export interface FileRoutesByFullPath { '/api/adapter-input-modalities': typeof ApiAdapterInputModalitiesRoute '/api/anthropic-auth-wire': typeof ApiAnthropicAuthWireRoute '/api/anthropic-bug-test': typeof ApiAnthropicBugTestRoute + '/api/anthropic-computer-toolset-wire': typeof ApiAnthropicComputerToolsetWireRoute '/api/anthropic-computer-use-wire': typeof ApiAnthropicComputerUseWireRoute '/api/anthropic-haiku-5-5-wire': typeof ApiAnthropicHaiku55WireRoute '/api/anthropic-max-tokens-usage': typeof ApiAnthropicMaxTokensUsageRoute @@ -1327,6 +1335,7 @@ export interface FileRoutesByTo { '/api/adapter-input-modalities': typeof ApiAdapterInputModalitiesRoute '/api/anthropic-auth-wire': typeof ApiAnthropicAuthWireRoute '/api/anthropic-bug-test': typeof ApiAnthropicBugTestRoute + '/api/anthropic-computer-toolset-wire': typeof ApiAnthropicComputerToolsetWireRoute '/api/anthropic-computer-use-wire': typeof ApiAnthropicComputerUseWireRoute '/api/anthropic-haiku-5-5-wire': typeof ApiAnthropicHaiku55WireRoute '/api/anthropic-max-tokens-usage': typeof ApiAnthropicMaxTokensUsageRoute @@ -1505,6 +1514,7 @@ export interface FileRoutesById { '/api/adapter-input-modalities': typeof ApiAdapterInputModalitiesRoute '/api/anthropic-auth-wire': typeof ApiAnthropicAuthWireRoute '/api/anthropic-bug-test': typeof ApiAnthropicBugTestRoute + '/api/anthropic-computer-toolset-wire': typeof ApiAnthropicComputerToolsetWireRoute '/api/anthropic-computer-use-wire': typeof ApiAnthropicComputerUseWireRoute '/api/anthropic-haiku-5-5-wire': typeof ApiAnthropicHaiku55WireRoute '/api/anthropic-max-tokens-usage': typeof ApiAnthropicMaxTokensUsageRoute @@ -1684,6 +1694,7 @@ export interface FileRouteTypes { | '/api/adapter-input-modalities' | '/api/anthropic-auth-wire' | '/api/anthropic-bug-test' + | '/api/anthropic-computer-toolset-wire' | '/api/anthropic-computer-use-wire' | '/api/anthropic-haiku-5-5-wire' | '/api/anthropic-max-tokens-usage' @@ -1861,6 +1872,7 @@ export interface FileRouteTypes { | '/api/adapter-input-modalities' | '/api/anthropic-auth-wire' | '/api/anthropic-bug-test' + | '/api/anthropic-computer-toolset-wire' | '/api/anthropic-computer-use-wire' | '/api/anthropic-haiku-5-5-wire' | '/api/anthropic-max-tokens-usage' @@ -2038,6 +2050,7 @@ export interface FileRouteTypes { | '/api/adapter-input-modalities' | '/api/anthropic-auth-wire' | '/api/anthropic-bug-test' + | '/api/anthropic-computer-toolset-wire' | '/api/anthropic-computer-use-wire' | '/api/anthropic-haiku-5-5-wire' | '/api/anthropic-max-tokens-usage' @@ -2216,6 +2229,7 @@ export interface RootRouteChildren { ApiAdapterInputModalitiesRoute: typeof ApiAdapterInputModalitiesRoute ApiAnthropicAuthWireRoute: typeof ApiAnthropicAuthWireRoute ApiAnthropicBugTestRoute: typeof ApiAnthropicBugTestRoute + ApiAnthropicComputerToolsetWireRoute: typeof ApiAnthropicComputerToolsetWireRoute ApiAnthropicComputerUseWireRoute: typeof ApiAnthropicComputerUseWireRoute ApiAnthropicHaiku55WireRoute: typeof ApiAnthropicHaiku55WireRoute ApiAnthropicMaxTokensUsageRoute: typeof ApiAnthropicMaxTokensUsageRoute @@ -2661,6 +2675,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiAnthropicBugTestRouteImport parentRoute: typeof rootRouteImport } + '/api/anthropic-computer-toolset-wire': { + id: '/api/anthropic-computer-toolset-wire' + path: '/api/anthropic-computer-toolset-wire' + fullPath: '/api/anthropic-computer-toolset-wire' + preLoaderRoute: typeof ApiAnthropicComputerToolsetWireRouteImport + parentRoute: typeof rootRouteImport + } '/api/anthropic-computer-use-wire': { id: '/api/anthropic-computer-use-wire' path: '/api/anthropic-computer-use-wire' @@ -3677,6 +3698,7 @@ const rootRouteChildren: RootRouteChildren = { ApiAdapterInputModalitiesRoute: ApiAdapterInputModalitiesRoute, ApiAnthropicAuthWireRoute: ApiAnthropicAuthWireRoute, ApiAnthropicBugTestRoute: ApiAnthropicBugTestRoute, + ApiAnthropicComputerToolsetWireRoute: ApiAnthropicComputerToolsetWireRoute, ApiAnthropicComputerUseWireRoute: ApiAnthropicComputerUseWireRoute, ApiAnthropicHaiku55WireRoute: ApiAnthropicHaiku55WireRoute, ApiAnthropicMaxTokensUsageRoute: ApiAnthropicMaxTokensUsageRoute, diff --git a/testing/e2e/src/routes/api.anthropic-computer-toolset-wire.ts b/testing/e2e/src/routes/api.anthropic-computer-toolset-wire.ts new file mode 100644 index 0000000000..0fbba1e94b --- /dev/null +++ b/testing/e2e/src/routes/api.anthropic-computer-toolset-wire.ts @@ -0,0 +1,162 @@ +import { createFileRoute } from '@tanstack/react-router' +import { chat, createChatOptions } from '@tanstack/ai' +import { createAnthropicChat } from '@tanstack/ai-anthropic' +import { computerUseTool } from '@tanstack/ai-anthropic/tools' + +const DUMMY_KEY = 'sk-ant-e2e-test-dummy-key' + +/** + * Drives one computer toolset loop on `claude-opus-5-5`. A custom `fetch` + * captures each outgoing request. The first answer has two toolset calls + * (`left_click` and `screenshot`). The second answer ends the turn. + * + * The companion spec reads the captured requests and the inputs that + * `execute` got, so it can check the toolset wire format without an API key. + */ +function sseStream(events: Array<{ type: string }>) { + const encoder = new TextEncoder() + return new ReadableStream({ + start(controller) { + for (const event of events) { + controller.enqueue( + encoder.encode( + `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`, + ), + ) + } + controller.close() + }, + }) +} + +const messageStart = { + type: 'message_start', + message: { + id: 'msg_computer_toolset_wire', + type: 'message', + role: 'assistant', + content: [], + model: 'claude-opus-5-5', + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 5, output_tokens: 0 }, + }, +} + +const end = (stopReason: string) => [ + { + type: 'message_delta', + delta: { stop_reason: stopReason, stop_sequence: null }, + usage: { output_tokens: 1 }, + }, + { type: 'message_stop' }, +] + +const toolsetCalls = [ + messageStart, + { + type: 'content_block_start', + index: 0, + content_block: { + type: 'tool_use', + id: 'toolu_click', + name: 'left_click', + toolset_name: 'computer', + input: {}, + }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { + type: 'input_json_delta', + partial_json: '{"coordinate":[500,300]}', + }, + }, + { type: 'content_block_stop', index: 0 }, + { + type: 'content_block_start', + index: 1, + content_block: { + type: 'tool_use', + id: 'toolu_shot', + name: 'screenshot', + toolset_name: 'computer', + input: {}, + }, + }, + { type: 'content_block_stop', index: 1 }, + ...end('tool_use'), +] + +export const Route = createFileRoute('/api/anthropic-computer-toolset-wire')({ + server: { + handlers: { + POST: async () => { + const requests: Array<{ anthropicBeta: string | null; body: unknown }> = + [] + const capturingFetch: typeof fetch = async (input, init) => { + const req = + input instanceof Request ? input : new Request(input, init) + const rawBody = await req.text() + requests.push({ + anthropicBeta: req.headers.get('anthropic-beta'), + body: rawBody ? JSON.parse(rawBody) : null, + }) + const events = + requests.length === 1 + ? toolsetCalls + : [messageStart, ...end('end_turn')] + return new Response(sseStream(events), { + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + }) + } + + const calls: Array = [] + const computer = { + ...computerUseTool({ + type: 'computer_toolset_20260801', + configs: { zoom: { enabled: false } }, + }), + execute: (input: { action: string }) => { + calls.push(input) + if (input.action !== 'screenshot') return 'OK' + return [ + { + type: 'image' as const, + source: { + type: 'data' as const, + value: 'AAAA', + mimeType: 'image/png', + }, + }, + ] + }, + } + + const adapter = createAnthropicChat('claude-opus-5-5', DUMMY_KEY, { + fetch: capturingFetch, + }) + + try { + for await (const _ of chat({ + ...createChatOptions({ adapter }), + messages: [{ role: 'user', content: 'Click the button.' }], + tools: [computer], + toolExecution: 'sequential', + })) { + // Drain the stream. + } + } catch (error) { + return Response.json({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }) + } + + return Response.json({ ok: true, calls, requests }) + }, + }, + }, +}) diff --git a/testing/e2e/tests/anthropic-computer-toolset-wire.spec.ts b/testing/e2e/tests/anthropic-computer-toolset-wire.spec.ts new file mode 100644 index 0000000000..3a982a686b --- /dev/null +++ b/testing/e2e/tests/anthropic-computer-toolset-wire.spec.ts @@ -0,0 +1,97 @@ +import { test, expect } from './fixtures' +import type { APIRequestContext } from '@playwright/test' + +/** + * Wire format of the Anthropic computer toolset (`computer_toolset_20260801`). + * + * The model calls each action as its own tool (`left_click`, `screenshot`) + * with `toolset_name: 'computer'`. The adapter runs each call as one + * `computer` call with the action in `input.action`. On the next request it + * sends each call back in the form that the model sent, and each result + * names the toolset. + * + * `/api/anthropic-computer-toolset-wire` runs one loop against a custom + * `fetch`, and returns the captured requests and the inputs that `execute` + * got. + */ +interface WireResult { + ok: boolean + error?: string + calls: Array + requests: Array<{ + anthropicBeta: string | null + body: { + tools: Array> + messages: Array<{ role: string; content: Array> }> + } + }> +} + +async function runLoop(request: APIRequestContext) { + const res = await request.post('/api/anthropic-computer-toolset-wire') + expect(res.ok()).toBe(true) + const result = (await res.json()) as WireResult + if (!result.ok) { + throw new Error(`Route failed: ${result.error}`) + } + return result +} + +test.describe('anthropic — computer toolset wire format', () => { + test('sends the toolset entry with no name and no computer use beta', async ({ + request, + }) => { + const { requests } = await runLoop(request) + const [first] = requests + expect(first?.body.tools[0]).toMatchObject({ + type: 'computer_toolset_20260801', + configs: { zoom: { enabled: false } }, + }) + expect(first?.body.tools[0]).not.toHaveProperty('name') + expect(first?.anthropicBeta ?? '').not.toContain('computer-use') + }) + + test('runs each toolset call as one computer call', async ({ request }) => { + const { calls } = await runLoop(request) + expect(calls).toEqual([ + { action: 'left_click', coordinate: [500, 300] }, + { action: 'screenshot' }, + ]) + }) + + test('sends the calls back as toolset calls, and the results name the toolset', async ({ + request, + }) => { + const { requests } = await runLoop(request) + const second = requests[1] + expect(second).toBeDefined() + const assistant = second?.body.messages.find((m) => m.role === 'assistant') + expect(assistant?.content).toMatchObject([ + { + type: 'tool_use', + id: 'toolu_click', + name: 'left_click', + toolset_name: 'computer', + input: { coordinate: [500, 300] }, + }, + { + type: 'tool_use', + id: 'toolu_shot', + name: 'screenshot', + toolset_name: 'computer', + input: {}, + }, + ]) + const results = second?.body.messages + .flatMap((m) => m.content) + .filter((block) => block['type'] === 'tool_result') + expect(results).toMatchObject([ + { tool_use_id: 'toolu_click', toolset_name: 'computer', content: 'OK' }, + { + tool_use_id: 'toolu_shot', + toolset_name: 'computer', + content: [{ type: 'image', source: { type: 'base64', data: 'AAAA' } }], + }, + ]) + }) +})