From 8384355ab9497c27b603f0e5ef4b65241d679d8a Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 19 Oct 2026 18:43:10 +0200 Subject: [PATCH 1/5] feat(ai-anthropic): computer use on every Claude model Upgrade @anthropic-ai/sdk to 0.135.0. Add computer_20251124 with its computer-use-2025-11-24 beta, and computer_toolset_20260801 for the Claude 5.5 models. A toolset call runs as one computer call with the action in input.action, and goes back to Claude in the form that it sent. Each model now takes only the computer use versions that it supports. --- docs/adapters/anthropic.md | 82 +++++-- docs/config.json | 4 +- docs/tools/provider-tools.md | 8 +- examples/ts-react-chat/package.json | 2 +- packages/ai-anthropic/package.json | 3 +- packages/ai-anthropic/src/adapters/files.ts | 4 +- packages/ai-anthropic/src/adapters/text.ts | 132 ++++++++--- packages/ai-anthropic/src/model-meta.ts | 43 ++-- packages/ai-anthropic/src/prompt-cache.ts | 4 +- .../src/tools/computer-use-tool.ts | 101 ++++++-- .../client-injection-type-safety.test.ts | 11 - .../tests/computer-toolset.test.ts | 222 ++++++++++++++++++ .../tests/tools-per-model-type-safety.test.ts | 76 ++++-- pnpm-lock.yaml | 45 ++-- scripts/model-sync/provider-supports.test.ts | 3 +- scripts/model-sync/provider-supports.ts | 3 +- testing/e2e/package.json | 2 +- 17 files changed, 594 insertions(+), 151 deletions(-) create mode 100644 packages/ai-anthropic/tests/computer-toolset.test.ts diff --git a/docs/adapters/anthropic.md b/docs/adapters/anthropic.md index 8b8b2b73a8..579f998c51 100644 --- a/docs/adapters/anthropic.md +++ b/docs/adapters/anthropic.md @@ -618,30 +618,84 @@ equivalent — see [Provider Skills](../tools/provider-skills.md). ### `computerUseTool` -Allows Claude to observe a virtual desktop (screenshots) and interact with it -via keyboard and mouse events. Provide the screen resolution so Claude can -calculate accurate coordinates. +Claude can look at a screen and use the mouse and the keyboard. Your app owns the screen: it runs each action that Claude asks for and sends back a screenshot. + +1. Pick the version that your model takes: + + | Version | Models | + | --- | --- | + | `computer_toolset_20260801` | Claude Opus 5.5, Sonnet 5.5, Haiku 5.5, Fable 5.1, Fable 5, Opus 5, Sonnet 5, Opus 4.8 | + | `computer_20251124` | Claude Fable 5.1, Fable 5, Opus 5, Sonnet 5, Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 4.6, Opus 4.5 | + | `computer_20250124` | Claude Sonnet 4.5, Haiku 4.5, Opus 4.1 | + +2. Give the tool an `execute` that runs each action on your screen. +3. Return a screenshot for the `screenshot` action, and `"OK"` for the other actions. ```typescript import { chat } from "@tanstack/ai"; import { anthropicText } from "@tanstack/ai-anthropic"; import { computerUseTool } from "@tanstack/ai-anthropic/tools"; +// Your screen, for example a virtual machine. Replace these two functions. +async function runAction(input: { action: string }): Promise { + console.log("run", input); +} +async function takeScreenshot(): Promise { + return ""; // A PNG, as base64. +} + +const computer = { + ...computerUseTool({ + type: "computer_toolset_20260801", + configs: { zoom: { enabled: false } }, + }), + // Every version calls `execute` with the action in `input.action`. + execute: async (input: { action: string }) => { + if (input.action === "screenshot") { + const value = await takeScreenshot(); + return [ + { + type: "image" as const, + source: { type: "data" as const, value, mimeType: "image/png" }, + }, + ]; + } + await runAction(input); + return "OK"; + }, +}; + const stream = chat({ - adapter: anthropicText("claude-sonnet-4-6"), - messages: [{ role: "user", content: "Open the browser and go to example.com" }], - tools: [ - computerUseTool({ - type: "computer_20250124", - name: "computer", - display_width_px: 1024, - display_height_px: 768, - }), - ], + adapter: anthropicText("claude-opus-5-5"), + messages: [{ role: "user", content: "Open the display settings." }], + tools: [computer], + // Claude can ask for several actions in one turn. Run them in order. + toolExecution: "sequential", }); ``` -**Supported models:** Claude Sonnet 3.5 and above, with two exceptions. `claude-opus-5-fast` takes no provider tools. Claude Opus 5.5, Claude Sonnet 5.5, and Claude Haiku 5.5 accept only the `computer_toolset_20260801` toolset, which the adapter does not offer yet. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). +The earlier versions also take a name and the screen size: + +```typescript +import { computerUseTool } from "@tanstack/ai-anthropic/tools"; + +const computer = computerUseTool({ + type: "computer_20251124", + name: "computer", + display_width_px: 1280, + display_height_px: 800, +}); +``` + +Good to know: + +- The adapter sends the beta header that an earlier version needs. The toolset needs no beta header. +- With the toolset, Claude calls each action as its own tool. The adapter gives you each call as one `computer` call, and sends it back in the form that Claude sent. So one `execute` works for every version. +- The toolset turns on `zoom` by default. To let Claude zoom, remove `configs` and return an image for the `zoom` action too. +- Send screenshots that fit the image limits of the model. The toolset does not make a large screenshot smaller. It refuses it. Claude gives coordinates in the pixels of your screenshots. +- If an action fails, answer each later action of the same turn with `Not executed: an earlier computer action in this turn failed.` + +A version that your model does not take is a type error. `claude-opus-5-fast` takes no provider tools. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). ### `bashTool` diff --git a/docs/config.json b/docs/config.json index 5fed936ceb..c235f5f524 100644 --- a/docs/config.json +++ b/docs/config.json @@ -323,7 +323,7 @@ "label": "Provider Tools", "to": "tools/provider-tools", "addedAt": "2026-04-21", - "updatedAt": "2026-09-24" + "updatedAt": "2026-10-19" }, { "label": "Provider Skills", @@ -1215,7 +1215,7 @@ "label": "Anthropic", "to": "adapters/anthropic", "addedAt": "2026-04-15", - "updatedAt": "2026-10-09" + "updatedAt": "2026-10-19" }, { "label": "Google Gemini", diff --git a/docs/tools/provider-tools.md b/docs/tools/provider-tools.md index 25da0275c4..64504bf5ac 100644 --- a/docs/tools/provider-tools.md +++ b/docs/tools/provider-tools.md @@ -149,10 +149,10 @@ Each adapter's `supports.tools` array is the source of truth. The comparison matrix is maintained alongside `model-meta.ts` and reflected here: - **Anthropic**: every registered model supports the full tool superset, - except four. `claude-opus-5-5`, `claude-sonnet-5-5`, and - `claude-haiku-5-5` support every tool but `computerUseTool`, and - `claude-opus-5-fast` supports none. The - retired Claude 3.x models with narrower support were removed. + except `claude-opus-5-fast`, which supports none. For `computerUseTool`, + each model takes only some versions. The Claude 5.5 models take only + `computer_toolset_20260801`. See the version table in + [`computerUseTool`](../adapters/anthropic.md#computerusetool). - **OpenAI**: GPT-5 family and reasoning models (O-series) support the full superset. GPT-6 Astra, Sol, and Luna support every tool except `webSearchPreviewTool` and `localShellTool`. GPT-4-series supports diff --git a/examples/ts-react-chat/package.json b/examples/ts-react-chat/package.json index dace1d669a..13ddbcc526 100644 --- a/examples/ts-react-chat/package.json +++ b/examples/ts-react-chat/package.json @@ -12,7 +12,7 @@ "test:types": "tsc --noEmit" }, "dependencies": { - "@anthropic-ai/sdk": "^0.97.1", + "@anthropic-ai/sdk": "^0.135.0", "@elevenlabs/elevenlabs-js": "^2.68.0", "@mcp-ui/client": "^7.1.1", "@modelcontextprotocol/sdk": "^1.29.0", diff --git a/packages/ai-anthropic/package.json b/packages/ai-anthropic/package.json index 1afd093b13..ff946b18c2 100644 --- a/packages/ai-anthropic/package.json +++ b/packages/ai-anthropic/package.json @@ -67,7 +67,7 @@ "test:types": "tsc" }, "dependencies": { - "@anthropic-ai/sdk": "^0.97.1", + "@anthropic-ai/sdk": "^0.135.0", "@tanstack/ai-utils": "workspace:^" }, "peerDependencies": { @@ -80,7 +80,6 @@ } }, "devDependencies": { - "@anthropic-ai/sdk-v112": "npm:@anthropic-ai/sdk@0.112.4", "@anthropic-ai/vertex-sdk": "^0.19.0", "@tanstack/ai": "workspace:*", "@vitest/coverage-v8": "4.1.10", diff --git a/packages/ai-anthropic/src/adapters/files.ts b/packages/ai-anthropic/src/adapters/files.ts index 37394d5fb2..133859ddad 100644 --- a/packages/ai-anthropic/src/adapters/files.ts +++ b/packages/ai-anthropic/src/adapters/files.ts @@ -8,7 +8,7 @@ import { getAnthropicCredentialFromEnv, } from '../utils/client' import type Anthropic_SDK from '@anthropic-ai/sdk' -import type { FileMetadata } from '@anthropic-ai/sdk/resources/beta/files' +import type { BetaFileMetadata } from '@anthropic-ai/sdk/resources/beta/files' import type { FileHandle, FileUploadInput } from '@tanstack/ai/adapters' import type { AnthropicClientConfig } from '../utils/client' @@ -55,7 +55,7 @@ export class AnthropicFilesAdapter extends BaseFilesAdapter<'anthropic'> { } } -function toFileHandle(file: FileMetadata): FileHandle<'anthropic'> { +function toFileHandle(file: BetaFileMetadata): FileHandle<'anthropic'> { return { id: file.id, provider: 'anthropic', diff --git a/packages/ai-anthropic/src/adapters/text.ts b/packages/ai-anthropic/src/adapters/text.ts index dd3dec4dbc..1e5f32c3c6 100644 --- a/packages/ai-anthropic/src/adapters/text.ts +++ b/packages/ai-anthropic/src/adapters/text.ts @@ -16,7 +16,13 @@ import { import { BaseTextAdapter } from '@tanstack/ai/adapters' import { convertToolsToProviderFormat } from '../tools/tool-converter' import { getAnthropicProviderToolKind } from '../tools/anthropic-provider-tool' -import { convertComputerUseToolToAdapterFormat } from '../tools/computer-use-tool' +import { + COMPUTER_TOOL_NAME, + convertComputerUseToolToAdapterFormat, + isComputerToolset, + joinComputerToolsetCall, + splitComputerToolsetCall, +} from '../tools/computer-use-tool' import { readCodeExecutionConfig, readCodeExecutionSkills, @@ -106,6 +112,7 @@ import type { AnthropicClientConfig, AnthropicMessagesClient, } from '../utils/client' +import type { ComputerUseSingleToolConfig } from '../tools/computer-use-tool' /** * The block type carried by an Anthropic provider-executed (server) tool's @@ -212,13 +219,20 @@ function toAnthropicToolChoice( return { type: 'tool' as const, name: choice.name } } +/** The beta header of each single `computer` tool version. */ +const COMPUTER_USE_BETAS = { + computer_20241022: 'computer-use-2024-10-22', + computer_20250124: 'computer-use-2025-01-24', + computer_20251124: 'computer-use-2025-11-24', +} as const satisfies Record + /** * Computes the `betas` array for a Messages request. Unions: * - `interleaved-thinking-2025-05-14` when interleaved thinking is enabled, * - `code-execution-2025-08-25` when a `code_execution` tool is present, * - `skills-2025-10-02` when that tool carries skills, - * - `computer-use-2024-10-22` or `computer-use-2025-01-24` for the version - * of a `computer` tool, + * - the `computer-use-*` beta of a single `computer` tool version (the + * computer toolset needs none), * - `context-management-2025-06-27` when `context_management` is set, * - `files-api-2025-04-14` when a message references an uploaded file handle, * - `mid-conversation-tool-changes-2026-07-01` in mid-conversation tool mode. @@ -299,12 +313,10 @@ export function computeAnthropicBetas( (tool) => getAnthropicProviderToolKind(tool) === 'computer_use', ) if (computerTool) { - betas.add( - convertComputerUseToolToAdapterFormat(computerTool).type === - 'computer_20241022' - ? 'computer-use-2024-10-22' - : 'computer-use-2025-01-24', - ) + const { type } = convertComputerUseToolToAdapterFormat(computerTool) + if (type !== 'computer_toolset_20260801') { + betas.add(COMPUTER_USE_BETAS[type]) + } } return betas.size > 0 ? Array.from(betas) : undefined @@ -323,18 +335,6 @@ const DEFERRED_TOOL_PLACEHOLDER: BetaTool = { defer_loading: true, } -/** - * A mid-conversation `system` message. SDK 0.97.1 types only `user` and - * `assistant` messages and has no `tool_addition` block, so this type is local. - */ -interface AnthropicMidConversationSystemMessage { - role: 'system' - content: Array< - | TextBlockParam - | { type: 'tool_addition'; tool: { type: 'tool_reference'; name: string } } - > -} - /** * Configuration for Anthropic text adapter */ @@ -607,7 +607,8 @@ export class AnthropicTextAdapter< messagesHaveFileSource(options.messages), // Mid-conversation tool mode puts the deferred placeholder in `tools`. requestParams.tools?.some( - (tool) => tool.name === DEFERRED_TOOL_PLACEHOLDER.name, + (tool) => + 'name' in tool && tool.name === DEFERRED_TOOL_PLACEHOLDER.name, ), ), ) @@ -821,6 +822,7 @@ export class AnthropicTextAdapter< const formattedMessages = this.formatMessages( replay.messages, mid?.systemMessages, + options.tools?.some(isComputerToolset) ?? false, ) const tools = options.tools ? (mid?.tools ?? convertToolsToProviderFormat(options.tools)) @@ -1064,7 +1066,7 @@ export class AnthropicTextAdapter< ) const systemMessages = new Map() for (const [before, change] of split.at) { - const content: AnthropicMidConversationSystemMessage['content'] = [ + const content: Array = [ ...(channels.systemPrompts ? change.systemPrompts.map( (p): TextBlockParam => ({ type: 'text', text: p.content }), @@ -1078,12 +1080,7 @@ export class AnthropicTextAdapter< : []), ] if (content.length === 0) continue - const message: AnthropicMidConversationSystemMessage = { - role: 'system', - content, - } - // oxlint-disable-next-line eslint-js/no-restricted-syntax -- SDK 0.97.1 types no `system` message in `messages`; the models in the channel map accept one (pi 0.87.1). - systemMessages.set(before, message as unknown as BetaMessageParam) + systemMessages.set(before, { role: 'system', content }) } return { ...(channels.systemPrompts && { @@ -1218,8 +1215,12 @@ export class AnthropicTextAdapter< messages: Array, /** Mid-conversation `system` messages by message index (`before`). */ systemMessages?: ReadonlyMap, + /** The tools have the computer toolset, so `computer` calls are its calls. */ + computerToolset = false, ): InternalTextProviderOptions['messages'] { const formattedMessages: InternalTextProviderOptions['messages'] = [] + // The ids of the computer toolset calls. Their results name the toolset. + const toolsetCalls = computerToolset ? new Set() : undefined // A system message waits for the next assistant message (or the end), // so a `tool_result` still follows its `tool_use` directly. const pendingSystemMessages: InternalTextProviderOptions['messages'] = [] @@ -1240,6 +1241,9 @@ export class AnthropicTextAdapter< { type: 'tool_result', tool_use_id: message.toolCallId, + ...(toolsetCalls?.has(message.toolCallId) && { + toolset_name: COMPUTER_TOOL_NAME, + }), content: Array.isArray(toolContent) ? toolContent.map((part) => this.convertContentPartToAnthropic(part), @@ -1264,7 +1268,7 @@ export class AnthropicTextAdapter< if (block.type === 'thinking') { this.appendThinkingBlocks(contentBlocks, [block.thinking]) } else if (block.type === 'tool-call') { - this.appendToolCallBlocks(contentBlocks, block.toolCall) + this.appendToolCallBlocks(contentBlocks, block.toolCall, toolsetCalls) } else { contentBlocks.push({ type: 'text', text: block.text }) } @@ -1292,7 +1296,7 @@ export class AnthropicTextAdapter< } for (const toolCall of message.toolCalls) { - this.appendToolCallBlocks(contentBlocks, toolCall) + this.appendToolCallBlocks(contentBlocks, toolCall, toolsetCalls) } formattedMessages.push({ @@ -1389,10 +1393,15 @@ export class AnthropicTextAdapter< } } - /** A tool call as a `tool_use` block, or a server tool as its two blocks. */ + /** + * A tool call as a `tool_use` block, or a server tool as its two blocks. + * With `toolsetCalls`, a `computer` call goes back as the computer toolset + * call that the model sent, and its id goes in `toolsetCalls`. + */ private appendToolCallBlocks( contentBlocks: Array, toolCall: ToolCall, + toolsetCalls?: Set, ): void { let parsedInput: unknown = {} try { @@ -1421,6 +1430,25 @@ export class AnthropicTextAdapter< return } + const member = + toolsetCalls && + toolCall.function.name === COMPUTER_TOOL_NAME && + typeof parsedInput === 'object' && + parsedInput !== null + ? splitComputerToolsetCall(parsedInput) + : undefined + if (member) { + toolsetCalls?.add(toolCall.id) + contentBlocks.push({ + type: 'tool_use', + id: toolCall.id, + name: member.name, + toolset_name: COMPUTER_TOOL_NAME, + input: member.input, + }) + return + } + const toolUseBlock: ToolUseBlockParam = { type: 'tool_use', id: toolCall.id, @@ -1534,6 +1562,8 @@ export class AnthropicTextAdapter< input: string started: boolean hasInputDelta: boolean + /** The action of a computer toolset call (`left_click`, …). */ + member?: string } >() let currentToolIndex = -1 @@ -1596,9 +1626,16 @@ export class AnthropicTextAdapter< currentBlockType = event.content_block.type if (event.content_block.type === 'tool_use') { currentToolIndex++ + // A computer toolset call names its action. It runs as one + // `computer` call with the action in `input.action`. + const isComputerCall = + event.content_block.toolset_name === COMPUTER_TOOL_NAME toolCallsMap.set(currentToolIndex, { id: event.content_block.id, - name: event.content_block.name, + name: isComputerCall + ? COMPUTER_TOOL_NAME + : event.content_block.name, + ...(isComputerCall && { member: event.content_block.name }), // The initial input snapshot. The first `input_json_delta` // replaces it. input: JSON.stringify(event.content_block.input) ?? '', @@ -1860,7 +1897,14 @@ export class AnthropicTextAdapter< // never appends onto the prior client tool's buffer. if (currentBlockType === 'tool_use') { const existing = toolCallsMap.get(currentToolIndex) - if (existing) { + // A computer toolset call sends its joined arguments at the end. + if (existing?.member !== undefined) { + if (!existing.hasInputDelta) { + existing.input = '' + existing.hasInputDelta = true + } + existing.input += event.delta.partial_json + } else if (existing) { // Emit TOOL_CALL_START on first args delta if (!existing.started) { existing.started = true @@ -1971,6 +2015,26 @@ export class AnthropicTextAdapter< // Keep `parsedInput` undefined for the tool error. } + if ( + existing.member !== undefined && + typeof parsedInput === 'object' && + parsedInput !== null + ) { + parsedInput = joinComputerToolsetCall( + existing.member, + parsedInput, + ) + existing.input = JSON.stringify(parsedInput) + yield { + type: EventType.TOOL_CALL_ARGS, + toolCallId: existing.id, + model, + timestamp: Date.now(), + delta: existing.input, + args: existing.input, + } + } + yield { type: EventType.TOOL_CALL_END, toolCallId: existing.id, diff --git a/packages/ai-anthropic/src/model-meta.ts b/packages/ai-anthropic/src/model-meta.ts index 4534d13c97..80dc5e69ce 100644 --- a/packages/ai-anthropic/src/model-meta.ts +++ b/packages/ai-anthropic/src/model-meta.ts @@ -92,11 +92,19 @@ interface ModelMeta< * every request, with or without thinking. Absent: it accepts one. */ forced_tool_choice?: false + /** + * Computer use has three kinds, one for each version that a model + * accepts: `computer_use` (`computer_20241022` and `computer_20250124`), + * `computer_use_20251124`, and `computer_toolset` + * (`computer_toolset_20260801`). + */ tools?: Array< | 'web_search' | 'web_fetch' | 'code_execution' | 'computer_use' + | 'computer_use_20251124' + | 'computer_toolset' | 'bash' | 'text_editor' | 'memory' @@ -155,7 +163,7 @@ const CLAUDE_OPUS_4_6 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', 'bash', 'text_editor', 'memory', @@ -195,7 +203,7 @@ const CLAUDE_OPUS_4_5 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', 'bash', 'text_editor', 'memory', @@ -238,7 +246,7 @@ const CLAUDE_SONNET_4_6 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', 'bash', 'text_editor', 'memory', @@ -393,7 +401,7 @@ const CLAUDE_OPUS_4_7 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', 'bash', 'text_editor', 'memory', @@ -437,7 +445,8 @@ const CLAUDE_OPUS_4_8 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -484,7 +493,8 @@ const CLAUDE_FABLE_5 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -529,7 +539,8 @@ const CLAUDE_SONNET_5 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -577,7 +588,8 @@ const CLAUDE_OPUS_5 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -645,7 +657,8 @@ const CLAUDE_FABLE_5_1 = { 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_use_20251124', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -681,12 +694,12 @@ const CLAUDE_OPUS_5_5 = { input: ['text', 'image', 'document'], forced_tool_choice: false, mid_conversation_channels: { tools: true, systemPrompts: true }, - // No 'computer_use': this model accepts only `computer_toolset_20260801`, - // and `computerUseTool()` sends the older versions, which return a 400. + // Computer use only through `computer_toolset_20260801`. tools: [ 'web_search', 'web_fetch', 'code_execution', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -728,12 +741,12 @@ const CLAUDE_SONNET_5_5 = { forced_tool_choice: false, extended_thinking: false, adaptive_thinking: true, - // No 'computer_use': this model accepts only `computer_toolset_20260801`, - // and `computerUseTool()` sends the older versions, which return a 400. + // Computer use only through `computer_toolset_20260801`. tools: [ 'web_search', 'web_fetch', 'code_execution', + 'computer_toolset', 'bash', 'text_editor', 'memory', @@ -771,12 +784,12 @@ const CLAUDE_HAIKU_5_5 = { input: ['text', 'image', 'document'], extended_thinking: false, adaptive_thinking: true, - // No 'computer_use': this model accepts only `computer_toolset_20260801`, - // and `computerUseTool()` sends the older versions, which return a 400. + // Computer use only through `computer_toolset_20260801`. tools: [ 'web_search', 'web_fetch', 'code_execution', + 'computer_toolset', 'bash', 'text_editor', 'memory', diff --git a/packages/ai-anthropic/src/prompt-cache.ts b/packages/ai-anthropic/src/prompt-cache.ts index 60f9fde70c..dee08e3bb7 100644 --- a/packages/ai-anthropic/src/prompt-cache.ts +++ b/packages/ai-anthropic/src/prompt-cache.ts @@ -153,7 +153,9 @@ export function applyAnthropicPromptCache( // last tool, as before. const placeholder = request.tools?.findIndex( - (tool) => tool.name === ANTHROPIC_DEFERRED_TOOL_PLACEHOLDER_NAME, + // The computer toolset entry has no name. + (tool) => + 'name' in tool && tool.name === ANTHROPIC_DEFERRED_TOOL_PLACEHOLDER_NAME, ) ?? -1 const tools = request.tools?.map((tool, index, all) => index === (placeholder > 0 ? placeholder - 1 : all.length - 1) diff --git a/packages/ai-anthropic/src/tools/computer-use-tool.ts b/packages/ai-anthropic/src/tools/computer-use-tool.ts index d37f5ce07b..8959ca6546 100644 --- a/packages/ai-anthropic/src/tools/computer-use-tool.ts +++ b/packages/ai-anthropic/src/tools/computer-use-tool.ts @@ -1,22 +1,50 @@ -import { - brandAnthropicProviderTool, - getAnthropicProviderToolMetadata, -} from './anthropic-provider-tool' +import { brandProviderTool } from '@tanstack/ai' +import { getAnthropicProviderToolMetadata } from './anthropic-provider-tool' import type { + BetaComputerToolset20260801, BetaToolComputerUse20241022, BetaToolComputerUse20250124, + BetaToolComputerUse20251124, } from '@anthropic-ai/sdk/resources/beta' import type { ProviderTool, Tool } from '@tanstack/ai' -export type ComputerUseToolConfig = +/** The single `computer` tool that sends its action in `input.action`. */ +export type ComputerUseSingleToolConfig = | BetaToolComputerUse20241022 | BetaToolComputerUse20250124 + | BetaToolComputerUse20251124 + +/** + * The computer toolset: one entry with no name. The model calls each action + * as its own tool (`left_click`, `screenshot`, …). + */ +export type ComputerToolsetConfig = BetaComputerToolset20260801 + +export type ComputerUseToolConfig = + | ComputerUseSingleToolConfig + | ComputerToolsetConfig /** @deprecated Renamed to `ComputerUseToolConfig`. Will be removed in a future release. */ export type ComputerUseTool = ComputerUseToolConfig +/** `computer_20241022` or `computer_20250124`. */ export type AnthropicComputerUseTool = ProviderTool<'anthropic', 'computer_use'> +/** `computer_20251124`. */ +export type AnthropicComputerUse20251124Tool = ProviderTool< + 'anthropic', + 'computer_use_20251124' +> + +/** `computer_toolset_20260801`. */ +export type AnthropicComputerToolset = ProviderTool< + 'anthropic', + 'computer_toolset' +> + +/** The name of the tool that every computer use version runs as. */ +export const COMPUTER_TOOL_NAME = 'computer' + export function convertComputerUseToolToAdapterFormat( tool: Tool, ): ComputerUseToolConfig { @@ -26,15 +54,58 @@ export function convertComputerUseToolToAdapterFormat( return metadata } +/** True for the `computer_toolset_20260801` entry. */ +export function isComputerToolset(tool: Tool) { + return tool.metadata?.['type'] === 'computer_toolset_20260801' +} + +/** + * One toolset call (`left_click` with `{ coordinate }`) as one `computer` + * call: `{ action: 'left_click', coordinate }`. So one `execute` works for + * every version. + */ +export function joinComputerToolsetCall(member: string, input: object) { + return { action: member, ...input } +} + +/** A `computer` call as the toolset call that the model sent. */ +export function splitComputerToolsetCall(input: object) { + if (!('action' in input) || typeof input.action !== 'string') { + return undefined + } + const { action, ...memberInput } = input + return { name: action, input: memberInput } +} + +/** + * Computer use. Every version runs as one tool named `computer`, with the + * action in `input.action`: + * + * - `computer_toolset_20260801`: Claude Opus 5.5, Sonnet 5.5, Haiku 5.5, and + * the other models that support the toolset. No beta header. + * - `computer_20251124`: the earlier tool of Claude Opus 4.5 to Claude Opus 5, + * Sonnet 4.6, and Sonnet 5. + * - `computer_20250124`: Claude Sonnet 4.5, Haiku 4.5, and Opus 4.1. + */ +export function computerUseTool( + config: ComputerToolsetConfig, +): AnthropicComputerToolset +export function computerUseTool( + config: BetaToolComputerUse20251124, +): AnthropicComputerUse20251124Tool export function computerUseTool( - config: ComputerUseToolConfig, -): AnthropicComputerUseTool { - return brandAnthropicProviderTool( - { - name: 'computer', - description: '', - metadata: config, - }, - 'computer_use', - ) + config: BetaToolComputerUse20241022 | BetaToolComputerUse20250124, +): AnthropicComputerUseTool +export function computerUseTool(config: ComputerUseToolConfig) { + return brandProviderTool< + | AnthropicComputerToolset + | AnthropicComputerUse20251124Tool + | AnthropicComputerUseTool + >({ + name: COMPUTER_TOOL_NAME, + description: '', + // Every version has the same runtime kind. The type kind picks the + // models that accept it. + metadata: { ...config, __kind: 'anthropic.computer_use' }, + }) } diff --git a/packages/ai-anthropic/tests/client-injection-type-safety.test.ts b/packages/ai-anthropic/tests/client-injection-type-safety.test.ts index 57392a90c4..18d3f73fcf 100644 --- a/packages/ai-anthropic/tests/client-injection-type-safety.test.ts +++ b/packages/ai-anthropic/tests/client-injection-type-safety.test.ts @@ -1,18 +1,7 @@ import { expectTypeOf, it } from 'vitest' -import type AnthropicSdkV112 from '@anthropic-ai/sdk-v112' import type { AnthropicVertex } from '@anthropic-ai/vertex-sdk' import type { AnthropicMessagesClient } from '../src' -type V112MessagesClient = { - readonly beta: { - readonly messages: Pick - } -} - it('accepts the official Vertex client', () => { expectTypeOf().toExtend() }) - -it('accepts clients backed by a newer Anthropic SDK version', () => { - expectTypeOf().toExtend() -}) diff --git a/packages/ai-anthropic/tests/computer-toolset.test.ts b/packages/ai-anthropic/tests/computer-toolset.test.ts new file mode 100644 index 0000000000..9edfbe1fc9 --- /dev/null +++ b/packages/ai-anthropic/tests/computer-toolset.test.ts @@ -0,0 +1,222 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { chat } from '@tanstack/ai' +import { AnthropicTextAdapter } from '../src/adapters/text' +import { computerUseTool } from '../src/tools' + +const mocks = vi.hoisted(() => { + const betaMessagesCreate = vi.fn() + const client = { beta: { messages: { create: betaMessagesCreate } } } + return { betaMessagesCreate, client } +}) + +vi.mock('@anthropic-ai/sdk', () => { + const { client } = mocks + class MockAnthropic { + beta = client.beta + constructor(_: { apiKey: string }) {} + } + return { default: MockAnthropic } +}) + +function stream(events: Array) { + mocks.betaMessagesCreate.mockResolvedValueOnce( + (async function* () { + yield* events + })(), + ) +} + +/** A computer toolset call, as the model streams it. */ +function memberCall(index: number, id: string, name: string, json?: string) { + return [ + { + type: 'content_block_start', + index, + content_block: { + type: 'tool_use', + id, + name, + toolset_name: 'computer', + input: {}, + }, + }, + ...(json === undefined + ? [] + : [ + { + type: 'content_block_delta', + index, + delta: { type: 'input_json_delta', partial_json: json }, + }, + ]), + { type: 'content_block_stop', index }, + ] +} + +const end = (stopReason: string) => [ + { + type: 'message_delta', + delta: { stop_reason: stopReason }, + usage: { output_tokens: 1 }, + }, + { type: 'message_stop' }, +] + +const screenshot = [ + { + type: 'image' as const, + source: { type: 'data' as const, value: 'AAAA', mimeType: 'image/png' }, + }, +] + +beforeEach(() => { + mocks.betaMessagesCreate.mockReset() +}) + +describe('anthropic computer toolset', () => { + it('sends the toolset entry with no name and no beta', async () => { + stream(end('end_turn')) + const adapter = new AnthropicTextAdapter({ apiKey: 'k' }, 'claude-opus-5-5') + + for await (const _ of chat({ + adapter, + messages: [{ role: 'user', content: 'Open the settings.' }], + tools: [ + computerUseTool({ + type: 'computer_toolset_20260801', + configs: { zoom: { enabled: false } }, + }), + ], + })) { + // consume + } + + const [payload] = mocks.betaMessagesCreate.mock.calls[0]! + // The prompt cache marker can also sit on the entry. + expect(payload.tools).toMatchObject([ + { type: 'computer_toolset_20260801', configs: { zoom: { enabled: false } } }, + ]) + expect(payload.tools[0]).not.toHaveProperty('name') + expect(payload.tools[0]).not.toHaveProperty('__kind') + expect(payload.betas).toBeUndefined() + }) + + it('runs each toolset call as one computer call and sends it back', async () => { + stream([ + ...memberCall(0, 'toolu_click', 'left_click', '{"coordinate":[500,300]}'), + ...memberCall(1, 'toolu_shot', 'screenshot'), + ...end('tool_use'), + ]) + stream(end('end_turn')) + const calls: Array = [] + const tool = { + ...computerUseTool({ type: 'computer_toolset_20260801' }), + execute: (input: unknown) => { + calls.push(input) + return calls.length === 1 ? 'OK' : screenshot + }, + } + const adapter = new AnthropicTextAdapter({ apiKey: 'k' }, 'claude-opus-5-5') + + for await (const _ of chat({ + adapter, + messages: [{ role: 'user', content: 'Click the button.' }], + tools: [tool], + toolExecution: 'sequential', + })) { + // consume + } + + expect(calls).toEqual([ + { action: 'left_click', coordinate: [500, 300] }, + { action: 'screenshot' }, + ]) + const [payload] = mocks.betaMessagesCreate.mock.calls[1]! + const [, assistant, results] = payload.messages + expect(assistant.content).toEqual([ + { + type: 'tool_use', + id: 'toolu_click', + name: 'left_click', + toolset_name: 'computer', + input: { coordinate: [500, 300] }, + }, + { + type: 'tool_use', + id: 'toolu_shot', + name: 'screenshot', + toolset_name: 'computer', + input: {}, + }, + ]) + expect(results.content).toMatchObject([ + { + type: 'tool_result', + tool_use_id: 'toolu_click', + toolset_name: 'computer', + content: 'OK', + }, + { + type: 'tool_result', + tool_use_id: 'toolu_shot', + toolset_name: 'computer', + content: [ + { + type: 'image', + source: { type: 'base64', data: 'AAAA', media_type: 'image/png' }, + }, + ], + }, + ]) + }) + + it('sends a computer call back as the single tool when the toolset is not in tools', async () => { + stream(end('end_turn')) + const adapter = new AnthropicTextAdapter({ apiKey: 'k' }, 'claude-opus-4-6') + + for await (const _ of chat({ + adapter, + messages: [ + { role: 'user', content: 'Click the button.' }, + { + role: 'assistant', + content: '', + toolCalls: [ + { + id: 'toolu_click', + type: 'function', + function: { + name: 'computer', + arguments: '{"action":"left_click","coordinate":[1,2]}', + }, + }, + ], + }, + { role: 'tool', toolCallId: 'toolu_click', content: 'OK' }, + ], + tools: [ + computerUseTool({ + type: 'computer_20251124', + name: 'computer', + display_width_px: 1024, + display_height_px: 768, + }), + ], + })) { + // consume + } + + const [payload] = mocks.betaMessagesCreate.mock.calls[0]! + expect(payload.betas).toContain('computer-use-2025-11-24') + const [, assistant, results] = payload.messages + expect(assistant.content).toEqual([ + { + type: 'tool_use', + id: 'toolu_click', + name: 'computer', + input: { action: 'left_click', coordinate: [1, 2] }, + }, + ]) + expect(results.content[0]).not.toHaveProperty('toolset_name') + }) +}) diff --git a/packages/ai-anthropic/tests/tools-per-model-type-safety.test.ts b/packages/ai-anthropic/tests/tools-per-model-type-safety.test.ts index d46b14bf07..abfca22c0f 100644 --- a/packages/ai-anthropic/tests/tools-per-model-type-safety.test.ts +++ b/packages/ai-anthropic/tests/tools-per-model-type-safety.test.ts @@ -58,7 +58,7 @@ describe('Anthropic per-model tool gating', () => { type: 'code_execution_20250825', }), computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -83,7 +83,7 @@ describe('Anthropic per-model tool gating', () => { type: 'code_execution_20250825', }), computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -108,7 +108,7 @@ describe('Anthropic per-model tool gating', () => { type: 'code_execution_20250825', }), computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -133,7 +133,7 @@ describe('Anthropic per-model tool gating', () => { type: 'code_execution_20250825', }), computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -158,7 +158,7 @@ describe('Anthropic per-model tool gating', () => { type: 'code_execution_20250825', }), computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -172,7 +172,7 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-opus-5-5 accepts every provider tool except computer use', () => { + it('claude-opus-5-5 accepts the full tool superset with the computer toolset', () => { const adapter = anthropicText('claude-opus-5-5') typedTools(adapter, [ userTool, @@ -182,6 +182,7 @@ describe('Anthropic per-model tool gating', () => { name: 'code_execution', type: 'code_execution_20250825', }), + computerUseTool({ type: 'computer_toolset_20260801' }), bashTool({ name: 'bash', type: 'bash_20250124' }), textEditorTool({ type: 'text_editor_20250124', @@ -191,12 +192,12 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-opus-5-5 rejects computerUseTool (only computer_toolset_20260801 works)', () => { + it('claude-opus-5-5 rejects computer_20251124 (only the toolset works)', () => { const adapter = anthropicText('claude-opus-5-5') typedTools(adapter, [ - // @ts-expect-error - computer_20250124 returns a 400 on claude-opus-5-5 + // @ts-expect-error - computer_20251124 returns a 400 on claude-opus-5-5 computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -204,7 +205,7 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-sonnet-5-5 accepts every provider tool except computer use', () => { + it('claude-sonnet-5-5 accepts the full tool superset with the computer toolset', () => { const adapter = anthropicText('claude-sonnet-5-5') typedTools(adapter, [ userTool, @@ -214,6 +215,7 @@ describe('Anthropic per-model tool gating', () => { name: 'code_execution', type: 'code_execution_20250825', }), + computerUseTool({ type: 'computer_toolset_20260801' }), bashTool({ name: 'bash', type: 'bash_20250124' }), textEditorTool({ type: 'text_editor_20250124', @@ -223,12 +225,12 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-sonnet-5-5 rejects computerUseTool (only computer_toolset_20260801 works)', () => { + it('claude-sonnet-5-5 rejects computer_20251124 (only the toolset works)', () => { const adapter = anthropicText('claude-sonnet-5-5') typedTools(adapter, [ - // @ts-expect-error - computer_20250124 returns a 400 on claude-sonnet-5-5 + // @ts-expect-error - computer_20251124 returns a 400 on claude-sonnet-5-5 computerUseTool({ - type: 'computer_20250124', + type: 'computer_20251124', name: 'computer', display_width_px: 1024, display_height_px: 768, @@ -236,7 +238,7 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-haiku-5-5 accepts every provider tool except computer use', () => { + it('claude-haiku-5-5 accepts the full tool superset with the computer toolset', () => { const adapter = anthropicText('claude-haiku-5-5') typedTools(adapter, [ userTool, @@ -246,6 +248,7 @@ describe('Anthropic per-model tool gating', () => { name: 'code_execution', type: 'code_execution_20250825', }), + computerUseTool({ type: 'computer_toolset_20260801' }), bashTool({ name: 'bash', type: 'bash_20250124' }), textEditorTool({ type: 'text_editor_20250124', @@ -255,16 +258,57 @@ describe('Anthropic per-model tool gating', () => { ]) }) - it('claude-haiku-5-5 rejects computerUseTool (only computer_toolset_20260801 works)', () => { + it('claude-haiku-5-5 rejects computer_20251124 (only the toolset works)', () => { const adapter = anthropicText('claude-haiku-5-5') typedTools(adapter, [ - // @ts-expect-error - computer_20250124 returns a 400 on claude-haiku-5-5 + // @ts-expect-error - computer_20251124 returns a 400 on claude-haiku-5-5 + computerUseTool({ + type: 'computer_20251124', + name: 'computer', + display_width_px: 1024, + display_height_px: 768, + }), + ]) + }) + + it('claude-opus-4-6 rejects computer_20250124 and the computer toolset', () => { + const adapter = anthropicText('claude-opus-4-6') + typedTools(adapter, [ + // @ts-expect-error - claude-opus-4-6 takes only computer_20251124 computerUseTool({ type: 'computer_20250124', name: 'computer', display_width_px: 1024, display_height_px: 768, }), + // @ts-expect-error - claude-opus-4-6 takes only computer_20251124 + computerUseTool({ type: 'computer_toolset_20260801' }), + ]) + }) + + it('claude-haiku-4-5 accepts computer_20250124 and rejects computer_20251124', () => { + const adapter = anthropicText('claude-haiku-4-5') + typedTools(adapter, [ + computerUseTool({ + type: 'computer_20250124', + name: 'computer', + display_width_px: 1024, + display_height_px: 768, + }), + // @ts-expect-error - claude-haiku-4-5 takes only computer_20250124 + computerUseTool({ + type: 'computer_20251124', + name: 'computer', + display_width_px: 1024, + display_height_px: 768, + }), + ]) + }) + + it('claude-opus-5 accepts the computer toolset', () => { + const adapter = anthropicText('claude-opus-5') + typedTools(adapter, [ + computerUseTool({ type: 'computer_toolset_20260801' }), ]) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bb0a8207db..7f90ae25c0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1065,8 +1065,8 @@ importers: examples/ts-react-chat: dependencies: '@anthropic-ai/sdk': - specifier: ^0.97.1 - version: 0.97.1(zod@4.2.1) + specifier: ^0.135.0 + version: 0.135.0(zod@4.2.1) '@elevenlabs/elevenlabs-js': specifier: ^2.68.0 version: 2.68.0 @@ -2120,15 +2120,12 @@ importers: packages/ai-anthropic: dependencies: '@anthropic-ai/sdk': - specifier: ^0.97.1 - version: 0.97.1(zod@4.2.1) + specifier: ^0.135.0 + version: 0.135.0(zod@4.2.1) '@tanstack/ai-utils': specifier: workspace:^ version: link:../ai-utils devDependencies: - '@anthropic-ai/sdk-v112': - specifier: npm:@anthropic-ai/sdk@0.112.4 - version: '@anthropic-ai/sdk@0.112.4(zod@4.2.1)' '@anthropic-ai/vertex-sdk': specifier: ^0.19.0 version: 0.19.0(zod@4.2.1) @@ -3456,7 +3453,7 @@ importers: devDependencies: '@google/genai': specifier: ^2.10.0 - version: 2.10.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6)) + version: 2.10.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.6.5)) '@tanstack/ai': specifier: workspace:* version: link:../ai @@ -3640,8 +3637,8 @@ importers: testing/e2e: dependencies: '@anthropic-ai/sdk': - specifier: ^0.97.1 - version: 0.97.1(zod@4.3.6) + specifier: ^0.135.0 + version: 0.135.0(zod@4.3.6) '@copilotkit/aimock': specifier: ^1.34.0 version: 1.34.0(vitest@4.1.11(@opentelemetry/api@1.9.1)(@types/node@24.10.3)(@vitest/coverage-v8@4.1.10(vitest@4.1.10))(happy-dom@20.11.2)(jsdom@27.4.0(@noble/hashes@2.3.0))(vite@8.2.1(@types/node@24.10.3)(esbuild@0.28.1)(jiti@2.7.0)(less@4.6.6)(sass@1.101.0)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0))) @@ -4155,17 +4152,8 @@ packages: '@angular/animations': optional: true - '@anthropic-ai/sdk@0.112.4': - resolution: {integrity: sha512-7eXJJnrmBI5GMC6drrCiSkycVsT7crRZX3qv5HusLSm+qiILjmtqP7gf+UiT7ASu/7Gdj+Zfl4f2haV8wATKUg==} - hasBin: true - peerDependencies: - zod: ^3.25.0 || ^4.0.0 - peerDependenciesMeta: - zod: - optional: true - - '@anthropic-ai/sdk@0.97.1': - resolution: {integrity: sha512-wOf7AUeJPitcVpvKO4UMu63mWH5SaVipkGd7OOQJt/G6VYGlV8D2Gp9dLxOrttDJh/9gqPqdaBwDGcBevumeAg==} + '@anthropic-ai/sdk@0.135.0': + resolution: {integrity: sha512-WWrdL57JhU5KoynpTjt9GXK6k7ud9nPw2uw/Trn+oBXh6s0KpjhAccWZm2Qxw08ARYG1UDQj5iC+JO42j1rt/g==} hasBin: true peerDependencies: zod: ^3.25.0 || ^4.0.0 @@ -19221,22 +19209,17 @@ snapshots: '@angular/core': 21.2.20(@angular/compiler@21.2.20)(rxjs@7.8.2)(zone.js@0.15.1) tslib: 2.8.1 - '@anthropic-ai/sdk@0.112.4(zod@4.2.1)': - dependencies: - json-schema-to-ts: 3.1.1 - standardwebhooks: 1.0.0 - optionalDependencies: - zod: 4.2.1 - - '@anthropic-ai/sdk@0.97.1(zod@4.2.1)': + '@anthropic-ai/sdk@0.135.0(zod@4.2.1)': dependencies: + '@opentelemetry/api': 1.9.1 json-schema-to-ts: 3.1.1 standardwebhooks: 1.0.0 optionalDependencies: zod: 4.2.1 - '@anthropic-ai/sdk@0.97.1(zod@4.3.6)': + '@anthropic-ai/sdk@0.135.0(zod@4.3.6)': dependencies: + '@opentelemetry/api': 1.9.1 json-schema-to-ts: 3.1.1 standardwebhooks: 1.0.0 optionalDependencies: @@ -19244,7 +19227,7 @@ snapshots: '@anthropic-ai/vertex-sdk@0.19.0(zod@4.2.1)': dependencies: - '@anthropic-ai/sdk': 0.112.4(zod@4.2.1) + '@anthropic-ai/sdk': 0.135.0(zod@4.2.1) google-auth-library: 10.5.0 transitivePeerDependencies: - supports-color diff --git a/scripts/model-sync/provider-supports.test.ts b/scripts/model-sync/provider-supports.test.ts index 355c75eae4..c311313371 100644 --- a/scripts/model-sync/provider-supports.test.ts +++ b/scripts/model-sync/provider-supports.test.ts @@ -38,7 +38,8 @@ describe('buildProviderSupportsBody', () => { expect(body).toContain("'web_search'") expect(body).toContain("'web_fetch'") expect(body).toContain("'code_execution'") - expect(body).toContain("'computer_use'") + expect(body).toContain("'computer_toolset'") + expect(body).not.toContain("'computer_use'") expect(body).toContain('extended_thinking: true') expect(body).toContain('priority_tier: true') expect(body).not.toContain('adaptive_thinking') diff --git a/scripts/model-sync/provider-supports.ts b/scripts/model-sync/provider-supports.ts index d27c86b0af..127b9c2c8d 100644 --- a/scripts/model-sync/provider-supports.ts +++ b/scripts/model-sync/provider-supports.ts @@ -17,11 +17,12 @@ export interface ProviderSupportsInput { reasoningMandatory?: boolean } +// New Claude models take computer use only through the computer toolset. const ANTHROPIC_SERVER_TOOLS = [ 'web_search', 'web_fetch', 'code_execution', - 'computer_use', + 'computer_toolset', 'bash', 'text_editor', 'memory', diff --git a/testing/e2e/package.json b/testing/e2e/package.json index 7001fbcc32..78f353b467 100644 --- a/testing/e2e/package.json +++ b/testing/e2e/package.json @@ -12,7 +12,7 @@ "postinstall": "playwright install chromium" }, "dependencies": { - "@anthropic-ai/sdk": "^0.97.1", + "@anthropic-ai/sdk": "^0.135.0", "@copilotkit/aimock": "^1.34.0", "@modelcontextprotocol/sdk": "^1.29.0", "@openrouter/sdk": "0.13.20", From 575855a7ebc50142af0511347bb8a98c96bb7894 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 19 Oct 2026 18:53:20 +0200 Subject: [PATCH 2/5] test(e2e): check the computer toolset loop on the wire Also add the changeset and format the changed files. --- .changeset/anthropic-computer-toolset.md | 10 ++ packages/ai-anthropic/src/adapters/text.ts | 6 +- packages/ai-anthropic/src/prompt-cache.ts | 3 +- .../tests/computer-toolset.test.ts | 5 +- testing/e2e/src/routeTree.gen.ts | 22 +++ .../api.anthropic-computer-toolset-wire.ts | 162 ++++++++++++++++++ .../anthropic-computer-toolset-wire.spec.ts | 97 +++++++++++ 7 files changed, 302 insertions(+), 3 deletions(-) create mode 100644 .changeset/anthropic-computer-toolset.md create mode 100644 testing/e2e/src/routes/api.anthropic-computer-toolset-wire.ts create mode 100644 testing/e2e/tests/anthropic-computer-toolset-wire.spec.ts diff --git a/.changeset/anthropic-computer-toolset.md b/.changeset/anthropic-computer-toolset.md new file mode 100644 index 0000000000..94212459f0 --- /dev/null +++ b/.changeset/anthropic-computer-toolset.md @@ -0,0 +1,10 @@ +--- +'@tanstack/ai-anthropic': minor +--- + +Run computer use on every Claude model that supports it. + +- `computerUseTool()` takes `computer_toolset_20260801`, the only computer use version of Claude Opus 5.5, Sonnet 5.5, and Haiku 5.5. The model calls each action as its own tool. The adapter runs each call as one `computer` call with the action in `input.action`, and sends it back to Claude in the form that Claude sent. So one `execute` works for every version. +- `computerUseTool()` takes `computer_20251124`, with its `computer-use-2025-11-24` beta header. +- Each model now takes only the computer use versions that it supports. For example, `computer_20250124` on `claude-opus-4-6` is a type error, because Anthropic rejects that pair. +- `@anthropic-ai/sdk` is now `^0.135.0`. diff --git a/packages/ai-anthropic/src/adapters/text.ts b/packages/ai-anthropic/src/adapters/text.ts index 1e5f32c3c6..4c906aedef 100644 --- a/packages/ai-anthropic/src/adapters/text.ts +++ b/packages/ai-anthropic/src/adapters/text.ts @@ -1268,7 +1268,11 @@ export class AnthropicTextAdapter< if (block.type === 'thinking') { this.appendThinkingBlocks(contentBlocks, [block.thinking]) } else if (block.type === 'tool-call') { - this.appendToolCallBlocks(contentBlocks, block.toolCall, toolsetCalls) + this.appendToolCallBlocks( + contentBlocks, + block.toolCall, + toolsetCalls, + ) } else { contentBlocks.push({ type: 'text', text: block.text }) } diff --git a/packages/ai-anthropic/src/prompt-cache.ts b/packages/ai-anthropic/src/prompt-cache.ts index dee08e3bb7..6874d4cb3a 100644 --- a/packages/ai-anthropic/src/prompt-cache.ts +++ b/packages/ai-anthropic/src/prompt-cache.ts @@ -155,7 +155,8 @@ export function applyAnthropicPromptCache( request.tools?.findIndex( // The computer toolset entry has no name. (tool) => - 'name' in tool && tool.name === ANTHROPIC_DEFERRED_TOOL_PLACEHOLDER_NAME, + 'name' in tool && + tool.name === ANTHROPIC_DEFERRED_TOOL_PLACEHOLDER_NAME, ) ?? -1 const tools = request.tools?.map((tool, index, all) => index === (placeholder > 0 ? placeholder - 1 : all.length - 1) diff --git a/packages/ai-anthropic/tests/computer-toolset.test.ts b/packages/ai-anthropic/tests/computer-toolset.test.ts index 9edfbe1fc9..1f952f1507 100644 --- a/packages/ai-anthropic/tests/computer-toolset.test.ts +++ b/packages/ai-anthropic/tests/computer-toolset.test.ts @@ -94,7 +94,10 @@ describe('anthropic computer toolset', () => { const [payload] = mocks.betaMessagesCreate.mock.calls[0]! // The prompt cache marker can also sit on the entry. expect(payload.tools).toMatchObject([ - { type: 'computer_toolset_20260801', configs: { zoom: { enabled: false } } }, + { + type: 'computer_toolset_20260801', + configs: { zoom: { enabled: false } }, + }, ]) expect(payload.tools[0]).not.toHaveProperty('name') expect(payload.tools[0]).not.toHaveProperty('__kind') diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index 70bc3a477a..3d2109a65d 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -54,6 +54,7 @@ import { Route as ApiActivityTestRouteImport } from './routes/api.activity-test' import { Route as ApiAdapterInputModalitiesRouteImport } from './routes/api.adapter-input-modalities' import { Route as ApiAnthropicAuthWireRouteImport } from './routes/api.anthropic-auth-wire' import { Route as ApiAnthropicBugTestRouteImport } from './routes/api.anthropic-bug-test' +import { Route as ApiAnthropicComputerToolsetWireRouteImport } from './routes/api.anthropic-computer-toolset-wire' import { Route as ApiAnthropicComputerUseWireRouteImport } from './routes/api.anthropic-computer-use-wire' import { Route as ApiAnthropicHaiku55WireRouteImport } from './routes/api.anthropic-haiku-5-5-wire' import { Route as ApiAnthropicMaxTokensUsageRouteImport } from './routes/api.anthropic-max-tokens-usage' @@ -414,6 +415,12 @@ const ApiAnthropicBugTestRoute = ApiAnthropicBugTestRouteImport.update({ path: '/api/anthropic-bug-test', getParentRoute: () => rootRouteImport, } as any) +const ApiAnthropicComputerToolsetWireRoute = + ApiAnthropicComputerToolsetWireRouteImport.update({ + id: '/api/anthropic-computer-toolset-wire', + path: '/api/anthropic-computer-toolset-wire', + getParentRoute: () => rootRouteImport, + } as any) const ApiAnthropicComputerUseWireRoute = ApiAnthropicComputerUseWireRouteImport.update({ id: '/api/anthropic-computer-use-wire', @@ -1150,6 +1157,7 @@ export interface FileRoutesByFullPath { '/api/adapter-input-modalities': typeof ApiAdapterInputModalitiesRoute '/api/anthropic-auth-wire': typeof ApiAnthropicAuthWireRoute '/api/anthropic-bug-test': typeof ApiAnthropicBugTestRoute + '/api/anthropic-computer-toolset-wire': typeof ApiAnthropicComputerToolsetWireRoute '/api/anthropic-computer-use-wire': typeof ApiAnthropicComputerUseWireRoute '/api/anthropic-haiku-5-5-wire': typeof ApiAnthropicHaiku55WireRoute '/api/anthropic-max-tokens-usage': typeof ApiAnthropicMaxTokensUsageRoute @@ -1327,6 +1335,7 @@ export interface FileRoutesByTo { '/api/adapter-input-modalities': typeof ApiAdapterInputModalitiesRoute '/api/anthropic-auth-wire': typeof ApiAnthropicAuthWireRoute '/api/anthropic-bug-test': typeof ApiAnthropicBugTestRoute + '/api/anthropic-computer-toolset-wire': typeof ApiAnthropicComputerToolsetWireRoute '/api/anthropic-computer-use-wire': typeof ApiAnthropicComputerUseWireRoute '/api/anthropic-haiku-5-5-wire': typeof ApiAnthropicHaiku55WireRoute '/api/anthropic-max-tokens-usage': typeof ApiAnthropicMaxTokensUsageRoute @@ -1505,6 +1514,7 @@ export interface FileRoutesById { '/api/adapter-input-modalities': typeof ApiAdapterInputModalitiesRoute '/api/anthropic-auth-wire': typeof ApiAnthropicAuthWireRoute '/api/anthropic-bug-test': typeof ApiAnthropicBugTestRoute + '/api/anthropic-computer-toolset-wire': typeof ApiAnthropicComputerToolsetWireRoute '/api/anthropic-computer-use-wire': typeof ApiAnthropicComputerUseWireRoute '/api/anthropic-haiku-5-5-wire': typeof ApiAnthropicHaiku55WireRoute '/api/anthropic-max-tokens-usage': typeof ApiAnthropicMaxTokensUsageRoute @@ -1684,6 +1694,7 @@ export interface FileRouteTypes { | '/api/adapter-input-modalities' | '/api/anthropic-auth-wire' | '/api/anthropic-bug-test' + | '/api/anthropic-computer-toolset-wire' | '/api/anthropic-computer-use-wire' | '/api/anthropic-haiku-5-5-wire' | '/api/anthropic-max-tokens-usage' @@ -1861,6 +1872,7 @@ export interface FileRouteTypes { | '/api/adapter-input-modalities' | '/api/anthropic-auth-wire' | '/api/anthropic-bug-test' + | '/api/anthropic-computer-toolset-wire' | '/api/anthropic-computer-use-wire' | '/api/anthropic-haiku-5-5-wire' | '/api/anthropic-max-tokens-usage' @@ -2038,6 +2050,7 @@ export interface FileRouteTypes { | '/api/adapter-input-modalities' | '/api/anthropic-auth-wire' | '/api/anthropic-bug-test' + | '/api/anthropic-computer-toolset-wire' | '/api/anthropic-computer-use-wire' | '/api/anthropic-haiku-5-5-wire' | '/api/anthropic-max-tokens-usage' @@ -2216,6 +2229,7 @@ export interface RootRouteChildren { ApiAdapterInputModalitiesRoute: typeof ApiAdapterInputModalitiesRoute ApiAnthropicAuthWireRoute: typeof ApiAnthropicAuthWireRoute ApiAnthropicBugTestRoute: typeof ApiAnthropicBugTestRoute + ApiAnthropicComputerToolsetWireRoute: typeof ApiAnthropicComputerToolsetWireRoute ApiAnthropicComputerUseWireRoute: typeof ApiAnthropicComputerUseWireRoute ApiAnthropicHaiku55WireRoute: typeof ApiAnthropicHaiku55WireRoute ApiAnthropicMaxTokensUsageRoute: typeof ApiAnthropicMaxTokensUsageRoute @@ -2661,6 +2675,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiAnthropicBugTestRouteImport parentRoute: typeof rootRouteImport } + '/api/anthropic-computer-toolset-wire': { + id: '/api/anthropic-computer-toolset-wire' + path: '/api/anthropic-computer-toolset-wire' + fullPath: '/api/anthropic-computer-toolset-wire' + preLoaderRoute: typeof ApiAnthropicComputerToolsetWireRouteImport + parentRoute: typeof rootRouteImport + } '/api/anthropic-computer-use-wire': { id: '/api/anthropic-computer-use-wire' path: '/api/anthropic-computer-use-wire' @@ -3677,6 +3698,7 @@ const rootRouteChildren: RootRouteChildren = { ApiAdapterInputModalitiesRoute: ApiAdapterInputModalitiesRoute, ApiAnthropicAuthWireRoute: ApiAnthropicAuthWireRoute, ApiAnthropicBugTestRoute: ApiAnthropicBugTestRoute, + ApiAnthropicComputerToolsetWireRoute: ApiAnthropicComputerToolsetWireRoute, ApiAnthropicComputerUseWireRoute: ApiAnthropicComputerUseWireRoute, ApiAnthropicHaiku55WireRoute: ApiAnthropicHaiku55WireRoute, ApiAnthropicMaxTokensUsageRoute: ApiAnthropicMaxTokensUsageRoute, diff --git a/testing/e2e/src/routes/api.anthropic-computer-toolset-wire.ts b/testing/e2e/src/routes/api.anthropic-computer-toolset-wire.ts new file mode 100644 index 0000000000..0fbba1e94b --- /dev/null +++ b/testing/e2e/src/routes/api.anthropic-computer-toolset-wire.ts @@ -0,0 +1,162 @@ +import { createFileRoute } from '@tanstack/react-router' +import { chat, createChatOptions } from '@tanstack/ai' +import { createAnthropicChat } from '@tanstack/ai-anthropic' +import { computerUseTool } from '@tanstack/ai-anthropic/tools' + +const DUMMY_KEY = 'sk-ant-e2e-test-dummy-key' + +/** + * Drives one computer toolset loop on `claude-opus-5-5`. A custom `fetch` + * captures each outgoing request. The first answer has two toolset calls + * (`left_click` and `screenshot`). The second answer ends the turn. + * + * The companion spec reads the captured requests and the inputs that + * `execute` got, so it can check the toolset wire format without an API key. + */ +function sseStream(events: Array<{ type: string }>) { + const encoder = new TextEncoder() + return new ReadableStream({ + start(controller) { + for (const event of events) { + controller.enqueue( + encoder.encode( + `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`, + ), + ) + } + controller.close() + }, + }) +} + +const messageStart = { + type: 'message_start', + message: { + id: 'msg_computer_toolset_wire', + type: 'message', + role: 'assistant', + content: [], + model: 'claude-opus-5-5', + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 5, output_tokens: 0 }, + }, +} + +const end = (stopReason: string) => [ + { + type: 'message_delta', + delta: { stop_reason: stopReason, stop_sequence: null }, + usage: { output_tokens: 1 }, + }, + { type: 'message_stop' }, +] + +const toolsetCalls = [ + messageStart, + { + type: 'content_block_start', + index: 0, + content_block: { + type: 'tool_use', + id: 'toolu_click', + name: 'left_click', + toolset_name: 'computer', + input: {}, + }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { + type: 'input_json_delta', + partial_json: '{"coordinate":[500,300]}', + }, + }, + { type: 'content_block_stop', index: 0 }, + { + type: 'content_block_start', + index: 1, + content_block: { + type: 'tool_use', + id: 'toolu_shot', + name: 'screenshot', + toolset_name: 'computer', + input: {}, + }, + }, + { type: 'content_block_stop', index: 1 }, + ...end('tool_use'), +] + +export const Route = createFileRoute('/api/anthropic-computer-toolset-wire')({ + server: { + handlers: { + POST: async () => { + const requests: Array<{ anthropicBeta: string | null; body: unknown }> = + [] + const capturingFetch: typeof fetch = async (input, init) => { + const req = + input instanceof Request ? input : new Request(input, init) + const rawBody = await req.text() + requests.push({ + anthropicBeta: req.headers.get('anthropic-beta'), + body: rawBody ? JSON.parse(rawBody) : null, + }) + const events = + requests.length === 1 + ? toolsetCalls + : [messageStart, ...end('end_turn')] + return new Response(sseStream(events), { + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + }) + } + + const calls: Array = [] + const computer = { + ...computerUseTool({ + type: 'computer_toolset_20260801', + configs: { zoom: { enabled: false } }, + }), + execute: (input: { action: string }) => { + calls.push(input) + if (input.action !== 'screenshot') return 'OK' + return [ + { + type: 'image' as const, + source: { + type: 'data' as const, + value: 'AAAA', + mimeType: 'image/png', + }, + }, + ] + }, + } + + const adapter = createAnthropicChat('claude-opus-5-5', DUMMY_KEY, { + fetch: capturingFetch, + }) + + try { + for await (const _ of chat({ + ...createChatOptions({ adapter }), + messages: [{ role: 'user', content: 'Click the button.' }], + tools: [computer], + toolExecution: 'sequential', + })) { + // Drain the stream. + } + } catch (error) { + return Response.json({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }) + } + + return Response.json({ ok: true, calls, requests }) + }, + }, + }, +}) diff --git a/testing/e2e/tests/anthropic-computer-toolset-wire.spec.ts b/testing/e2e/tests/anthropic-computer-toolset-wire.spec.ts new file mode 100644 index 0000000000..3a982a686b --- /dev/null +++ b/testing/e2e/tests/anthropic-computer-toolset-wire.spec.ts @@ -0,0 +1,97 @@ +import { test, expect } from './fixtures' +import type { APIRequestContext } from '@playwright/test' + +/** + * Wire format of the Anthropic computer toolset (`computer_toolset_20260801`). + * + * The model calls each action as its own tool (`left_click`, `screenshot`) + * with `toolset_name: 'computer'`. The adapter runs each call as one + * `computer` call with the action in `input.action`. On the next request it + * sends each call back in the form that the model sent, and each result + * names the toolset. + * + * `/api/anthropic-computer-toolset-wire` runs one loop against a custom + * `fetch`, and returns the captured requests and the inputs that `execute` + * got. + */ +interface WireResult { + ok: boolean + error?: string + calls: Array + requests: Array<{ + anthropicBeta: string | null + body: { + tools: Array> + messages: Array<{ role: string; content: Array> }> + } + }> +} + +async function runLoop(request: APIRequestContext) { + const res = await request.post('/api/anthropic-computer-toolset-wire') + expect(res.ok()).toBe(true) + const result = (await res.json()) as WireResult + if (!result.ok) { + throw new Error(`Route failed: ${result.error}`) + } + return result +} + +test.describe('anthropic — computer toolset wire format', () => { + test('sends the toolset entry with no name and no computer use beta', async ({ + request, + }) => { + const { requests } = await runLoop(request) + const [first] = requests + expect(first?.body.tools[0]).toMatchObject({ + type: 'computer_toolset_20260801', + configs: { zoom: { enabled: false } }, + }) + expect(first?.body.tools[0]).not.toHaveProperty('name') + expect(first?.anthropicBeta ?? '').not.toContain('computer-use') + }) + + test('runs each toolset call as one computer call', async ({ request }) => { + const { calls } = await runLoop(request) + expect(calls).toEqual([ + { action: 'left_click', coordinate: [500, 300] }, + { action: 'screenshot' }, + ]) + }) + + test('sends the calls back as toolset calls, and the results name the toolset', async ({ + request, + }) => { + const { requests } = await runLoop(request) + const second = requests[1] + expect(second).toBeDefined() + const assistant = second?.body.messages.find((m) => m.role === 'assistant') + expect(assistant?.content).toMatchObject([ + { + type: 'tool_use', + id: 'toolu_click', + name: 'left_click', + toolset_name: 'computer', + input: { coordinate: [500, 300] }, + }, + { + type: 'tool_use', + id: 'toolu_shot', + name: 'screenshot', + toolset_name: 'computer', + input: {}, + }, + ]) + const results = second?.body.messages + .flatMap((m) => m.content) + .filter((block) => block['type'] === 'tool_result') + expect(results).toMatchObject([ + { tool_use_id: 'toolu_click', toolset_name: 'computer', content: 'OK' }, + { + tool_use_id: 'toolu_shot', + toolset_name: 'computer', + content: [{ type: 'image', source: { type: 'base64', data: 'AAAA' } }], + }, + ]) + }) +}) From 854320e01a956d25e671c03b296716912758bf5d Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 19 Oct 2026 18:54:56 +0200 Subject: [PATCH 3/5] feat(ai-anthropic): export the computer use config and tool types --- packages/ai-anthropic/src/tools/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/ai-anthropic/src/tools/index.ts b/packages/ai-anthropic/src/tools/index.ts index 9ad77c0431..bc6d3d9354 100644 --- a/packages/ai-anthropic/src/tools/index.ts +++ b/packages/ai-anthropic/src/tools/index.ts @@ -26,7 +26,11 @@ export { } from './code-execution-tool' export { computerUseTool, + type AnthropicComputerToolset, + type AnthropicComputerUse20251124Tool, type AnthropicComputerUseTool, + type ComputerToolsetConfig, + type ComputerUseSingleToolConfig, type ComputerUseToolConfig, type ComputerUseTool, } from './computer-use-tool' From 0eec128a8e2aa7a753f61256499a37903ffceca5 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 19 Oct 2026 19:35:12 +0200 Subject: [PATCH 4/5] fix(ai-anthropic): keep the vertex genai lock entry and expect one files beta header The SDK install moved the @google/genai of ai-vertex to a copy with zod 4.6.5, so its test mock missed the copy that ai-gemini loads. SDK 0.135.0 sends the files-api beta header once, where 0.97.1 sent it twice. --- packages/ai-anthropic/tests/auth-parity.test.ts | 2 +- pnpm-lock.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ai-anthropic/tests/auth-parity.test.ts b/packages/ai-anthropic/tests/auth-parity.test.ts index da1f951822..25af71ff17 100644 --- a/packages/ai-anthropic/tests/auth-parity.test.ts +++ b/packages/ai-anthropic/tests/auth-parity.test.ts @@ -473,7 +473,7 @@ describe('files factories', () => { authorization: null, xApiKey: 'files-key', xApp: null, - anthropicBeta: 'files-api-2025-04-14,files-api-2025-04-14', + anthropicBeta: 'files-api-2025-04-14', }) expect(file).toEqual(FILE) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7f90ae25c0..1e072c4214 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3453,7 +3453,7 @@ importers: devDependencies: '@google/genai': specifier: ^2.10.0 - version: 2.10.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.6.5)) + version: 2.10.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6)) '@tanstack/ai': specifier: workspace:* version: link:../ai From 7fac2ac4ca5e884868448bb0c57e192f1f9e845d Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Tue, 20 Oct 2026 10:21:57 +0200 Subject: [PATCH 5/5] docs(ai-anthropic): drop the as const in the computer use sample --- docs/adapters/anthropic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/adapters/anthropic.md b/docs/adapters/anthropic.md index 579f998c51..f603d0f027 100644 --- a/docs/adapters/anthropic.md +++ b/docs/adapters/anthropic.md @@ -655,8 +655,8 @@ const computer = { const value = await takeScreenshot(); return [ { - type: "image" as const, - source: { type: "data" as const, value, mimeType: "image/png" }, + type: "image", + source: { type: "data", value, mimeType: "image/png" }, }, ]; }