diff --git a/.agent/domain-map.json b/.agent/domain-map.json new file mode 100644 index 0000000000..db20795f76 --- /dev/null +++ b/.agent/domain-map.json @@ -0,0 +1,161 @@ +{ + "version": 1, + "domains": [ + { + "id": "core", + "owns": ["packages/ai", "packages/ai-utils", "packages/ai-event-client"], + "talksTo": [] + }, + { + "id": "provider-adapters", + "owns": [ + "packages/openai-base", + "packages/ai-openai", + "packages/ai-anthropic", + "packages/ai-gemini", + "packages/ai-ollama", + "packages/ai-bedrock", + "packages/ai-byteplus", + "packages/ai-cloudflare", + "packages/ai-cohere", + "packages/ai-elevenlabs", + "packages/ai-fal", + "packages/ai-grok", + "packages/ai-groq", + "packages/ai-llmgateway", + "packages/ai-lovable", + "packages/ai-mistral", + "packages/ai-ollaya", + "packages/ai-openrouter", + "packages/ai-perplexity", + "packages/ai-reactor", + "packages/ai-typesafe", + "packages/ai-vercel-gateway", + "packages/ai-vertex", + "packages/ai-worldlabs" + ], + "talksTo": ["core"] + }, + { + "id": "harness", + "owns": [ + "packages/ai-acp", + "packages/ai-claude-code", + "packages/ai-codex", + "packages/ai-opencode", + "packages/ai-grok-build" + ], + "talksTo": ["core", "sandbox"] + }, + { + "id": "client", + "owns": [ + "packages/ai-client", + "packages/ai-react", + "packages/ai-solid", + "packages/ai-vue", + "packages/ai-svelte", + "packages/ai-angular", + "packages/ai-preact", + "packages/ai-remix", + "packages/ai-octane", + "packages/ai-react-ui", + "packages/ai-solid-ui", + "packages/ai-vue-ui" + ], + "talksTo": ["core"] + }, + { + "id": "code-mode", + "owns": [ + "packages/ai-code-mode", + "packages/ai-code-mode-snippets", + "packages/ai-isolate-cloudflare", + "packages/ai-isolate-daytona", + "packages/ai-isolate-e2b", + "packages/ai-isolate-node", + "packages/ai-isolate-quickjs", + "packages/ai-isolate-quickjs-bun" + ], + "talksTo": ["core"] + }, + { + "id": "sandbox", + "owns": [ + "packages/ai-sandbox", + "packages/ai-sandbox-blaxel", + "packages/ai-sandbox-boxd", + "packages/ai-sandbox-cloudflare", + "packages/ai-sandbox-daytona", + "packages/ai-sandbox-docker", + "packages/ai-sandbox-e2b", + "packages/ai-sandbox-local-process", + "packages/ai-sandbox-railway", + "packages/ai-sandbox-sprites", + "packages/ai-sandbox-upstash-box", + "packages/ai-sandbox-vercel" + ], + "talksTo": ["core", "middleware", "persistence"] + }, + { + "id": "middleware", + "owns": [ + "packages/ai-compaction", + "packages/ai-memory", + "packages/ai-skills", + "packages/ai-mcp" + ], + "talksTo": ["core"] + }, + { + "id": "persistence", + "owns": ["packages/ai-persistence", "packages/ai-durable-stream"], + "talksTo": ["core"] + }, + { + "id": "devtools", + "owns": [ + "packages/ai-devtools", + "packages/react-ai-devtools", + "packages/solid-ai-devtools", + "packages/svelte-ai-devtools", + "packages/preact-ai-devtools" + ], + "talksTo": ["core"] + }, + { + "id": "computer-use", + "owns": [ + "packages/ai-computer-use", + "packages/ai-computer-playwright", + "packages/ai-computer-sandbox" + ], + "talksTo": ["core"] + }, + { + "id": "e2e", + "owns": ["testing/e2e"], + "talksTo": [ + "client", + "code-mode", + "computer-use", + "core", + "harness", + "middleware", + "persistence", + "provider-adapters", + "sandbox" + ] + }, + { + "id": "docs", + "owns": ["docs"], + "talksTo": [] + }, + { + "id": "model-sync", + "owns": ["scripts/model-sync"], + "talksTo": ["provider-adapters"] + } + ] +} diff --git a/.changeset/ai-computer-use.md b/.changeset/ai-computer-use.md new file mode 100644 index 0000000000..931e6ceb0b --- /dev/null +++ b/.changeset/ai-computer-use.md @@ -0,0 +1,45 @@ +--- +'@tanstack/ai-computer-use': minor +'@tanstack/ai-computer-playwright': minor +'@tanstack/ai-computer-sandbox': minor +'@tanstack/ai-computer-local': minor +'@tanstack/ai': minor +'@tanstack/openai-base': minor +'@tanstack/ai-openai': minor +'@tanstack/ai-anthropic': minor +--- + +Add computer use that runs the same code on OpenAI and on Anthropic. A driver controls the machine. The Playwright and sandbox drivers give each thread its own machine. + +New packages: + +- `@tanstack/ai-computer-use` adds `createComputerUse()`. It returns `tools`, a middleware, a safety-check interrupt, and `close`. Each `threadId` gets its own machine, action queue, and approvals. A thread that stays idle for `idleTimeout` (10 minutes by default) closes. +- `@tanstack/ai-computer-playwright` adds `playwrightComputer()`, which opens one browser page for each thread. `createPlaywrightComputerDriver()` controls one page that you give it. Both add a `navigate` tool that opens `http:` and `https:` URLs. +- `@tanstack/ai-computer-sandbox` adds `sandboxComputer()`, which creates one Linux desktop sandbox for each thread from any `@tanstack/ai-sandbox` provider, like `e2bSandbox({ template: 'desktop' })`. `createSandboxComputerDriver()` controls one sandbox that you give it. +- `@tanstack/ai-computer-local` adds `localComputer()`, which lets the model use the mouse, keyboard, and main screen of the machine that runs `chat()`. It works on Windows, macOS, and Linux with `@jitsi/robotjs`. + +Changes in existing packages: + +- `@tanstack/ai` adds `threadId` to `ToolExecutionContext`. It also adds the computer tool metadata reader in `@tanstack/ai/adapter-internals`. +- `@tanstack/ai-anthropic` and `@tanstack/openai-base` send the generic computer tool to the model as the native computer tool of the provider. +- `@tanstack/ai-openai` marks the models that support the `computer` tool. + +```ts +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: 'https://tanstack.com' }), +}) + +chat({ + adapter, + messages, + threadId, + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], +}) + +await computerUse.close(threadId) // one thread +await computerUse.close() // every thread, and the browser +``` + +The `ai-core` agent skill routes computer use to `@tanstack/ai-computer-use` and its three drivers. diff --git a/.changeset/anthropic-computer-toolset.md b/.changeset/anthropic-computer-toolset.md new file mode 100644 index 0000000000..94212459f0 --- /dev/null +++ b/.changeset/anthropic-computer-toolset.md @@ -0,0 +1,10 @@ +--- +'@tanstack/ai-anthropic': minor +--- + +Run computer use on every Claude model that supports it. + +- `computerUseTool()` takes `computer_toolset_20260801`, the only computer use version of Claude Opus 5.5, Sonnet 5.5, and Haiku 5.5. The model calls each action as its own tool. The adapter runs each call as one `computer` call with the action in `input.action`, and sends it back to Claude in the form that Claude sent. So one `execute` works for every version. +- `computerUseTool()` takes `computer_20251124`, with its `computer-use-2025-11-24` beta header. +- Each model now takes only the computer use versions that it supports. For example, `computer_20250124` on `claude-opus-4-6` is a type error, because Anthropic rejects that pair. +- `@anthropic-ai/sdk` is now `^0.135.0`. diff --git a/.changeset/anthropic-computer-use-beta.md b/.changeset/anthropic-computer-use-beta.md new file mode 100644 index 0000000000..780f25cd02 --- /dev/null +++ b/.changeset/anthropic-computer-use-beta.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-anthropic': patch +--- + +Send the `anthropic-beta` header that `computerUseTool()` needs: `computer-use-2025-01-24` for `computer_20250124`, and `computer-use-2024-10-22` for `computer_20241022`. Before, a request with the tool had no computer use beta, so Anthropic rejected it. diff --git a/.changeset/daytona-spawn-unhandled-rejection.md b/.changeset/daytona-spawn-unhandled-rejection.md new file mode 100644 index 0000000000..af8180ad8f --- /dev/null +++ b/.changeset/daytona-spawn-unhandled-rejection.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-sandbox-daytona': patch +--- + +A background process from `spawn()` no longer crashes Node with an unhandled rejection when its status poll fails, for example after `destroy()`. `wait()` still rejects with the error. diff --git a/.changeset/openai-computer-call-loop.md b/.changeset/openai-computer-call-loop.md new file mode 100644 index 0000000000..afce6105da --- /dev/null +++ b/.changeset/openai-computer-call-loop.md @@ -0,0 +1,9 @@ +--- +'@tanstack/openai-base': patch +--- + +The computer use loop of `computerUseTool()` runs. A `computer_call` from the model is now a tool call to `computer_use_preview`, with the `action` and the `pending_safety_checks` as its input. The tool result goes back as `computer_call_output`, with the first image of the result as the screenshot. When the call does not fail, the output acknowledges the safety checks of the call. + +A failed computer call sends the newest screenshot of the conversation, because the failed call has no screenshot of its own. A computer call output has no text, so the reason for the failure goes to the model in a user message after the call outputs. + +A computer tool result with no image fails the run with an error. Before, the adapter sent an empty screenshot. diff --git a/.changeset/openai-computer-tool.md b/.changeset/openai-computer-tool.md new file mode 100644 index 0000000000..40fbc8cab4 --- /dev/null +++ b/.changeset/openai-computer-tool.md @@ -0,0 +1,6 @@ +--- +'@tanstack/openai-base': minor +'@tanstack/ai-openai': minor +--- + +`computerUseTool({ type: 'computer' })` adds the computer tool of the newer OpenAI models, for example `gpt-6.1-sol`. The tool name is `computer`. Each call has the batched `actions` of the model and the `pending_safety_checks`, and the replay sends the actions back as they came. `computerUseTool({ type: 'computer_use_preview', ... })` for the `computer-use-preview` model works as before. The Responses tool choice type also accepts `computer`. Computer screenshots now go out with `detail: 'original'`, as in OpenAI's computer use guide. diff --git a/.changeset/sandbox-skill-computer-use.md b/.changeset/sandbox-skill-computer-use.md new file mode 100644 index 0000000000..5b6bc96e7b --- /dev/null +++ b/.changeset/sandbox-skill-computer-use.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-sandbox': patch +--- + +The `ai-sandbox` skill now says how to use a sandbox provider for computer use, with `sandboxComputer()` from `@tanstack/ai-computer-sandbox`. diff --git a/docs/adapters/anthropic.md b/docs/adapters/anthropic.md index 8b8b2b73a8..f0bfd88b17 100644 --- a/docs/adapters/anthropic.md +++ b/docs/adapters/anthropic.md @@ -618,30 +618,75 @@ equivalent — see [Provider Skills](../tools/provider-skills.md). ### `computerUseTool` -Allows Claude to observe a virtual desktop (screenshots) and interact with it -via keyboard and mouse events. Provide the screen resolution so Claude can -calculate accurate coordinates. +Claude can look at a screen and use the mouse and the keyboard. Your app runs each action and sends back a screenshot. `createComputerUse` from `@tanstack/ai-computer-use` does this on a Playwright browser or an E2B desktop. Each thread gets its own machine: ```typescript import { chat } from "@tanstack/ai"; import { anthropicText } from "@tanstack/ai-anthropic"; -import { computerUseTool } from "@tanstack/ai-anthropic/tools"; +import { createComputerUse } from "@tanstack/ai-computer-use"; +import { playwrightComputer } from "@tanstack/ai-computer-playwright"; + +declare const threadId: string; + +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: "https://tanstack.com" }), +}); const stream = chat({ - adapter: anthropicText("claude-sonnet-4-6"), - messages: [{ role: "user", content: "Open the browser and go to example.com" }], - tools: [ - computerUseTool({ - type: "computer_20250124", - name: "computer", - display_width_px: 1024, - display_height_px: 768, - }), - ], + adapter: anthropicText("claude-opus-5-5"), + messages: [{ role: "user", content: "Find the TanStack AI docs" }], + threadId, + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], }); ``` -**Supported models:** Claude Sonnet 3.5 and above, with two exceptions. `claude-opus-5-fast` takes no provider tools. Claude Opus 5.5, Claude Sonnet 5.5, and Claude Haiku 5.5 accept only the `computer_toolset_20260801` toolset, which the adapter does not offer yet. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). +The adapter sends the `computer` tool to Claude as the newest native computer tool that your model takes. + +- For the setup, the threads, and how to close the machines, see [Computer use](../computer-use/computer-use.md). +- For your own machine, see [Custom driver](../computer-use/custom-driver.md). + +#### The raw tool + +`computerUseTool` from `@tanstack/ai-anthropic/tools` is the native tool with no `execute`. Use it when your app runs the actions itself. Pick the version that your model takes: + +| Version | Models | +| --- | --- | +| `computer_toolset_20260801` | Claude Opus 5.5, Sonnet 5.5, Haiku 5.5, Fable 5.1, Fable 5, Opus 5, Sonnet 5, Opus 4.8 | +| `computer_20251124` | Claude Fable 5.1, Fable 5, Opus 5, Sonnet 5, Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 4.6, Opus 4.5 | +| `computer_20250124` | Claude Sonnet 4.5, Haiku 4.5, Opus 4.1 | + +```typescript +import { computerUseTool } from "@tanstack/ai-anthropic/tools"; + +const computer = computerUseTool({ type: "computer_toolset_20260801" }); +``` + +The earlier versions also take a name and the screen size: + +```typescript +import { computerUseTool } from "@tanstack/ai-anthropic/tools"; + +const computer = computerUseTool({ + type: "computer_20251124", + name: "computer", + display_width_px: 1280, + display_height_px: 800, +}); +``` + +Add an `execute` that runs the action and returns a result: + +- Every version calls `execute` with the action in `input.action`. With the toolset, Claude calls each action as its own tool. The adapter gives you each call as one `computer` call. +- Return a screenshot as an image part for the `screenshot` action, and `"OK"` for the other actions. +- Send screenshots that fit the image limits of the model. The toolset refuses a large screenshot. It does not make it smaller. +- The toolset turns on `zoom` by default. Return an image for the `zoom` action, or turn it off with `configs: { zoom: { enabled: false } }`. +- The adapter sends the beta header that an earlier version needs. The toolset needs no beta header. + +Claude gives coordinates in the pixels of your screenshots. + +A version that your model does not take is a type error. `claude-opus-5-fast` takes no provider tools. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). ### `bashTool` diff --git a/docs/adapters/openai.md b/docs/adapters/openai.md index 2812e181cd..d6a0985b7c 100644 --- a/docs/adapters/openai.md +++ b/docs/adapters/openai.md @@ -992,30 +992,67 @@ const stream = chat({ ### `computerUseTool` -Lets the model observe a virtual desktop via screenshots and interact with -it using keyboard and mouse events. Provide the display dimensions and the -execution environment type. +The model can look at a screen and use the mouse and the keyboard. Your app runs each action and sends back a screenshot. `createComputerUse` from `@tanstack/ai-computer-use` does this on a Playwright browser or an E2B desktop. Each thread gets its own machine: ```typescript import { chat } from "@tanstack/ai"; import { openaiText } from "@tanstack/ai-openai"; -import { computerUseTool } from "@tanstack/ai-openai/tools"; +import { createComputerUse } from "@tanstack/ai-computer-use"; +import { playwrightComputer } from "@tanstack/ai-computer-playwright"; + +declare const threadId: string; + +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: "https://tanstack.com" }), +}); const stream = chat({ - adapter: openaiText("computer-use-preview"), - messages: [{ role: "user", content: "Open Chrome and navigate to example.com" }], - tools: [ - computerUseTool({ - type: "computer_use_preview", - display_width: 1024, - display_height: 768, - environment: "browser", - }), - ], + adapter: openaiText("gpt-6.1-sol"), + messages: [{ role: "user", content: "Find the TanStack AI docs" }], + threadId, + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], +}); +``` + +The adapter sends the `computer` tool to OpenAI as its native computer tool. + +- For the setup, the threads, and how to close the machines, see [Computer use](../computer-use/computer-use.md). +- For the approval of OpenAI safety checks, see [Safety checks](../computer-use/safety-checks.md). +- For your own machine, see [Custom driver](../computer-use/custom-driver.md). + +#### The raw tool + +`computerUseTool` from `@tanstack/ai-openai/tools` is the native tool with no `execute`. Use it when your app runs the actions itself: + +```typescript +import { computerUseTool } from "@tanstack/ai-openai/tools"; + +const computer = computerUseTool({ type: "computer" }); +``` + +Add an `execute` that runs the actions and returns a screenshot as an image part. The input of each call has two fields: + +- `actions`: the actions to run, in order, as OpenAI sends them. For example `[{ type: "click", button: "left", x: 120, y: 340 }, { type: "type", text: "ada@example.com" }]`. +- `pending_safety_checks`: the safety checks that OpenAI attached to the actions. + +The adapter sends the first image of the result back as the screenshot. When the call does not fail, the adapter also tells OpenAI that the safety checks are accepted. When the call fails, they stay unaccepted. + +The `computer-use-preview` model takes the older tool type. Give it the display size and the environment. Each call then has one `action` field in place of `actions`: + +```typescript +import { computerUseTool } from "@tanstack/ai-openai/tools"; + +const previewTool = computerUseTool({ + type: "computer_use_preview", + display_width: 1024, + display_height: 768, + environment: "browser", }); ``` -**Supported models:** `computer-use-preview`. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). +**Supported models:** `type: "computer"` works with the models that list computer use, for example `gpt-6.1-sol`. `type: "computer_use_preview"` works with `computer-use-preview`. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). ### `localShellTool` diff --git a/docs/api/ai.md b/docs/api/ai.md index 4e125f4ee4..6f377a4075 100644 --- a/docs/api/ai.md +++ b/docs/api/ai.md @@ -664,6 +664,7 @@ interface Tool { ```typescript ignore type ToolExecutionContext = { toolCallId?: string; + threadId?: string; emitCustomEvent: ( eventName: string, value: Record, @@ -674,6 +675,8 @@ type ToolExecutionContext = { `context` is the runtime value from `chat({ context })` for server tools, or from `ChatClient` / framework hook options for client tools. It is required when a tool declares a concrete `TContext` and optional for untyped tools where the context type is `unknown`. +`threadId` is the thread id of the chat run. `chat()` sets it to the same value as `ctx.threadId` in middleware. A tool can use it to keep its own state for each conversation. + ### `ChatMiddleware` ```typescript diff --git a/docs/computer-use/computer-use.md b/docs/computer-use/computer-use.md new file mode 100644 index 0000000000..8ae53df9d4 --- /dev/null +++ b/docs/computer-use/computer-use.md @@ -0,0 +1,236 @@ +--- +title: Computer Use +id: computer-use +order: 1 +description: "Let the model use a web page for the user. It sees screenshots and sends clicks and key presses. Each conversation gets its own browser page, and the same code runs on OpenAI and Anthropic." +keywords: + - tanstack ai + - computer use + - createComputerUse + - playwrightComputer + - browser agent + - playwright + - openai + - anthropic +--- + +You have a chat app. You want the model to do a task on a web page for the user, for example fill in a form or find a price. Computer use gives the model a screen, a mouse, and a keyboard. Each conversation gets its own browser page. The same code runs on OpenAI and on Anthropic. + +To build a full app step by step, with the screenshots in the chat, open the [Computer Use tutorial](../tutorials/computer-use). + +## Install + + + +react: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +vue: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +solid: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +svelte: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +preact: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +angular: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +vanilla: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +octane: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod + + + +Then download Chromium one time: + +```bash +npx playwright-core install chromium +``` + +## Give the model a browser + +1. Create the computer with `playwrightComputer()`, one time, at module scope. +2. Give `tools`, `middleware`, and `interrupt` to `chat()`. +3. Give the `threadId` of the request to `chat()`. + +```ts +// app/api/chat/route.ts +import { chat, chatParamsFromRequest, toServerSentEventsResponse } from '@tanstack/ai' +import { openaiText } from '@tanstack/ai-openai' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { playwrightComputer } from '@tanstack/ai-computer-playwright' + +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: 'https://tanstack.com' }), +}) + +export async function POST(request: Request) { + const params = await chatParamsFromRequest(request) + const stream = chat({ + adapter: openaiText('gpt-6.1-sol'), + messages: params.messages, + threadId: params.threadId, + runId: params.runId, + ...(params.parentRunId ? { parentRunId: params.parentRunId } : {}), + ...(params.resume ? { resume: params.resume } : {}), + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], + }) + return toServerSentEventsResponse(stream) +} +``` + +- `tools`: the `computer` tool and the `navigate` tool. With `computer`, the model clicks, types, and takes screenshots. With `navigate`, it opens a URL. +- `middleware`: stops the rest of a turn after a failed action. It also runs the OpenAI safety checks. +- `interrupt`: asks a person to approve an action that has a safety check. + +Chromium starts on the first action. Each thread gets a new page on its own first action, and the page opens `startUrl`. + +To use Anthropic, change only the adapter: + +```ts +import { anthropicText } from '@tanstack/ai-anthropic' + +const adapter = anthropicText('claude-opus-5-5') +``` + +Need a full Linux desktop or your own computer, and not a browser page? See [Drivers](./drivers). + +## One page for each thread + +Each `threadId` gets its own machine. With `playwrightComputer()`, the machine is a browser context with one page: + +- Each thread has its own cookies and storage. +- Each thread has its own action queue and its own safety-check approvals. +- All threads share one Chromium process. + +Give a stable `threadId` to `chat()`. If `chat()` gets no `threadId`, it makes a new id for each request. Then each request gets a new page. + +`useChat` sends the same thread id with each request of a chat. `chatParamsFromRequest` reads it, as in the route above. + +## Close the pages + +Each open page uses memory on your server. Close the page of a thread when its conversation ends. Close all pages when the server stops: + +```ts +import { createComputerUse } from '@tanstack/ai-computer-use' +import { playwrightComputer } from '@tanstack/ai-computer-playwright' + +declare const threadId: string + +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: 'https://tanstack.com' }), + idleTimeout: 30 * 60_000, +}) + +// The conversation ended. Close the page of this thread. +await computerUse.close(threadId) + +// The server stops. Close every page, then Chromium. +await computerUse.close() +``` + +A thread that stays idle closes by itself. `idleTimeout` sets this time in ms: + +- The default is 10 minutes. +- When the time ends, the page of the thread closes. The next action of that thread opens a new page at `startUrl`. +- `false` keeps every thread open until you call `close`. + +## Keep the browser away from private addresses + +CAUTION: Run the browser in a network where it cannot reach private services. The browser can open every address that your server can reach. + +For example, a page can link to an internal admin page or to a cloud metadata URL. The `navigate` tool accepts only `http:` and `https:` URLs. It does not block private addresses. + +To keep the machine off your network, use an [E2B sandbox](./drivers#e2b). + +## Register the interrupt on the client + +The client imports `computerSafetyCheck`. It is the same object as `interrupt`, so the client does not load the driver or Playwright. + +```tsx +// app/chat.tsx +import { useState } from 'react' +import { fetchServerSentEvents, useChat } from '@tanstack/ai-react' +import { computerSafetyCheck } from '@tanstack/ai-computer-use' + +export function Chat() { + const [input, setInput] = useState('') + const { messages, sendMessage } = useChat({ + connection: fetchServerSentEvents('/api/chat'), + interrupts: [computerSafetyCheck], + }) + + return ( + <> + {messages.map((message) => ( +
+ {message.parts.map((part, index) => + part.type === 'text' ?

{part.content}

: null, + )} +
+ ))} +
{ + event.preventDefault() + sendMessage(input) + setInput('') + }} + > + setInput(event.target.value)} /> + +
+ + ) +} +``` + +To show the approval card for a safety check, see [Safety checks](./safety-checks). + +## What the model can do + +The model clicks, double-clicks, drags, moves the mouse, scrolls, types text, and presses key combos. It can also wait. + +- A normal action returns a screenshot 500 ms later, so the page can settle. An OpenAI batch returns one screenshot after the last action. +- `zoom` returns an image of one region. `cursor_position` returns the mouse position as text. Neither takes another screenshot. +- `navigate` opens a URL, then returns a screenshot. + +To change the delay, set `screenshotDelay` in ms: + +```ts +import { createComputerUse } from '@tanstack/ai-computer-use' +import { playwrightComputer } from '@tanstack/ai-computer-playwright' + +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: 'https://tanstack.com' }), + screenshotDelay: 1000, +}) +``` + +## Order and failures + +In one thread, actions run one at a time, in the order that the model calls them. A driver factory gives each thread its own machine, and those threads can run at the same time. A driver that you pass in is one machine. Actions on that machine run one at a time, even from two threads. + +If one action fails, the other actions of that model turn do not run. The model gets this result for each of them: + +```text +Not executed: an earlier computer action in this turn failed. +``` + +The next model turn starts clean. The model sees the error and can try again. + +## Limits + +- **One server process.** The pages live in the memory of one process. If the next request of a thread goes to another server, the thread gets a new page there. +- **Time to approve.** If a person answers a safety check after `idleTimeout`, the page is already closed. The approved action then runs on a new page. Set `idleTimeout` above the time that a person takes to answer. + +## Model support + +- A model with no computer use is a type error at `chat()`. +- The Anthropic adapter picks the newest computer tool version that the model supports. If the model supports no version, the adapter throws an error. +- `openaiText` uses the Responses API, which has the computer tool. The Chat Completions adapters throw an error, because that API has no computer tool. + +## Try it + +Start the app and send this message: + +```text +Go to https://tanstack.com/ai and tell me the title of the page. +``` + +The model calls `navigate`, reads the screenshot, and answers in the chat. You now have a browser agent. + +To let the model use your own computer, see [Your own computer](./drivers#your-own-computer). For a machine that no driver controls, see [Custom driver](./custom-driver). diff --git a/docs/computer-use/custom-driver.md b/docs/computer-use/custom-driver.md new file mode 100644 index 0000000000..3ba89194a6 --- /dev/null +++ b/docs/computer-use/custom-driver.md @@ -0,0 +1,191 @@ +--- +title: Custom Computer Use Driver +id: computer-use-custom-driver +order: 4 +description: "Let the model use your own machine. Write a ComputerDriver with a small set of mouse, keyboard, and screenshot methods, and a ComputerDriverFactory to give each thread its own machine." +keywords: + - tanstack ai + - computer use + - ComputerDriver + - ComputerDriverFactory + - custom driver + - virtual machine +--- + +You have your own machine: a VM, a remote desktop, or a device in a lab. No driver package controls it. Write a `ComputerDriver`, and [computer use](./computer-use) runs on it with no other change. + +## Write the driver + +A driver is an object with a screen size and a small set of methods. This example sends each method to an HTTP agent that runs on the machine. + +```ts +// app/my-driver.ts +import type { ComputerDriver } from '@tanstack/ai-computer-use' + +export function createMyDriver(base: string) { + async function send(path: string, body: object) { + const response = await fetch(`${base}${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + if (!response.ok) throw new Error(`${path} failed: ${response.status}`) + } + + const driver: ComputerDriver = { + display: { width: 1920, height: 1080 }, + environment: 'windows', + async screenshot() { + const response = await fetch(`${base}/screenshot`) + if (!response.ok) throw new Error(`/screenshot failed: ${response.status}`) + return new Uint8Array(await response.arrayBuffer()) + }, + click: (x, y, { button, count }) => send('/click', { x, y, button, count }), + move: (x, y) => send('/move', { x, y }), + mouseDown: (button) => send('/mouse-down', { button }), + mouseUp: (button) => send('/mouse-up', { button }), + scroll: (x, y, deltaX, deltaY) => send('/scroll', { x, y, deltaX, deltaY }), + keyDown: (key) => send('/key-down', { key }), + keyUp: (key) => send('/key-up', { key }), + press: (keys) => send('/press', { keys }), + type: (text) => send('/type', { text }), + } + return driver +} +``` + +Then give it to `createComputerUse`: + +```ts +import { createComputerUse } from '@tanstack/ai-computer-use' +import type { ComputerDriver } from '@tanstack/ai-computer-use' + +// From app/my-driver.ts above. +declare function createMyDriver(base: string): ComputerDriver + +const computerUse = createComputerUse({ + driver: createMyDriver('http://localhost:8000'), +}) +``` + +A driver is one machine. Every thread shares it, so it serves one conversation at a time. `createComputerUse` never closes it. To give each thread its own machine, write a [driver factory](#one-machine-for-each-thread). + +## The contract + +Coordinates are in pixels, in `display` space. Key names are `KeyboardEvent.key` names, for example `'Enter'`, `'Control'`, or `'a'`. + +| Member | What it does | +| --- | --- | +| `display` | The screen size in pixels. Screenshots have this size. | +| `environment` | `'browser'`, `'linux'`, `'mac'`, or `'windows'`. | +| `screenshot()` | Returns PNG bytes as a `Uint8Array`. | +| `click(x, y, { button, count })` | Clicks `count` times. `button` is `left`, `right`, `middle`, `back`, or `forward`. | +| `move(x, y)` | Moves the mouse. | +| `mouseDown(button)`, `mouseUp(button)` | Presses or releases a button at the mouse position. | +| `scroll(x, y, deltaX, deltaY)` | Scrolls at the point. The deltas are in pixels. | +| `keyDown(key)`, `keyUp(key)` | Presses or releases one key. | +| `press(keys)` | Presses one combo, like `['Control', 's']`. | +| `type(text)` | Types text as a user does. | + +## Add the optional methods + +Four methods are optional: + +- `zoom(region)`: returns PNG bytes of one region at full size. The region has `x`, `y`, `width`, and `height`. `computer_toolset_20260801` and `computer_20251124` turn on `zoom` when the driver has this method. `computer_20250124` does not. +- `cursorPosition()`: returns the mouse position as `{ x, y }`. Only `computer_toolset_20260801` turns on `cursor_position`, and only when the driver has this method. +- `navigate(url)`: opens the URL. `computerUse.tools` then has the `navigate` tool. The tool accepts only `http:` and `https:` URLs. +- `close()`: frees the machine. `createComputerUse` calls it only on a driver that a factory made. + +Add this function to `app/my-driver.ts`. It narrows the JSON from your agent before it returns it: + +```ts +export async function readCursorPosition(base: string) { + const response = await fetch(`${base}/cursor`) + const data: unknown = await response.json() + const isPoint = + typeof data === 'object' && + data !== null && + 'x' in data && + 'y' in data && + typeof data.x === 'number' && + typeof data.y === 'number' + if (!isPoint) throw new Error('The agent sent no cursor position.') + return { x: data.x, y: data.y } +} +``` + +Then add `cursorPosition: () => readCursorPosition(base)` to `driver` in `createMyDriver`. + +## One machine for each thread + +One shared machine serves one conversation at a time. To give each thread its own machine, write a `ComputerDriverFactory`. `createComputerUse` calls `create` on the first action of each thread. + +This example asks a pool service for one VM for each thread, and stops the VM when the thread closes: + +```ts +// app/my-machines.ts +import type { + ComputerDriver, + ComputerDriverFactory, +} from '@tanstack/ai-computer-use' + +// From app/my-driver.ts above. +declare function createMyDriver(base: string): ComputerDriver + +const pool = 'http://localhost:9000' + +export const myMachines: ComputerDriverFactory = { + display: { width: 1920, height: 1080 }, + environment: 'windows', + capabilities: { zoom: false, cursorPosition: false, navigate: false }, + async create({ threadId }) { + const machine = `${pool}/machines/${encodeURIComponent(threadId)}` + const response = await fetch(machine, { method: 'PUT' }) + if (!response.ok) throw new Error(`Start failed: ${response.status}`) + return { + ...createMyDriver(`${machine}/agent`), + close: async () => { + await fetch(machine, { method: 'DELETE' }) + }, + } + }, +} +``` + +Then give the factory to `createComputerUse`: + +```ts +import { createComputerUse } from '@tanstack/ai-computer-use' +import type { ComputerDriverFactory } from '@tanstack/ai-computer-use' + +// From app/my-machines.ts above. +declare const myMachines: ComputerDriverFactory + +const computerUse = createComputerUse({ driver: myMachines }) +``` + +The factory has these members: + +| Member | What it does | +| --- | --- | +| `display` | The screen size of every machine. Each driver from `create` has this `display`. | +| `environment` | The kind of every machine. | +| `capabilities` | `{ zoom, cursorPosition, navigate }`: the optional methods that every driver from `create` has. The tool is built before any machine exists, so the factory tells it. | +| `create({ threadId })` | Makes the driver of one thread. If it throws, the model gets the error. The next model turn calls `create` again. | +| `close()` | Optional. Frees what the threads share. `computerUse.close()` calls it after it closes every thread. | + +When a thread closes or goes idle, `createComputerUse` calls `close()` on the driver of that thread. + +## What the core builds for you + +Your driver does only the small steps. The core builds the rest from them: + +- **Drags**: `move`, `mouseDown`, more `move` calls, then `mouseUp`. +- **Modifier clicks and scrolls**: `keyDown` for each modifier, the action, then `keyUp` in reverse order. +- **`hold_key` and `wait`**: the core holds the keys or waits, then continues. +- **Key names**: the core changes OpenAI and Anthropic key names to `KeyboardEvent.key` names. +- **Screenshots**: after a normal action, the core waits `screenshotDelay` ms and calls `screenshot()`. An OpenAI batch waits once, after the last action, then takes one screenshot. `zoom` and `cursor_position` return their own result. The core does not call `screenshot()` for those two. + +## Try it + +Start your agent on the machine, and send a task in the chat. Your agent gets the first `/screenshot` request, then the clicks and the key presses. The model now uses your own machine. diff --git a/docs/computer-use/drivers.md b/docs/computer-use/drivers.md new file mode 100644 index 0000000000..e88f40a66a --- /dev/null +++ b/docs/computer-use/drivers.md @@ -0,0 +1,266 @@ +--- +title: Computer Use Drivers +id: computer-use-drivers +order: 2 +description: "Pick the machine that the model uses: a browser page for each thread with Playwright, a full Linux desktop for each thread in a sandbox, or your own computer." +keywords: + - tanstack ai + - computer use + - driver + - playwrightComputer + - sandboxComputer + - localComputer + - playwright + - e2b + - daytona + - linux desktop + - sandbox + - robotjs + - local desktop +--- + +A browser page is not always enough. Sometimes the model must use a desktop app, a file manager, or a terminal. A driver decides which machine the model uses. Change the driver, and the rest of your [computer use](./computer-use) code stays the same. + +## Pick a driver + +| | Playwright | Sandbox | Your own computer | +| --------------------- | ----------------------------------------- | --------------------------------------- | ------------------------------------------ | +| **Machine** | A browser page for each thread | A Linux desktop sandbox for each thread | Your mouse, keyboard, and main screen | +| **Environment** | `browser` | `linux` | `windows`, `mac`, or `linux` | +| **Runs on** | Your server | Any sandbox provider | The machine that runs `chat()` | +| **`navigate` tool** | Yes | No | No | +| **`zoom`** | Yes | No | Yes | +| **`cursor_position`** | No | Yes | Yes | +| **Setup** | `playwright-core`, then download Chromium | A provider and a desktop image | `@jitsi/robotjs`, and permissions on macOS | + +The Anthropic adapter turns on `zoom` and `cursor_position` only on the computer versions that have those flags. + +- `computer_toolset_20260801` turns on both when the driver has the method. +- `computer_20251124` turns on `zoom` only, and only when the driver has `zoom`. +- `computer_20250124` turns on neither. + +## Playwright + +If the task stays inside a web page, use Playwright. + + + +react: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +vue: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +solid: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +svelte: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +preact: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +angular: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +vanilla: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod +octane: @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod + + + +Then download Chromium one time: + +```bash +npx playwright-core install chromium +``` + +```ts +import { createComputerUse } from '@tanstack/ai-computer-use' +import { playwrightComputer } from '@tanstack/ai-computer-playwright' + +const computerUse = createComputerUse({ + driver: playwrightComputer({ + startUrl: 'https://tanstack.com', + display: { width: 1280, height: 800 }, + }), +}) +``` + +`playwrightComputer()` takes these options. All of them are optional: + +- `startUrl`: the URL that each new page opens. Without it, the page stays blank. +- `display`: the viewport of each page in pixels. The default is 1280 x 800. +- `browser`: a Playwright `Browser` that makes the pages. `createComputerUse` never closes it. +- `launch`: the options for `chromium.launch()`. When you give `browser`, the factory ignores them. + +Without `browser`, Chromium starts on the first action. `computerUse.close()` closes it. + +The `back` and `forward` mouse buttons go back and forward in the page history. + +### Share one page + +You already have a Playwright page, for example in a test. Give it to `createPlaywrightComputerDriver`. Use the Playwright package that your app already has. + +```ts +import { createComputerUse } from '@tanstack/ai-computer-use' +import { createPlaywrightComputerDriver } from '@tanstack/ai-computer-playwright' +import type { PlaywrightPageLike } from '@tanstack/ai-computer-playwright' + +// Your Playwright page. It must have a viewport size. +declare const page: PlaywrightPageLike + +const computerUse = createComputerUse({ + driver: createPlaywrightComputerDriver({ page }), +}) +``` + +- Every thread shares this page, so it serves one conversation at a time. +- The page must have a viewport size. If it has none, the driver throws an error. +- The driver reads the viewport size one time, when you create it. +- A page from `playwright`, `playwright-core`, or `@playwright/test` works. + +Your app owns this page. `createComputerUse` never closes it. + +## Linux desktop in a sandbox + +If the model must use any app on a Linux desktop, use a sandbox. Each thread gets its own desktop, away from your server. The driver works with every [sandbox provider](../sandbox/overview): E2B, Daytona, Docker, and the others. + + + +react: @tanstack/ai-computer-use @tanstack/ai-computer-sandbox @tanstack/ai-sandbox @tanstack/ai-sandbox-e2b zod +vue: @tanstack/ai-computer-use @tanstack/ai-computer-sandbox @tanstack/ai-sandbox @tanstack/ai-sandbox-e2b zod +solid: @tanstack/ai-computer-use @tanstack/ai-computer-sandbox @tanstack/ai-sandbox @tanstack/ai-sandbox-e2b zod +svelte: @tanstack/ai-computer-use @tanstack/ai-computer-sandbox @tanstack/ai-sandbox @tanstack/ai-sandbox-e2b zod +preact: @tanstack/ai-computer-use @tanstack/ai-computer-sandbox @tanstack/ai-sandbox @tanstack/ai-sandbox-e2b zod +angular: @tanstack/ai-computer-use @tanstack/ai-computer-sandbox @tanstack/ai-sandbox @tanstack/ai-sandbox-e2b zod +vanilla: @tanstack/ai-computer-use @tanstack/ai-computer-sandbox @tanstack/ai-sandbox @tanstack/ai-sandbox-e2b zod +octane: @tanstack/ai-computer-use @tanstack/ai-computer-sandbox @tanstack/ai-sandbox @tanstack/ai-sandbox-e2b zod + + + +1. Set `E2B_API_KEY` in your server environment. +2. Give `sandboxComputer()` a provider that makes desktop sandboxes. The E2B `desktop` template is one. + +```ts +import { createComputerUse } from '@tanstack/ai-computer-use' +import { sandboxComputer } from '@tanstack/ai-computer-sandbox' +import { e2bSandbox } from '@tanstack/ai-sandbox-e2b' + +const computerUse = createComputerUse({ + driver: sandboxComputer({ + provider: e2bSandbox({ template: 'desktop', timeoutMs: 60 * 60_000 }), + }), +}) +``` + +`sandboxComputer()` takes these options: + +- `provider`: the sandbox provider. It makes one sandbox for each thread. +- `display`: the screen size of each desktop in pixels. The default is 1024 x 768. + +CAUTION: Set the provider timeout above `idleTimeout` plus the longest time that one thread stays in use. E2B kills a sandbox at its `timeoutMs`, also when the thread is busy. + +- There is no `navigate` tool, because a desktop has no address. The model opens the browser on the desktop. +- When a thread closes or goes idle, `createComputerUse` destroys its sandbox. +- One scroll step is 100 pixels. +- The driver quotes every key name and all typed text before they go to the shell. + +### The sandbox image + +The image must have `xdotool`, `xdpyinfo`, and `scrot`. Without `scrot`, the driver takes screenshots with ImageMagick `import`, but they do not show the mouse pointer. When no X display runs, the driver starts one with `Xvfb`. It also starts `startxfce4` when the image has it. + +The E2B `desktop` template has all of these. For Docker, Daytona, and the other providers, use an image with the tools. On Debian or Ubuntu: + +```dockerfile +RUN apt-get update && apt-get install -y xvfb xdotool scrot x11-utils +``` + +If a tool is missing, the driver throws an error with its name. + +### Share one sandbox + +You already have a desktop sandbox. Give it to `createSandboxComputerDriver`: + +```ts +import { createComputerUse } from '@tanstack/ai-computer-use' +import { createSandboxComputerDriver } from '@tanstack/ai-computer-sandbox' +import { e2bSandbox } from '@tanstack/ai-sandbox-e2b' + +const sandbox = await e2bSandbox({ template: 'desktop' }).create({}) + +const computerUse = createComputerUse({ + driver: await createSandboxComputerDriver({ sandbox }), +}) +``` + +- Every thread shares this sandbox, so it serves one conversation at a time. +- `createSandboxComputerDriver` is async. It starts the desktop if needed, and reads the screen size one time. + +Your app owns this sandbox. `createComputerUse` never destroys it. When you are done, call `await sandbox.destroy()`. + +## Your own computer + +If the model must use the apps on the machine that runs `chat()`, use `localComputer()`. The model moves your real mouse, presses your real keys, and reads your main screen. + +CAUTION: Watch the model while it uses your computer. It can do what you can do there: read files, send messages, and delete data. If you cannot watch it, use a [sandbox](#linux-desktop-in-a-sandbox) or a VM. + + + +react: @tanstack/ai-computer-use @tanstack/ai-computer-local @jitsi/robotjs zod +vue: @tanstack/ai-computer-use @tanstack/ai-computer-local @jitsi/robotjs zod +solid: @tanstack/ai-computer-use @tanstack/ai-computer-local @jitsi/robotjs zod +svelte: @tanstack/ai-computer-use @tanstack/ai-computer-local @jitsi/robotjs zod +preact: @tanstack/ai-computer-use @tanstack/ai-computer-local @jitsi/robotjs zod +angular: @tanstack/ai-computer-use @tanstack/ai-computer-local @jitsi/robotjs zod +vanilla: @tanstack/ai-computer-use @tanstack/ai-computer-local @jitsi/robotjs zod +octane: @tanstack/ai-computer-use @tanstack/ai-computer-local @jitsi/robotjs zod + + + +`@jitsi/robotjs` has prebuilt binaries for Windows, macOS, and Linux. If pnpm asks to allow its install script, you can deny it. + +1. On macOS, open System Settings > Privacy & Security. Give the Accessibility and Screen Recording permissions to the app that runs Node, for example your terminal. +2. On Linux, use an X11 session. Wayland is not supported. +3. Give `localComputer()` to `createComputerUse`. + +```ts +import { createComputerUse } from '@tanstack/ai-computer-use' +import { localComputer } from '@tanstack/ai-computer-local' + +const computerUse = createComputerUse({ + driver: localComputer(), +}) +``` + +`localComputer()` takes these options. Both are optional: + +- `display`: the size of the screenshots that the model sees. The driver scales each screenshot down to this size, and scales each click up to the screen. The default is your screen size, scaled down to fit in 1280 x 800. Keep the aspect ratio of your screen. +- `robot`: the object that moves the mouse and presses the keys. The default is `@jitsi/robotjs`. A test can give a fake with the same methods. + +Your computer is one machine: + +- Every thread shares it, so it serves one conversation at a time. +- The model sees only the main screen. +- There is no `navigate` tool. The model opens the browser on your desktop. +- There are no `back` and `forward` mouse buttons. + +## Use it in chat + +With every driver, `createComputerUse` gives `tools`, `middleware`, and `interrupt`. Give them to `chat()` with the thread id: + +```ts +import { chat } from '@tanstack/ai' +import type { ModelMessage } from '@tanstack/ai' +import { anthropicText } from '@tanstack/ai-anthropic' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { sandboxComputer } from '@tanstack/ai-computer-sandbox' +import { e2bSandbox } from '@tanstack/ai-sandbox-e2b' + +declare const messages: Array +declare const threadId: string + +const computerUse = createComputerUse({ + driver: sandboxComputer({ + provider: e2bSandbox({ template: 'desktop', timeoutMs: 60 * 60_000 }), + }), +}) + +const stream = chat({ + adapter: anthropicText('claude-opus-5-5'), + messages, + threadId, + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], +}) +``` + +The model now uses a browser page or a full Linux desktop for each thread, or your own computer. For a machine that no driver controls, write a [custom driver](./custom-driver). diff --git a/docs/computer-use/safety-checks.md b/docs/computer-use/safety-checks.md new file mode 100644 index 0000000000..72b2858fff --- /dev/null +++ b/docs/computer-use/safety-checks.md @@ -0,0 +1,132 @@ +--- +title: Computer Use Safety Checks +id: computer-use-safety-checks +order: 3 +description: "Pause a computer action until a person approves it. OpenAI attaches safety checks to risky actions, and TanStack AI turns each one into an interrupt." +keywords: + - tanstack ai + - computer use + - safety checks + - pending_safety_checks + - computerSafetyCheck + - interrupt + - approval +--- + +Some computer actions are risky. The page can ask the model to do a thing the user did not ask for, or the action can send data. OpenAI attaches safety checks to these actions. The run pauses, and a person approves or declines the action. + +## How it works + +1. The model sends a computer call with `pending_safety_checks`. +2. The `middleware` pauses the run with one interrupt for each call that has checks. +3. A person approves or declines on the client. + +- If the person approves all calls, the calls run. The checks go back to OpenAI as acknowledged. +- If the person declines one call, no call of that turn runs. +- A call with checks that nobody approved does not run. The tool refuses it. + +Safety checks are an OpenAI feature. Anthropic models do not send them, so this code does nothing on Anthropic. + +## Server + +Give `tools`, `middleware`, and `interrupt` to `chat()`. Also give `parentRunId` and `resume`, so that the approval can continue the run. + +```ts +// app/api/chat/route.ts +import { chat, chatParamsFromRequest, toServerSentEventsResponse } from '@tanstack/ai' +import { openaiText } from '@tanstack/ai-openai' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { playwrightComputer } from '@tanstack/ai-computer-playwright' + +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: 'https://tanstack.com' }), + // A person has up to one hour to answer before the page closes. + idleTimeout: 60 * 60_000, +}) + +export async function POST(request: Request) { + const params = await chatParamsFromRequest(request) + const stream = chat({ + adapter: openaiText('gpt-6.1-sol'), + messages: params.messages, + threadId: params.threadId, + runId: params.runId, + ...(params.parentRunId ? { parentRunId: params.parentRunId } : {}), + ...(params.resume ? { resume: params.resume } : {}), + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], + }) + return toServerSentEventsResponse(stream) +} +``` + +`computerUse` keeps the approval for the thread until the resume request comes. To keep it there: + +- Create `computerUse` one time, at module scope. Every request uses the same one. +- Give `params.threadId` to `chat()`. The resume request has the same thread id as the request that paused. +- Set `idleTimeout` above the time that a person takes to answer. If the time ends first, the page closes, and the approved action runs on a new page. +- Send the resume request to the same server process. The approvals live in the memory of that process. + +## Client + +Register `computerSafetyCheck` with `useChat`. Each item has these fields: + +- `payload.checks`: the safety checks. Each check has an `id`, and can have a `code` and a `message`. +- `payload.action`: the action that waits. For a batch, this is the list of actions. + +```tsx +// app/chat.tsx +import { fetchServerSentEvents, useChat } from '@tanstack/ai-react' +import type { GenericInterrupt } from '@tanstack/ai-react' +import { computerSafetyCheck } from '@tanstack/ai-computer-use' + +function SafetyCheckCard({ + interrupt, +}: { + interrupt: GenericInterrupt +}) { + const checks = interrupt.payload?.checks ?? [] + return ( +
+

Approve this computer action?

+
    + {checks.map((check) => ( +
  • {check.message ?? check.code ?? check.id}
  • + ))} +
+
{JSON.stringify(interrupt.payload?.action, null, 2)}
+ + +
+ ) +} + +export function Chat() { + const { interrupts } = useChat({ + connection: fetchServerSentEvents('/api/chat'), + interrupts: [computerSafetyCheck], + }) + + return ( + <> + {interrupts.map((interrupt) => { + if (interrupt.kind !== 'generic') return null + if (!('definitionId' in interrupt)) return null + if (interrupt.definitionId !== computerSafetyCheck.id) return null + return + })} + + ) +} +``` + +`interrupt.cancel()` also declines the action. When every interrupt of the batch has an answer, the client sends the answers. For more about generic interrupts, see [Generic Interrupts](../interrupts/generic). + +## Try it + +Ask the model to do a task that sends data, for example a form submit. When OpenAI adds a safety check, the card shows. Click **Approve**, and the action runs. Click **Decline**, and the page stays as it was. diff --git a/docs/config.json b/docs/config.json index 5fed936ceb..3e6326867d 100644 --- a/docs/config.json +++ b/docs/config.json @@ -13,13 +13,13 @@ "label": "Overview", "to": "getting-started/overview", "addedAt": "2026-04-15", - "updatedAt": "2026-09-29" + "updatedAt": "2026-10-21" }, { "label": "Quick Start", "to": "getting-started/quick-start", "addedAt": "2026-04-15", - "updatedAt": "2026-09-17" + "updatedAt": "2026-10-21" }, { "label": "Quick Start: React Native", @@ -85,6 +85,11 @@ "label": "Build an MCP Server", "to": "tutorials/mcp-server", "addedAt": "2026-09-22" + }, + { + "label": "Computer Use", + "to": "tutorials/computer-use", + "addedAt": "2026-10-21" } ] }, @@ -126,6 +131,11 @@ "label": "MCP Server", "to": "framework/react/examples/mcp-server", "addedAt": "2026-09-22" + }, + { + "label": "Computer Use", + "to": "framework/react/examples/computer-use", + "addedAt": "2026-10-21" } ] }, @@ -323,7 +333,7 @@ "label": "Provider Tools", "to": "tools/provider-tools", "addedAt": "2026-04-21", - "updatedAt": "2026-09-24" + "updatedAt": "2026-10-20" }, { "label": "Provider Skills", @@ -919,6 +929,34 @@ ], "tab": "guides" }, + { + "label": "Computer Use", + "children": [ + { + "label": "Computer Use", + "to": "computer-use/computer-use", + "addedAt": "2026-10-20", + "updatedAt": "2026-10-21" + }, + { + "label": "Drivers", + "to": "computer-use/drivers", + "addedAt": "2026-10-20", + "updatedAt": "2026-10-21" + }, + { + "label": "Safety Checks", + "to": "computer-use/safety-checks", + "addedAt": "2026-10-20" + }, + { + "label": "Custom Driver", + "to": "computer-use/custom-driver", + "addedAt": "2026-10-20" + } + ], + "tab": "guides" + }, { "label": "Sandboxes", "tab": "guides", @@ -1209,13 +1247,13 @@ "label": "OpenAI", "to": "adapters/openai", "addedAt": "2026-04-15", - "updatedAt": "2026-10-12" + "updatedAt": "2026-10-20" }, { "label": "Anthropic", "to": "adapters/anthropic", "addedAt": "2026-04-15", - "updatedAt": "2026-10-09" + "updatedAt": "2026-10-20" }, { "label": "Google Gemini", @@ -1479,7 +1517,7 @@ "label": "@tanstack/ai", "to": "api/ai", "addedAt": "2026-04-15", - "updatedAt": "2026-10-13" + "updatedAt": "2026-10-20" }, { "label": "@tanstack/ai-client", diff --git a/docs/getting-started/overview.md b/docs/getting-started/overview.md index 7368445499..4807766aaf 100644 --- a/docs/getting-started/overview.md +++ b/docs/getting-started/overview.md @@ -26,6 +26,8 @@ Stream a typed table from `chat({ outputSchema })` in [Streaming Structured Tabl Add three named agents (researcher, SEO, and writer) to a blog-writing chat in [Subagents](../tutorials/subagents). +Let the model use a web page from screenshots in [Computer Use](../tutorials/computer-use). + ## Key Features - ✅ **Type-Safe** - Full TypeScript support with Zod schema inference diff --git a/docs/getting-started/quick-start.md b/docs/getting-started/quick-start.md index dc9f284006..9413b696f3 100644 --- a/docs/getting-started/quick-start.md +++ b/docs/getting-started/quick-start.md @@ -38,6 +38,8 @@ For a table that fills as JSON streams, open [Streaming Structured Table](../tut For a blog desk where a researcher and an SEO agent run together and a writer uses their notes, open [Subagents](../tutorials/subagents). +For a model that clicks through a web page from screenshots, open [Computer Use](../tutorials/computer-use). + > [!TIP] > If you do not want a key per provider, [OpenRouter](../adapters/openrouter) gives you 300+ models with one API key. diff --git a/docs/tools/provider-tools.md b/docs/tools/provider-tools.md index 25da0275c4..929e438b87 100644 --- a/docs/tools/provider-tools.md +++ b/docs/tools/provider-tools.md @@ -90,7 +90,31 @@ const followUp = chat({ The search/fetch call surfaces as a provider-executed `tool-call` part on the assistant message; the agent loop never tries to run it client-side. -OpenAI `applyPatchTool`, `localShellTool`, and `shellTool({ environment: { type: "local" } })` are different. The model returns the call, and your app runs it. See [OpenAI adapter](../adapters/openai.md#applypatchtool). +OpenAI `applyPatchTool`, `localShellTool`, `computerUseTool`, and `shellTool({ environment: { type: "local" } })` are different. The model returns the call, and your app runs it. See [OpenAI adapter](../adapters/openai.md#applypatchtool) and [`computerUseTool`](../adapters/openai.md#computerusetool). + +To run computer use actions on a browser or a desktop without your own action code, use `createComputerUse` from `@tanstack/ai-computer-use`. The same `tools` work with OpenAI and Anthropic, and each thread gets its own machine. See [Computer use](../computer-use/computer-use.md). + +```typescript +import { chat } from '@tanstack/ai' +import { openaiText } from '@tanstack/ai-openai' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { playwrightComputer } from '@tanstack/ai-computer-playwright' + +declare const threadId: string + +const computerUse = createComputerUse({ + driver: playwrightComputer({ startUrl: 'https://tanstack.com' }), +}) + +const stream = chat({ + adapter: openaiText('gpt-6.1-sol'), + messages: [{ role: 'user', content: 'Find the TanStack AI docs' }], + threadId, + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], +}) +``` ### Read web search sources @@ -149,10 +173,10 @@ Each adapter's `supports.tools` array is the source of truth. The comparison matrix is maintained alongside `model-meta.ts` and reflected here: - **Anthropic**: every registered model supports the full tool superset, - except four. `claude-opus-5-5`, `claude-sonnet-5-5`, and - `claude-haiku-5-5` support every tool but `computerUseTool`, and - `claude-opus-5-fast` supports none. The - retired Claude 3.x models with narrower support were removed. + except `claude-opus-5-fast`, which supports none. For `computerUseTool`, + each model takes only some versions. The Claude 5.5 models take only + `computer_toolset_20260801`. See the version table in + [`computerUseTool`](../adapters/anthropic.md#computerusetool). - **OpenAI**: GPT-5 family and reasoning models (O-series) support the full superset. GPT-6 Astra, Sol, and Luna support every tool except `webSearchPreviewTool` and `localShellTool`. GPT-4-series supports diff --git a/docs/tutorials/computer-use.md b/docs/tutorials/computer-use.md new file mode 100644 index 0000000000..8d88156d80 --- /dev/null +++ b/docs/tutorials/computer-use.md @@ -0,0 +1,229 @@ +--- +title: Computer Use +id: computer-use +order: 8 +description: "Add computer use to a React chat. The model opens tanstack.com in Chromium, reads each screenshot, and clicks until it finds the computer use docs. The chat shows each screenshot." +keywords: + - tanstack ai + - tutorial + - computer use + - createComputerUse + - playwrightComputer + - useChat + - openai + - tanstack start +--- + +You have a chat app. You want the model to do a task on a web page for your user. In this tutorial, the model opens tanstack.com and finds the TanStack AI computer use docs. The model sees only screenshots. It clicks, types, and scrolls on what it sees. + +This tutorial is React + Start, with OpenAI. + +## 1. Create a Start app + +```bash +npx @tanstack/cli@latest create +``` + +Pick React. Then install the packages: + + + +react: @tanstack/ai @tanstack/ai-react @tanstack/ai-openai @tanstack/ai-computer-use @tanstack/ai-computer-playwright playwright-core zod + + + +Download Chromium one time: + +```bash +npx playwright-core install chromium +``` + +Put your OpenAI key in `.env.local`: + +```bash +OPENAI_API_KEY=sk-... +``` + +## Client and server + +The **server** owns the browser. It runs each action that the model asks for, then sends a screenshot back to the model. + +The **client** sends the task. It shows each screenshot, so your user can see what the model sees. + +The next steps build the server. Then they build the client. + +## 2. Give the model a browser + +Create `src/routes/api.chat.ts`, next to `index.tsx`. Start maps that file name to the `/api/chat` path. + +```typescript ignore +import { createFileRoute } from '@tanstack/react-router' +import { + chat, + chatParamsFromRequest, + maxIterations, + toServerSentEventsResponse, +} from '@tanstack/ai' +import { openaiText } from '@tanstack/ai-openai' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { playwrightComputer } from '@tanstack/ai-computer-playwright' + +// One computer for the whole server. Each chat thread gets its own page. +const computerUse = createComputerUse({ + driver: playwrightComputer({ + startUrl: 'https://tanstack.com', + // Show the Chromium window, so you can watch the model work. + launch: { headless: false }, + }), + // tanstack.com renders in the browser. Give each page time to draw. + screenshotDelay: 1000, +}) + +export async function POST({ request }: { request: Request }) { + const params = await chatParamsFromRequest(request) + const stream = chat({ + adapter: openaiText('gpt-6.1-sol'), + messages: params.messages, + threadId: params.threadId, + runId: params.runId, + ...(params.parentRunId ? { parentRunId: params.parentRunId } : {}), + ...(params.resume ? { resume: params.resume } : {}), + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], + // Each action is one step. A search takes more than the default 5. + agentLoopStrategy: maxIterations(20), + }) + return toServerSentEventsResponse(stream) +} + +export const Route = createFileRoute('/api/chat')({ + server: { + handlers: { + POST, + }, + }, +}) +``` + +- `createComputerUse` is at module scope. Every request uses the same browser. +- `threadId` gives each chat its own page. The page stays open between requests of the same chat. +- `tools` gives the model the `computer` tool and the `navigate` tool. +- `middleware` and `interrupts` pause a risky action until a person approves it. +- `maxIterations(20)` lets the model take up to 20 actions in one reply. + +CAUTION: The browser can open every address that your server can reach. Run this app only on your own machine. + +## 3. Show the screenshots + +Each computer action returns a screenshot. On the client, the screenshot is an `image` part inside a `tool-result` part. Open `src/routes/index.tsx` and turn those parts into `` sources: + +```tsx ignore +import type { UIMessage } from '@tanstack/ai-react' + +type Part = UIMessage['parts'][number] + +/** The screenshots in one tool result, as `` sources. */ +function screenshots(part: Part) { + if (part.type !== 'tool-result' || typeof part.content === 'string') return [] + return part.content.flatMap((content) => { + if (content.type !== 'image') return [] + const { source } = content + if (source.type === 'data') { + return [`data:${source.mimeType};base64,${source.value}`] + } + return source.type === 'url' ? [source.value] : [] + }) +} +``` + +Then render text parts and screenshots for each message: + +```tsx ignore +{message.parts.map((part, index) => { + if (part.type === 'text' && part.content) { + return
{part.content}
+ } + return screenshots(part).map((src) => ( + What the model sees + )) +})} +``` + +## 4. Approve risky actions + +OpenAI can mark an action as risky, for example a form submit. The run then pauses. Register `computerSafetyCheck` with `useChat` and show a card for each pause: + +```tsx ignore +import { + fetchServerSentEvents, + useChat, + type GenericInterrupt, +} from '@tanstack/ai-react' +import { computerSafetyCheck } from '@tanstack/ai-computer-use' + +function SafetyCheckCard({ + interrupt, +}: { + interrupt: GenericInterrupt +}) { + const checks = interrupt.payload?.checks ?? [] + return ( +
+

Approve this computer action?

+
    + {checks.map((check) => ( +
  • {check.message ?? check.code ?? check.id}
  • + ))} +
+ + +
+ ) +} + +function ChatPage() { + const { interrupts } = useChat({ + connection: fetchServerSentEvents('/api/chat'), + interrupts: [computerSafetyCheck], + }) + + return ( + <> + {interrupts.map((interrupt) => { + if (interrupt.kind !== 'generic') return null + if (!('definitionId' in interrupt)) return null + if (interrupt.definitionId !== computerSafetyCheck.id) return null + return + })} + + ) +} +``` + +Put the cards above the input, next to the messages from step 3. + +- If you click Approve, the action runs. +- If you click Decline, no action of that turn runs. + +For more, see [Safety checks](../computer-use/safety-checks). + +## 5. Send the task + +Run the app and send this task: + +```text +You are on the TanStack home page. Find the TanStack AI documentation page about computer use. +Use the screenshots. Click, type, and scroll only on what you can see. Do not type a URL from memory. +When the page is the computer use docs, say the page title and stop. +``` + +A Chromium window opens on tanstack.com. The model clicks through the site, and each screenshot shows in the chat. At the end, the model says the title of the computer use page. + +The full example is on GitHub: [TanStack/ai `examples/react/computer-use`](https://github.com/TanStack/ai/tree/main/examples/react/computer-use). It has the same task in the input box. + +To use Anthropic, change the adapter only. To run the browser in a sandbox, see [Drivers](../computer-use/drivers). diff --git a/examples/README.md b/examples/README.md index 34a6460347..2e0660075f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -12,6 +12,7 @@ Choose an example based on your use case: - **Want typed ticket routing?** → [Evaluate](#evaluate) - **Want a blog desk with three agents?** → [Subagents](#subagents) - **Want your own MCP server?** → [MCP server](#mcp-server) +- **Want the model to use a web page?** → [Computer Use](#computer-use) - **Want a full-stack TypeScript app?** → [TanStack Chat (ts-react-chat)](#tanstack-chat-ts-react-chat) - **Need a live world stream?** → [World generation (ts-react-media)](#world-generation-ts-react-media) - **Need a vanilla JS frontend?** → [Vanilla Chat](#vanilla-chat) @@ -146,6 +147,25 @@ Open http://localhost:3100. Click Call the server. --- +### Computer Use + +A slim TanStack Start app. The model opens tanstack.com in Chromium and finds the computer use docs. One provider (OpenAI). Each screenshot shows in the chat. + +**Path:** `examples/react/computer-use` + +**Getting Started:** + +```bash +pnpm --filter computer-use exec playwright-core install chromium +pnpm --filter computer-use dev +``` + +Set `OPENAI_API_KEY` in `.env.local`. Open http://localhost:3100. Click Send. + +📖 [Full Documentation](react/computer-use/README.md) + +--- + ### TanStack Chat (ts-react-chat) A full-featured chat application built with the TanStack ecosystem. diff --git a/examples/react/computer-use/.env.example b/examples/react/computer-use/.env.example new file mode 100644 index 0000000000..00b69dfa39 --- /dev/null +++ b/examples/react/computer-use/.env.example @@ -0,0 +1,3 @@ +# The model reads the screenshots and picks each click. Get a key at +# https://platform.openai.com/api-keys +OPENAI_API_KEY= diff --git a/examples/react/computer-use/.gitignore b/examples/react/computer-use/.gitignore new file mode 100644 index 0000000000..b6ceee9913 --- /dev/null +++ b/examples/react/computer-use/.gitignore @@ -0,0 +1,7 @@ +node_modules +dist +.env +.env.local +.nitro +.tanstack +.output diff --git a/examples/react/computer-use/README.md b/examples/react/computer-use/README.md new file mode 100644 index 0000000000..1a8a09e0ae --- /dev/null +++ b/examples/react/computer-use/README.md @@ -0,0 +1,19 @@ +# Computer Use + +You want the model to do a task on a web page for you. This app opens tanstack.com in Chromium, and the model finds the TanStack AI computer use docs. It reads each screenshot, then clicks, types, and scrolls. Each screenshot shows in the chat. + +## Run it + +1. From the repo root, run `pnpm install`. +2. Download Chromium one time: `pnpm --filter computer-use exec playwright-core install chromium`. +3. Copy `.env.example` to `.env.local`, then set `OPENAI_API_KEY`. Get a key at https://platform.openai.com/api-keys. +4. Run `pnpm --filter computer-use dev`. +5. Open http://localhost:3100 and click Send. + +A Chromium window opens on your machine. Watch the model use it. + +If OpenAI marks an action as risky, an approval card shows. Click Approve to run the action, or Decline to skip it. + +The server route is `src/routes/api.chat.ts`. The chat UI is `src/routes/index.tsx`. + +CAUTION: The browser can open every address that your server can reach. Run this app only on your own machine. diff --git a/examples/react/computer-use/package.json b/examples/react/computer-use/package.json new file mode 100644 index 0000000000..52bbba0b84 --- /dev/null +++ b/examples/react/computer-use/package.json @@ -0,0 +1,34 @@ +{ + "name": "computer-use", + "private": true, + "type": "module", + "scripts": { + "dev": "vite dev --port 3100", + "build": "vite build", + "test:types": "tsc" + }, + "dependencies": { + "@tailwindcss/vite": "^4.1.18", + "@tanstack/ai": "workspace:*", + "@tanstack/ai-computer-playwright": "workspace:*", + "@tanstack/ai-computer-use": "workspace:*", + "@tanstack/ai-openai": "workspace:*", + "@tanstack/ai-react": "workspace:*", + "@tanstack/react-router": "^1.170.41", + "@tanstack/react-start": "^1.168.60", + "@tanstack/router-plugin": "^1.168.42", + "playwright-core": "^1.57.0", + "react": "^19.2.3", + "react-dom": "^19.2.3", + "tailwindcss": "^4.1.18", + "zod": "^4.2.0" + }, + "devDependencies": { + "@types/node": "^24.10.1", + "@types/react": "^19.2.7", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react": "^5.2.0", + "typescript": "5.9.3", + "vite": "^8.2.1" + } +} diff --git a/examples/react/computer-use/src/routeTree.gen.ts b/examples/react/computer-use/src/routeTree.gen.ts new file mode 100644 index 0000000000..861dc17e2b --- /dev/null +++ b/examples/react/computer-use/src/routeTree.gen.ts @@ -0,0 +1,86 @@ +/* eslint-disable */ + +// @ts-nocheck + +// noinspection JSUnusedGlobalSymbols + +// This file was automatically generated by TanStack Router. +// You should NOT make any changes in this file as it will be overwritten. +// Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. + +import { Route as rootRouteImport } from './routes/__root' +import { Route as IndexRouteImport } from './routes/index' +import { Route as ApiChatRouteImport } from './routes/api.chat' + +const IndexRoute = IndexRouteImport.update({ + id: '/', + path: '/', + getParentRoute: () => rootRouteImport, +} as any) +const ApiChatRoute = ApiChatRouteImport.update({ + id: '/api/chat', + path: '/api/chat', + getParentRoute: () => rootRouteImport, +} as any) + +export interface FileRoutesByFullPath { + '/': typeof IndexRoute + '/api/chat': typeof ApiChatRoute +} +export interface FileRoutesByTo { + '/': typeof IndexRoute + '/api/chat': typeof ApiChatRoute +} +export interface FileRoutesById { + __root__: typeof rootRouteImport + '/': typeof IndexRoute + '/api/chat': typeof ApiChatRoute +} +export interface FileRouteTypes { + fileRoutesByFullPath: FileRoutesByFullPath + fullPaths: '/' | '/api/chat' + fileRoutesByTo: FileRoutesByTo + to: '/' | '/api/chat' + id: '__root__' | '/' | '/api/chat' + fileRoutesById: FileRoutesById +} +export interface RootRouteChildren { + IndexRoute: typeof IndexRoute + ApiChatRoute: typeof ApiChatRoute +} + +declare module '@tanstack/react-router' { + interface FileRoutesByPath { + '/': { + id: '/' + path: '/' + fullPath: '/' + preLoaderRoute: typeof IndexRouteImport + parentRoute: typeof rootRouteImport + } + '/api/chat': { + id: '/api/chat' + path: '/api/chat' + fullPath: '/api/chat' + preLoaderRoute: typeof ApiChatRouteImport + parentRoute: typeof rootRouteImport + } + } +} + +const rootRouteChildren: RootRouteChildren = { + IndexRoute: IndexRoute, + ApiChatRoute: ApiChatRoute, +} +export const routeTree = rootRouteImport + ._addFileChildren(rootRouteChildren) + ._addFileTypes() + +import type { getRouter } from './router.tsx' +import type { createStart } from '@tanstack/react-start' +declare module '@tanstack/react-start' { + interface Register { + ssr: true + router: Awaited> + } +} diff --git a/examples/react/computer-use/src/router.tsx b/examples/react/computer-use/src/router.tsx new file mode 100644 index 0000000000..a595444648 --- /dev/null +++ b/examples/react/computer-use/src/router.tsx @@ -0,0 +1,10 @@ +import { createRouter } from '@tanstack/react-router' +import { routeTree } from './routeTree.gen' + +export const getRouter = () => { + return createRouter({ + routeTree, + scrollRestoration: true, + defaultPreloadStaleTime: 0, + }) +} diff --git a/examples/react/computer-use/src/routes/__root.tsx b/examples/react/computer-use/src/routes/__root.tsx new file mode 100644 index 0000000000..7486c94f8f --- /dev/null +++ b/examples/react/computer-use/src/routes/__root.tsx @@ -0,0 +1,41 @@ +import { HeadContent, Scripts, createRootRoute } from '@tanstack/react-router' +import appCss from '../styles.css?url' + +export const Route = createRootRoute({ + head: () => ({ + meta: [ + { + charSet: 'utf-8', + }, + { + name: 'viewport', + content: 'width=device-width, initial-scale=1', + }, + { + title: 'Computer Use', + }, + ], + links: [ + { + rel: 'stylesheet', + href: appCss, + }, + ], + }), + + shellComponent: RootDocument, +}) + +function RootDocument({ children }: { children: React.ReactNode }) { + return ( + + + + + + {children} + + + + ) +} diff --git a/examples/react/computer-use/src/routes/api.chat.ts b/examples/react/computer-use/src/routes/api.chat.ts new file mode 100644 index 0000000000..07c1aafa64 --- /dev/null +++ b/examples/react/computer-use/src/routes/api.chat.ts @@ -0,0 +1,47 @@ +import { createFileRoute } from '@tanstack/react-router' +import { + chat, + chatParamsFromRequest, + maxIterations, + toServerSentEventsResponse, +} from '@tanstack/ai' +import { openaiText } from '@tanstack/ai-openai' +import { createComputerUse } from '@tanstack/ai-computer-use' +import { playwrightComputer } from '@tanstack/ai-computer-playwright' + +// One computer for the whole server. Each chat thread gets its own page. +const computerUse = createComputerUse({ + driver: playwrightComputer({ + startUrl: 'https://tanstack.com', + // Show the Chromium window, so you can watch the model work. + launch: { headless: false }, + }), + // tanstack.com renders in the browser. Give each page time to draw. + screenshotDelay: 1000, +}) + +export async function POST({ request }: { request: Request }) { + const params = await chatParamsFromRequest(request) + const stream = chat({ + adapter: openaiText('gpt-6.1-sol'), + messages: params.messages, + threadId: params.threadId, + runId: params.runId, + ...(params.parentRunId ? { parentRunId: params.parentRunId } : {}), + ...(params.resume ? { resume: params.resume } : {}), + tools: computerUse.tools, + middleware: [computerUse.middleware], + interrupts: [computerUse.interrupt], + // Each action is one step. A search takes more than the default 5. + agentLoopStrategy: maxIterations(20), + }) + return toServerSentEventsResponse(stream) +} + +export const Route = createFileRoute('/api/chat')({ + server: { + handlers: { + POST, + }, + }, +}) diff --git a/examples/react/computer-use/src/routes/index.tsx b/examples/react/computer-use/src/routes/index.tsx new file mode 100644 index 0000000000..49c5e52ffe --- /dev/null +++ b/examples/react/computer-use/src/routes/index.tsx @@ -0,0 +1,200 @@ +import { useEffect, useRef, useState } from 'react' +import { createFileRoute } from '@tanstack/react-router' +import { + fetchServerSentEvents, + useChat, + type GenericInterrupt, + type UIMessage, +} from '@tanstack/ai-react' +import { computerSafetyCheck } from '@tanstack/ai-computer-use' + +const TASK = `You are on the TanStack home page. Find the TanStack AI documentation page about computer use. +Use the screenshots. Click, type, and scroll only on what you can see. Do not type a URL from memory. +When the page is the computer use docs, say the page title and stop.` + +type Part = UIMessage['parts'][number] + +/** The screenshots in one tool result, as `` sources. */ +function screenshots(part: Part) { + if (part.type !== 'tool-result' || typeof part.content === 'string') return [] + return part.content.flatMap((content) => { + if (content.type !== 'image') return [] + const { source } = content + if (source.type === 'data') { + return [`data:${source.mimeType};base64,${source.value}`] + } + return source.type === 'url' ? [source.value] : [] + }) +} + +function Messages({ messages }: { messages: ReadonlyArray }) { + const containerRef = useRef(null) + + useEffect(() => { + if (containerRef.current) { + containerRef.current.scrollTop = containerRef.current.scrollHeight + } + }, [messages]) + + if (!messages.length) { + return ( +
+
+

+ Computer Use +

+

+ Click Send. A Chromium window opens on tanstack.com, and the model + looks for the computer use docs. Each screenshot shows here. +

+
+
+ ) + } + + return ( +
+ {messages.map((message) => ( +
+ {message.parts.map((part, index) => { + if (part.type === 'text' && part.content) { + return ( +
+ {part.content} +
+ ) + } + return screenshots(part).map((src) => ( + What the model sees + )) + })} +
+ ))} +
+ ) +} + +/** OpenAI can mark an action as risky. A person approves it before it runs. */ +function SafetyCheckCard({ + interrupt, +}: { + interrupt: GenericInterrupt +}) { + const checks = interrupt.payload?.checks ?? [] + return ( +
+

Approve this computer action?

+
    + {checks.map((check) => ( +
  • {check.message ?? check.code ?? check.id}
  • + ))} +
+
+        {JSON.stringify(interrupt.payload?.action, null, 2)}
+      
+
+ + +
+
+ ) +} + +function ChatPage() { + const [input, setInput] = useState(TASK) + const { messages, sendMessage, isLoading, error, stop, interrupts } = useChat( + { + connection: fetchServerSentEvents('/api/chat'), + interrupts: [computerSafetyCheck], + }, + ) + + const handleSendMessage = () => { + if (!input.trim()) return + sendMessage(input.trim()) + setInput('') + } + + return ( +
+
+ + + {interrupts.map((interrupt) => { + if (interrupt.kind !== 'generic') return null + if (!('definitionId' in interrupt)) return null + if (interrupt.definitionId !== computerSafetyCheck.id) return null + return + })} + + {error ? ( +
+ {error.message} +
+ ) : null} + +
+
+ {isLoading ? ( +
+ +
+ ) : null} +
+