fix(openai-base): run the computer use loop

.changeset/openai-computer-call-loop.md

@@ -0,0 +1,5 @@
+---
+'@tanstack/openai-base': patch
+---
+
+The computer use loop of `computerUseTool()` runs. A `computer_call` from the model is now a tool call to `computer_use_preview`, with the `action` and the `pending_safety_checks` as its input. The tool result goes back as `computer_call_output`, with the first image of the result as the screenshot. When the call does not fail, the output acknowledges the safety checks of the call.

docs/adapters/openai.md

@@ -996,25 +996,63 @@ Lets the model observe a virtual desktop via screenshots and interact with
 it using keyboard and mouse events. Provide the display dimensions and the
 execution environment type.
 
+The model asks for one action at a time, for example a click. Your app runs the action, then returns a screenshot of the screen. Add an `execute` to the tool that does this:
+
 ```typescript
 import { chat } from "@tanstack/ai";
 import { openaiText } from "@tanstack/ai-openai";
 import { computerUseTool } from "@tanstack/ai-openai/tools";
 
+interface SafetyCheck {
+  id: string;
+  code?: string;
+  message?: string;
+}
+
+// Your code: run one action on the desktop, then return a PNG of the screen as base64.
+declare function runAction(action: unknown): Promise<string>;
+// Your code: show the checks to a person and return their answer.
+declare function askPerson(checks: Array<SafetyCheck>): Promise<boolean>;
+
 const stream = chat({
   adapter: openaiText("computer-use-preview"),
   messages: [{ role: "user", content: "Open Chrome and navigate to example.com" }],
   tools: [
-    computerUseTool({
-      type: "computer_use_preview",
-      display_width: 1024,
-      display_height: 768,
-      environment: "browser",
-    }),
+    {
+      ...computerUseTool({
+        type: "computer_use_preview",
+        display_width: 1024,
+        display_height: 768,
+        environment: "browser",
+      }),
+      execute: async (input: {
+        action: unknown;
+        pending_safety_checks: Array<SafetyCheck>;
+      }) => {
+        const checks = input.pending_safety_checks;
+        if (checks.length > 0 && !(await askPerson(checks))) {
+          throw new Error("A person did not allow this action.");
+        }
+        const screenshot = await runAction(input.action);
+        return [
+          {
+            type: "image",
+            source: { type: "data", value: screenshot, mimeType: "image/png" },
+          },
+        ];
+      },
+    },
   ],
 });
 ```
 
+The input of each call has two fields:
+
+- `action`: the action to run, as OpenAI sends it. For example `{ type: "click", button: "left", x: 120, y: 340 }`.
+- `pending_safety_checks`: the safety checks of the action. If the list is not empty, ask a person before you run the action.
+
+The adapter sends the first image of the result back as the screenshot. When the call does not fail, the adapter also tells OpenAI that the safety checks are accepted. When the call fails, they stay unaccepted.
+
 **Supported models:** `computer-use-preview`. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools).
 
 ### `localShellTool`

docs/tools/provider-tools.md

@@ -90,7 +90,7 @@ const followUp = chat({
 The search/fetch call surfaces as a provider-executed `tool-call` part on the
 assistant message; the agent loop never tries to run it client-side.
 
-OpenAI `applyPatchTool`, `localShellTool`, and `shellTool({ environment: { type: "local" } })` are different. The model returns the call, and your app runs it. See [OpenAI adapter](../adapters/openai.md#applypatchtool).
+OpenAI `applyPatchTool`, `localShellTool`, `computerUseTool`, and `shellTool({ environment: { type: "local" } })` are different. The model returns the call, and your app runs it. See [OpenAI adapter](../adapters/openai.md#applypatchtool) and [`computerUseTool`](../adapters/openai.md#computerusetool).
 
 ### Read web search sources
 

packages/openai-base/src/adapters/responses-text.ts

@@ -51,6 +51,7 @@ import type {
 } from '@tanstack/ai/adapters'
 import type {
   Response,
+  ResponseComputerToolCallOutputScreenshot,
   ResponseCreateParams,
   ResponseFunctionCallOutputItem,
   ResponseFunctionWebSearch,
@@ -400,7 +401,10 @@ export interface OpenAIResponsesToolCallMetadata extends ProviderExecutedToolMet
    * `additional_tools` has one, and the API needs it on the replayed call.
    */
   namespace?: string
-  /** Set for shell, local_shell, and apply_patch calls the app must run. */
+  /**
+   * Set for shell, local_shell, apply_patch, and computer calls the app must
+   * run.
+   */
   openaiUserTool?: OpenAIUserToolName
   /** Shell `action.max_output_length`, echoed on `shell_call_output`. */
   maxOutputLength?: number | null
@@ -2913,7 +2917,10 @@ export abstract class OpenAIBaseResponsesTextAdapter<
           ? userToolCalls.get(message.toolCallId)
           : undefined
         const userOutput = owner
-          ? userToolResultItem(owner, message.content)
+          ? userToolResultItem(owner, message.content, {
+              screenshot: () => this.screenshotOf(message.content),
+              failed: message.error !== undefined,
+            })
           : null
         if (userOutput) {
           result.push(userOutput)
@@ -3149,6 +3156,22 @@ export abstract class OpenAIBaseResponsesTextAdapter<
     return result
   }
 
+  /** The first image of a tool result, as a computer screenshot. */
+  private screenshotOf(
+    content: ModelMessage['content'],
+  ): ResponseComputerToolCallOutputScreenshot {
+    const image = Array.isArray(content)
+      ? content.find((part) => part.type === 'image')
+      : undefined
+    const input = image ? this.convertContentPartToInput(image) : undefined
+    if (input?.type !== 'input_image') return { type: 'computer_screenshot' }
+    return {
+      type: 'computer_screenshot',
+      ...(input.image_url ? { image_url: input.image_url } : {}),
+      ...(input.file_id ? { file_id: input.file_id } : {}),
+    }
+  }
+
   /**
    * Converts a ContentPart to Responses API input content item.
    * Handles text, image, audio, and document (PDF) content parts.

packages/openai-base/src/adapters/responses-user-tools.ts

@@ -1,11 +1,20 @@
-import type { ResponseInputItem } from 'openai/resources/responses/responses'
+import type {
+  ResponseComputerToolCall,
+  ResponseComputerToolCallOutputScreenshot,
+  ResponseInputItem,
+} from 'openai/resources/responses/responses'
 
 /**
  * OpenAI Responses tools the app must run.
  * A container `shell` runs on the provider. `apply_patch`, `local_shell`,
- * and `shell` with `environment.type: "local"` (or no environment) do not.
+ * `computer_use_preview`, and `shell` with `environment.type: "local"` (or
+ * no environment) do not.
  */
-export type OpenAIUserToolName = 'shell' | 'apply_patch' | 'local_shell'
+export type OpenAIUserToolName =
+  | 'shell'
+  | 'apply_patch'
+  | 'local_shell'
+  | 'computer_use_preview'
 
 export interface OpenAIUserExecutedCall {
   name: OpenAIUserToolName
@@ -71,12 +80,47 @@ export function hostedShellCallIds(
 export function readUserToolName(metadata: unknown): OpenAIUserToolName | null {
   if (!isRecord(metadata)) return null
   const name = metadata.openaiUserTool
-  if (name === 'shell' || name === 'apply_patch' || name === 'local_shell') {
+  if (
+    name === 'shell' ||
+    name === 'apply_patch' ||
+    name === 'local_shell' ||
+    name === 'computer_use_preview'
+  ) {
     return name
   }
   return null
 }
 
+/**
+ * A computer action. It goes back to the API as the model sent it, so only
+ * its `type` is checked.
+ */
+function isComputerAction(
+  value: unknown,
+): value is NonNullable<ResponseComputerToolCall['action']> {
+  return isRecord(value) && typeof value.type === 'string'
+}
+
+/** The safety checks of a computer call, with only their known fields. */
+function safetyChecks(
+  value: unknown,
+): Array<ResponseComputerToolCall.PendingSafetyCheck> {
+  if (!Array.isArray(value)) return []
+  return value.flatMap((check) =>
+    isRecord(check) && typeof check.id === 'string'
+      ? [
+          {
+            id: check.id,
+            ...(typeof check.code === 'string' ? { code: check.code } : {}),
+            ...(typeof check.message === 'string'
+              ? { message: check.message }
+              : {}),
+          },
+        ]
+      : [],
+  )
+}
+
 function readItemId(metadata: unknown): string | undefined {
   if (!isRecord(metadata)) return undefined
   return typeof metadata.itemId === 'string' && metadata.itemId.length > 0
@@ -135,6 +179,19 @@ export function readUserExecutedCall(
     }
   }
 
+  if (item.type === 'computer_call') {
+    if (!callId || !isComputerAction(item.action)) return null
+    return {
+      name: 'computer_use_preview',
+      callId,
+      ...(itemId ? { itemId } : {}),
+      input: {
+        action: item.action,
+        pending_safety_checks: safetyChecks(item.pending_safety_checks),
+      },
+    }
+  }
+
   if (item.type === 'local_shell_call') {
     if (!isRecord(item.action)) return null
     const command = stringList(item.action.command)
@@ -234,6 +291,18 @@ export function userToolRequestItem(
     }
   }
 
+  if (name === 'computer_use_preview') {
+    if (!isComputerAction(args.action)) return null
+    return {
+      type: 'computer_call',
+      id: itemId ?? toolCall.id,
+      call_id: toolCall.id,
+      status: 'completed',
+      action: args.action,
+      pending_safety_checks: safetyChecks(args.pending_safety_checks),
+    }
+  }
+
   const commands = stringList(args.commands)
   if (!commands) return null
   return {
@@ -294,13 +363,38 @@ function shellOutputList(content: unknown): Array<{
   return [shellEntry(JSON.stringify(parsed ?? ''))]
 }
 
-/** Replay the app's tool result as the matching Responses output item. */
+/**
+ * Replay the app's tool result as the matching Responses output item.
+ * `computer` is what a `computer_use_preview` result needs: the screenshot,
+ * and whether the call failed. Only a computer result reads the screenshot.
+ * The output of a call that did not fail acknowledges the safety checks of
+ * the call.
+ */
 export function userToolResultItem(
   toolCall: ToolCallLike,
   content: unknown,
+  computer?: {
+    screenshot: () => ResponseComputerToolCallOutputScreenshot
+    failed: boolean
+  },
 ): ResponseInputItem | null {
   const name = readUserToolName(toolCall.metadata)
   if (!name) return null
+
+  if (name === 'computer_use_preview') {
+    const checks = computer?.failed
+      ? []
+      : safetyChecks(
+          parseArguments(toolCall.function.arguments).pending_safety_checks,
+        )
+    return {
+      type: 'computer_call_output',
+      call_id: toolCall.id,
+      output: computer?.screenshot() ?? { type: 'computer_screenshot' },
+      ...(checks.length > 0 ? { acknowledged_safety_checks: checks } : {}),
+    }
+  }
+
   const parsed = parseContent(content)
 
   if (name === 'apply_patch') {