Skip to content

Commit b759cf8

Browse files
authored
Merge pull request #79 from Catapult291/fix/tool-output-image-parts
fix(translator): accept content-part arrays in function_call_output so tool images reach the model
2 parents 012fc96 + 31f4d6e commit b759cf8

3 files changed

Lines changed: 144 additions & 2 deletions

File tree

‎src/translator/openai-to-anthropic.test.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
translateResponseAnthropicToOpenAI,
88
anthropicUsageToOpenAI,
99
} from "./openai-to-anthropic.js";
10+
import { responsesToChatCompletions } from "./responses-to-chat.js";
1011
import {
1112
translateRequestAnthropicToOpenAI,
1213
translateResponseOpenAIToAnthropic,
@@ -304,6 +305,37 @@ describe("translateRequestOpenAIToAnthropic", () => {
304305
expect(blocks[0].content).toBe("62°F and sunny");
305306
});
306307

308+
it("end-to-end: Responses tool output with input_image reaches Anthropic as a tool_result image block", () => {
309+
// A client sending a read_file image result to /v1/responses goes through
310+
// the full chain Responses→Chat→Anthropic, and every hop must preserve
311+
// the image (previously dropped at the Responses→Chat hop).
312+
const dataUrl = "data:image/png;base64,aGVsbG8=";
313+
const responsesReq = {
314+
model: "glm-4.6",
315+
input: [
316+
{ type: "message", role: "user", content: "look at the screenshot" },
317+
{ type: "function_call", call_id: "call_img", name: "read_file", arguments: '{"path":"s.png"}' },
318+
{ type: "function_call_output", call_id: "call_img", output: [
319+
{ type: "input_text", text: "Read image file: s.png" },
320+
{ type: "input_image", image_url: dataUrl },
321+
] },
322+
],
323+
} as Parameters<typeof responsesToChatCompletions>[0];
324+
const chatReq = responsesToChatCompletions(responsesReq).chatRequest;
325+
const anthropicReq = translateRequestOpenAIToAnthropic(chatReq);
326+
const toolResultMsg = anthropicReq.messages[2];
327+
expect(toolResultMsg.role).toBe("user");
328+
const blocks = toolResultMsg.content as any[];
329+
expect(blocks).toHaveLength(1);
330+
expect(blocks[0].type).toBe("tool_result");
331+
expect(blocks[0].tool_use_id).toBe("call_img");
332+
const inner = blocks[0].content as any[];
333+
expect(inner).toEqual([
334+
{ type: "text", text: "Read image file: s.png" },
335+
{ type: "image", source: { type: "base64", media_type: "image/png", data: "aGVsbG8=" } },
336+
]);
337+
});
338+
307339
it("coalesces consecutive role:'tool' messages into a single user message with multiple tool_result blocks", () => {
308340
const req: OpenAIChatRequest = {
309341
model: "glm-4.6",

‎src/translator/responses-to-chat.test.ts‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,44 @@ describe("responsesToChatCompletions", () => {
158158
expect(msgs[2]).toEqual({ role: "tool", tool_call_id: "call_1", content: "sunny" });
159159
});
160160

161+
it("maps function_call_output content-part arrays (input_image) onto Chat tool-message image parts", () => {
162+
// The Responses spec allows `output` to be a content-part array (OpenAI's
163+
// own docs show {type:"input_image", image_url} inside a tool result so
164+
// the model can look at a screenshot the tool returned) — stringly-typed
165+
// handling dropped the image.
166+
const dataUrl = "data:image/png;base64,aGVsbG8=";
167+
const r = responsesToChatCompletions(baseReq({
168+
input: [
169+
{ type: "message", role: "user", content: "read the png" },
170+
{ type: "function_call", call_id: "call_img", name: "read_file", arguments: '{"path":"a.png"}' },
171+
{ type: "function_call_output", call_id: "call_img", output: [
172+
{ type: "input_text", text: "Read image file: a.png" },
173+
{ type: "input_image", image_url: dataUrl },
174+
] },
175+
],
176+
}));
177+
const toolMsg = r.chatRequest.messages[2];
178+
expect(toolMsg.role).toBe("tool");
179+
expect(toolMsg.content).toEqual([
180+
{ type: "text", text: "Read image file: a.png" },
181+
{ type: "image_url", image_url: { url: dataUrl } },
182+
]);
183+
});
184+
185+
it("stringifies structured (non-array, non-string) function_call_output output", () => {
186+
const r = responsesToChatCompletions(baseReq({
187+
input: [
188+
{ type: "function_call", call_id: "c1", name: "get_data", arguments: "{}" },
189+
{ type: "function_call_output", call_id: "c1", output: { rows: 3, ok: true } },
190+
],
191+
}));
192+
expect(r.chatRequest.messages[1]).toEqual({
193+
role: "tool",
194+
tool_call_id: "c1",
195+
content: '{"rows":3,"ok":true}',
196+
});
197+
});
198+
161199
it("translates custom_tool_call + output with {input:...} argument shape", () => {
162200
const r = responsesToChatCompletions(baseReq({
163201
input: [

‎src/translator/responses-to-chat.ts‎

Lines changed: 74 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -221,11 +221,11 @@ function buildMessagesFromItems(items: ResponsesInputItem[]): OpenAIMessage[] {
221221
}
222222
case "function_call_output":
223223
case "custom_tool_call_output": {
224-
const fco = item as { call_id: string; output?: string };
224+
const fco = item as { call_id: string; output?: unknown };
225225
out.push({
226226
role: "tool",
227227
tool_call_id: fco.call_id,
228-
content: fco.output ?? "",
228+
content: toolOutputToChat(fco.output),
229229
});
230230
pendingReasoning = "";
231231
continue;
@@ -302,6 +302,78 @@ function extractReasoningText(item: ResponsesInputItem): string {
302302
.join("\n");
303303
}
304304

305+
/**
306+
* Normalize a `function_call_output` / `custom_tool_call_output` `output` into
307+
* a Chat tool-message `content` value.
308+
*
309+
* The Responses spec allows the output to be a content-part array (OpenAI's
310+
* own docs show `{type:"input_image", image_url}` inside a tool result so the
311+
* model can look at a screenshot the tool returned); upstream Chat dialects
312+
* carry tool images as `image_url` parts on `role:"tool"` — and the downstream
313+
* Chat→Anthropic translator maps those onto Anthropic `tool_result` image
314+
* blocks. Clients sending read_file image results in the array form had every
315+
* image silently dropped when only strings were accepted here.
316+
*
317+
* Non-array, non-string outputs (structured JSON objects) stringify — matches
318+
* OpenAI Chat, where tool-message content is stringly typed.
319+
*/
320+
function toolOutputToChat(output: unknown): string | OpenAIContentPart[] {
321+
if (typeof output === "string") return output;
322+
if (Array.isArray(output)) {
323+
const chatParts = outputPartsToChat(output);
324+
if (chatParts.every((p) => p.type === "text")) return chatParts.map((p) => p.text ?? "").join("");
325+
return chatParts;
326+
}
327+
if (output === undefined || output === null) return "";
328+
try {
329+
return JSON.stringify(output);
330+
} catch {
331+
return String(output);
332+
}
333+
}
334+
335+
/**
336+
* Convert content-part objects appearing inside a tool output array to Chat
337+
* content parts. Accepts the Responses dialect (`input_image` with
338+
* `image_url: string | {url}`, `input_text`, `input_video`, `input_file`)
339+
* and the already-Chat-shaped parts some clients emit verbatim. Anything
340+
* unknown degrades to a text part carrying its JSON — never dropped, so the
341+
* model can still see what the tool returned.
342+
*/
343+
function outputPartsToChat(parts: unknown[]): OpenAIContentPart[] {
344+
const out: OpenAIContentPart[] = [];
345+
for (const part of parts) {
346+
if (part === null || typeof part !== "object") {
347+
out.push({ type: "text", text: String(part) });
348+
continue;
349+
}
350+
const p = part as ResponsesContentPart;
351+
if ((p.type === "input_text" || p.type === "output_text" || p.type === "text") && typeof p.text === "string") {
352+
out.push({ type: "text", text: p.text });
353+
} else if (p.type === "input_image" || p.type === "image_url") {
354+
const url = typeof p.image_url === "string" ? p.image_url : p.image_url?.url;
355+
if (!url) continue;
356+
out.push({ type: "image_url", image_url: { url } });
357+
} else if (p.type === "input_video" || p.type === "video_url") {
358+
const url = typeof p.video_url === "string"
359+
? p.video_url
360+
: p.video_url?.url ?? (typeof p.data === "string" ? p.data : undefined);
361+
if (!url) continue;
362+
out.push({ type: "video_url", video_url: { url } });
363+
} else if (p.type === "input_file" || p.type === "file") {
364+
const fileData = p.file_data ?? p.file?.file_data;
365+
if (!fileData) continue;
366+
out.push({
367+
type: "file",
368+
file: { filename: p.filename ?? p.file?.filename, file_data: fileData },
369+
});
370+
} else {
371+
out.push({ type: "text", text: JSON.stringify(p) });
372+
}
373+
}
374+
return out;
375+
}
376+
305377
/** Convert Responses content parts to a Chat `content` value (string or parts[]). */
306378
function contentPartsToChat(
307379
content: ResponsesContentPart[] | string | undefined,

0 commit comments

Comments
 (0)