diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7f9154ecc..e4e64c332 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2263,9 +2263,14 @@ Claude uses its restrictive profile without claiming those general capabilities. Idle and initial-input Session creation qualify the resolved configuration before persistence; saved Agent resources remain independent of engine restrictions. Claude additionally requires medium verbosity and explicit object-root function -schemas. Function-result batches normalize through the existing shared parser and -reject non-text content before any batch write, preserving pending calls and retry -identity. These are implementation limits, not changes to the upstream contract. +schemas. Function-result batches normalize through the existing shared parser. Claude accepts +text results and, on `none`, successful ordered inline PNG/JPEG results; +workspace images, failed image results and +invalid/remote references reject before any batch write, preserving pending calls +and retry identity. Public qualification receives the full neutral result so +success-dependent limitations remain in the profile. Image-bearing delivery alone +requires Runtime function-result image support; text results and function +declarations do not acquire that requirement. These are implementation limits, not changes to the upstream contract. Do not bypass them by dropping fields, changing model identity or fabricating usage. Operators may configure the daemon provider environment or the existing transient `AGENTS_API_EXECUTION_OPTIONS_FILE` with adapter-owned `claude_provider` @@ -2274,7 +2279,7 @@ persisting them in Session configuration. The adapter exclusively selects the provider environment and removes credentials from native tool environments. Product `claude_code` and product execution are unchanged. The `none` public profile accepts only text, explicit model/system instructions, managed state, exact native resume and -declared functions with ordered text results, and the HTTP MCP subset +declared functions with ordered text or successful inline PNG/JPEG results, and the HTTP MCP subset described above. It rejects unsupported request options and disables built-in tools and undeclared MCP discovery. `DisableExecutionEnvironment` and `DisableSubagents` are accepted assertions about @@ -2318,7 +2323,12 @@ permission checks. It grants no runtime-token business authority. Function results remain pending after stdin/MCP delivery. A matching live, root native user tool_result confirms application only when its Session/call identity, -error flag and returned text match the submission. Ignore replayed, synthetic and +error flag and ordered content match the submission. Text matches exactly; each +submitted image position must remain a valid native base64 image. Native resizing +or re-encoding may change image bytes. This acknowledges incorporation into native +history, not byte/pixel fidelity or completed provider consumption. Public Items +retain the original caller content; real image-dependent model responses separately +qualify usability. Ignore replayed, synthetic and subagent messages. Native error text joins the submitted text parts with newlines; neutral observations retain their original order and separate failure status. Missing/mismatched receipts fail the execution; do not replay unknown delivery. @@ -2327,7 +2337,7 @@ execution on timeout. Invalid or unsupported image results fail before consuming a pending call. Function state belongs to one live Run and ends with it; the existing router owns receipt retry/conflict handling. This does not establish crash recovery or exactly-once effects. Public schemas outside MCP's object-root -contract and image result mapping remain admission/execution gaps. +contract, failed image results and remote image references remain admission/execution gaps. Each SDK result supplies one native usage snapshot, including reported failures. `Usage.Raw.claude_sdk_result` holds the latest; queries with multiple native results @@ -2375,7 +2385,7 @@ recovery remain separate work. Daemon registration alone does not establish publ Public text/function execution, active input, pending-call cancellation and cold continuation are accepted for the registered restrictive profile. Environment -provisioning, broader tools/verbosity, complete public Usage, image results and +provisioning, broader tools/verbosity, complete public Usage, failed image results and process-loss recovery remain gaps. Managed installation and release publication remain separate tasks. `make check-cli` also builds and tests the SDK package, including native output draining; CI selects that check diff --git a/apps/parsar-daemon/internal/agent/claudesdk/functions.go b/apps/parsar-daemon/internal/agent/claudesdk/functions.go index 35ea91186..b135f8ec5 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/functions.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/functions.go @@ -107,10 +107,13 @@ func (s *session) SubmitFunctionResult(ctx context.Context, result proto.Functio return err } for _, part := range result.Content { - if part.Type != "input_text" { - return fmt.Errorf("claudesdk: image function results are not supported") + if part.Type == "input_image" && !result.Success { + return fmt.Errorf("claudesdk: native error results cannot retain images") } } + if err := (proto.MessageInput{{Content: result.Content}}).ValidateInlineImages(); err != nil { + return err + } data, err := json.Marshal(struct { Type string `json:"type"` proto.FunctionResultPayload diff --git a/apps/parsar-daemon/internal/agent/claudesdk/readiness.go b/apps/parsar-daemon/internal/agent/claudesdk/readiness.go index f1852d0c4..b485c4a6a 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/readiness.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/readiness.go @@ -25,6 +25,10 @@ type RuntimeInfo struct { Features []string `json:"features"` } +func (info RuntimeInfo) SupportsFunctionResultImages() bool { + return slices.Contains(info.Features, "function_result_images") +} + func (info RuntimeInfo) SupportsMessageImages() bool { return slices.Contains(info.Features, "message_images") } diff --git a/apps/parsar-daemon/internal/cli/agent_discovery.go b/apps/parsar-daemon/internal/cli/agent_discovery.go index 5f0c4b925..281ced11e 100644 --- a/apps/parsar-daemon/internal/cli/agent_discovery.go +++ b/apps/parsar-daemon/internal/cli/agent_discovery.go @@ -89,6 +89,7 @@ func discoverAgentCLIs(rc *runContext, profile string, checks agentCLIChecks) (a DurableTurns: true, DurableInputReceipts: true, MessageImages: true, + FunctionResultImages: true, FunctionTools: true, MCPHTTPTools: true, MCPHTTPBearerAuth: true, diff --git a/apps/parsar-daemon/internal/cli/claude_sdk.go b/apps/parsar-daemon/internal/cli/claude_sdk.go index adc035a4f..1cfb22748 100644 --- a/apps/parsar-daemon/internal/cli/claude_sdk.go +++ b/apps/parsar-daemon/internal/cli/claude_sdk.go @@ -99,6 +99,7 @@ func discoverClaudeSDK(rc *runContext, profile string, check func(context.Contex } out.Info.Available, out.Info.Version = true, info.SDK out.Info.Capabilities.MessageImages = info.SupportsMessageImages() + out.Info.Capabilities.FunctionResultImages = info.SupportsFunctionResultImages() out.Info.Capabilities.ToolSearch = out.Config.Workspace == nil && info.SupportsToolSearch() out.Info.Capabilities.StructuredOutput = out.Config.Workspace == nil && info.SupportsStructuredOutput() out.Info.Capabilities.SubagentObservations = info.SupportsSubagents() diff --git a/contracts/agents-api/README.md b/contracts/agents-api/README.md index 75ad01bf8..ae9b8f68e 100644 --- a/contracts/agents-api/README.md +++ b/contracts/agents-api/README.md @@ -653,8 +653,9 @@ the immutable Session configuration and creation retry identity. Saved-Agent inheritance uses the same resolved tools. The bounded [deferred discovery path](tool-search.md) adds type-only `tool_search` for its qualified profile. Other discovery combinations, other tool kinds, the native 64-definition cap and unique nonblank names of at most 512 bytes remain -compatibility gaps. Claude SDK additionally requires object-root schemas and -text-only results. Codex internal Goal/Skills/user-input/discovery semantics need +compatibility gaps. Claude SDK additionally requires object-root schemas. It accepts text and +successful inline PNG/JPEG function results on `none`; failed/workspace images and remote references +remain gaps. See [function image coverage](function-result-images.md). Codex internal Goal/Skills/user-input/discovery semantics need upstream evidence; their presence alone does not prove a tool-set mismatch. The worker selects a same-tenant host advertising `function_tools` for configured diff --git a/contracts/agents-api/function-result-images.md b/contracts/agents-api/function-result-images.md new file mode 100644 index 000000000..61e468a56 --- /dev/null +++ b/contracts/agents-api/function-result-images.md @@ -0,0 +1,56 @@ +# Function-result image coverage + +The pinned official function result accepts a string or ordered text/image content, +independently of success. Our qualified Claude subset is successful inline PNG/JPEG +on `environment:none`. Error images, workspace images and remote URLs reject before +batch persistence, without consuming the pending call. These are implementation +gaps, not narrower official types. MiniMax public functions remain unqualified. + +Core retains the original ordered output, error field presence and retry identity. +It passes the existing neutral `FunctionResultPayload` through Runtime. Profile +validation sees placement and success; adapters own native conversion. Only actual +image-result delivery requires `function_result_images` from the selected Runtime. +Ordinary function declarations and text results do not acquire an image requirement. +A Runtime refusal after durable admission fails execution without fabricating +application; the original result remains available for recovery queries. + +Claude uses native MCP text/image blocks. A live root tool result acknowledges the +once-only pending call only with matching Session/call identity, success, exact text, +block count/order and a native base64 image at every image position. Replay, +synthetic and subagent records cannot acknowledge it. Native image resizing or +re-encoding is allowed; this is incorporation into native history, not unchanged +bytes/pixels or a guarantee that the provider has already consumed the image. +Public Items preserve the caller's bytes. Native decode failure, text fallback or +missing images fails receipt validation. The existing uncertain-delivery timeout +and cancellation behavior remain unchanged; no replay mechanism is added. + +Codex's existing result acknowledgement follows a successful transport write. It +must not be described as a native consumption receipt. Real model image use and +native completion provide separate execution evidence. The submitted result remains +durable; process loss between write and native consumption is still unqualified +and tracked as `FUNCTION-RECEIPT-NATIVE-001`. + +## Validation + +`TestNativeFunctionImagePublicExecution` and `tests/official_function_images.py` +exercise the same independent Core/PostgreSQL/daemon/native path with each selected +real model and the pinned SDK 3.13.0 plus raw HTTP. The workflow covers mixed +text/PNG/text, a 6000x2100 PNG requiring native preprocessing, image-only JPEG, +failed text, pending-call cancellation and cold daemon continuation with unchanged +native Session identity. The real answer must read visual information absent from +the tool description and text content. Recovery reads must retain original content. +Retries admit one result; changed retries conflict; foreign tenants cannot read or +submit it. Claude invalid/remote/error image batches leave the call and history +untouched, then a valid result succeeds on that same pending call. + +Direct native feasibility probes separately establish Claude's successful image +preprocessing and lossy error-image path. They do not replace public acceptance. +Controlled tests cover malformed/missing/reordered receipts, wrong identity, +unsupported placement, operation-specific Runtime support and batch atomicity. +The bundled JPEG fixture has yellow, blue, red and green vertical bands; it contains +no metadata or credentials. PNG markers are generated with randomized band order. + +The accepted combinations and run evidence are recorded in the task board. This +batch does not qualify workspace image results, all native image limits, provider +parity, arbitrary managed output rewrites, crash recovery or full Agents API +compatibility. No downloader, image converter or second tool loop belongs in Core. diff --git a/contracts/agents-api/harnesses.md b/contracts/agents-api/harnesses.md index 62566f4e5..0dba1093b 100644 --- a/contracts/agents-api/harnesses.md +++ b/contracts/agents-api/harnesses.md @@ -71,12 +71,12 @@ syntactically or everything either upstream harness can theoretically perform. | --- | --- | --- | | Docker hosted text execution, native local tools | Qualified | Qualified | | Files, immutable Artifacts, cancellation, restart/history recovery | Qualified | Qualified | -| Public functions in `none` | Qualified | Qualified; object-root schemas and text results | +| Public functions in `none` | Qualified | Qualified; object-root schemas; text or successful inline PNG/JPEG results | | Public functions alongside hosted workspace tools | Qualified | Qualified; object-root schemas and text results | | HTTP MCP and static-bearer Vault credentials in `none` | Qualified | Qualified subset | | Required MCP initialization | Qualified | Qualified on `none`; native readiness before initial input | | Hosted HTTP MCP | Gap | Gap | -| Function image results | Supported subset | Gap; currently rejected | +| Function image results | Supported subset; early acknowledgement is transport-only | Successful inline PNG/JPEG on `none`; native resizing allowed, error/workspace images rejected | | Non-default verbosity | Native/model-dependent support | No equivalent qualified; medium only | | Public detailed Usage | Supported native counters | Native raw usage retained; public breakdown gap | | V1 `self_hosted` daemon enrollment at `/workspace` | [Qualified deployment scope](user-managed-runtime-v1.md) | [Qualified deployment scope](user-managed-runtime-v1.md) | diff --git a/contracts/agents-api/openapi.yaml b/contracts/agents-api/openapi.yaml index 1baa5cd1d..ecf80aaba 100644 --- a/contracts/agents-api/openapi.yaml +++ b/contracts/agents-api/openapi.yaml @@ -3239,10 +3239,13 @@ paths: executor URL settings. Disconnecting the waiting HTTP request does not cancel retained work or restart its deadline. Retry keys identify the whole ordered batch. Function output accepts text or ordered text/image parts subject to - engine support; Claude SDK currently accepts text results only. Codex and - Claude SDK on none accept ordered inline PNG/JPEG image messages. Workspace - profiles and other engines remain text-only; remote image URLs are unsupported. - Image references are retained unchanged without service-side downloads. + engine support; Claude SDK accepts text results and, on none, successful inline + PNG/JPEG results, preserving ordered content; error images and remote references + reject before admission. Native image resizing may change bytes. Runtime image-result + support is checked only for image-bearing delivery. Codex and Claude SDK on + none accept ordered inline PNG/JPEG image messages. Workspace profiles and + other engines remain text-only; remote image URLs are unsupported. Image references + are retained unchanged without service-side downloads. parameters: - description: agents=v1 in: header diff --git a/internal/agentdaemon/device/state.go b/internal/agentdaemon/device/state.go index 4cd991e85..4d9ac1a6a 100644 --- a/internal/agentdaemon/device/state.go +++ b/internal/agentdaemon/device/state.go @@ -84,6 +84,7 @@ type KindCapabilities struct { StructuredOutput bool `json:"structured_output,omitempty"` ToolSearch bool `json:"tool_search,omitempty"` MessageImages bool `json:"message_images,omitempty"` + FunctionResultImages bool `json:"function_result_images,omitempty"` SubagentControl bool `json:"subagent_control,omitempty"` FunctionTools bool `json:"function_tools,omitempty"` MCPHTTPTools bool `json:"mcp_http_tools,omitempty"` diff --git a/internal/agentdaemon/gateway/functions_test.go b/internal/agentdaemon/gateway/functions_test.go index 1de61472c..c77c9daf1 100644 --- a/internal/agentdaemon/gateway/functions_test.go +++ b/internal/agentdaemon/gateway/functions_test.go @@ -7,11 +7,11 @@ import ( func TestFunctionCapabilitySurvivesHeartbeatMapping(t *testing.T) { for _, supported := range []bool{false, true} { - kinds := deviceKindsFromHeartbeat(proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: proto.AgentKindCapabilities{FunctionTools: supported}}}}) + kinds := deviceKindsFromHeartbeat(proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: proto.AgentKindCapabilities{FunctionTools: supported, FunctionResultImages: supported}}}}) s := &Session{} s.setSupportedAgentKinds(kinds) info, found, known := s.AgentKindStatus("codex") - if !found || !known || info.Capabilities.FunctionTools != supported { + if !found || !known || info.Capabilities.FunctionTools != supported || info.Capabilities.FunctionResultImages != supported { t.Fatal(info, found, known) } } diff --git a/internal/agentdaemon/gateway/session.go b/internal/agentdaemon/gateway/session.go index 4188fa311..47fd77dc5 100644 --- a/internal/agentdaemon/gateway/session.go +++ b/internal/agentdaemon/gateway/session.go @@ -564,6 +564,7 @@ func deviceKindsFromHeartbeat(p proto.HeartbeatPayload) []device.SupportedAgentK StructuredOutput: info.Capabilities.StructuredOutput, ToolSearch: info.Capabilities.ToolSearch, MessageImages: info.Capabilities.MessageImages, + FunctionResultImages: info.Capabilities.FunctionResultImages, ExecutionControls: info.Capabilities.ExecutionControls, SubagentControl: info.Capabilities.SubagentControl, SubagentObservations: info.Capabilities.SubagentObservations, diff --git a/internal/agentdaemon/proto/inbound.go b/internal/agentdaemon/proto/inbound.go index b88ae9052..20dc08183 100644 --- a/internal/agentdaemon/proto/inbound.go +++ b/internal/agentdaemon/proto/inbound.go @@ -276,6 +276,7 @@ type AgentKindCapabilities struct { StructuredOutput bool `json:"structured_output,omitempty"` ToolSearch bool `json:"tool_search,omitempty"` MessageImages bool `json:"message_images,omitempty"` + FunctionResultImages bool `json:"function_result_images,omitempty"` SubagentControl bool `json:"subagent_control,omitempty"` DurableInputReceipts bool `json:"durable_input_receipts,omitempty"` // DurableTurns includes strict resume, completion release and cancellation snapshots. diff --git a/internal/agentdaemon/proto/message_images.go b/internal/agentdaemon/proto/message_images.go index 8d440dc36..233c472de 100644 --- a/internal/agentdaemon/proto/message_images.go +++ b/internal/agentdaemon/proto/message_images.go @@ -10,8 +10,8 @@ import ( "strings" ) -// ValidateInlineImages checks the bounded user-message image profile. It does -// not download references, rewrite bytes or change function-result support. +// ValidateInlineImages checks inline PNG/JPEG content without downloading or +// rewriting it. Callers separately qualify message/function-result support. func (m MessageInput) ValidateInlineImages() error { for _, message := range m { for _, part := range message.Content { @@ -23,15 +23,15 @@ func (m MessageInput) ValidateInlineImages() error { } header, encoded, ok := strings.Cut(*part.ImageURL, ",") if !ok || (header != "data:image/png;base64" && header != "data:image/jpeg;base64") { - return errors.New("user image requires inline PNG or JPEG") + return errors.New("image requires inline PNG or JPEG") } data, err := base64.StdEncoding.Strict().DecodeString(encoded) if err != nil || base64.StdEncoding.EncodeToString(data) != encoded { - return errors.New("user image requires valid base64") + return errors.New("image requires valid base64") } _, format, err := image.DecodeConfig(bytes.NewReader(data)) if err != nil || header != "data:image/"+format+";base64" { - return errors.New("user image format does not match its media type") + return errors.New("image format does not match its media type") } } } diff --git a/packages/claude-sdk-adapter/src/function_bridge.ts b/packages/claude-sdk-adapter/src/function_bridge.ts index 8e43a3a38..d849288c6 100644 --- a/packages/claude-sdk-adapter/src/function_bridge.ts +++ b/packages/claude-sdk-adapter/src/function_bridge.ts @@ -2,13 +2,14 @@ import type { SDKMessage } from "@anthropic-ai/claude-agent-sdk"; import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import { isDeepStrictEqual } from "node:util"; import type { FunctionCall, FunctionHandler } from "./functions.js"; +import { imageSource, nativeContent, parseInputContent, type InputContent } from "./message_input.js"; export type FunctionResult = { type: "function_result"; call_id: string; delivery_id: string; success: boolean; - content: { type: "input_text"; text: string }[]; + content: InputContent[]; }; export type FunctionEvent = | { type: "function_call"; call: { call_id: string; name: string; arguments: Record } } @@ -51,15 +52,21 @@ export class FunctionBridge { const result = value as FunctionResult; if (Object.keys(result).some(key => !["type", "call_id", "delivery_id", "success", "content"].includes(key)) || result.type !== "function_result" || typeof result.call_id !== "string" || !result.call_id || - typeof result.delivery_id !== "string" || !result.delivery_id || typeof result.success !== "boolean" || - !Array.isArray(result.content) || result.content.some(part => !part || part.type !== "input_text" || - typeof part.text !== "string" || Object.keys(part).some(key => key !== "type" && key !== "text"))) { + typeof result.delivery_id !== "string" || !result.delivery_id || typeof result.success !== "boolean") { throw new Error("Invalid function result."); } + parseInputContent(result.content); + if (!result.success && result.content.some(part => part.type === "input_image")) { + throw new Error("Native error results cannot retain images."); + } const pending = this.pending.get(result.call_id); if (!pending || pending.result) throw new Error("Function result is not pending."); pending.result = result; - pending.resolve({ content: result.content.map(part => ({ type: "text", text: part.text })), isError: !result.success }); + pending.resolve({ content: result.content.map(part => { + if (part.type === "input_text") return { type: "text" as const, text: part.text }; + const source = imageSource(part.image_url); + return { type: "image" as const, mimeType: source.media_type, data: source.data }; + }), isError: !result.success }); } async consume(message: SDKMessage, sessionID: string): Promise { @@ -72,9 +79,9 @@ export class FunctionBridge { const result = pending.result; if (!result) throw new Error("Native response preceded host result."); // Native error results join MCP text blocks; retain the original parts in the host. - const expected = result.success ? result.content.map(part => ({ type: "text", text: part.text })) : - result.content.map(part => part.text).join("\n"); - if (!!block.is_error !== !result.success || !isDeepStrictEqual(block.content, expected)) { + const matches = result.success ? matchesNativeContent(result.content, block.content) : + block.content === result.content.map(part => part.type === "input_text" ? part.text : "").join("\n"); + if (!!block.is_error !== !result.success || !matches) { throw new Error("Native function response differs from submitted result."); } await this.emit({ type: "function_applied", call_id: result.call_id, delivery_id: result.delivery_id }); @@ -95,3 +102,20 @@ export class FunctionBridge { this.pending.clear(); } } + +// A matching root call confirms incorporation into native history. Native image +// decoding/resizing may change bytes; preserve text and ordered image positions. +function matchesNativeContent(content: InputContent[], actual: unknown): boolean { + const expected = nativeContent({ content }); + if (!Array.isArray(expected) || !Array.isArray(actual) || actual.length !== expected.length) return false; + return expected.every((part, index) => { + const received = actual[index]; + if (part.type !== "image") return isDeepStrictEqual(part, received); + if (!received || received.type !== "image" || received.source?.type !== "base64" || + typeof received.source.media_type !== "string" || typeof received.source.data !== "string") return false; + try { + imageSource(`data:${received.source.media_type};base64,${received.source.data}`); + return true; + } catch { return false; } + }); +} diff --git a/packages/claude-sdk-adapter/src/message_input.ts b/packages/claude-sdk-adapter/src/message_input.ts index 41fa19b16..c8add18d8 100644 --- a/packages/claude-sdk-adapter/src/message_input.ts +++ b/packages/claude-sdk-adapter/src/message_input.ts @@ -9,23 +9,29 @@ export function parseMessageInput(value: unknown): MessageInput { for (const message of value) { if (!message || typeof message !== "object" || Object.keys(message).some(key => key !== "content") || !Array.isArray(message.content) || !message.content.length) throw new Error("Invalid user message."); - let meaningful = false; - for (const part of message.content) { - if (!part || typeof part !== "object") throw new Error("Invalid user content."); - if (part.type === "input_text" && typeof part.text === "string" && Object.keys(part).every(key => key === "type" || key === "text")) { - meaningful ||= Boolean(part.text.trim()); - } else if (part.type === "input_image" && typeof part.image_url === "string" && - Object.keys(part).every(key => key === "type" || key === "image_url")) { - imageSource(part.image_url); - meaningful = true; - } else throw new Error("Invalid user content."); - } + const content = parseInputContent(message.content); + const meaningful = content.some(part => part.type === "input_image" || Boolean(part.text.trim())); if (!meaningful) throw new Error("Empty user message."); } return value as MessageInput; } -function imageSource(url: string): { type: "base64"; media_type: "image/png" | "image/jpeg"; data: string } { +// Function results share content validation but permit empty arrays and empty text. +export function parseInputContent(content: unknown): InputContent[] { + if (!Array.isArray(content)) throw new Error("Invalid input content."); + for (const part of content) { + if (!part || typeof part !== "object") throw new Error("Invalid user content."); + if (part.type === "input_text" && typeof part.text === "string" && Object.keys(part).every(key => key === "type" || key === "text")) { + continue; + } else if (part.type === "input_image" && typeof part.image_url === "string" && + Object.keys(part).every(key => key === "type" || key === "image_url")) { + imageSource(part.image_url); + } else throw new Error("Invalid user content."); + } + return content as InputContent[]; +} + +export function imageSource(url: string): { type: "base64"; media_type: "image/png" | "image/jpeg"; data: string } { const match = /^data:(image\/png|image\/jpeg);base64,([A-Za-z0-9+/]+={0,2})$/.exec(url); if (!match || Buffer.from(match[2]!, "base64").toString("base64") !== match[2]) throw new Error("Unsupported image reference."); return { type: "base64", media_type: match[1] as "image/png" | "image/jpeg", data: match[2]! }; diff --git a/packages/claude-sdk-adapter/src/runtime_check.ts b/packages/claude-sdk-adapter/src/runtime_check.ts index ad2463b27..5db1b0a51 100644 --- a/packages/claude-sdk-adapter/src/runtime_check.ts +++ b/packages/claude-sdk-adapter/src/runtime_check.ts @@ -45,7 +45,7 @@ try { assert.equal(smoke.error, undefined, "bridge_unavailable"); assert.equal(smoke.status, 0, "bridge_unavailable"); assert.deepEqual(JSON.parse(smoke.stdout), { type: "error", code: "invalid_request" }); - process.stdout.write(JSON.stringify({ type: "runtime_ready", protocol: 2, features: [...(process.platform === "linux" ? ["workspace_directory", "local_runtime_v1", "workspace_functions"] : []), "message_images", "tool_search", "structured_output", "subagent_resources", "mcp_http_tools", "mcp_http_bearer_auth", "mcp_http_required", "workspace_tools", "workspace_prepare", "workspace_read", "workspace_command_observations"], node: process.versions.node, sdk: sdk.version, mcp: mcp.version, native: nativeVersion }) + "\n"); + process.stdout.write(JSON.stringify({ type: "runtime_ready", protocol: 2, features: [...(process.platform === "linux" ? ["workspace_directory", "local_runtime_v1", "workspace_functions"] : []), "message_images", "function_result_images", "tool_search", "structured_output", "subagent_resources", "mcp_http_tools", "mcp_http_bearer_auth", "mcp_http_required", "workspace_tools", "workspace_prepare", "workspace_read", "workspace_command_observations"], node: process.versions.node, sdk: sdk.version, mcp: mcp.version, native: nativeVersion }) + "\n"); } catch { // Native diagnostics can include operator environment; never forward them. process.stdout.write(JSON.stringify({ type: "runtime_unavailable" }) + "\n"); diff --git a/packages/claude-sdk-adapter/tests/function_bridge.test.mjs b/packages/claude-sdk-adapter/tests/function_bridge.test.mjs index 08f97db79..fac8458d9 100644 --- a/packages/claude-sdk-adapter/tests/function_bridge.test.mjs +++ b/packages/claude-sdk-adapter/tests/function_bridge.test.mjs @@ -65,3 +65,50 @@ test("an aborted submitted result never becomes a native application receipt", { await bridge.consume(native(result("a", false)), "native"); assert.deepEqual(events.map(event => event.type), ["function_call"]); }); + + +test("successful image results confirm native preprocessing with unchanged identity and block order", async () => { + const events = []; + const bridge = new FunctionBridge(async event => { events.push(event); }); + const value = { ...result("visual"), content: [ + { type: "input_text", text: "before" }, + { type: "input_image", image_url: "data:image/png;base64,AQID" }, + { type: "input_text", text: "after" }, + ] }; + const waiting = bridge.invoke(call("visual"), new AbortController().signal); + bridge.submit(JSON.stringify(value)); + assert.deepEqual((await waiting).content, [ + { type: "text", text: "before" }, { type: "image", mimeType: "image/png", data: "AQID" }, + { type: "text", text: "after" }, + ]); + const converted = [ + { type: "text", text: "before" }, + { type: "image", source: { type: "base64", media_type: "image/jpeg", data: "BAUG" } }, + { type: "text", text: "after" }, + ]; + for (const bad of [converted.slice(1), [...converted, converted[0]], + [converted[0], { type: "text", text: "Image could not be processed" }, converted[2]], + [converted[2], converted[1], converted[0]], + [converted[0], { type: "image", source: { type: "base64", media_type: "image/jpeg", data: "!!" } }, converted[2]]]) { + await assert.rejects(bridge.consume(native(value, bad), "native"), /differs/); + } + await bridge.consume({ ...native(value, converted), isSynthetic: true }, "native"); + await bridge.consume({ ...native(value, converted), isReplay: true }, "native"); + await bridge.consume(native(value, converted), "other"); + assert.equal(events.length, 1); + await bridge.consume(native(value, converted), "native"); + assert.deepEqual(events[1], { type: "function_applied", call_id: "visual", delivery_id: "delivery-visual" }); + bridge.assertComplete(); +}); + +test("native error-image rejection leaves the pending call available for a supported result", async () => { + const events = []; + const bridge = new FunctionBridge(async event => { events.push(event); }); + const waiting = bridge.invoke(call("error"), new AbortController().signal); + const value = { ...result("error", false), content: [{ type: "input_image", image_url: "data:image/png;base64,AQID" }] }; + assert.throws(() => bridge.submit(JSON.stringify(value)), /cannot retain images/); + bridge.submit(JSON.stringify(result("error", false))); + assert.equal((await waiting).isError, true); + await bridge.consume(native(result("error", false)), "native"); + bridge.assertComplete(); +}); diff --git a/scripts/check-claude-sdk-runtime.mjs b/scripts/check-claude-sdk-runtime.mjs index 115f19533..716c4c226 100644 --- a/scripts/check-claude-sdk-runtime.mjs +++ b/scripts/check-claude-sdk-runtime.mjs @@ -23,7 +23,7 @@ assert.equal(probe.status, 0, "Exported runtime is unavailable"); const report = JSON.parse(probe.stdout); assert.equal(report.type, "runtime_ready"); assert.equal(report.protocol, 2); -assert.deepEqual(report.features, [...(process.platform === "linux" ? ["workspace_directory", "local_runtime_v1", "workspace_functions"] : []), "message_images", "tool_search", "structured_output", "subagent_resources", "mcp_http_tools", "mcp_http_bearer_auth", "mcp_http_required", "workspace_tools", "workspace_prepare", "workspace_read", "workspace_command_observations"]); +assert.deepEqual(report.features, [...(process.platform === "linux" ? ["workspace_directory", "local_runtime_v1", "workspace_functions"] : []), "message_images", "function_result_images", "tool_search", "structured_output", "subagent_resources", "mcp_http_tools", "mcp_http_bearer_auth", "mcp_http_required", "workspace_tools", "workspace_prepare", "workspace_read", "workspace_command_observations"]); assert.equal(report.sdk, source.dependencies["@anthropic-ai/claude-agent-sdk"]); assert.equal(report.mcp, source.dependencies["@modelcontextprotocol/sdk"]); console.log(`Verified exported SDK ${report.sdk}, MCP ${report.mcp}, ${report.native}`); diff --git a/services/agents-api/internal/api/inputs.go b/services/agents-api/internal/api/inputs.go index ca4ab3bf9..23183ecd1 100644 --- a/services/agents-api/internal/api/inputs.go +++ b/services/agents-api/internal/api/inputs.go @@ -24,7 +24,7 @@ type Option func(*Handler) func WithExecution(s InputSubmitter) Option { return func(h *Handler) { h.inputs = s } } // @Summary Submit Session input events -// @Description For environment none, atomically accepts text messages, cancellation and function results. Messages steer active work or start a queued Turn. The supported self_hosted and openai_hosted profiles accept text-only batches. Under the Session lock, matching retries retain their original target; new active messages append to the current Turn, while idle messages reserve work and wait up to the original five-minute connection/admission deadline. Return 204 only after durable admission, without claiming native application; active messages create no Turn or reservation. Cancellation-only prepared-environment batches use existing durable cancellation admission and return 204 without waiting for native exit; a new cancellation conflicts while a pre-Turn reservation is pending. Homogeneous tool_result-only prepared-environment batches reuse existing scoped result admission and application receipts without creating a Turn or bypassing a pending reservation. Mixed prepared-environment batches remain unsupported. HTTP expiry/cancellation use local 409 environment_input_expired/environment_input_cancelled errors; exact hosted failure mapping is unverified. Losing execution ownership returns 503. The response write deadline accommodates the admission window for either prepared Environment, independently of new-hosted-admission and executor URL settings. Disconnecting the waiting HTTP request does not cancel retained work or restart its deadline. Retry keys identify the whole ordered batch. Function output accepts text or ordered text/image parts subject to engine support; Claude SDK currently accepts text results only. Codex and Claude SDK on none accept ordered inline PNG/JPEG image messages. Workspace profiles and other engines remain text-only; remote image URLs are unsupported. Image references are retained unchanged without service-side downloads. +// @Description For environment none, atomically accepts text messages, cancellation and function results. Messages steer active work or start a queued Turn. The supported self_hosted and openai_hosted profiles accept text-only batches. Under the Session lock, matching retries retain their original target; new active messages append to the current Turn, while idle messages reserve work and wait up to the original five-minute connection/admission deadline. Return 204 only after durable admission, without claiming native application; active messages create no Turn or reservation. Cancellation-only prepared-environment batches use existing durable cancellation admission and return 204 without waiting for native exit; a new cancellation conflicts while a pre-Turn reservation is pending. Homogeneous tool_result-only prepared-environment batches reuse existing scoped result admission and application receipts without creating a Turn or bypassing a pending reservation. Mixed prepared-environment batches remain unsupported. HTTP expiry/cancellation use local 409 environment_input_expired/environment_input_cancelled errors; exact hosted failure mapping is unverified. Losing execution ownership returns 503. The response write deadline accommodates the admission window for either prepared Environment, independently of new-hosted-admission and executor URL settings. Disconnecting the waiting HTTP request does not cancel retained work or restart its deadline. Retry keys identify the whole ordered batch. Function output accepts text or ordered text/image parts subject to engine support; Claude SDK accepts text results and, on none, successful inline PNG/JPEG results, preserving ordered content; error images and remote references reject before admission. Native image resizing may change bytes. Runtime image-result support is checked only for image-bearing delivery. Codex and Claude SDK on none accept ordered inline PNG/JPEG image messages. Workspace profiles and other engines remain text-only; remote image URLs are unsupported. Image references are retained unchanged without service-side downloads. // @Tags Sessions // @Accept json // @Security BearerAuth diff --git a/services/agents-api/internal/engine/claude.go b/services/agents-api/internal/engine/claude.go index aae4ac8ce..f08a13353 100644 --- a/services/agents-api/internal/engine/claude.go +++ b/services/agents-api/internal/engine/claude.go @@ -18,12 +18,15 @@ func claudeProfile() Profile { Placements: []string{"none", "openai_hosted", "self_hosted"}, MCPBearer: true, ValidateConfiguration: validateClaudeConfiguration, ValidateTools: validateClaudeTools, - ValidateFunctionResult: func(content []proto.InputContent) error { - for _, part := range content { - if part.Type != "input_text" { + ValidateFunctionResult: func(placement string, result proto.FunctionResultPayload) error { + for _, part := range result.Content { + if part.Type == "input_image" && (!result.Success || placement != "none") { return ErrInvalidInput } } + if (proto.MessageInput{{Content: result.Content}}).ValidateInlineImages() != nil { + return ErrInvalidInput + } return nil }, } diff --git a/services/agents-api/internal/engine/profile.go b/services/agents-api/internal/engine/profile.go index d4d290198..ad9d5f7a7 100644 --- a/services/agents-api/internal/engine/profile.go +++ b/services/agents-api/internal/engine/profile.go @@ -20,7 +20,7 @@ type Profile struct { MessageImagePlacements []string ValidateConfiguration func(agent v1.Agent, environment *v1.Environment, hasDaemon bool) error ValidateTools func(environment *v1.Environment, hasDaemon bool, functions []proto.FunctionTool, mcp []proto.MCPHTTPServer) error - ValidateFunctionResult func([]proto.InputContent) error + ValidateFunctionResult func(placement string, result proto.FunctionResultPayload) error } func (p Profile) Accepts(placement string) bool { diff --git a/services/agents-api/internal/execution/delivery.go b/services/agents-api/internal/execution/delivery.go index 27e7ea111..52b6d59dd 100644 --- a/services/agents-api/internal/execution/delivery.go +++ b/services/agents-api/internal/execution/delivery.go @@ -97,7 +97,7 @@ func (d *Dispatcher) deliver(ctx context.Context, tenantID, sessionID string, pe var pending *pendingInput var cancelSent time.Time var cancelReply <-chan cancellationResult - functions := &functionExchange{store: d.Store, tenant: tenantID, session: sessionID, turn: request.RunID, tools: request.FunctionTools} + functions := &functionExchange{kind: request.AgentKind, store: d.Store, tenant: tenantID, session: sessionID, turn: request.RunID, tools: request.FunctionTools} done := false cancelCtx, stopCancellation := context.WithCancel(ctx) defer stopCancellation() diff --git a/services/agents-api/internal/execution/engine_profile.go b/services/agents-api/internal/execution/engine_profile.go index d29865cd6..75af24f68 100644 --- a/services/agents-api/internal/execution/engine_profile.go +++ b/services/agents-api/internal/execution/engine_profile.go @@ -71,7 +71,7 @@ func validateProfileInputs(profile engine.Profile, placement string, inputs []st if err != nil { return store.ErrInvalidInput } - if err := profile.ValidateFunctionResult(result.Content); err != nil { + if err := profile.ValidateFunctionResult(placement, result); err != nil { return profileError(err) } } diff --git a/services/agents-api/internal/execution/engine_profile_test.go b/services/agents-api/internal/execution/engine_profile_test.go index 139cb840d..80bf58033 100644 --- a/services/agents-api/internal/execution/engine_profile_test.go +++ b/services/agents-api/internal/execution/engine_profile_test.go @@ -45,9 +45,10 @@ func TestAdditionalProfileUsesCommonAdmission(t *testing.T) { } return nil }, - ValidateFunctionResult: func(content []proto.InputContent) error { + ValidateFunctionResult: func(placement string, result proto.FunctionResultPayload) error { + content := result.Content resultChecked = true - if len(content) != 1 || content[0].Text == nil || *content[0].Text != "response" { + if placement != "none" || !result.Success || len(content) != 1 || content[0].Text == nil || *content[0].Text != "response" { t.Fatal("common result decoding did not reach profile") } return engine.ErrInvalidInput diff --git a/services/agents-api/internal/execution/function_images_test.go b/services/agents-api/internal/execution/function_images_test.go new file mode 100644 index 000000000..e32ef8c0d --- /dev/null +++ b/services/agents-api/internal/execution/function_images_test.go @@ -0,0 +1,52 @@ +package execution + +import ( + "encoding/json" + "errors" + "testing" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/engine" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" +) + +func TestFunctionImageAdmission(t *testing.T) { + image := "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aXioAAAAASUVORK5CYII=" + profile, _ := (engine.Catalog{}).Lookup("claude_sdk") + for _, tc := range []struct { + name, url string + success, valid bool + }{ + {"success", image, true, true}, + {"native error drops images", image, false, false}, + {"remote", "https://example.test/image.png", true, false}, + {"invalid base64", "data:image/png;base64,?", true, false}, + {"invalid image", "data:image/png;base64,AQID", true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + output := []any{map[string]any{"type": "input_text", "text": "before"}, map[string]any{"type": "input_image", "image_url": tc.url}, map[string]any{"type": "input_text", "text": "after"}} + raw, _ := json.Marshal(map[string]any{"call_id": "call", "result": map[string]any{"success": tc.success, "output": output}}) + err := validateProfileInputs(profile, "none", []store.Input{{Kind: "tool_result", Payload: raw}}) + if tc.valid && err != nil || !tc.valid && !errors.Is(err, store.ErrInvalidInput) { + t.Fatal(err) + } + }) + } + for _, placement := range []string{"openai_hosted", "self_hosted"} { + url := image + if err := profile.ValidateFunctionResult(placement, proto.FunctionResultPayload{Success: true, Content: []proto.InputContent{{Type: "input_image", ImageURL: &url}}}); !errors.Is(err, engine.ErrInvalidInput) { + t.Fatal("unqualified image placement", placement, err) + } + if err := profile.ValidateFunctionResult(placement, proto.FunctionResultPayload{Success: true, Content: proto.TextInput("text")[0].Content}); err != nil { + t.Fatal("workspace text regressed", err) + } + } + // Readiness for image-bearing results cannot become a global function gate. + if err := requireFunctionResultImages(nil, "unknown", proto.FunctionResultPayload{Content: proto.TextInput("text")[0].Content}); err != nil { + t.Fatal(err) + } + if err := requireFunctionResultImages(&gateway.Session{}, "unknown", proto.FunctionResultPayload{Content: []proto.InputContent{{Type: "input_image", ImageURL: &image}}}); err == nil { + t.Fatal("image delivery accepted without Runtime support") + } +} diff --git a/services/agents-api/internal/execution/functions.go b/services/agents-api/internal/execution/functions.go index 83f316bca..6a7e9a9ec 100644 --- a/services/agents-api/internal/execution/functions.go +++ b/services/agents-api/internal/execution/functions.go @@ -51,6 +51,7 @@ type functionReply struct { type functionExchange struct { store *store.Store tenant, session, turn string + kind string tools []proto.FunctionTool callID string reply <-chan functionReply @@ -90,6 +91,9 @@ func (f *functionExchange) start(ctx context.Context, peer *gateway.Session) err if err != nil { return err } + if err := requireFunctionResultImages(peer, f.kind, result); err != nil { + return err + } env, err := proto.NewEnvelope(proto.TypeFunctionResult, f.turn, result) if err != nil { return err @@ -178,3 +182,15 @@ func functionResult(call store.FunctionCall) (proto.FunctionResultPayload, error } return result, result.ValidateContent() } + +// Check only this result, not ordinary function declarations or text delivery. +func requireFunctionResultImages(peer *gateway.Session, kind string, result proto.FunctionResultPayload) error { + if !(proto.MessageInput{{Content: result.Content}}).HasImages() { + return nil + } + info, found, known := peer.AgentKindStatus(kind) + if !found || !known || !info.Available || !info.Capabilities.FunctionResultImages { + return errors.New("Runtime does not support function result images") + } + return nil +} diff --git a/services/agents-api/internal/store/claude_execution_test.go b/services/agents-api/internal/store/claude_execution_test.go index abda0543f..f6eb75005 100644 --- a/services/agents-api/internal/store/claude_execution_test.go +++ b/services/agents-api/internal/store/claude_execution_test.go @@ -122,7 +122,7 @@ func TestClaudeDispatcherRejectsUnsupportedConfigurationBeforeClaim(t *testing.T } } -func TestClaudeImageResultRejectsWholeBatchBeforePersistence(t *testing.T) { +func TestClaudeInvalidImageResultRejectsWholeBatchBeforePersistence(t *testing.T) { h := newDispatchHarness(t) claudeSession(t, h, functionConfiguration, false) worker, err := execution.StartWorker(t.Context(), h.d) diff --git a/services/agents-api/internal/store/function_execution_test.go b/services/agents-api/internal/store/function_execution_test.go index 8b567dce0..19416593d 100644 --- a/services/agents-api/internal/store/function_execution_test.go +++ b/services/agents-api/internal/store/function_execution_test.go @@ -26,7 +26,7 @@ func newFunctionHarness(t *testing.T) *dispatchHarness { if err := h.s.BindSessionDevice(t.Context(), h.tenant, h.session.ID, h.device.ID); err != nil { t.Fatal(err) } - h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: proto.AgentKindCapabilities{Streaming: true, Steering: true, Resume: true, DurableTurns: true, DurableInputReceipts: true, WebSearchControl: true, TextVerbosity: true, ExecutionControls: true, SubagentControl: true, ToolObservations: true, EnvironmentNone: true, FunctionTools: true}}}}) + h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: proto.AgentKindCapabilities{Streaming: true, Steering: true, Resume: true, DurableTurns: true, DurableInputReceipts: true, WebSearchControl: true, TextVerbosity: true, ExecutionControls: true, SubagentControl: true, ToolObservations: true, EnvironmentNone: true, FunctionTools: true, FunctionResultImages: true}}}}) deadline := time.Now().Add(time.Second) for { peer, _ := h.registry.LookupDevice(h.device.ID) diff --git a/services/agents-api/internal/store/function_images_native_test.go b/services/agents-api/internal/store/function_images_native_test.go new file mode 100644 index 000000000..5aac51894 --- /dev/null +++ b/services/agents-api/internal/store/function_images_native_test.go @@ -0,0 +1,117 @@ +package store_test + +import ( + "context" + "encoding/json" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/api" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" +) + +func TestNativeFunctionImagePublicExecution(t *testing.T) { + python, binary, root, optionsFile := os.Getenv("PARSAR_OFFICIAL_SDK_PYTHON"), os.Getenv("PARSAR_NATIVE_DAEMON_BIN"), os.Getenv("PARSAR_NATIVE_PROOF_DIR"), os.Getenv("PARSAR_FUNCTION_IMAGE_REAL_OPTIONS") + if python == "" || binary == "" || root == "" || optionsFile == "" { + t.Skip("native daemon, fixed SDK, real model options and evidence directory required") + } + raw, err := os.ReadFile(optionsFile) + if err != nil { + t.Fatal(err) + } + var options map[string]any + if json.Unmarshal(raw, &options) != nil { + t.Fatal("invalid private options") + } + model, _ := options["model"].(string) + if model == "" { + t.Fatal("real model required") + } + kind := os.Getenv("PARSAR_FUNCTION_IMAGE_ENGINE") + if kind != "codex" && kind != "claude_sdk" { + t.Fatal("image acceptance requires a specified native engine") + } + h := newDispatchHarness(t) + h.d.Options = func(context.Context, store.Session) (map[string]any, error) { return options, nil } + home, err := os.MkdirTemp(root, "function-image-public-") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute) + defer cancel() + worker, err := execution.StartWorker(ctx, h.d) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- worker.Run(ctx) }() + defer func() { + cancel() + select { + case <-done: + case <-time.After(20 * time.Second): + t.Error("worker did not stop") + } + }() + token, foreign := uuid.NewString(), uuid.NewString() + auth, err := api.NewAuthenticator([]api.APIKey{ + {OrganizationID: "test", ProjectID: h.tenant, SubjectKind: "service_account", SubjectID: "owner", TokenSHA256: device.HashCredential(token), TenantID: h.tenant}, + {OrganizationID: "test", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "other", TokenSHA256: device.HashCredential(foreign), TenantID: uuid.NewString()}, + }) + if err != nil { + t.Fatal(err) + } + handler, err := api.NewHandler(h.s, auth, kind, api.WithExecution(worker), api.WithExecutionPolicy(h.d.Policy)) + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(handler) + defer server.Close() + stop := startNativeEngineDaemon(t, h, home, binary, kind) + defer func() { stop() }() + evidence := filepath.Join(home, "public.json") + run := func(stage string) { + cmd := exec.CommandContext(ctx, python, "../../tests/official_function_images.py", server.URL, token, foreign, model, stage, evidence) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("function images %s: %v %s; evidence %s", stage, err, output, home) + } + } + run("initial") + var proof struct { + Session string `json:"session"` + Calls []struct{ Turn, Call string } `json:"calls"` + } + raw, err = os.ReadFile(evidence) + if err != nil || json.Unmarshal(raw, &proof) != nil || len(proof.Calls) != 4 { + t.Fatal("invalid evidence", err) + } + for _, item := range proof.Calls { + call, err := h.s.GetFunctionCall(ctx, h.tenant, proof.Session, item.Turn, item.Call) + if err != nil || !call.Applied { + t.Fatal("function delivery acknowledgement missing", err) + } + inputs, err := h.s.ListTurnInputs(ctx, h.tenant, proof.Session, item.Turn, 0, 100) + if err != nil || len(inputs) != 2 || inputs[0].Kind != "message" || inputs[1].Kind != "tool_result" { + t.Fatal("function result admission duplicated or mutated", err) + } + } + before, err := h.s.GetSessionExecutionBinding(ctx, h.tenant, proof.Session) + if err != nil || before.NativeSessionID == "" { + t.Fatal("native binding missing", err) + } + stop() + stop = startNativeEngineDaemon(t, h, home, binary, kind) + run("resume") + after, err := h.s.GetSessionExecutionBinding(ctx, h.tenant, proof.Session) + if err != nil || before.NativeSessionID != after.NativeSessionID { + t.Fatal("native history changed", err) + } + t.Logf("Real function image SDK/raw HTTP, delivery, cold daemon recovery and isolation passed: %s", home) +} diff --git a/services/agents-api/tests/image_fixture.py b/services/agents-api/tests/image_fixture.py index 02e7e45fe..23276e737 100644 --- a/services/agents-api/tests/image_fixture.py +++ b/services/agents-api/tests/image_fixture.py @@ -4,12 +4,12 @@ import zlib -def picture(names): +def picture(names, scale=1): colors = {"red": (255, 0, 0), "blue": (0, 0, 255), "green": (0, 170, 0), "yellow": (255, 255, 0)} - rows = b"".join(b"\0" + b"".join(bytes(colors[n]) * 100 for n in names) for _ in range(140)) + rows = b"".join(b"\0" + b"".join(bytes(colors[n]) * (100 * scale) for n in names) for _ in range(140 * scale)) def chunk(kind, value): return struct.pack(">I", len(value)) + kind + value + struct.pack(">I", zlib.crc32(kind + value) & 0xffffffff) - image = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 400, 140, 8, 2, 0, 0, 0)) + chunk(b"IDAT", zlib.compress(rows)) + chunk(b"IEND", b"") + image = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 400 * scale, 140 * scale, 8, 2, 0, 0, 0)) + chunk(b"IDAT", zlib.compress(rows)) + chunk(b"IEND", b"") return "data:image/png;base64," + base64.b64encode(image).decode() diff --git a/services/agents-api/tests/official_function_images.py b/services/agents-api/tests/official_function_images.py new file mode 100644 index 000000000..c07945bf4 --- /dev/null +++ b/services/agents-api/tests/official_function_images.py @@ -0,0 +1,145 @@ +"""Function images through pinned SDK/raw HTTP and a real native Runtime/model.""" +import base64 +import importlib.metadata +import json +import secrets +import sys +from pathlib import Path + +import httpx2 +from openai import OpenAI +from image_fixture import picture + +base, token, foreign, model, stage, evidence = sys.argv[1:] +pin = json.loads((Path(__file__).resolve().parents[3] / "contracts/agents-api/upstream.json").read_text()) +dist = importlib.metadata.distribution("openai") +assert dist.version == pin["sdk_version"] +assert json.loads(dist.read_text("direct_url.json"))["vcs_info"]["commit_id"] == pin["commit"] +http = httpx2.Client(trust_env=False, timeout=180) +client = OpenAI(base_url=base + "/v1", api_key=token, max_retries=0, + _strict_response_validation=True, http_client=http) +sessions = client.beta.agents.sessions +headers = {"Authorization": "Bearer " + token, "OpenAI-Beta": "agents=v1"} +proof = {"calls": [], "runs": []} if stage == "initial" else json.loads(Path(evidence).read_text()) + + +def save(): + Path(evidence).write_text(json.dumps(proof, indent=2)) + + +def text(value): + return {"type": "input_text", "text": value} + + +def post(sid, events, key=None, foreign_request=False): + hdr = {**headers} + if key: + hdr["Idempotency-Key"] = key + if foreign_request: + hdr["Authorization"] = "Bearer " + foreign + return http.post(base + "/v1/agents/sessions/" + sid + "/events", headers=hdr, json={"events": events}) + + +def items(sid): + return [i.to_dict() for i in sessions.items.list(sid, order="asc", limit=100).data] + + +def answer(sid, expected): + answers = [i for i in items(sid) if i["type"] == "message" and i["role"] == "assistant"] + assert answers, "missing model answer" + result = " ".join(c["text"] for c in answers[-1]["content"] if c["type"] == "output_text").lower() + positions = [result.find(name) for name in expected] + assert all(p >= 0 for p in positions) and positions == sorted(positions), result + + +def run(sid, output=None, expected=None, cancel=False, failed_text=False, validate=False, recall=False): + events, handled = [], False + prompt = "Call get_visual exactly once. Read the image returned by that tool and reply with its four band colors from left to right. Do not call it again." + if recall: + prompt = "Without calling tools, recall the most recent image from get_visual and repeat its four band colors from left to right." + with sessions.events.stream(sid, timeout=180) as stream: + sessions.events.create(sid, events=[{"type": "agent.session.input.message", "input": [{"role": "user", "content": [text(prompt)]}]}]) + for event in stream: + events.append(event.to_dict()) + if event.type == "agent.session.requires_action": + assert not handled and not recall + handled = True + action = event.session.required_actions[0] + assert action.name == "get_visual" and len(event.session.required_actions) == 1 + if cancel: + sessions.events.create(sid, events=[{"type": "agent.session.input.cancel"}]) + else: + result = {"type": "agent.session.input.tool_result", "turn_id": action.turn_id, + "call_id": action.call_id, "success": not failed_text, "output": output} + if failed_text: + result["error"] = "No image available; reply only TOOL-ERROR-RECEIVED." + key = "result-" + action.call_id + if validate: + before = items(sid) + invalid_urls = ["https://example.test/image.png", "data:image/png;base64,?", "data:image/png;base64,AQID"] + # Claude rejects malformed/remote and error images before consuming a call. + if proof["kind"] == "claude_sdk": + invalid = [{**result, "output": [{"type": "input_image", "image_url": u}]} for u in invalid_urls] + invalid.append({**result, "success": False}) + for bad in invalid: + assert post(sid, [{"type": "agent.session.input.message", "input": "must not persist"}, bad], key).status_code == 400 + assert items(sid) == before + assert sessions.retrieve(sid).required_actions[0].call_id == action.call_id + assert post(sid, [result], foreign_request=True).status_code == 404 + assert post(sid, [{**result, "call_id": "unknown-call"}]).status_code in {400, 404, 409} + assert items(sid) == before + assert sessions.events.create(sid, events=[result], idempotency_key=key) is None + assert post(sid, [result], key).status_code == 204 + assert post(sid, [{**result, "output": "different"}], key).status_code == 409 + proof["calls"].append({"turn": action.turn_id, "call": action.call_id, "output": output, "success": not failed_text}) + assert event.type not in {"agent.session.failed", "agent.session.turn.failed"}, event.to_dict() + if event.type == "agent.session.idle": + break + else: + raise AssertionError("stream ended without idle") + proof["runs"].append(events) + save() + types = [e["type"] for e in events] + terminal = "agent.session.turn." + ("cancelled" if cancel else "completed") + assert types.index("agent.session.turn.created") < types.index(terminal) < types.index("agent.session.idle") + assert len({e["event_id"] for e in events}) == len(events) + assert handled != recall + assert sessions.turns.list(sid, order="desc").data[0].status == ("cancelled" if cancel else "completed") + if expected: + answer(sid, expected) + recovered = {i["call_id"]: i for i in items(sid) if i["type"] == "function_call_output"} + for call in proof["calls"]: + assert recovered[call["call"]]["output"] == call["output"] + assert ("error" not in recovered[call["call"]]) == call["success"] + + +try: + if stage == "initial": + import os + proof["kind"] = os.environ["PARSAR_FUNCTION_IMAGE_ENGINE"] + colors = ["red", "green", "blue", "yellow"] + secrets.SystemRandom().shuffle(colors) + proof["colors"] = colors + session = sessions.create(agent={"model": model, "tools": [{"type": "function", "name": "get_visual", + "description": "Return a visual to inspect.", "parameters": {"type": "object", "properties": {}, "additionalProperties": False}}]}, environment={"type": "none"}) + proof["session"] = sid = session.id + save() + for scale in [1, 15]: + output = [text("Read this visual."), {"type": "input_image", "image_url": picture(colors, scale)}, text("Return its four band colors in order.")] + run(sid, output, colors, validate=scale == 1) + jpeg = base64.b64encode((Path(__file__).parent / "testdata/function-bands.jpg").read_bytes()).decode() + proof["colors"] = ["yellow", "blue", "red", "green"] + run(sid, [{"type": "input_image", "image_url": "data:image/jpeg;base64," + jpeg}], proof["colors"]) + # Failed text remains supported and must not be mistaken for failed images. + run(sid, [text("No image was returned.")], failed_text=True) + for suffix in ["", "/items", "/turns"]: + response = http.get(base + "/v1/agents/sessions/" + sid + suffix, headers={**headers, "Authorization": "Bearer " + foreign}) + assert response.status_code == 404 + else: + sid = proof["session"] + run(sid, expected=proof["colors"], recall=True) + run(sid, cancel=True) + assert len(sessions.turns.list(sid, limit=100).data) == 6 + save() +finally: + client.close() diff --git a/services/agents-api/tests/testdata/function-bands.jpg b/services/agents-api/tests/testdata/function-bands.jpg new file mode 100644 index 000000000..bdd3d6a79 Binary files /dev/null and b/services/agents-api/tests/testdata/function-bands.jpg differ