From 18bb0f7e2c3105396d835a9ed40459bf8c1269e7 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Tue, 22 Sep 2026 12:10:03 +0800 Subject: [PATCH 1/3] Support ordered message images through the shared Runtime contract --- CONTRIBUTING.md | 41 +++- .../internal/agent/claudecode/export_test.go | 4 - .../agent/claudecode/message_input.go | 41 ++++ .../agent/claudecode/message_input_test.go | 40 ++++ .../internal/agent/claudecode/options.go | 62 ------ .../internal/agent/claudecode/options_test.go | 120 ------------ .../internal/agent/claudecode/session.go | 13 +- .../claudecode/session_knowledge_test.go | 2 +- .../internal/agent/claudecode/session_test.go | 6 +- .../claudesdk/cancellation_live_linux_test.go | 2 +- .../agent/claudesdk/cancellation_test.go | 4 +- .../agent/claudesdk/commands_session_test.go | 2 +- .../claudesdk/execution_controls_test.go | 10 +- .../agent/claudesdk/functions_test.go | 6 +- .../agent/claudesdk/live_linux_test.go | 6 +- .../agent/claudesdk/mcp_bearer_test.go | 4 +- .../internal/agent/claudesdk/mcp_test.go | 2 +- .../internal/agent/claudesdk/messages_test.go | 2 +- .../internal/agent/claudesdk/options.go | 8 +- .../internal/agent/claudesdk/options_test.go | 2 +- .../internal/agent/claudesdk/preparation.go | 8 +- .../claudesdk/preparation_fixture_test.go | 6 +- .../agent/claudesdk/preparation_test.go | 23 ++- .../internal/agent/claudesdk/readiness.go | 4 + .../agent/claudesdk/readiness_test.go | 6 +- .../agent/claudesdk/restrictions_test.go | 2 +- .../internal/agent/claudesdk/session.go | 12 +- .../internal/agent/claudesdk/session_test.go | 6 +- .../internal/agent/claudesdk/steering.go | 10 +- .../internal/agent/claudesdk/steering_test.go | 17 +- .../internal/agent/claudesdk/usage_test.go | 2 +- .../claudesdk/workspace_directory_test.go | 4 +- .../claudesdk/workspace_live_linux_test.go | 6 +- .../agent/claudesdk/workspace_read_test.go | 4 +- .../agent/claudesdk/workspace_test.go | 2 +- .../internal/agent/codex/functions_test.go | 4 +- .../agent/codex/mcp_http_preflight_test.go | 2 +- .../internal/agent/codex/mcp_required_test.go | 2 +- .../internal/agent/codex/message_input.go | 25 +++ .../agent/codex/message_input_test.go | 20 ++ .../internal/agent/codex/options.go | 12 -- .../internal/agent/codex/options_test.go | 12 +- .../internal/agent/codex/preparation.go | 6 +- .../agent/codex/preparation_router_test.go | 2 +- .../internal/agent/codex/prepared.go | 8 +- .../agent/codex/prepared_cancel_test.go | 10 +- .../internal/agent/codex/prepared_test.go | 8 +- .../agent/codex/protocol_wire_test.go | 2 +- .../internal/agent/codex/recovery_test.go | 2 +- .../internal/agent/codex/session_run.go | 4 +- .../internal/agent/codex/session_steering.go | 6 +- .../codex/session_steering_lifecycle_test.go | 8 +- .../codex/session_steering_receipt_test.go | 4 +- .../agent/codex/session_steering_test.go | 8 +- .../internal/agent/codex/tool_observations.go | 8 +- .../agent/mcode/environment_mcp_test.go | 2 +- .../agent/mcode/native_history_test.go | 2 +- .../internal/agent/mcode/native_test.go | 4 +- .../internal/agent/mcode/options.go | 2 +- .../internal/agent/mcode/options_test.go | 4 +- .../internal/agent/mcode/preparation.go | 13 +- .../internal/agent/mcode/preparation_test.go | 10 +- .../internal/agent/mcode/session.go | 6 +- .../internal/agent/mcode/session_test.go | 2 +- .../internal/agent/mcode/steering.go | 5 +- .../internal/agent/mcode/steering_test.go | 2 +- .../internal/agent/opencode/session.go | 6 +- .../internal/agent/opencode/session_test.go | 6 +- .../internal/agent/pi/session.go | 6 +- .../agent/pi/session_provider_test.go | 2 +- .../internal/agent/pi/session_skills_test.go | 2 +- .../internal/agent/pi/session_test.go | 6 +- .../internal/agent/preparation.go | 2 +- .../internal/cli/agent_discovery.go | 1 + apps/parsar-daemon/internal/cli/claude_sdk.go | 1 + .../cli/claude_sdk_live_linux_test.go | 4 +- .../internal/cli/claude_sdk_test.go | 2 +- .../dispatch/functions_native_test.go | 4 +- .../internal/dispatch/functions_test.go | 8 +- .../internal/dispatch/local_directory.go | 2 +- .../internal/dispatch/local_directory_test.go | 2 +- .../internal/dispatch/preparation.go | 2 +- .../dispatch/preparation_cancel_test.go | 12 +- .../dispatch/preparation_cleanup_test.go | 8 +- .../internal/dispatch/preparation_start.go | 4 +- .../internal/dispatch/preparation_test.go | 32 ++-- .../prepared_handoff_mutation_test.go | 12 +- .../dispatch/prepared_handoff_test.go | 18 +- .../parsar-daemon/internal/dispatch/prompt.go | 2 +- .../internal/dispatch/receipt_order_test.go | 6 +- .../dispatch/receipt_shutdown_test.go | 2 +- .../internal/dispatch/router_test.go | 4 +- .../internal/dispatch/steering.go | 6 +- .../dispatch/steering_lifetime_test.go | 6 +- .../internal/dispatch/steering_test.go | 18 +- .../dispatch/workspace_directory_test.go | 4 +- .../internal/localworkspace/binding.go | 2 +- .../parsar-daemon/testdata/onboarding/main.go | 11 +- contracts/agents-api/README.md | 13 +- contracts/agents-api/harness-onboarding.md | 7 + contracts/agents-api/message-input.md | 84 +++++++++ contracts/agents-api/openapi.yaml | 70 +++---- contracts/agents-api/v1/inputs.go | 5 +- internal/agentdaemon/device/state.go | 1 + .../gateway/mcp_bearer_live_linux_test.go | 2 +- internal/agentdaemon/gateway/session.go | 1 + internal/agentdaemon/gateway/session_test.go | 2 +- internal/agentdaemon/proto/envelope_test.go | 2 +- internal/agentdaemon/proto/functions.go | 27 +-- internal/agentdaemon/proto/functions_test.go | 2 +- internal/agentdaemon/proto/inbound.go | 1 + internal/agentdaemon/proto/message_images.go | 39 ++++ .../agentdaemon/proto/message_images_test.go | 45 +++++ internal/agentdaemon/proto/message_input.go | 99 ++++++++++ .../agentdaemon/proto/message_input_test.go | 54 ++++++ internal/agentdaemon/proto/outbound.go | 27 +-- internal/agentdaemon/proto/preparation.go | 6 +- internal/agentdaemon/proto/steering.go | 10 +- .../agentdaemon/proto/tool_observations.go | 26 +-- internal/agentdaemon/proto/version.go | 2 +- .../proto/workspace_read_preparation.go | 4 +- packages/claude-sdk-adapter/src/adapter.ts | 6 +- packages/claude-sdk-adapter/src/inputs.ts | 41 ++-- packages/claude-sdk-adapter/src/main.ts | 6 +- .../claude-sdk-adapter/src/message_input.ts | 40 ++++ packages/claude-sdk-adapter/src/request.ts | 26 ++- .../claude-sdk-adapter/src/runtime_check.ts | 2 +- .../tests/execution.test.mjs | 6 +- .../claude-sdk-adapter/tests/inputs.test.mjs | 59 +++++- .../claude-sdk-adapter/tests/mcp.test.mjs | 2 +- .../tests/mcp_bearer.test.mjs | 2 +- .../tests/mcp_required.test.mjs | 4 +- .../tests/mcp_workspace.test.mjs | 6 +- .../tests/messages.test.mjs | 2 +- .../tests/preparation.test.mjs | 28 +-- .../tests/structured_output.test.mjs | 2 +- .../tests/workspace.test.mjs | 2 +- .../tests/workspace_execution.test.mjs | 2 +- .../internal/api/function_inputs.go | 17 ++ services/agents-api/internal/api/handler.go | 2 +- services/agents-api/internal/api/inputs.go | 13 +- .../internal/api/session_initial_input.go | 2 +- services/agents-api/internal/engine/claude.go | 7 +- services/agents-api/internal/engine/codex.go | 5 +- .../agents-api/internal/engine/profile.go | 5 +- .../agents-api/internal/execution/delivery.go | 12 +- .../internal/execution/dispatcher.go | 5 +- .../internal/execution/engine_profile.go | 17 +- .../internal/execution/engine_profile_test.go | 4 +- .../execution/environment_admission.go | 3 + .../internal/execution/functions.go | 6 +- .../internal/execution/functions_test.go | 14 +- .../internal/execution/input_text.go | 73 -------- .../internal/execution/message_input.go | 66 +++++++ .../internal/execution/message_input_test.go | 16 ++ .../internal/execution/message_support.go | 51 +++++ .../execution/message_support_test.go | 42 +++++ .../internal/execution/preparation.go | 2 +- .../internal/execution/prepared_dispatch.go | 14 +- .../agents-api/internal/execution/support.go | 12 +- .../agents-api/internal/execution/worker.go | 6 +- .../internal/execution/worker_device.go | 20 +- .../internal/execution/worker_schedule.go | 2 +- .../internal/store/dispatch_test.go | 6 +- .../store/environment_admission_test.go | 2 +- .../internal/store/environment_worker_test.go | 2 +- .../store/function_input_execution_test.go | 2 +- .../store/local_environment_worker_test.go | 2 +- .../store/message_image_admission_test.go | 56 ++++++ .../store/message_images_native_test.go | 124 ++++++++++++ .../store/message_input_helpers_test.go | 15 ++ .../internal/store/prepared_dispatch_test.go | 6 +- .../internal/store/public_execution_test.go | 2 +- .../tests/official_message_images.py | 176 ++++++++++++++++++ 174 files changed, 1708 insertions(+), 763 deletions(-) create mode 100644 apps/parsar-daemon/internal/agent/claudecode/message_input.go create mode 100644 apps/parsar-daemon/internal/agent/claudecode/message_input_test.go create mode 100644 apps/parsar-daemon/internal/agent/codex/message_input.go create mode 100644 apps/parsar-daemon/internal/agent/codex/message_input_test.go create mode 100644 contracts/agents-api/message-input.md create mode 100644 internal/agentdaemon/proto/message_images.go create mode 100644 internal/agentdaemon/proto/message_images_test.go create mode 100644 internal/agentdaemon/proto/message_input.go create mode 100644 internal/agentdaemon/proto/message_input_test.go create mode 100644 packages/claude-sdk-adapter/src/message_input.ts delete mode 100644 services/agents-api/internal/execution/input_text.go create mode 100644 services/agents-api/internal/execution/message_input.go create mode 100644 services/agents-api/internal/execution/message_input_test.go create mode 100644 services/agents-api/internal/execution/message_support.go create mode 100644 services/agents-api/internal/execution/message_support_test.go create mode 100644 services/agents-api/internal/store/message_image_admission_test.go create mode 100644 services/agents-api/internal/store/message_images_native_test.go create mode 100644 services/agents-api/internal/store/message_input_helpers_test.go create mode 100644 services/agents-api/tests/official_message_images.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 58b311ac6..639cd1197 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -102,6 +102,11 @@ databases, credentials and migrations. The product uses Core exclusively; it has merely for new names or directories. Replacements may retire obsolete private interfaces and history backfills in bounded PRs; this does not authorize deleting product data or changing unrelated product behavior. +- Every concrete harness interaction goes through the common Runtime contract + and its adapter. Extend that contract minimally when a current operation cannot + be expressed; never put native capability logic or transport conversion into + Core handlers, storage or scheduling. Qualify public workflows through the same + shared chain; direct native probes establish feasibility only. - Keep engine-specific types, process management and protocol translation inside execution adapters. The public API and persistence/application core must not interpret Parsar product payloads or depend on one engine's native item types. @@ -114,10 +119,14 @@ databases, credentials and migrations. The product uses Core exclusively; it has - Maintain tasks, priorities and evidence in the Feishu board. Register issues discovered during a task without switching work or automatically selecting them next. Only a direct acceptance blocker justifies a minimal in-scope fix. After - each bounded task passes checks/review and merges, mark it done, reread the full - board and choose the next task by value, dependencies, risk and effort. Agent API - protocol and atomic execution work takes priority over product integration, UI - work and business Team orchestration. Prioritize a sound architecture skeleton + each bounded task passes checks/review and merges, mark its child entry done. + Select large Core tasks in order from the concise TODO, then use the full board + to choose bounded children by value, dependencies, risk and effort. Finish the + selected large task before switching to the next one. The main agent selects + tasks; subagents are for technical design, scoped collaboration and review, + not prioritization. Completing a child or milestone does not stop an explicitly + active long-term goal. Product integration, UI and business Team work remain + outside Core delivery. Prioritize a sound architecture skeleton and correct principal workflows with real API validation. Record and defer low-frequency corner cases when risk and ROI permit; do not let minor details delay the main work. Required checks and material correctness guarantees apply. @@ -158,11 +167,14 @@ The current MVP covers Codex, Claude Code and MiniMax Code through the shared single-Agent path: Session creation, environment preparation, native execution, files/artifacts, cancellation, reconnection/recovery queries, and standalone deployment acceptance. Select each -bounded task from the complete board; nonblocking local improvements stay queued. +bounded child from the complete board within the selected concise-TODO task; +nonblocking local improvements stay queued. Authentication, tenant/credential isolation, state consistency and data loss remain material acceptance requirements. Optional feature equality is not required. After the three profiles pass merged-main validation, publish the results, limitations -and backlog, then stop development until new user direction. Additional harness implementations remain queued. The separately authorized +and backlog. The user has since renewed continuous Core delivery: select and +finish large tasks from the concise TODO, retaining bounded child acceptance. +Additional harness implementations remain queued. The separately authorized Subagent batch targets the six read operations across these three harnesses and does not change the complete pinned protocol target. @@ -204,9 +216,22 @@ forwarding. The explicit daemon-executor decision supersedes the previous native executor interoperability requirement. The superseded execution route is removed; retain reusable filesystem helpers, necessary regressions and historical evidence without a compatibility layer. -The private daemon wire protocol is 0.3.0 after removal of remote execution fields. +The private daemon wire protocol is 0.4.0. Initial, prepared and active input use +the same ordered MessageInput contract, replacing scalar prompts and attachments. +User-message boundaries and text/image order remain intact through Core and the +Runtime wire; adapters own native conversion and receipt aggregation. Text-only +transports reject image content rather than dropping it. Codex has a flat native +input list and uses blank-line separators between messages; this does not preserve +independent native user-message boundaries. No old wire fallback is maintained. Deploy Core and daemon together; the existing major/minor WebSocket check rejects -0.2 peers before dispatch rather than ignoring their removed configuration. +older major/minor peers before dispatch rather than ignoring removed fields. +Image-bearing messages require a qualified profile/placement before persistence +and image support from the selected Runtime before native delivery. These checks +apply to that operation only; ordinary text retains offline queueing. Initial, +prepared and active paths use the same content and preserve receipt ownership. +The qualified public profile is inline PNG/JPEG on Codex/Claude `none`; workspace +images, MiniMax images and remote URLs remain explicit implementation gaps. Core +does not fetch or transform media. See [message input coverage](contracts/agents-api/message-input.md). User-managed onboarding creates a `self_hosted` Session first, then passes its Environment ID and unchanged `remote_url` to our Runtime with connect-only diff --git a/apps/parsar-daemon/internal/agent/claudecode/export_test.go b/apps/parsar-daemon/internal/agent/claudecode/export_test.go index be900dd72..a3cfe05bb 100644 --- a/apps/parsar-daemon/internal/agent/claudecode/export_test.go +++ b/apps/parsar-daemon/internal/agent/claudecode/export_test.go @@ -78,10 +78,6 @@ type ( Envelope = proto.Envelope ) -func BuildUserMessageForTest(prompt string, attachments []proto.PromptAttachment) ([]byte, error) { - return buildUserMessageWithAttachments(prompt, attachments) -} - // BuildAskUserToolResultForTest exposes the daemon-side tool_result // builder so ask_test.go can lock in the JSON shape claude's stdin // expects. diff --git a/apps/parsar-daemon/internal/agent/claudecode/message_input.go b/apps/parsar-daemon/internal/agent/claudecode/message_input.go new file mode 100644 index 000000000..47a176f29 --- /dev/null +++ b/apps/parsar-daemon/internal/agent/claudecode/message_input.go @@ -0,0 +1,41 @@ +package claudecode + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "strings" +) + +func buildOrderedUserMessages(input proto.MessageInput) ([]byte, error) { + if err := input.Validate(); err != nil { + return nil, err + } + var output bytes.Buffer + encoder := json.NewEncoder(&output) + encoder.SetEscapeHTML(false) + for _, message := range input { + var blocks []userContentBlock + for _, part := range message.Content { + if part.Type == "input_text" { + blocks = append(blocks, userContentBlock{Type: "text", Text: *part.Text}) + continue + } + header, data, ok := strings.Cut(*part.ImageURL, ",") + if !ok || (header != "data:image/png;base64" && header != "data:image/jpeg;base64") { + return nil, fmt.Errorf("claudecode: unsupported image reference") + } + mime := strings.TrimSuffix(strings.TrimPrefix(header, "data:"), ";base64") + blocks = append(blocks, userContentBlock{Type: "image", Source: &userContentSource{Type: "base64", MediaType: mime, Data: data}}) + } + var content any = blocks + if len(blocks) == 1 && blocks[0].Type == "text" { + content = blocks[0].Text + } + if err := encoder.Encode(userMessage{Type: "user", Message: userMessageContent{Role: "user", Content: content}}); err != nil { + return nil, err + } + } + return output.Bytes(), nil +} diff --git a/apps/parsar-daemon/internal/agent/claudecode/message_input_test.go b/apps/parsar-daemon/internal/agent/claudecode/message_input_test.go new file mode 100644 index 000000000..249875502 --- /dev/null +++ b/apps/parsar-daemon/internal/agent/claudecode/message_input_test.go @@ -0,0 +1,40 @@ +package claudecode + +import ( + "bytes" + "encoding/json" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "testing" +) + +func TestOrderedMessageInputNativeConversion(t *testing.T) { + image := "data:image/png;base64,aW1hZ2U=" + input := proto.TextInput(" before ") + input[0].Content = append(input[0].Content, proto.InputContent{Type: "input_image", ImageURL: &image}, proto.TextInput(" after ")[0].Content[0]) + input = append(input, proto.TextInput("next")...) + raw, err := buildOrderedUserMessages(input) + if err != nil { + t.Fatal(err) + } + lines := bytes.Split(bytes.TrimSpace(raw), []byte("\n")) + if len(lines) != 2 { + t.Fatalf("message boundary lost: %s", raw) + } + var first struct { + Message struct{ Content []userContentBlock } + } + if err := json.Unmarshal(lines[0], &first); err != nil { + t.Fatal(err) + } + content := first.Message.Content + if len(content) != 3 || content[0].Text != " before " || content[1].Source == nil || content[1].Source.Data != "aW1hZ2U=" || content[2].Text != " after " { + t.Fatalf("native order changed: %s", raw) + } + image = "https://unsupported.example/image.png" + if _, err := buildOrderedUserMessages(input); err == nil { + t.Fatal("unsupported image reference accepted") + } + if _, err := buildOrderedUserMessages(proto.TextInput("")); err == nil { + t.Fatal("empty message accepted") + } +} diff --git a/apps/parsar-daemon/internal/agent/claudecode/options.go b/apps/parsar-daemon/internal/agent/claudecode/options.go index e8679a5b8..e3a7e910f 100644 --- a/apps/parsar-daemon/internal/agent/claudecode/options.go +++ b/apps/parsar-daemon/internal/agent/claudecode/options.go @@ -6,16 +6,12 @@ package claudecode import ( - "bytes" "encoding/json" - "errors" "fmt" "os" "path/filepath" "sort" "strings" - - "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" ) // BuildResult is the output of BuildArgs. Cleanup is always non-nil @@ -286,61 +282,3 @@ type userContentSource struct { MediaType string `json:"media_type"` Data string `json:"data"` } - -func buildUserMessage(prompt string) ([]byte, error) { - return buildUserMessageWithAttachments(prompt, nil) -} - -// buildUserMessageWithAttachments is the multimodal-aware variant. With -// no attachments, the output is byte-identical to the bare-string -// Content path so existing log greps for prompt content keep working. -// Non-image attachments are dropped — Claude Code SDK only understands -// the image block shape on stdin. -func buildUserMessageWithAttachments(prompt string, attachments []proto.PromptAttachment) ([]byte, error) { - if prompt == "" && len(attachments) == 0 { - return nil, errors.New("claudecode: empty prompt") - } - var content any - if len(attachments) == 0 { - content = prompt - } else { - blocks := make([]userContentBlock, 0, len(attachments)+1) - if prompt != "" { - blocks = append(blocks, userContentBlock{Type: "text", Text: prompt}) - } - for _, att := range attachments { - if att.Kind != "image" || att.DataBase64 == "" { - continue - } - mime := att.MIME - if mime == "" { - mime = "image/png" - } - blocks = append(blocks, userContentBlock{ - Type: "image", - Source: &userContentSource{ - Type: "base64", - MediaType: mime, - Data: att.DataBase64, - }, - }) - } - if len(blocks) == 0 { - return nil, errors.New("claudecode: empty prompt after dropping unsupported attachments") - } - content = blocks - } - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - enc.SetEscapeHTML(false) - if err := enc.Encode(userMessage{ - Type: "user", - Message: userMessageContent{ - Role: "user", - Content: content, - }, - }); err != nil { - return nil, fmt.Errorf("claudecode: marshal user message: %w", err) - } - return buf.Bytes(), nil -} diff --git a/apps/parsar-daemon/internal/agent/claudecode/options_test.go b/apps/parsar-daemon/internal/agent/claudecode/options_test.go index 4140b1ae3..5b9250495 100644 --- a/apps/parsar-daemon/internal/agent/claudecode/options_test.go +++ b/apps/parsar-daemon/internal/agent/claudecode/options_test.go @@ -8,7 +8,6 @@ import ( "testing" "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent/claudecode" - "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" ) func TestBuildArgsBaseHasStreamFlags(t *testing.T) { @@ -284,122 +283,3 @@ func containsPair(args []string, flag, value string) bool { } return false } - -func TestBuildUserMessage_TextOnlyKeepsBareStringContent(t *testing.T) { - // Backwards compat: no attachments → Content stays a bare string. - raw, err := claudecode.BuildUserMessageForTest("hello world", nil) - if err != nil { - t.Fatalf("BuildUserMessageForTest: %v", err) - } - var msg struct { - Type string `json:"type"` - Message struct { - Role string `json:"role"` - Content json.RawMessage `json:"content"` - } `json:"message"` - } - if err := json.Unmarshal(raw, &msg); err != nil { - t.Fatalf("unmarshal: %v\nbody=%s", err, raw) - } - if msg.Type != "user" || msg.Message.Role != "user" { - t.Fatalf("unexpected envelope: %+v", msg) - } - if string(msg.Message.Content) != `"hello world"` { - t.Fatalf("Content not bare string: %s", msg.Message.Content) - } -} - -func TestBuildUserMessage_WithImageEmitsContentBlocks(t *testing.T) { - att := []proto.PromptAttachment{ - {Kind: "image", MIME: "image/png", DataBase64: "AAAA"}, - {Kind: "image", MIME: "image/jpeg", DataBase64: "BBBB"}, - } - raw, err := claudecode.BuildUserMessageForTest("look at this", att) - if err != nil { - t.Fatalf("BuildUserMessageForTest: %v", err) - } - var msg struct { - Message struct { - Content []struct { - Type string `json:"type"` - Text string `json:"text"` - Source *struct { - Type string `json:"type"` - MediaType string `json:"media_type"` - Data string `json:"data"` - } `json:"source"` - } `json:"content"` - } `json:"message"` - } - if err := json.Unmarshal(raw, &msg); err != nil { - t.Fatalf("unmarshal: %v\nbody=%s", err, raw) - } - if len(msg.Message.Content) != 3 { - t.Fatalf("expected 3 blocks (text+2 images), got %d: %s", len(msg.Message.Content), raw) - } - if msg.Message.Content[0].Type != "text" || msg.Message.Content[0].Text != "look at this" { - t.Errorf("block 0 = %+v", msg.Message.Content[0]) - } - for i, want := range []string{"image/png", "image/jpeg"} { - b := msg.Message.Content[i+1] - if b.Type != "image" || b.Source == nil { - t.Errorf("block %d not image: %+v", i+1, b) - continue - } - if b.Source.Type != "base64" || b.Source.MediaType != want { - t.Errorf("block %d source = %+v", i+1, b.Source) - } - } -} - -func TestBuildUserMessage_EmptyPromptWithImageStillValid(t *testing.T) { - // Pure-image-no-caption is a valid message — user pastes a - // screenshot without typing anything. - att := []proto.PromptAttachment{ - {Kind: "image", MIME: "image/png", DataBase64: "AAAA"}, - } - raw, err := claudecode.BuildUserMessageForTest("", att) - if err != nil { - t.Fatalf("BuildUserMessageForTest: %v", err) - } - var msg struct { - Message struct { - Content []struct { - Type string `json:"type"` - } `json:"content"` - } `json:"message"` - } - if err := json.Unmarshal(raw, &msg); err != nil { - t.Fatalf("unmarshal: %v\nbody=%s", err, raw) - } - if len(msg.Message.Content) != 1 || msg.Message.Content[0].Type != "image" { - t.Fatalf("expected single image block, got %+v", msg.Message.Content) - } -} - -func TestBuildUserMessage_UnsupportedAttachmentsDropped(t *testing.T) { - // Non-image kinds aren't representable on stdin; dropped silently. - att := []proto.PromptAttachment{ - {Kind: "file", MIME: "text/plain", DataBase64: "AAAA"}, - {Kind: "image", DataBase64: ""}, - } - raw, err := claudecode.BuildUserMessageForTest("hi", att) - if err != nil { - t.Fatalf("BuildUserMessageForTest: %v", err) - } - if !strings.Contains(string(raw), `"hi"`) { - t.Errorf("prompt text missing: %s", raw) - } -} - -func TestBuildUserMessage_EmptyPromptAndNoImagesErrors(t *testing.T) { - if _, err := claudecode.BuildUserMessageForTest("", nil); err == nil { - t.Fatal("expected error for empty prompt + no attachments") - } - att := []proto.PromptAttachment{ - {Kind: "file", DataBase64: "X"}, - } - if _, err := claudecode.BuildUserMessageForTest("", att); err == nil { - t.Fatal("expected error when all attachments dropped + empty prompt") - } -} diff --git a/apps/parsar-daemon/internal/agent/claudecode/session.go b/apps/parsar-daemon/internal/agent/claudecode/session.go index 4591e224e..3a3d8fa73 100644 --- a/apps/parsar-daemon/internal/agent/claudecode/session.go +++ b/apps/parsar-daemon/internal/agent/claudecode/session.go @@ -114,11 +114,8 @@ func newSession(parent context.Context, req proto.PromptRequestPayload, out chan if out == nil { return nil, errors.New("claudecode: nil out channel") } - if req.Prompt == "" && len(req.Attachments) == 0 { - // A pure-image inbound (Feishu user pastes a screenshot - // without typing) is a valid prompt — Attachments alone - // drives the turn — and must not 400 here. - return nil, errors.New("claudecode: empty prompt and no attachments") + if err := req.Input.Validate(); err != nil { + return nil, err } if cfg.logger == nil { cfg.logger = obslog.Bg() @@ -132,7 +129,7 @@ func newSession(parent context.Context, req proto.PromptRequestPayload, out chan cfg.logger.Info("claudecode: newSession start", "run_id", req.RunID, "agent_kind", req.AgentKind, - "prompt_len", len(req.Prompt), "work_dir", req.WorkDir, + "prompt_len", len(req.Input), "work_dir", req.WorkDir, "has_agent_options", req.AgentOptions != nil, "agent_session_id", req.AgentSessionID, "claude_binary", cfg.claudeBinary) @@ -262,9 +259,9 @@ func newSession(parent context.Context, req proto.PromptRequestPayload, out chan // first stdout line corresponds to the prompt we just sent. Best // effort: write failure → pump sees EOF and synthesises // error+done. - if msg, err := buildUserMessageWithAttachments(req.Prompt, req.Attachments); err == nil { + if msg, err := buildOrderedUserMessages(req.Input); err == nil { cfg.logger.Info("claudecode: writing initial user message to stdin", - "run_id", req.RunID, "msg_bytes", len(msg), "attachments", len(req.Attachments)) + "run_id", req.RunID, "msg_bytes", len(msg)) if _, werr := s.writeStdin(msg); werr != nil { cfg.logger.Warn("claudecode: write initial user message", "run_id", req.RunID, "err", werr) diff --git a/apps/parsar-daemon/internal/agent/claudecode/session_knowledge_test.go b/apps/parsar-daemon/internal/agent/claudecode/session_knowledge_test.go index 25e7f7e1d..94d1ce9f8 100644 --- a/apps/parsar-daemon/internal/agent/claudecode/session_knowledge_test.go +++ b/apps/parsar-daemon/internal/agent/claudecode/session_knowledge_test.go @@ -15,7 +15,7 @@ func TestStartupLogsDoNotContainReferenceDocuments(t *testing.T) { var output bytes.Buffer const privateDocument = "PRIVATE-REFERENCE-9481" _, err := newSession(t.Context(), proto.PromptRequestPayload{ - RunID: "knowledge-log-check", WorkDir: t.TempDir(), Prompt: "Answer from the reference.", + RunID: "knowledge-log-check", WorkDir: t.TempDir(), Input: proto.TextInput("Answer from the reference."), AgentOptions: map[string]any{"system_prompt": privateDocument}, }, make(chan proto.Envelope, 8), sessionConfig{ claudeBinary: filepath.Join(t.TempDir(), "missing-claude"), diff --git a/apps/parsar-daemon/internal/agent/claudecode/session_test.go b/apps/parsar-daemon/internal/agent/claudecode/session_test.go index 4326cf14f..d38055f1c 100644 --- a/apps/parsar-daemon/internal/agent/claudecode/session_test.go +++ b/apps/parsar-daemon/internal/agent/claudecode/session_test.go @@ -195,8 +195,8 @@ func helperConfig() claudecode.SessionConfigForTest { // helperReq points the helper at a specific role via env passthrough. func helperReq(runID, prompt, role string) proto.PromptRequestPayload { return proto.PromptRequestPayload{ - RunID: runID, - Prompt: prompt, + RunID: runID, + Input: proto.TextInput(prompt), AgentOptions: map[string]any{ "env": map[string]any{ helperEnvKey: role, @@ -465,7 +465,7 @@ func TestSessionRejectsEmptyPrompt(t *testing.T) { // valid prompt today and must NOT be rejected. out := make(chan proto.Envelope, 4) _, err := claudecode.NewSessionForTest(context.Background(), - proto.PromptRequestPayload{RunID: "r0", Prompt: ""}, + proto.PromptRequestPayload{RunID: "r0", Input: proto.TextInput("")}, out, helperConfig()) if err == nil { t.Fatal("expected error on empty prompt + no attachments") diff --git a/apps/parsar-daemon/internal/agent/claudesdk/cancellation_live_linux_test.go b/apps/parsar-daemon/internal/agent/claudesdk/cancellation_live_linux_test.go index 64860c7cf..c0ce0724c 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/cancellation_live_linux_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/cancellation_live_linux_test.go @@ -65,7 +65,7 @@ func TestLiveClaudeSDKCancelResume(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) defer cancel() out := make(chan proto.Envelope, 64) - request := proto.PromptRequestPayload{RunID: uuid.NewString(), Prompt: prompt, AgentSessionID: resume, StrictResume: true, ReleaseOnCompletion: true, ObserveMessages: true, DisableExecutionEnvironment: true, DisableSubagents: true, ExecutionControls: &proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "medium"}, AgentOptions: map[string]any{"model": "MiniMax-M3", "system_prompt": "Follow the user's requested format. Preserve the exact verification value in conversation history. Use no tools."}} + request := proto.PromptRequestPayload{RunID: uuid.NewString(), Input: proto.TextInput(prompt), AgentSessionID: resume, StrictResume: true, ReleaseOnCompletion: true, ObserveMessages: true, DisableExecutionEnvironment: true, DisableSubagents: true, ExecutionControls: &proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "medium"}, AgentOptions: map[string]any{"model": "MiniMax-M3", "system_prompt": "Follow the user's requested format. Preserve the exact verification value in conversation history. Use no tools."}} running, err := NewFactory(config)(ctx, request, out) if err != nil { t.Fatal(err) diff --git a/apps/parsar-daemon/internal/agent/claudesdk/cancellation_test.go b/apps/parsar-daemon/internal/agent/claudesdk/cancellation_test.go index 63c13936b..f6e1ee7c3 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/cancellation_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/cancellation_test.go @@ -49,7 +49,7 @@ func TestCancellationWaitsForDrainAndPublishesOutcome(t *testing.T) { if got := provider.CancellationOutcome(); !reflect.DeepEqual(got, proto.DonePayload{}) { t.Fatal("unsettled outcome was exposed", got) } - if err := running.(*session).Steer(ctx, proto.PromptSteerPayload{InputID: "later", Text: "later"}); !errors.Is(err, agent.ErrSteeringInactive) { + if err := running.(*session).Steer(ctx, proto.PromptSteerPayload{InputID: "later", Input: proto.TextInput("later")}); !errors.Is(err, agent.ErrSteeringInactive) { t.Fatal("cancelled execution accepted steering", err) } if err := os.WriteFile(filepath.Join(config.StateDir, "release"), nil, 0o600); err != nil { @@ -168,7 +168,7 @@ func cancellationConfig(root, mode string) Config { } func cancellationRequest() proto.PromptRequestPayload { - return proto.PromptRequestPayload{RunID: "run", Prompt: "hello", AgentSessionID: "native-session", AgentOptions: map[string]any{"model": "fake-model", "system_prompt": "instructions"}} + return proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("hello"), AgentSessionID: "native-session", AgentOptions: map[string]any{"model": "fake-model", "system_prompt": "instructions"}} } func runCancellationHelper(request startRequest, mode string, emit func(bridgeEvent)) { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/commands_session_test.go b/apps/parsar-daemon/internal/agent/claudesdk/commands_session_test.go index 10df4cee6..f87ed4969 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/commands_session_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/commands_session_test.go @@ -54,7 +54,7 @@ func TestWorkspaceCommandFramesKeepStartIdentityAndObservedOutput(t *testing.T) t.Fatal("preparation submitted a command") } out := make(chan proto.Envelope, 16) - s, err := resource.Start(t.Context(), "actual-command-run", "hello", out) + s, err := resource.Start(t.Context(), "actual-command-run", proto.TextInput("hello"), out) if err != nil { t.Fatal(err) } diff --git a/apps/parsar-daemon/internal/agent/claudesdk/execution_controls_test.go b/apps/parsar-daemon/internal/agent/claudesdk/execution_controls_test.go index f3b1c00b0..0768159b6 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/execution_controls_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/execution_controls_test.go @@ -17,7 +17,7 @@ func TestExecutionControlsPreserveNativeDefaultsAndInstructions(t *testing.T) { root := t.TempDir() t.Setenv("PARSAR_HOME", root) config := Config{Entrypoint: filepath.Join(root, "worker"), StateDir: filepath.Join(root, "state")} - request := proto.PromptRequestPayload{RunID: "run", Prompt: "Original input.", AgentSessionID: "native-session", AgentOptions: map[string]any{"model": "native-model", "system_prompt": "Keep these exact instructions.\nDo not replace them."}} + request := proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("Original input."), AgentSessionID: "native-session", AgentOptions: map[string]any{"model": "native-model", "system_prompt": "Keep these exact instructions.\nDo not replace them."}} ordinary, _, err := prepare(config, request) if err != nil { t.Fatal(err) @@ -49,7 +49,7 @@ func TestExecutionControlsRejectUnsupportedProfilesBeforeLaunch(t *testing.T) { root := t.TempDir() t.Setenv("PARSAR_HOME", root) config := Config{Node: "must-not-run", Entrypoint: filepath.Join(root, "worker"), StateDir: filepath.Join(root, "state")} - request := proto.PromptRequestPayload{RunID: "run", Prompt: "Original input.", ExecutionControls: &controls, AgentOptions: map[string]any{"model": "native-model"}} + request := proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("Original input."), ExecutionControls: &controls, AgentOptions: map[string]any{"model": "native-model"}} _, err := NewFactory(config)(t.Context(), request, make(chan proto.Envelope, 1)) if err == nil || !strings.Contains(err.Error(), "execution controls require") { t.Fatal("unsupported controls did not fail at admission", err) @@ -66,7 +66,7 @@ func TestMCPWithoutEnvironmentNoneRejectedBeforeSetup(t *testing.T) { t.Setenv("PARSAR_HOME", root) config := Config{Node: "must-not-run", Entrypoint: filepath.Join(root, "worker"), StateDir: filepath.Join(root, "state")} servers := []proto.MCPHTTPServer{} - request := proto.PromptRequestPayload{RunID: "run", Prompt: "Input", MCPHTTPServers: &servers} + request := proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("Input"), MCPHTTPServers: &servers} _, err := NewFactory(config)(t.Context(), request, make(chan proto.Envelope, 1)) if err == nil || !strings.Contains(err.Error(), "HTTP MCP requires environment:none") { t.Fatal("MCP reached an unsupported environment", err) @@ -81,12 +81,12 @@ func TestStructuredOutputConfigurationReachesNativeUnchanged(t *testing.T) { t.Setenv("PARSAR_HOME", root) config := Config{Entrypoint: filepath.Join(root, "worker"), StateDir: filepath.Join(root, "state")} schema := json.RawMessage(`{"type":"object","properties":{"n":{"const":9007199254740992}}}`) - request := proto.PromptRequestPayload{RunID: "run", Prompt: "Original input.", ObserveMessages: true, DisableSubagents: true, AgentOptions: map[string]any{"model": "model", "system_prompt": "Original instructions."}, ExecutionControls: &proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "medium", OutputFormat: &proto.OutputFormat{Type: "json_schema", Schema: schema}}} + request := proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("Original input."), ObserveMessages: true, DisableSubagents: true, AgentOptions: map[string]any{"model": "model", "system_prompt": "Original instructions."}, ExecutionControls: &proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "medium", OutputFormat: &proto.OutputFormat{Type: "json_schema", Schema: schema}}} start, _, err := prepare(config, request) if err != nil { t.Fatal(err) } - if start.OutputFormat == nil || string(start.OutputFormat.Schema) != string(schema) || start.Prompt != request.Prompt || start.SystemPrompt != "Original instructions." { + if start.OutputFormat == nil || string(start.OutputFormat.Schema) != string(schema) || *start.Input[0].Content[0].Text != *request.Input[0].Content[0].Text || start.SystemPrompt != "Original instructions." { t.Fatal("native configuration changed") } request.ExecutionControls.OutputFormat.Schema = json.RawMessage(`{"type":"object","const":9007199254740993}`) diff --git a/apps/parsar-daemon/internal/agent/claudesdk/functions_test.go b/apps/parsar-daemon/internal/agent/claudesdk/functions_test.go index c0b92f809..99c4abb9f 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/functions_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/functions_test.go @@ -22,7 +22,7 @@ func TestFunctionFactoryNativeReceipts(t *testing.T) { root := t.TempDir() t.Setenv("PARSAR_HOME", root) config := Config{Node: os.Args[0], Entrypoint: filepath.Join(root, "worker"), StateDir: filepath.Join(root, "state"), Env: []string{"GO_CLAUDE_SDK_HELPER=1", "SDK_HELPER_MODE=" + mode, "GORACE=atexit_sleep_ms=0"}} - request := proto.PromptRequestPayload{RunID: "run", Prompt: "hello", AgentSessionID: "native-session", ObserveToolObservations: true, AgentOptions: map[string]any{"model": "fake-model", "system_prompt": "instructions"}, FunctionTools: []proto.FunctionTool{{Name: "lookup", Description: "Lookup.", Parameters: json.RawMessage(`{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"}}}}`)}}} + request := proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("hello"), AgentSessionID: "native-session", ObserveToolObservations: true, AgentOptions: map[string]any{"model": "fake-model", "system_prompt": "instructions"}, FunctionTools: []proto.FunctionTool{{Name: "lookup", Description: "Lookup.", Parameters: json.RawMessage(`{"type":"object","properties":{"ids":{"type":"array","items":{"type":"string"}}}}`)}}} ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() out := make(chan proto.Envelope, 16) @@ -52,12 +52,12 @@ func TestFunctionFactoryNativeReceipts(t *testing.T) { t.Fatal("missing content consumed call") } image := "https://example.invalid/image" - invalid.Content = []proto.FunctionResultContent{{Type: "input_image", ImageURL: &image}} + invalid.Content = []proto.InputContent{{Type: "input_image", ImageURL: &image}} if err := submitter.SubmitFunctionResult(ctx, invalid); err == nil { t.Fatal("image should fail before delivery") } first, second := "first-"+call.CallID, "second-"+call.CallID - value := proto.FunctionResultPayload{DeliveryID: "delivery-" + call.CallID, CallID: call.CallID, Success: call.CallID == "b", Content: []proto.FunctionResultContent{{Type: "input_text", Text: &first}, {Type: "input_text", Text: &second}}} + value := proto.FunctionResultPayload{DeliveryID: "delivery-" + call.CallID, CallID: call.CallID, Success: call.CallID == "b", Content: []proto.InputContent{{Type: "input_text", Text: &first}, {Type: "input_text", Text: &second}}} go func() { submissions <- submitter.SubmitFunctionResult(ctx, value) }() case proto.TypeToolCall: var tool proto.ToolCallPayload diff --git a/apps/parsar-daemon/internal/agent/claudesdk/live_linux_test.go b/apps/parsar-daemon/internal/agent/claudesdk/live_linux_test.go index 6b9124feb..98b6c0bea 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/live_linux_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/live_linux_test.go @@ -137,7 +137,7 @@ func TestLiveClaudeSDKTextResume(t *testing.T) { requestStart := len(requests) mu.Unlock() out := make(chan proto.Envelope, 64) - request := proto.PromptRequestPayload{RunID: uuid.NewString(), Prompt: prompt, AgentSessionID: resume, StrictResume: true, ReleaseOnCompletion: true, ObserveMessages: true, DisableExecutionEnvironment: true, DisableSubagents: true, ExecutionControls: &proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "medium"}, AgentOptions: map[string]any{"model": "MiniMax-M3", "system_prompt": "Answer briefly and preserve the exact verification value in the conversation. Use no tools."}} + request := proto.PromptRequestPayload{RunID: uuid.NewString(), Input: proto.TextInput(prompt), AgentSessionID: resume, StrictResume: true, ReleaseOnCompletion: true, ObserveMessages: true, DisableExecutionEnvironment: true, DisableSubagents: true, ExecutionControls: &proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "medium"}, AgentOptions: map[string]any{"model": "MiniMax-M3", "system_prompt": "Answer briefly and preserve the exact verification value in the conversation. Use no tools."}} if success != nil { request.ObserveToolObservations = true request.AgentOptions["system_prompt"] = "Call lookup exactly once as requested, then report both result parts and any prior verification value. Never retry a failed tool." @@ -174,7 +174,7 @@ func TestLiveClaudeSDKTextResume(t *testing.T) { timer := time.AfterFunc(10*time.Second, cancel) defer timer.Stop() written := false - err := s.SteerWithReceipt(callCtx, proto.PromptSteerPayload{InputID: uuid.NewString(), Text: proof.SteeringText}, func() { written = timer.Stop() }) + err := s.SteerWithReceipt(callCtx, proto.PromptSteerPayload{InputID: uuid.NewString(), Input: proto.TextInput(proof.SteeringText)}, func() { written = timer.Stop() }) steeringReply <- steeringResult{err: err, elapsed: time.Since(steeringAt).Milliseconds(), written: written} }() } @@ -252,7 +252,7 @@ func TestLiveClaudeSDKTextResume(t *testing.T) { if !*success { first, second = "synthetic-current-failure", "do-not-retry" } - value := proto.FunctionResultPayload{CallID: call.CallID, DeliveryID: uuid.NewString(), Success: *success, Content: []proto.FunctionResultContent{{Type: "input_text", Text: &first}, {Type: "input_text", Text: &second}}} + value := proto.FunctionResultPayload{CallID: call.CallID, DeliveryID: uuid.NewString(), Success: *success, Content: []proto.InputContent{{Type: "input_text", Text: &first}, {Type: "input_text", Text: &second}}} if err := s.SubmitFunctionResult(ctx, value); err != nil { t.Fatalf("live native result receipt failed: %v; proof root %s", err, root) } diff --git a/apps/parsar-daemon/internal/agent/claudesdk/mcp_bearer_test.go b/apps/parsar-daemon/internal/agent/claudesdk/mcp_bearer_test.go index b0d71e183..3a172369f 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/mcp_bearer_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/mcp_bearer_test.go @@ -22,7 +22,7 @@ func TestMCPBearerUsesFreshOwnedEnvironmentReferences(t *testing.T) { {ServerLabel: "second", ServerURL: "https://second.example/mcp", BearerToken: &tokens[1]}, {ServerLabel: "anonymous", ServerURL: "http://anonymous.example/mcp"}, } - req := proto.PromptRequestPayload{RunID: "run", Prompt: "hello", DisableExecutionEnvironment: true, MCPHTTPServers: &servers, AgentOptions: map[string]any{"model": "fixture"}} + req := proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("hello"), DisableExecutionEnvironment: true, MCPHTTPServers: &servers, AgentOptions: map[string]any{"model": "fixture"}} seen := map[string]bool{} for range 2 { start, env, err := prepare(config, req) @@ -66,7 +66,7 @@ func TestMCPBearerRejectsInvalidCredentialBeforeStateCreation(t *testing.T) { t.Setenv("PARSAR_HOME", root) config := Config{Entrypoint: filepath.Join(root, "main.js"), StateDir: filepath.Join(root, "state")} servers := []proto.MCPHTTPServer{{ServerLabel: "fixture", ServerURL: "https://example.invalid/mcp", BearerToken: &token}} - req := proto.PromptRequestPayload{RunID: "run", Prompt: "hello", DisableExecutionEnvironment: true, MCPHTTPServers: &servers, AgentOptions: map[string]any{"model": "fixture"}} + req := proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("hello"), DisableExecutionEnvironment: true, MCPHTTPServers: &servers, AgentOptions: map[string]any{"model": "fixture"}} if _, _, err := prepare(config, req); err == nil || err.Error() != "claudesdk: unsupported HTTPS MCP bearer credential" { t.Fatal("invalid bearer accepted or unsafe error returned") } diff --git a/apps/parsar-daemon/internal/agent/claudesdk/mcp_test.go b/apps/parsar-daemon/internal/agent/claudesdk/mcp_test.go index 2653e4401..99ef03ae0 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/mcp_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/mcp_test.go @@ -16,7 +16,7 @@ func TestHTTPMCPDeclaration(t *testing.T) { t.Setenv("PARSAR_HOME", root) config := Config{Entrypoint: filepath.Join(root, "main.js"), StateDir: filepath.Join(root, "state")} servers := []proto.MCPHTTPServer{{ServerLabel: "fixture", ServerURL: "https://example.invalid/mcp"}} - req := proto.PromptRequestPayload{RunID: "run", Prompt: "hello", DisableExecutionEnvironment: true, MCPHTTPServers: &servers, AgentOptions: map[string]any{"model": "fixture"}} + req := proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("hello"), DisableExecutionEnvironment: true, MCPHTTPServers: &servers, AgentOptions: map[string]any{"model": "fixture"}} tools := []string{"echo"} switch mode { case "selected": diff --git a/apps/parsar-daemon/internal/agent/claudesdk/messages_test.go b/apps/parsar-daemon/internal/agent/claudesdk/messages_test.go index 6a3b10f7f..f41fc60cb 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/messages_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/messages_test.go @@ -20,7 +20,7 @@ func TestMessageObservations(t *testing.T) { root := t.TempDir() t.Setenv("PARSAR_HOME", root) config := Config{Node: os.Args[0], Entrypoint: filepath.Join(root, "worker"), StateDir: filepath.Join(root, "state"), Env: []string{"GO_CLAUDE_SDK_HELPER=1", "SDK_HELPER_MODE=" + mode, "GORACE=atexit_sleep_ms=0"}} - request := proto.PromptRequestPayload{RunID: "run", Prompt: "hello", ObserveMessages: mode != "messages-unrequested", AgentSessionID: "native-session", AgentOptions: map[string]any{"model": "fake-model", "system_prompt": "instructions"}} + request := proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("hello"), ObserveMessages: mode != "messages-unrequested", AgentSessionID: "native-session", AgentOptions: map[string]any{"model": "fake-model", "system_prompt": "instructions"}} ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() out := make(chan proto.Envelope, 16) diff --git a/apps/parsar-daemon/internal/agent/claudesdk/options.go b/apps/parsar-daemon/internal/agent/claudesdk/options.go index 5a551da65..48368ee77 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/options.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/options.go @@ -26,7 +26,7 @@ type startRequest struct { Subagents *subagentOptions `json:"subagents,omitempty"` OutputFormat *proto.OutputFormat `json:"output_format,omitempty"` Type string `json:"type"` - Prompt string `json:"prompt,omitempty"` + Input proto.MessageInput `json:"input,omitempty"` Model string `json:"model"` SystemPrompt string `json:"system_prompt"` Cwd string `json:"cwd"` @@ -40,14 +40,14 @@ type startRequest struct { } func prepare(config Config, req proto.PromptRequestPayload) (startRequest, []string, error) { - if req.RunID == "" || strings.TrimSpace(req.Prompt) == "" { + if req.RunID == "" || req.Input.Validate() != nil { return startRequest{}, nil, fmt.Errorf("claudesdk: run id and prompt are required") } start, env, err := prepareConfiguration(config, req) if err != nil { return startRequest{}, nil, err } - start.Prompt = req.Prompt + start.Input = req.Input return start, env, nil } @@ -56,7 +56,7 @@ func prepareConfiguration(config Config, req proto.PromptRequestPayload) (startR fail := func(reason string) (startRequest, []string, error) { return startRequest{}, nil, fmt.Errorf("claudesdk: %s", reason) } - if len(req.Attachments) > 0 || req.WorkspaceAuthoring || req.ObserveTools { + if req.WorkspaceAuthoring || req.ObserveTools { return fail("requested capability is not available in the private SDK adapter") } if err := validateMCP(req); err != nil { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/options_test.go b/apps/parsar-daemon/internal/agent/claudesdk/options_test.go index 8607300d5..33344b656 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/options_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/options_test.go @@ -25,7 +25,7 @@ func TestNullableSystemPrompt(t *testing.T) { root := t.TempDir() t.Setenv("PARSAR_HOME", root) config := Config{Entrypoint: filepath.Join(root, "main.js"), StateDir: filepath.Join(root, "state")} - start, _, err := prepare(config, proto.PromptRequestPayload{RunID: "run", Prompt: "hello", AgentOptions: tc.options}) + start, _, err := prepare(config, proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("hello"), AgentOptions: tc.options}) if (err != nil) != tc.reject || (!tc.reject && start.SystemPrompt != tc.want) { t.Fatalf("system prompt %q, error %v", start.SystemPrompt, err) } diff --git a/apps/parsar-daemon/internal/agent/claudesdk/preparation.go b/apps/parsar-daemon/internal/agent/claudesdk/preparation.go index 3a39f2d44..91166a81e 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/preparation.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/preparation.go @@ -24,7 +24,7 @@ type prepared struct { type preparedStart struct { runID string - prompt string + prompt proto.MessageInput out chan<- proto.Envelope } @@ -36,7 +36,7 @@ func NewPreparationFactory(config Config) agent.PreparationFactory { if owner == nil { owner = context.Background() } - if config.Workspace == nil || req.RunID != "" || req.Prompt != "" || req.ConversationID != "" { + if config.Workspace == nil || req.RunID != "" || len(req.Input) != 0 || req.ConversationID != "" { return nil, fmt.Errorf("claudesdk: preparation requires workspace configuration without input or conversation") } snapshot := config @@ -84,7 +84,7 @@ func NewPreparationFactory(config Config) agent.PreparationFactory { } // Start transfers ownership once; ctx bounds this operation, not the Session lifetime. -func (p *prepared) Start(ctx context.Context, runID, prompt string, out chan<- proto.Envelope) (agent.Session, error) { +func (p *prepared) Start(ctx context.Context, runID string, prompt proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { if ctx == nil { ctx = context.Background() } @@ -94,7 +94,7 @@ func (p *prepared) Start(ctx context.Context, runID, prompt string, out chan<- p return nil, fmt.Errorf("claudesdk: preparation is no longer available") } var err error - if strings.TrimSpace(runID) == "" || strings.TrimSpace(prompt) == "" || out == nil { + if strings.TrimSpace(runID) == "" || prompt.Validate() != nil || out == nil { err = fmt.Errorf("claudesdk: start requires a run identity, prompt and output channel") } else if ctx.Err() != nil { err = ctx.Err() diff --git a/apps/parsar-daemon/internal/agent/claudesdk/preparation_fixture_test.go b/apps/parsar-daemon/internal/agent/claudesdk/preparation_fixture_test.go index fd39af809..c0d187d5c 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/preparation_fixture_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/preparation_fixture_test.go @@ -30,7 +30,7 @@ func preparationFixture(t *testing.T, mode string) Config { func preparationRequest() proto.PromptRequestPayload { req := workspaceRequest() - req.RunID, req.Prompt = "", "" + req.RunID, req.Input = "", nil req.AgentSessionID = "native-session" return req } @@ -57,7 +57,7 @@ func runPreparationHelper() { _ = os.WriteFile(filepath.Join(state, "prepare.json"), raw, 0o600) var fields map[string]json.RawMessage var request startRequest - if json.Unmarshal(raw, &fields) != nil || json.Unmarshal(raw, &request) != nil || request.Type != "prepare" || fields["prompt"] != nil || fields["run_id"] != nil || request.Workspace == nil { + if json.Unmarshal(raw, &fields) != nil || json.Unmarshal(raw, &request) != nil || request.Type != "prepare" || fields["input"] != nil || fields["run_id"] != nil || request.Workspace == nil { os.Exit(3) } emit := func(event bridgeEvent) { _ = json.NewEncoder(os.Stdout).Encode(event) } @@ -93,7 +93,7 @@ func runPreparationHelper() { raw = append([]byte{}, scanner.Bytes()...) _ = os.WriteFile(filepath.Join(state, "start.json"), raw, 0o600) fields = nil - if json.Unmarshal(raw, &fields) != nil || len(fields) != 2 || string(fields["type"]) != `"start"` || string(fields["prompt"]) != `"hello"` { + if json.Unmarshal(raw, &fields) != nil || len(fields) != 2 || string(fields["type"]) != `"start"` || string(fields["input"]) != `[{"content":[{"type":"input_text","text":"hello"}]}]` { os.Exit(4) } if mode == "cancellation" { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/preparation_test.go b/apps/parsar-daemon/internal/agent/claudesdk/preparation_test.go index ae25a7991..610c9154f 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/preparation_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/preparation_test.go @@ -63,7 +63,7 @@ func TestPreparationWaitsForReceiptAndRetainsConfiguration(t *testing.T) { if err := json.Unmarshal(raw, &frozen); err != nil { t.Fatal(err) } - if frozen.Model != "fixture" || frozen.Resume != "native-session" || frozen.Workspace == nil || frozen.Prompt != "" { + if frozen.Model != "fixture" || frozen.Resume != "native-session" || frozen.Workspace == nil || len(frozen.Input) != 0 { t.Fatal("configuration-only request was not retained") } config.Env[0] = "ANTHROPIC_AUTH_TOKEN=changed" @@ -73,7 +73,7 @@ func TestPreparationWaitsForReceiptAndRetainsConfiguration(t *testing.T) { req.AgentSessionID = "changed" out := make(chan proto.Envelope, 16) operation, stopOperation := context.WithCancel(ctx) - s, err := p.Start(operation, "actual-run", "hello", out) + s, err := p.Start(operation, "actual-run", proto.TextInput("hello"), out) stopOperation() if err != nil { t.Fatal(err) @@ -84,7 +84,7 @@ func TestPreparationWaitsForReceiptAndRetainsConfiguration(t *testing.T) { if err := p.Close(); err != nil { t.Fatal(err) } - if _, err := p.Start(ctx, "duplicate", "hello", make(chan proto.Envelope, 8)); err == nil { + if _, err := p.Start(ctx, "duplicate", proto.TextInput("hello"), make(chan proto.Envelope, 8)); err == nil { t.Fatal("duplicate Start was accepted") } var done proto.DonePayload @@ -115,11 +115,11 @@ func TestPreparationRejectsInputAndUnavailableProfilesBeforeLaunch(t *testing.T) case "run": req.RunID = "unexpected" case "prompt": - req.Prompt = "unexpected" + req.Input = proto.TextInput("unexpected") case "conversation": req.ConversationID = "product" case "attachments": - req.Attachments = []proto.PromptAttachment{{Kind: "image"}} + req.Input = proto.MessageInput{{Content: []proto.InputContent{{Type: "input_image"}}}} case "authoring": req.WorkspaceAuthoring = true case "subagents": @@ -179,7 +179,7 @@ func TestPreparationFailureAndUnusedRelease(t *testing.T) { prompt = "hello" cancel() } - _, startErr := p.Start(operation, "run", prompt, make(chan proto.Envelope, 8)) + _, startErr := p.Start(operation, "run", proto.TextInput(prompt), make(chan proto.Envelope, 8)) cancel() if startErr == nil { t.Fatal("invalid or cancelled Start succeeded") @@ -193,7 +193,7 @@ func TestPreparationFailureAndUnusedRelease(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("unused process was not settled") } - if _, err := p.Start(t.Context(), "late", "hello", make(chan proto.Envelope, 8)); err == nil { + if _, err := p.Start(t.Context(), "late", proto.TextInput("hello"), make(chan proto.Envelope, 8)); err == nil { t.Fatal("released preparation was reusable") } if got := p.CancellationOutcome(); !reflect.DeepEqual(got, proto.DonePayload{}) { @@ -215,7 +215,7 @@ func TestPreparedCancellationKeepsOwnershipUntilOutputDrain(t *testing.T) { defer p.Cancel(context.Background()) out := make(chan proto.Envelope) operation, stopOperation := context.WithCancel(owner) - if _, err := p.Start(operation, "run", "hello", out); err != nil { + if _, err := p.Start(operation, "run", proto.TextInput("hello"), out); err != nil { t.Fatal(err) } stopOperation() @@ -259,7 +259,10 @@ func TestPreparedStartRacesCloseAndCancellation(t *testing.T) { var startErr error var wg sync.WaitGroup wg.Add(2) - go func() { defer wg.Done(); running, startErr = p.Start(t.Context(), "run", "hello", out) }() + go func() { + defer wg.Done() + running, startErr = p.Start(t.Context(), "run", proto.TextInput("hello"), out) + }() go func() { defer wg.Done() if cancelResource { @@ -303,7 +306,7 @@ func TestPreparedConcurrentStartTransfersOnlyOnce(t *testing.T) { go func() { defer wg.Done() out := make(chan proto.Envelope, 16) - _, err := p.Start(t.Context(), "run", "hello", out) + _, err := p.Start(t.Context(), "run", proto.TextInput("hello"), out) results <- err == nil if err == nil { for event := range out { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/readiness.go b/apps/parsar-daemon/internal/agent/claudesdk/readiness.go index 3d01a10eb..d799b2fef 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/readiness.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/readiness.go @@ -25,6 +25,10 @@ type RuntimeInfo struct { Features []string `json:"features"` } +func (info RuntimeInfo) SupportsMessageImages() bool { + return slices.Contains(info.Features, "message_images") +} + func (info RuntimeInfo) SupportsStructuredOutput() bool { return slices.Contains(info.Features, "structured_output") } diff --git a/apps/parsar-daemon/internal/agent/claudesdk/readiness_test.go b/apps/parsar-daemon/internal/agent/claudesdk/readiness_test.go index 7301aa6d7..7236ca8ea 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/readiness_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/readiness_test.go @@ -23,7 +23,7 @@ func TestRequiredMCPNeedsQualifiedRuntime(t *testing.T) { config := Config{Node: os.Args[0], Entrypoint: filepath.Join(root, "main.js"), StateDir: filepath.Join(root, "state"), Env: []string{ "GO_CLAUDE_READINESS_HELPER=1", "READINESS_MODE=ready-http-mcp", "GORACE=atexit_sleep_ms=0", }} - req := proto.PromptRequestPayload{RunID: "run", Prompt: "hello", DisableExecutionEnvironment: true, + req := proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("hello"), DisableExecutionEnvironment: true, AgentOptions: map[string]any{"model": "fixture"}, MCPHTTPServers: &[]proto.MCPHTTPServer{{ServerLabel: "fixture", ServerURL: "https://example.invalid/mcp", Required: true}}} if _, err := NewFactory(config)(t.Context(), req, make(chan proto.Envelope, 1)); err == nil || err.Error() != "claudesdk: packaged runtime does not support required HTTP MCP" { t.Fatalf("unqualified runtime executed required MCP: %v", err) @@ -44,7 +44,7 @@ func TestHTTPMCPRejectsOldPackagedRuntime(t *testing.T) { if !info.SupportsHTTPMCP() { t.Fatal("runtime feature not recognized") } - req := proto.PromptRequestPayload{RunID: "run", Prompt: "hello", DisableExecutionEnvironment: true, + req := proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("hello"), DisableExecutionEnvironment: true, AgentOptions: map[string]any{"model": "fixture"}, MCPHTTPServers: &[]proto.MCPHTTPServer{{ServerLabel: "fixture", ServerURL: "https://example.invalid/mcp"}}} if _, err := NewFactory(config)(t.Context(), req, make(chan proto.Envelope, 1)); err == nil || !strings.Contains(err.Error(), "packaged runtime does not support HTTP MCP") { t.Fatalf("old runtime was not rejected before execution: %v", err) @@ -81,7 +81,7 @@ func TestMCPBearerRejectsAnonymousOnlyRuntimeWithoutProbeSecrets(t *testing.T) { "GO_CLAUDE_READINESS_HELPER=1", "READINESS_MODE=ready-http-mcp", "GORACE=atexit_sleep_ms=0", }} token := "private-fixture-token" - req := proto.PromptRequestPayload{RunID: "run", Prompt: "hello", DisableExecutionEnvironment: true, + req := proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("hello"), DisableExecutionEnvironment: true, AgentOptions: map[string]any{"model": "fixture"}, MCPHTTPServers: &[]proto.MCPHTTPServer{{ServerLabel: "fixture", ServerURL: "https://example.invalid/mcp", BearerToken: &token}}} if _, err := NewFactory(config)(t.Context(), req, make(chan proto.Envelope, 1)); err == nil || err.Error() != "claudesdk: packaged runtime does not support authenticated HTTP MCP" { t.Fatalf("old runtime executed authenticated request or readiness received its secret: %v", err) diff --git a/apps/parsar-daemon/internal/agent/claudesdk/restrictions_test.go b/apps/parsar-daemon/internal/agent/claudesdk/restrictions_test.go index 0d621380a..f0571d66c 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/restrictions_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/restrictions_test.go @@ -25,7 +25,7 @@ func TestTextFactoryAcceptsRestrictiveCapabilities(t *testing.T) { root := t.TempDir() t.Setenv("PARSAR_HOME", root) config := Config{Node: os.Args[0], Entrypoint: filepath.Join(root, "worker"), StateDir: filepath.Join(root, "state"), Env: []string{"GO_CLAUDE_SDK_HELPER=1", "SDK_HELPER_MODE=success", "GORACE=atexit_sleep_ms=0"}} - request := proto.PromptRequestPayload{RunID: "restricted-run", Prompt: "hello", AgentSessionID: "native-session", DisableExecutionEnvironment: test.environment, DisableSubagents: test.subagents, ExecutionControls: test.controls, AgentOptions: map[string]any{"model": "fake-model", "system_prompt": "instructions"}} + request := proto.PromptRequestPayload{RunID: "restricted-run", Input: proto.TextInput("hello"), AgentSessionID: "native-session", DisableExecutionEnvironment: test.environment, DisableSubagents: test.subagents, ExecutionControls: test.controls, AgentOptions: map[string]any{"model": "fake-model", "system_prompt": "instructions"}} ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() out := make(chan proto.Envelope, 16) diff --git a/apps/parsar-daemon/internal/agent/claudesdk/session.go b/apps/parsar-daemon/internal/agent/claudesdk/session.go index 13697df8b..267811f80 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/session.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/session.go @@ -33,11 +33,11 @@ func NewFactory(config Config) agent.Factory { return nil, fmt.Errorf("claudesdk: output channel is required") } if config.Workspace != nil { - runID, prompt := req.RunID, req.Prompt - if strings.TrimSpace(runID) == "" || strings.TrimSpace(prompt) == "" { + runID, prompt := req.RunID, req.Input + if strings.TrimSpace(runID) == "" || prompt.Validate() != nil { return nil, fmt.Errorf("claudesdk: run id and prompt are required") } - req.RunID, req.Prompt = "", "" + req.RunID, req.Input = "", nil prepared, err := NewPreparationFactory(config)(ctx, req) if err != nil { return nil, err @@ -148,9 +148,9 @@ func (s *session) run(ctx context.Context, runID string, start startRequest, out runID, out = binding.runID, binding.out s.writeMu.Lock() err = json.NewEncoder(s.process.Stdin).Encode(struct { - Type string `json:"type"` - Prompt string `json:"prompt"` - }{Type: "start", Prompt: binding.prompt}) + Type string `json:"type"` + Input proto.MessageInput `json:"input"` + }{Type: "start", Input: binding.prompt}) s.writeMu.Unlock() if err != nil { failure = fmt.Errorf("claudesdk: cannot submit SDK input") diff --git a/apps/parsar-daemon/internal/agent/claudesdk/session_test.go b/apps/parsar-daemon/internal/agent/claudesdk/session_test.go index 94a964076..a8e108538 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/session_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/session_test.go @@ -22,7 +22,7 @@ func TestTextFactoryCompletionAndFailures(t *testing.T) { root := t.TempDir() t.Setenv("PARSAR_HOME", root) config := Config{Node: os.Args[0], Entrypoint: filepath.Join(root, "worker"), StateDir: filepath.Join(root, "state"), Env: []string{"GO_CLAUDE_SDK_HELPER=1", "SDK_HELPER_MODE=" + mode, "GORACE=atexit_sleep_ms=0"}} - request := proto.PromptRequestPayload{RunID: "run", Prompt: "hello", AgentSessionID: "native-session", AgentOptions: map[string]any{"model": "fake-model", "system_prompt": "instructions"}} + request := proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("hello"), AgentSessionID: "native-session", AgentOptions: map[string]any{"model": "fake-model", "system_prompt": "instructions"}} ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() out := make(chan proto.Envelope, 16) @@ -79,7 +79,7 @@ func TestTextFactoryRejectsUnsupportedInput(t *testing.T) { root := t.TempDir() t.Setenv("PARSAR_HOME", root) config := Config{Node: "must-not-run", Entrypoint: filepath.Join(root, "worker"), StateDir: filepath.Join(root, "state")} - request := proto.PromptRequestPayload{RunID: "run", Prompt: "hello", AgentOptions: map[string]any{"model": "fake"}} + request := proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("hello"), AgentOptions: map[string]any{"model": "fake"}} switch kind { case "execution-controls": request.ExecutionControls = &proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "low"} @@ -122,7 +122,7 @@ func runSDKHelper() { os.Exit(2) } var request startRequest - if json.Unmarshal(scanner.Bytes(), &request) != nil || request.Type != "start" || request.Prompt != "hello" || request.Model != "fake-model" || request.SystemPrompt != "instructions" { + if json.Unmarshal(scanner.Bytes(), &request) != nil || request.Type != "start" || *request.Input[0].Content[0].Text != "hello" || request.Model != "fake-model" || request.SystemPrompt != "instructions" { os.Exit(3) } encode := func(event bridgeEvent) { _ = json.NewEncoder(os.Stdout).Encode(event) } diff --git a/apps/parsar-daemon/internal/agent/claudesdk/steering.go b/apps/parsar-daemon/internal/agent/claudesdk/steering.go index bf7ceea9e..b87b16f1d 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/steering.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/steering.go @@ -37,14 +37,14 @@ func (s *session) SteerWithReceipt(ctx context.Context, input proto.PromptSteerP if ctx == nil { ctx = context.Background() } - if strings.TrimSpace(input.InputID) == "" || len(input.InputID) > 256 || strings.TrimSpace(input.Text) == "" { + if strings.TrimSpace(input.InputID) == "" || len(input.InputID) > 256 || input.Input.Validate() != nil { return fmt.Errorf("%w: input identity and text are required", agent.ErrSteeringRejected) } data, err := json.Marshal(struct { - Type string `json:"type"` - InputID string `json:"input_id"` - Text string `json:"text"` - }{Type: "steer", InputID: input.InputID, Text: input.Text}) + Type string `json:"type"` + InputID string `json:"input_id"` + Input proto.MessageInput `json:"input"` + }{Type: "steer", InputID: input.InputID, Input: input.Input}) if err != nil || len(data) > 1024*1024 { return fmt.Errorf("%w: input exceeds bridge limit", agent.ErrSteeringRejected) } diff --git a/apps/parsar-daemon/internal/agent/claudesdk/steering_test.go b/apps/parsar-daemon/internal/agent/claudesdk/steering_test.go index fc9c5ea47..df07a4542 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/steering_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/steering_test.go @@ -27,7 +27,7 @@ func TestSteeringReceiptsAndLifecycle(t *testing.T) { if mode == "phased" { config.Env[1] = "SDK_HELPER_MODE=steering-timeout" } - request := proto.PromptRequestPayload{RunID: "run", Prompt: "hello", AgentSessionID: "native", AgentOptions: map[string]any{"model": "fake-model", "system_prompt": "instructions"}} + request := proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("hello"), AgentSessionID: "native", AgentOptions: map[string]any{"model": "fake-model", "system_prompt": "instructions"}} ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() out := make(chan proto.Envelope, 16) @@ -40,9 +40,9 @@ func TestSteeringReceiptsAndLifecycle(t *testing.T) { if frame := <-out; frame.Type != proto.TypeDelta { t.Fatal("missing ready barrier", frame.Type) } - input := proto.PromptSteerPayload{InputID: "extra", Text: "additional"} + input := proto.PromptSteerPayload{InputID: "extra", Input: proto.TextInput("additional")} if mode == "blocked-write" { - input.Text = strings.Repeat("x", 512*1024) + input.Input = proto.TextInput(strings.Repeat("x", 512*1024)) } receiptCtx, receiptCancel := context.WithTimeout(ctx, time.Second) if mode == "timeout" { @@ -158,11 +158,11 @@ func TestSteeringReceiptsAndLifecycle(t *testing.T) { func TestSteeringDoesNotSendBeforeReadiness(t *testing.T) { s := &session{process: &clirunner.Process{}} - if err := s.Steer(context.Background(), proto.PromptSteerPayload{InputID: "one", Text: "hello"}); !errors.Is(err, agent.ErrSteeringNotReady) { + if err := s.Steer(context.Background(), proto.PromptSteerPayload{InputID: "one", Input: proto.TextInput("hello")}); !errors.Is(err, agent.ErrSteeringNotReady) { t.Fatal(err) } s.stopSteering() - if err := s.Steer(context.Background(), proto.PromptSteerPayload{InputID: "one", Text: "hello"}); !errors.Is(err, agent.ErrSteeringInactive) { + if err := s.Steer(context.Background(), proto.PromptSteerPayload{InputID: "one", Input: proto.TextInput("hello")}); !errors.Is(err, agent.ErrSteeringInactive) { t.Fatal(err) } } @@ -178,10 +178,11 @@ func runSteeringHelper(request startRequest, mode string, scanner *bufio.Scanner os.Exit(2) } var input struct { - Type, Text string - InputID string `json:"input_id"` + Type string + Input proto.MessageInput + InputID string `json:"input_id"` } - if json.Unmarshal(scanner.Bytes(), &input) != nil || input.Type != "steer" || input.Text != "additional" || input.InputID != "extra" { + if json.Unmarshal(scanner.Bytes(), &input) != nil || input.Type != "steer" || *input.Input[0].Content[0].Text != "additional" || input.InputID != "extra" { os.Exit(3) } if mode == "steering-cancel" { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/usage_test.go b/apps/parsar-daemon/internal/agent/claudesdk/usage_test.go index 06af3e84a..7c409f4e2 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/usage_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/usage_test.go @@ -23,7 +23,7 @@ func TestUsageTransportPreservesSnapshotOnFailureAndDone(t *testing.T) { root := t.TempDir() t.Setenv("PARSAR_HOME", root) config := Config{Node: os.Args[0], Entrypoint: filepath.Join(root, "worker"), StateDir: filepath.Join(root, "state"), Env: []string{"GO_CLAUDE_SDK_HELPER=1", "SDK_HELPER_MODE=usage-" + mode, "GORACE=atexit_sleep_ms=0"}} - request := proto.PromptRequestPayload{RunID: "usage-run", Prompt: "hello", AgentSessionID: "native-session", AgentOptions: map[string]any{"model": "fake-model", "system_prompt": "instructions"}} + request := proto.PromptRequestPayload{RunID: "usage-run", Input: proto.TextInput("hello"), AgentSessionID: "native-session", AgentOptions: map[string]any{"model": "fake-model", "system_prompt": "instructions"}} ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() out := make(chan proto.Envelope, 16) diff --git a/apps/parsar-daemon/internal/agent/claudesdk/workspace_directory_test.go b/apps/parsar-daemon/internal/agent/claudesdk/workspace_directory_test.go index ecd35fcf2..f31003a9c 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/workspace_directory_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/workspace_directory_test.go @@ -127,7 +127,7 @@ func TestWorkspaceDirectoryDetachAndTransfer(t *testing.T) { t.Fatal(err) } out := make(chan proto.Envelope, 16) - running, err := p.Start(t.Context(), "run", "hello", out) + running, err := p.Start(t.Context(), "run", proto.TextInput("hello"), out) if err != nil { t.Fatal(err) } @@ -203,7 +203,7 @@ func TestWorkspaceDirectoryDeadlineStopsOwnerBeforeUnknown(t *testing.T) { if p.session.process.Context().Err() == nil { t.Fatal("uncertain deadline returned before owner cancellation") } - if _, err := p.Start(t.Context(), "late", "hello", make(chan proto.Envelope, 8)); err == nil { + if _, err := p.Start(t.Context(), "late", proto.TextInput("hello"), make(chan proto.Envelope, 8)); err == nil { t.Fatal("unknown owner accepted a new Start") } } diff --git a/apps/parsar-daemon/internal/agent/claudesdk/workspace_live_linux_test.go b/apps/parsar-daemon/internal/agent/claudesdk/workspace_live_linux_test.go index 5939b4e08..a18dbfe8c 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/workspace_live_linux_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/workspace_live_linux_test.go @@ -103,7 +103,7 @@ func testLiveClaudeWorkspace(t *testing.T, explicitPreparation bool) { defer cancel() out := make(chan proto.Envelope, 64) req := workspaceRequest() - req.RunID, req.Prompt, req.AgentSessionID = uuid.NewString(), prompt, resume + req.RunID, req.Input, req.AgentSessionID = uuid.NewString(), proto.TextInput(prompt), resume req.StrictResume, req.ReleaseOnCompletion, req.ObserveMessages, req.ObserveToolObservations = true, true, true, true req.AgentOptions = map[string]any{"model": "MiniMax-M3", "system_prompt": "Follow the exact verification instructions using the requested native tools. Preserve conversation facts. No other files, network operations or background work."} proof := evidence{RunID: req.RunID} @@ -111,7 +111,7 @@ func testLiveClaudeWorkspace(t *testing.T, explicitPreparation bool) { var owner agent.PreparedCancellation if explicitPreparation { preparation := req - preparation.RunID, preparation.Prompt = "", "" + preparation.RunID, preparation.Input = "", nil var resource agent.Prepared resource, err = NewPreparationFactory(config)(ctx, preparation) if err == nil { @@ -127,7 +127,7 @@ func testLiveClaudeWorkspace(t *testing.T, explicitPreparation bool) { } proof.Reads = append(proof.Reads, liveWorkspaceReads(t, ctx, resource.(agent.WorkspaceReader), config.Workspace.Directory, "prepared", "read-binary.bin", "read-empty.bin", "read-large.bin")...) operation, stopOperation := context.WithCancel(ctx) - running, err = resource.Start(operation, req.RunID, req.Prompt, out) + running, err = resource.Start(operation, req.RunID, req.Input, out) stopOperation() proof.StartContextCancelled = true if err == nil { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/workspace_read_test.go b/apps/parsar-daemon/internal/agent/claudesdk/workspace_read_test.go index 919497dec..61209b63b 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/workspace_read_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/workspace_read_test.go @@ -121,7 +121,7 @@ func TestWorkspaceReadDetachAndTransfer(t *testing.T) { t.Fatal(err) } out := make(chan proto.Envelope, 16) - running, err := p.Start(t.Context(), "run", "hello", out) + running, err := p.Start(t.Context(), "run", proto.TextInput("hello"), out) if err != nil { t.Fatal(err) } @@ -197,7 +197,7 @@ func TestWorkspaceReadDeadlineStopsOwnerBeforeUnknown(t *testing.T) { if p.session.process.Context().Err() == nil { t.Fatal("uncertain deadline returned before owner cancellation") } - if _, err := p.Start(t.Context(), "late", "hello", make(chan proto.Envelope, 8)); err == nil { + if _, err := p.Start(t.Context(), "late", proto.TextInput("hello"), make(chan proto.Envelope, 8)); err == nil { t.Fatal("unknown owner accepted a new Start") } } diff --git a/apps/parsar-daemon/internal/agent/claudesdk/workspace_test.go b/apps/parsar-daemon/internal/agent/claudesdk/workspace_test.go index 419f0af7a..70d57e69e 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/workspace_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/workspace_test.go @@ -35,7 +35,7 @@ func workspaceFixture(t *testing.T) Config { } func workspaceRequest() proto.PromptRequestPayload { - return proto.PromptRequestPayload{RunID: "run", Prompt: "hello", DisableSubagents: true, + return proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("hello"), DisableSubagents: true, AgentOptions: map[string]any{"model": "fixture"}} } diff --git a/apps/parsar-daemon/internal/agent/codex/functions_test.go b/apps/parsar-daemon/internal/agent/codex/functions_test.go index 4611c10ec..2b86b8d04 100644 --- a/apps/parsar-daemon/internal/agent/codex/functions_test.go +++ b/apps/parsar-daemon/internal/agent/codex/functions_test.go @@ -38,8 +38,8 @@ func TestFunctionCallWaitsAndRepliesOnce(t *testing.T) { t.Fatal("missing function call") } text, image, empty := "answer", "https://example.com/result.png", "" - content := []proto.FunctionResultContent{{Type: "input_text", Text: &text}, {Type: "input_image", ImageURL: &image}, {Type: "input_text", Text: &empty}} - if err := s.SubmitFunctionResult(t.Context(), proto.FunctionResultPayload{CallID: "call", Content: []proto.FunctionResultContent{{Type: "input_audio"}}}); err == nil { + content := []proto.InputContent{{Type: "input_text", Text: &text}, {Type: "input_image", ImageURL: &image}, {Type: "input_text", Text: &empty}} + if err := s.SubmitFunctionResult(t.Context(), proto.FunctionResultPayload{CallID: "call", Content: []proto.InputContent{{Type: "input_audio"}}}); err == nil { t.Fatal("invalid result consumed the pending call") } finished := make(chan error, 1) diff --git a/apps/parsar-daemon/internal/agent/codex/mcp_http_preflight_test.go b/apps/parsar-daemon/internal/agent/codex/mcp_http_preflight_test.go index ce2045d43..34bf9e43c 100644 --- a/apps/parsar-daemon/internal/agent/codex/mcp_http_preflight_test.go +++ b/apps/parsar-daemon/internal/agent/codex/mcp_http_preflight_test.go @@ -169,7 +169,7 @@ func TestPublicMCPHTTPPreparationChecksBeforeNewAndResumedThread(t *testing.T) { } defer p.Close() assertPreparationOnly(t, root) - s, err := p.start(t.Context(), "actual-run", "actual prompt", make(chan proto.Envelope, 8)) + s, err := p.start(t.Context(), "actual-run", proto.TextInput("actual prompt"), make(chan proto.Envelope, 8)) if err != nil { t.Fatal(err) } diff --git a/apps/parsar-daemon/internal/agent/codex/mcp_required_test.go b/apps/parsar-daemon/internal/agent/codex/mcp_required_test.go index aa9bd2da6..53b1ad816 100644 --- a/apps/parsar-daemon/internal/agent/codex/mcp_required_test.go +++ b/apps/parsar-daemon/internal/agent/codex/mcp_required_test.go @@ -41,7 +41,7 @@ func TestRequiredMCPWaitsForNativeThreadAndNeverRestartsFailedResume(t *testing. } defer p.Close() out := make(chan proto.Envelope, 16) - s, err := p.start(t.Context(), "required-run", "actual prompt", out) + s, err := p.start(t.Context(), "required-run", proto.TextInput("actual prompt"), out) if err != nil { t.Fatal(err) } diff --git a/apps/parsar-daemon/internal/agent/codex/message_input.go b/apps/parsar-daemon/internal/agent/codex/message_input.go new file mode 100644 index 000000000..f8bc41a18 --- /dev/null +++ b/apps/parsar-daemon/internal/agent/codex/message_input.go @@ -0,0 +1,25 @@ +package codex + +import "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + +// nativeInput keeps content order. Codex accepts a flat input list, so message +// boundaries use the same blank-line separator as the existing text path. +func nativeInput(messages proto.MessageInput) ([]UserInput, error) { + if err := messages.Validate(); err != nil { + return nil, err + } + var input []UserInput + for i, message := range messages { + if i > 0 { + input = append(input, UserInput{Type: UserInputText, Text: "\n\n"}) + } + for _, part := range message.Content { + if part.Type == "input_text" { + input = append(input, UserInput{Type: UserInputText, Text: *part.Text}) + } else { + input = append(input, UserInput{Type: UserInputRemoteImg, URL: *part.ImageURL}) + } + } + } + return input, nil +} diff --git a/apps/parsar-daemon/internal/agent/codex/message_input_test.go b/apps/parsar-daemon/internal/agent/codex/message_input_test.go new file mode 100644 index 000000000..6438506c6 --- /dev/null +++ b/apps/parsar-daemon/internal/agent/codex/message_input_test.go @@ -0,0 +1,20 @@ +package codex + +import ( + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "testing" +) + +func TestNativeInputRetainsImageOrderAndMessageSeparator(t *testing.T) { + image := "data:image/png;base64,aW1hZ2U=" + messages := proto.TextInput(" before ") + messages[0].Content = append(messages[0].Content, proto.InputContent{Type: "input_image", ImageURL: &image}, proto.TextInput(" after ")[0].Content[0]) + messages = append(messages, proto.TextInput("next")...) + input, err := nativeInput(messages) + if err != nil { + t.Fatal(err) + } + if len(input) != 5 || input[0].Text != " before " || input[1].Type != UserInputRemoteImg || input[1].URL != image || input[2].Text != " after " || input[3].Text != "\n\n" || input[4].Text != "next" { + t.Fatalf("native input changed: %+v", input) + } +} diff --git a/apps/parsar-daemon/internal/agent/codex/options.go b/apps/parsar-daemon/internal/agent/codex/options.go index b30d320de..67d3f1edd 100644 --- a/apps/parsar-daemon/internal/agent/codex/options.go +++ b/apps/parsar-daemon/internal/agent/codex/options.go @@ -213,18 +213,6 @@ func BuildSessionPlan(runID, agentStateKey, workDir string, opts map[string]any) return plan, nil } -// FirstUserInput translates the prompt text + attachments into the -// turn/start payload. Today only text is honoured; image / file -// attachments arrive as proto.PromptAttachment but aren't surfaced to -// the codex CLI yet — TODO once the daemon writes them to disk. -func FirstUserInput(prompt string) []UserInput { - prompt = strings.TrimSpace(prompt) - if prompt == "" { - return nil - } - return []UserInput{{Type: UserInputText, Text: prompt}} -} - // --------------------------------------------------------------------------- // helpers // --------------------------------------------------------------------------- diff --git a/apps/parsar-daemon/internal/agent/codex/options_test.go b/apps/parsar-daemon/internal/agent/codex/options_test.go index 19bb72737..cfafc7195 100644 --- a/apps/parsar-daemon/internal/agent/codex/options_test.go +++ b/apps/parsar-daemon/internal/agent/codex/options_test.go @@ -1,6 +1,7 @@ package codex import ( + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" "os" "path/filepath" "strings" @@ -254,18 +255,21 @@ func TestBuildSessionPlan_MissingMCPCommandErrors(t *testing.T) { } } -func TestFirstUserInput_TrimsAndWrapsAsText(t *testing.T) { - inputs := FirstUserInput(" hello world ") +func TestNativeInputPreservesText(t *testing.T) { + inputs, err := nativeInput(proto.TextInput(" hello world ")) + if err != nil { + t.Fatal(err) + } if len(inputs) != 1 { t.Fatalf("len = %d", len(inputs)) } - if inputs[0].Type != UserInputText || inputs[0].Text != "hello world" { + if inputs[0].Type != UserInputText || inputs[0].Text != " hello world " { t.Fatalf("input = %+v", inputs[0]) } } func TestFirstUserInput_EmptyReturnsNil(t *testing.T) { - if got := FirstUserInput(" "); got != nil { + if got, err := nativeInput(proto.TextInput(" ")); err == nil { t.Fatalf("empty prompt must return nil, got %+v", got) } } diff --git a/apps/parsar-daemon/internal/agent/codex/preparation.go b/apps/parsar-daemon/internal/agent/codex/preparation.go index 14217bff5..cf4bd4980 100644 --- a/apps/parsar-daemon/internal/agent/codex/preparation.go +++ b/apps/parsar-daemon/internal/agent/codex/preparation.go @@ -24,9 +24,9 @@ func newSession(parent context.Context, req proto.PromptRequestPayload, out chan if out == nil { return nil, errors.New("codex: nil out channel") } - runID, prompt := req.RunID, req.Prompt + runID, prompt := req.RunID, req.Input req.AgentStateKey = effectiveAgentStateKey(req) - req.RunID, req.Prompt = "", "" + req.RunID, req.Input = "", nil prepared, err := newPreparation(parent, req, cfg) if err != nil { return nil, err @@ -45,7 +45,7 @@ func newPreparation(parent context.Context, req proto.PromptRequestPayload, cfg if req.RequireExistingNativeSession && (!req.StrictResume || req.AgentStateKey == "" || req.WorkspaceReadOnly) { return nil, errors.New("codex: native-session recovery requires strict private state") } - if req.RunID != "" || req.Prompt != "" { + if req.RunID != "" || len(req.Input) != 0 { return nil, errors.New("codex: preparation does not accept a run identity or prompt") } if cfg.logger == nil { diff --git a/apps/parsar-daemon/internal/agent/codex/preparation_router_test.go b/apps/parsar-daemon/internal/agent/codex/preparation_router_test.go index 3e6b3d607..9d1fe4acc 100644 --- a/apps/parsar-daemon/internal/agent/codex/preparation_router_test.go +++ b/apps/parsar-daemon/internal/agent/codex/preparation_router_test.go @@ -119,7 +119,7 @@ func TestPreparationRouterRetainsActualNativeChild(t *testing.T) { t.Fatal("preparation became a Run") } if start { - input := proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "actual-run", Prompt: "actual input"} + input := proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "actual-run", Input: proto.TextInput("actual input")} send(proto.TypeExecutionStart, "prepare-request", input) await("started") frames := waitPreparationMethod(t, root, "turn/start") diff --git a/apps/parsar-daemon/internal/agent/codex/prepared.go b/apps/parsar-daemon/internal/agent/codex/prepared.go index 7b71ce36a..a83cf6ffe 100644 --- a/apps/parsar-daemon/internal/agent/codex/prepared.go +++ b/apps/parsar-daemon/internal/agent/codex/prepared.go @@ -30,7 +30,7 @@ var _ agent.PreparedCancellation = (*Prepared)(nil) // Start consumes the preparation once. ctx bounds only this start operation; // cancellation after return does not cancel the transferred Session. The original // owner context remains its lifetime context. On success the Session owns out. -func (p *Prepared) Start(ctx context.Context, runID, prompt string, out chan<- proto.Envelope) (agent.Session, error) { +func (p *Prepared) Start(ctx context.Context, runID string, prompt proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { session, err := p.start(ctx, runID, prompt, out) if err != nil { return nil, err @@ -38,8 +38,8 @@ func (p *Prepared) Start(ctx context.Context, runID, prompt string, out chan<- p return session, nil } -func (p *Prepared) start(ctx context.Context, runID, prompt string, out chan<- proto.Envelope) (*Session, error) { - if out == nil || strings.TrimSpace(runID) == "" || strings.TrimSpace(prompt) == "" { +func (p *Prepared) start(ctx context.Context, runID string, prompt proto.MessageInput, out chan<- proto.Envelope) (*Session, error) { + if out == nil || strings.TrimSpace(runID) == "" || prompt.Validate() != nil { return nil, errors.New("codex: start requires a run identity, prompt and output channel") } p.mu.Lock() @@ -70,7 +70,7 @@ func (p *Prepared) start(ctx context.Context, runID, prompt string, out chan<- p p.started = true close(p.transferred) transferred = true - req := proto.PromptRequestPayload{RunID: runID, Prompt: prompt, AgentSessionID: p.resumeID, StrictResume: p.strictResume, RequireExistingNativeSession: p.requireExistingNativeSession} + req := proto.PromptRequestPayload{RunID: runID, Input: prompt, AgentSessionID: p.resumeID, StrictResume: p.strictResume, RequireExistingNativeSession: p.requireExistingNativeSession} go s.run(p.plan, req) return s, nil } diff --git a/apps/parsar-daemon/internal/agent/codex/prepared_cancel_test.go b/apps/parsar-daemon/internal/agent/codex/prepared_cancel_test.go index 9f475c817..0169591dd 100644 --- a/apps/parsar-daemon/internal/agent/codex/prepared_cancel_test.go +++ b/apps/parsar-daemon/internal/agent/codex/prepared_cancel_test.go @@ -40,7 +40,7 @@ func TestPreparedCancelUnusedWaitsForCleanup(t *testing.T) { t.Fatal("cancellation did not begin cleanup") } out := make(chan proto.Envelope, 8) - if s, err := p.Start(t.Context(), "late", "must not execute", out); err == nil || s != nil { + if s, err := p.Start(t.Context(), "late", proto.TextInput("must not execute"), out); err == nil || s != nil { t.Fatal("cancellation did not fence Start") } select { @@ -83,7 +83,7 @@ func TestPreparedCancelTransferredPreservesObservedOutcome(t *testing.T) { t.Fatal(err) } defer p.Cancel(context.Background()) - started, err := p.Start(t.Context(), "run", "prompt", make(chan proto.Envelope, 16)) + started, err := p.Start(t.Context(), "run", proto.TextInput("prompt"), make(chan proto.Envelope, 16)) if err != nil { t.Fatal(err) } @@ -150,7 +150,7 @@ func TestPreparedCancelTransferredWaitsForCleanup(t *testing.T) { p.session.cleanup = sync.OnceFunc(func() { close(entered); <-release; cleanup() }) p.plan.Cleanup = p.session.cleanup out := make(chan proto.Envelope, 16) - if _, err := p.Start(t.Context(), "run", "prompt", out); err != nil { + if _, err := p.Start(t.Context(), "run", proto.TextInput("prompt"), out); err != nil { t.Fatal(err) } waitPreparationMethod(t, root, "turn/start") @@ -201,7 +201,7 @@ func TestPreparedCancelRacingTransfer(t *testing.T) { var calls sync.WaitGroup calls.Go(func() { <-begin - _, _ = p.Start(t.Context(), "run", "prompt", make(chan proto.Envelope, 16)) + _, _ = p.Start(t.Context(), "run", proto.TextInput("prompt"), make(chan proto.Envelope, 16)) }) calls.Go(func() { <-begin; _ = p.Cancel(context.Background()) }) calls.Go(func() { <-begin; _ = p.Close() }) @@ -221,7 +221,7 @@ func TestPreparedCancelRacingTransfer(t *testing.T) { assertUnstartedCancellation(t, p) } waitPreparedRelease(t, p, root) - if s, err := p.Start(t.Context(), "again", "must not execute", make(chan proto.Envelope, 8)); err == nil || s != nil { + if s, err := p.Start(t.Context(), "again", proto.TextInput("must not execute"), make(chan proto.Envelope, 8)); err == nil || s != nil { t.Fatal("cancelled preparation started again") } } diff --git a/apps/parsar-daemon/internal/agent/codex/prepared_test.go b/apps/parsar-daemon/internal/agent/codex/prepared_test.go index 6e65d87d4..adbc0a30f 100644 --- a/apps/parsar-daemon/internal/agent/codex/prepared_test.go +++ b/apps/parsar-daemon/internal/agent/codex/prepared_test.go @@ -36,7 +36,7 @@ func TestPreparedSessionTransfersSameResourceOnce(t *testing.T) { copy(req.FunctionTools[0].Parameters, strings.ReplaceAll(string(req.FunctionTools[0].Parameters), "integer", "boolean")) out := make(chan proto.Envelope, 8) startCtx, stopStart := context.WithCancel(t.Context()) - started, err := p.Start(startCtx, "actual-run", "actual prompt", out) + started, err := p.Start(startCtx, "actual-run", proto.TextInput("actual prompt"), out) stopStart() if err != nil { t.Fatal(err) @@ -49,7 +49,7 @@ func TestPreparedSessionTransfersSameResourceOnce(t *testing.T) { if err := p.Close(); err != nil || !session.rpc.Alive() { t.Fatal("close cancelled transferred resource", err) } - if again, err := p.Start(t.Context(), "second", "second prompt", out); err == nil || again != nil { + if again, err := p.Start(t.Context(), "second", proto.TextInput("second prompt"), out); err == nil || again != nil { t.Fatal("preparation started twice", err) } frames := waitPreparationMethod(t, root, "turn/start") @@ -134,7 +134,7 @@ func TestPreparedSessionAbandonmentAndFailedStart(t *testing.T) { waitPreparedRelease(t, p, root) } out := make(chan proto.Envelope, 8) - if started, err := p.Start(startCtx, "late-run", "must not start", out); err == nil || started != nil { + if started, err := p.Start(startCtx, "late-run", proto.TextInput("must not start"), out); err == nil || started != nil { t.Fatal("abandoned preparation started", err) } waitPreparedRelease(t, p, root) @@ -170,7 +170,7 @@ func TestPreparedSessionConcurrentStartAndClose(t *testing.T) { go func() { defer wg.Done() <-begin - s, err := p.start(t.Context(), "run", "prompt", make(chan proto.Envelope, 8)) + s, err := p.start(t.Context(), "run", proto.TextInput("prompt"), make(chan proto.Envelope, 8)) if err == nil { started <- s } diff --git a/apps/parsar-daemon/internal/agent/codex/protocol_wire_test.go b/apps/parsar-daemon/internal/agent/codex/protocol_wire_test.go index d42b070bc..0b383c724 100644 --- a/apps/parsar-daemon/internal/agent/codex/protocol_wire_test.go +++ b/apps/parsar-daemon/internal/agent/codex/protocol_wire_test.go @@ -110,7 +110,7 @@ func TestTurnStartParams_CollaborationModeUsesPlanWireShape(t *testing.T) { developerInstructions := "stay within the configured workspace" params := TurnStartParams{ ThreadID: "thread-1", - Input: FirstUserInput("ask me a question"), + Input: []UserInput{{Type: UserInputText, Text: "ask me a question"}}, CollaborationMode: &CollaborationMode{ Mode: CollaborationModePlan, Settings: CollaborationModeSettings{ diff --git a/apps/parsar-daemon/internal/agent/codex/recovery_test.go b/apps/parsar-daemon/internal/agent/codex/recovery_test.go index 98dfdaff7..372102bc3 100644 --- a/apps/parsar-daemon/internal/agent/codex/recovery_test.go +++ b/apps/parsar-daemon/internal/agent/codex/recovery_test.go @@ -148,7 +148,7 @@ func TestPreparedRecoveryCannotStartWithoutExistingHistory(t *testing.T) { defer p.Close() assertPreparationOnly(t, root) out := make(chan proto.Envelope, 16) - session, err := p.Start(t.Context(), "recovery-run", "continue", out) + session, err := p.Start(t.Context(), "recovery-run", proto.TextInput("continue"), out) if err != nil { t.Fatal(err) } diff --git a/apps/parsar-daemon/internal/agent/codex/session_run.go b/apps/parsar-daemon/internal/agent/codex/session_run.go index a5e47d626..170708625 100644 --- a/apps/parsar-daemon/internal/agent/codex/session_run.go +++ b/apps/parsar-daemon/internal/agent/codex/session_run.go @@ -21,8 +21,8 @@ func (s *Session) run(plan SessionPlan, req proto.PromptRequestPayload) { return } - input := FirstUserInput(req.Prompt) - if len(input) == 0 { + input, err := nativeInput(req.Input) + if err != nil { s.emitTerminal("codex: empty prompt", true) return } diff --git a/apps/parsar-daemon/internal/agent/codex/session_steering.go b/apps/parsar-daemon/internal/agent/codex/session_steering.go index 9ed03b3d1..4ad2716fe 100644 --- a/apps/parsar-daemon/internal/agent/codex/session_steering.go +++ b/apps/parsar-daemon/internal/agent/codex/session_steering.go @@ -58,7 +58,11 @@ func (s *Session) steer(ctx context.Context, input proto.PromptSteerPayload, wri if turnID == "" || threadID == "" { return agent.ErrSteeringNotReady } - params := TurnSteerParams{ThreadID: threadID, ExpectedTurnID: turnID, Input: FirstUserInput(input.Text)} + content, err := nativeInput(input.Input) + if err != nil { + return agent.ErrSteeringRejected + } + params := TurnSteerParams{ThreadID: threadID, ExpectedTurnID: turnID, Input: content} timeout := s.rpc.cfg.RequestTimeout if written != nil { timeout = 0 diff --git a/apps/parsar-daemon/internal/agent/codex/session_steering_lifecycle_test.go b/apps/parsar-daemon/internal/agent/codex/session_steering_lifecycle_test.go index dd10a238b..46bad9b45 100644 --- a/apps/parsar-daemon/internal/agent/codex/session_steering_lifecycle_test.go +++ b/apps/parsar-daemon/internal/agent/codex/session_steering_lifecycle_test.go @@ -26,7 +26,9 @@ func TestSteeringReceiptTimeoutAndCompletionKeepProcessAlive(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() done := make(chan error, 1) - go func() { done <- s.Steer(ctx, proto.PromptSteerPayload{InputID: "input", Text: "extra"}) }() + go func() { + done <- s.Steer(ctx, proto.PromptSteerPayload{InputID: "input", Input: proto.TextInput("extra")}) + }() var request JsonRpcRequest if err := json.NewDecoder(server.FromClient).Decode(&request); err != nil { t.Fatal(err) @@ -117,13 +119,13 @@ func TestBlockedSteeringWriteEndsRunWithTerminalFrames(t *testing.T) { } ready <- nil }() - go s.run(SessionPlan{Model: "synthetic"}, proto.PromptRequestPayload{Prompt: "first"}) + go s.run(SessionPlan{Model: "synthetic"}, proto.PromptRequestPayload{Input: proto.TextInput("first")}) if err := <-ready; err != nil { t.Fatal(err) } callCtx, callCancel := context.WithTimeout(ctx, 50*time.Millisecond) defer callCancel() - if err := s.Steer(callCtx, proto.PromptSteerPayload{InputID: "blocked", Text: "extra"}); err == nil { + if err := s.Steer(callCtx, proto.PromptSteerPayload{InputID: "blocked", Input: proto.TextInput("extra")}); err == nil { t.Fatal("blocked write accepted") } if client.Alive() { diff --git a/apps/parsar-daemon/internal/agent/codex/session_steering_receipt_test.go b/apps/parsar-daemon/internal/agent/codex/session_steering_receipt_test.go index 1c744d273..5f6fab215 100644 --- a/apps/parsar-daemon/internal/agent/codex/session_steering_receipt_test.go +++ b/apps/parsar-daemon/internal/agent/codex/session_steering_receipt_test.go @@ -23,7 +23,7 @@ func TestDurableSteeringBypassesOnlyNativeResponseDeadline(t *testing.T) { written := make(chan struct{}) reply := make(chan error, 1) go func() { - input := proto.PromptSteerPayload{InputID: "extra", Text: "text"} + input := proto.PromptSteerPayload{InputID: "extra", Input: proto.TextInput("text")} if durable { reply <- s.SteerWithReceipt(ctx, input, func() { close(written) }) } else { @@ -85,7 +85,7 @@ func TestDurableSteeringKeepsConfirmedReceiptAtCompletion(t *testing.T) { inWritten, releaseWritten := make(chan struct{}), make(chan struct{}) reply := make(chan error, 1) go func() { - reply <- s.SteerWithReceipt(ctx, proto.PromptSteerPayload{InputID: "extra", Text: "text"}, func() { close(inWritten); <-releaseWritten }) + reply <- s.SteerWithReceipt(ctx, proto.PromptSteerPayload{InputID: "extra", Input: proto.TextInput("text")}, func() { close(inWritten); <-releaseWritten }) }() var request JsonRpcRequest if err := json.NewDecoder(server.FromClient).Decode(&request); err != nil { diff --git a/apps/parsar-daemon/internal/agent/codex/session_steering_test.go b/apps/parsar-daemon/internal/agent/codex/session_steering_test.go index 651677311..fa1bd0746 100644 --- a/apps/parsar-daemon/internal/agent/codex/session_steering_test.go +++ b/apps/parsar-daemon/internal/agent/codex/session_steering_test.go @@ -33,7 +33,7 @@ func TestSteeringUsesNativeActiveTurnAndReceipt(t *testing.T) { s.onTurnStarted(json.RawMessage(`{"threadId":"native-thread","turn":{"id":"native-turn"}}`)) done := make(chan error, 1) go func() { - done <- s.Steer(ctx, proto.PromptSteerPayload{InputID: "input-1", Text: "追加输入"}) + done <- s.Steer(ctx, proto.PromptSteerPayload{InputID: "input-1", Input: proto.TextInput("追加输入")}) }() var request struct { ID string `json:"id"` @@ -77,7 +77,9 @@ func TestSteeringDeadlineReleasesBlockedNativeWrite(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() done := make(chan error, 1) - go func() { done <- s.Steer(ctx, proto.PromptSteerPayload{InputID: "blocked", Text: "extra"}) }() + go func() { + done <- s.Steer(ctx, proto.PromptSteerPayload{InputID: "blocked", Input: proto.TextInput("extra")}) + }() // No reader drains the pipe, so the request never reaches its response wait. select { case err := <-done: @@ -97,7 +99,7 @@ func TestSteeringDoesNotStartOrReviveTurns(t *testing.T) { defer cancel() s := &Session{cancelCtx: ctx} s.setThreadID("native-thread") - input := proto.PromptSteerPayload{InputID: "input-1", Text: "extra"} + input := proto.PromptSteerPayload{InputID: "input-1", Input: proto.TextInput("extra")} // A nil RPC client proves these states do not issue a request. for _, notification := range []json.RawMessage{nil, json.RawMessage(`{"threadId":"other-thread","turn":{"id":"other-turn"}}`)} { s.onTurnStarted(notification) diff --git a/apps/parsar-daemon/internal/agent/codex/tool_observations.go b/apps/parsar-daemon/internal/agent/codex/tool_observations.go index ecc494c01..96c3e68a7 100644 --- a/apps/parsar-daemon/internal/agent/codex/tool_observations.go +++ b/apps/parsar-daemon/internal/agent/codex/tool_observations.go @@ -63,14 +63,14 @@ func normalizeToolObservation(id, stage string, raw json.RawMessage) (*proto.Too out.Status = "failed" } if n.ContentItems != nil { - content := make([]proto.FunctionResultContent, 0, len(*n.ContentItems)) + content := make([]proto.InputContent, 0, len(*n.ContentItems)) for _, part := range *n.ContentItems { - var value proto.FunctionResultContent + var value proto.InputContent switch part.Type { case "inputText": - value = proto.FunctionResultContent{Type: "input_text", Text: part.Text} + value = proto.InputContent{Type: "input_text", Text: part.Text} case "inputImage": - value = proto.FunctionResultContent{Type: "input_image", ImageURL: part.ImageURL} + value = proto.InputContent{Type: "input_image", ImageURL: part.ImageURL} default: return nil, errors.New("unsupported function result content") } diff --git a/apps/parsar-daemon/internal/agent/mcode/environment_mcp_test.go b/apps/parsar-daemon/internal/agent/mcode/environment_mcp_test.go index cc82cb780..567376eac 100644 --- a/apps/parsar-daemon/internal/agent/mcode/environment_mcp_test.go +++ b/apps/parsar-daemon/internal/agent/mcode/environment_mcp_test.go @@ -110,7 +110,7 @@ func TestEnvironmentMCPCancelSettlesPendingObservationBeforeDone(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() out := make(chan proto.Envelope, 16) - session, err := resource.Start(ctx, "run", "invoke and wait", out) + session, err := resource.Start(ctx, "run", proto.TextInput("invoke and wait"), out) if err != nil { t.Fatal(err) } diff --git a/apps/parsar-daemon/internal/agent/mcode/native_history_test.go b/apps/parsar-daemon/internal/agent/mcode/native_history_test.go index 280bd5882..df2272ca6 100644 --- a/apps/parsar-daemon/internal/agent/mcode/native_history_test.go +++ b/apps/parsar-daemon/internal/agent/mcode/native_history_test.go @@ -28,7 +28,7 @@ func TestNativeMCodeHistoryIsolation(t *testing.T) { if json.Unmarshal(raw, &req.AgentOptions) != nil { t.Fatal("invalid private options") } - req.AgentSessionID, req.Prompt = id, "This input must never execute." + req.AgentSessionID, req.Input = id, proto.TextInput("This input must never execute.") ctx, cancel := context.WithTimeout(t.Context(), 90*time.Second) defer cancel() out := make(chan proto.Envelope, 64) diff --git a/apps/parsar-daemon/internal/agent/mcode/native_test.go b/apps/parsar-daemon/internal/agent/mcode/native_test.go index 896dcae5c..8fbc749e7 100644 --- a/apps/parsar-daemon/internal/agent/mcode/native_test.go +++ b/apps/parsar-daemon/internal/agent/mcode/native_test.go @@ -101,7 +101,7 @@ func TestNativeMCodeACP(t *testing.T) { req.AgentOptions["skills"] = []any{map[string]any{"name": "qa-mcode-skill", "version": "1", "download_url": skill.URL, "sha256": hex.EncodeToString(digest[:])}} req.AgentOptions["mcp_servers"] = map[string]any{"qa": map[string]any{"type": "http", "url": mcp.URL}} req.AgentOptions["system_prompt"] = "SP-MCODE-672: use the available tools when requested." - req.Prompt = "QA-CALL-MCP: call get_fixture, then reply PARSAR-MCODE-OK." + req.Input = proto.TextInput("QA-CALL-MCP: call get_fixture, then reply PARSAR-MCODE-OK.") run := func() proto.DonePayload { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) t.Cleanup(cancel) @@ -156,7 +156,7 @@ func TestNativeMCodeACP(t *testing.T) { models["fixture-new"] = models["fixture"] delete(models, "fixture") req.AgentOptions["model"] = "fixture-new" - req.Prompt = "Now reply PARSAR-MCODE-OK." + req.Input = proto.TextInput("Now reply PARSAR-MCODE-OK.") run() mu.Lock() resumed := strings.Join(requests, "\n") diff --git a/apps/parsar-daemon/internal/agent/mcode/options.go b/apps/parsar-daemon/internal/agent/mcode/options.go index 374f9ea79..9676879ff 100644 --- a/apps/parsar-daemon/internal/agent/mcode/options.go +++ b/apps/parsar-daemon/internal/agent/mcode/options.go @@ -33,7 +33,7 @@ func prepareOptionsWithSkills(ctx context.Context, req proto.PromptRequestPayloa return result, err } } - if len(req.Attachments) > 0 { + if req.Input.HasImages() { return result, fmt.Errorf("mcode: ACP does not support attachments") } root, err := agent.ManagedSkillsRoot("mcode", req.AgentStateKey, req.ConversationID, req.RunID) diff --git a/apps/parsar-daemon/internal/agent/mcode/options_test.go b/apps/parsar-daemon/internal/agent/mcode/options_test.go index a2508a052..0bc3c98e9 100644 --- a/apps/parsar-daemon/internal/agent/mcode/options_test.go +++ b/apps/parsar-daemon/internal/agent/mcode/options_test.go @@ -65,7 +65,9 @@ func TestOptionsRejectDroppedContext(t *testing.T) { }{ {"relative workdir", func(r *proto.PromptRequestPayload) { r.WorkDir = "relative" }}, {"oversized instructions", func(r *proto.PromptRequestPayload) { r.AgentOptions["system_prompt"] = strings.Repeat("x", 32*1024+1) }}, - {"attachment", func(r *proto.PromptRequestPayload) { r.Attachments = []proto.PromptAttachment{{Kind: "image"}} }}, + {"attachment", func(r *proto.PromptRequestPayload) { + r.Input = proto.MessageInput{{Content: []proto.InputContent{{Type: "input_image"}}}} + }}, {"missing model", func(r *proto.PromptRequestPayload) { delete(r.AgentOptions, "model") }}, {"missing provider", func(r *proto.PromptRequestPayload) { delete(r.AgentOptions, "mcode_provider") }}, {"invalid permission mode", func(r *proto.PromptRequestPayload) { r.AgentOptions["mode"] = "plan" }}, diff --git a/apps/parsar-daemon/internal/agent/mcode/preparation.go b/apps/parsar-daemon/internal/agent/mcode/preparation.go index fa8339262..97e614d82 100644 --- a/apps/parsar-daemon/internal/agent/mcode/preparation.go +++ b/apps/parsar-daemon/internal/agent/mcode/preparation.go @@ -21,8 +21,9 @@ type prepared struct { } type preparedStart struct { - runID, prompt string - out chan<- proto.Envelope + runID string + prompt proto.MessageInput + out chan<- proto.Envelope } func NewPreparationFactory(config WorkspaceConfig) agent.PreparationFactory { @@ -30,7 +31,7 @@ func NewPreparationFactory(config WorkspaceConfig) agent.PreparationFactory { if ctx == nil { ctx = context.Background() } - if req.RunID != "" || req.Prompt != "" || req.ConversationID != "" { + if req.RunID != "" || len(req.Input) != 0 || req.ConversationID != "" { return nil, fmt.Errorf("mcode: preparation cannot contain input or product context") } opts, err := prepareWorkspaceOptions(ctx, config, req) @@ -56,7 +57,7 @@ func NewPreparationFactory(config WorkspaceConfig) agent.PreparationFactory { } } -func (p *prepared) Start(ctx context.Context, runID, prompt string, out chan<- proto.Envelope) (agent.Session, error) { +func (p *prepared) Start(ctx context.Context, runID string, prompt proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { if ctx == nil { ctx = context.Background() } @@ -65,7 +66,7 @@ func (p *prepared) Start(ctx context.Context, runID, prompt string, out chan<- p if p.closed || p.binding != nil { return nil, fmt.Errorf("mcode: preparation is no longer available") } - if strings.TrimSpace(runID) == "" || strings.TrimSpace(prompt) == "" || out == nil || ctx.Err() != nil { + if strings.TrimSpace(runID) == "" || prompt.Validate() != nil || out == nil || ctx.Err() != nil { return nil, fmt.Errorf("mcode: start requires a live context, identity, prompt and output") } select { @@ -136,7 +137,7 @@ func (p *prepared) awaitStart(err error) error { } p.mu.Lock() if b := p.binding; b != nil { - p.session.req.RunID, p.session.req.Prompt, p.session.out = b.runID, b.prompt, b.out + p.session.req.RunID, p.session.req.Input, p.session.out = b.runID, b.prompt, b.out } else { p.closed = true } diff --git a/apps/parsar-daemon/internal/agent/mcode/preparation_test.go b/apps/parsar-daemon/internal/agent/mcode/preparation_test.go index 48732b868..754446e9d 100644 --- a/apps/parsar-daemon/internal/agent/mcode/preparation_test.go +++ b/apps/parsar-daemon/internal/agent/mcode/preparation_test.go @@ -16,7 +16,7 @@ import ( func workspaceFixture(t *testing.T) (WorkspaceConfig, proto.PromptRequestPayload, string) { t.Helper() r := executionRequest(t) - r.RunID, r.Prompt, r.ConversationID = "", "", "" + r.RunID, r.Input, r.ConversationID = "", nil, "" r.DisableExecutionEnvironment = false r.LocalEnvironment = &proto.LocalEnvironment{ID: "environment", NetworkAccess: "disabled"} r.WorkDir = t.TempDir() @@ -56,7 +56,11 @@ func TestPreparedWorkspaceHasOneInputAndOutputOwner(t *testing.T) { winners := make(chan bool, 8) for range 8 { wg.Add(1) - go func() { defer wg.Done(); _, err := p.Start(ctx, "run", "input", out); winners <- err == nil }() + go func() { + defer wg.Done() + _, err := p.Start(ctx, "run", proto.TextInput("input"), out) + winners <- err == nil + }() } wg.Wait() close(winners) @@ -114,7 +118,7 @@ func TestPreparedWorkspaceCloseBeforeStart(t *testing.T) { if err = resource.Close(); err != nil { t.Fatal(err) } - if _, err = resource.Start(ctx, "run", "input", make(chan proto.Envelope)); err == nil { + if _, err = resource.Start(ctx, "run", proto.TextInput("input"), make(chan proto.Envelope)); err == nil { t.Fatal("released preparation started") } if err = resource.Close(); err != nil { diff --git a/apps/parsar-daemon/internal/agent/mcode/session.go b/apps/parsar-daemon/internal/agent/mcode/session.go index a1521386e..6f9e6f615 100644 --- a/apps/parsar-daemon/internal/agent/mcode/session.go +++ b/apps/parsar-daemon/internal/agent/mcode/session.go @@ -248,11 +248,15 @@ func (s *Session) prepareNative() error { } func (s *Session) executePrompt() error { + prompt, err := s.req.Input.TextOnly() + if err != nil { + return err + } s.active = true var result struct { StopReason string `json:"stopReason"` } - err := s.call("session/prompt", map[string]any{"sessionId": s.sessionID, "prompt": promptContent(s.req.Prompt, s.req.StrictResume)}, &result, true) + err = s.call("session/prompt", map[string]any{"sessionId": s.sessionID, "prompt": promptContent(prompt, s.req.StrictResume)}, &result, true) s.active = false s.mu.Lock() s.steeringReady = false diff --git a/apps/parsar-daemon/internal/agent/mcode/session_test.go b/apps/parsar-daemon/internal/agent/mcode/session_test.go index 1240966c8..5082bfeeb 100644 --- a/apps/parsar-daemon/internal/agent/mcode/session_test.go +++ b/apps/parsar-daemon/internal/agent/mcode/session_test.go @@ -18,7 +18,7 @@ import ( func testRequest(t *testing.T) proto.PromptRequestPayload { t.Helper() t.Setenv("PARSAR_HOME", t.TempDir()) - return proto.PromptRequestPayload{RunID: "run-1", ConversationID: "conversation-1", AgentStateKey: "conversation-1/agent-1/mcode", Prompt: "Hello", AgentOptions: map[string]any{ + return proto.PromptRequestPayload{RunID: "run-1", ConversationID: "conversation-1", AgentStateKey: "conversation-1/agent-1/mcode", Input: proto.TextInput("Hello"), AgentOptions: map[string]any{ "model": "fixture", "mcode_provider": map[string]any{"kind": "custom", "enabled": true}, "system_prompt": "Current instructions", }} } diff --git a/apps/parsar-daemon/internal/agent/mcode/steering.go b/apps/parsar-daemon/internal/agent/mcode/steering.go index 994ae4a08..7479b2964 100644 --- a/apps/parsar-daemon/internal/agent/mcode/steering.go +++ b/apps/parsar-daemon/internal/agent/mcode/steering.go @@ -18,7 +18,8 @@ func (s *Session) Steer(ctx context.Context, input proto.PromptSteerPayload) err // Native acceptance belongs to the active ACP Turn; it does not promise model consumption. func (s *Session) SteerWithReceipt(ctx context.Context, input proto.PromptSteerPayload, written func()) error { - if strings.TrimSpace(input.InputID) == "" || strings.TrimSpace(input.Text) == "" { + text, err := input.Input.TextOnly() + if strings.TrimSpace(input.InputID) == "" || err != nil { return agent.ErrSteeringRejected } s.mu.Lock() @@ -32,7 +33,7 @@ func (s *Session) SteerWithReceipt(ctx context.Context, input proto.PromptSteerP } id, response := s.reserveResponse() defer s.removeResponse(id) - raw, err := json.Marshal(map[string]string{"sessionId": native, "text": input.Text, "clientRequestId": input.InputID}) + raw, err := json.Marshal(map[string]string{"sessionId": native, "text": text, "clientRequestId": input.InputID}) if err != nil { return err } diff --git a/apps/parsar-daemon/internal/agent/mcode/steering_test.go b/apps/parsar-daemon/internal/agent/mcode/steering_test.go index b5bce7af7..df5dfc1ab 100644 --- a/apps/parsar-daemon/internal/agent/mcode/steering_test.go +++ b/apps/parsar-daemon/internal/agent/mcode/steering_test.go @@ -28,7 +28,7 @@ func TestNativeSteeringReceipt(t *testing.T) { written := false reply := make(chan error, 1) go func() { - reply <- s.SteerWithReceipt(ctx, proto.PromptSteerPayload{InputID: "input-1", Text: "next"}, func() { written = true }) + reply <- s.SteerWithReceipt(ctx, proto.PromptSteerPayload{InputID: "input-1", Input: proto.TextInput("next")}, func() { written = true }) }() // Drain native output concurrently, as the router does. drained := make(chan struct{}) diff --git a/apps/parsar-daemon/internal/agent/opencode/session.go b/apps/parsar-daemon/internal/agent/opencode/session.go index f6f83c7d8..622b33abe 100644 --- a/apps/parsar-daemon/internal/agent/opencode/session.go +++ b/apps/parsar-daemon/internal/agent/opencode/session.go @@ -80,7 +80,11 @@ func newSession(parent context.Context, req proto.PromptRequestPayload, out chan return nil, err } - buildRes, err := BuildArgs(req.RunID, req.Prompt, req.WorkDir, opts) + prompt, err := req.Input.TextOnly() + if err != nil { + return nil, err + } + buildRes, err := BuildArgs(req.RunID, prompt, req.WorkDir, opts) if err != nil { return nil, fmt.Errorf("opencode: build args: %w", err) } diff --git a/apps/parsar-daemon/internal/agent/opencode/session_test.go b/apps/parsar-daemon/internal/agent/opencode/session_test.go index 72bfbbf6a..d9512a9b0 100644 --- a/apps/parsar-daemon/internal/agent/opencode/session_test.go +++ b/apps/parsar-daemon/internal/agent/opencode/session_test.go @@ -126,8 +126,8 @@ func opencodeHelperConfig() opencode.SessionConfigForTest { func opencodeHelperReq(runID, prompt, role string) proto.PromptRequestPayload { return proto.PromptRequestPayload{ - RunID: runID, - Prompt: prompt, + RunID: runID, + Input: proto.TextInput(prompt), AgentOptions: map[string]any{ "env": map[string]any{ opencodeHelperEnvKey: role, @@ -372,7 +372,7 @@ func TestSessionRejectsNilOut(t *testing.T) { func TestSessionRejectsEmptyPrompt(t *testing.T) { out := make(chan proto.Envelope, 4) _, err := opencode.NewSessionForTest(context.Background(), - proto.PromptRequestPayload{RunID: "run_empty", Prompt: ""}, out, opencodeHelperConfig()) + proto.PromptRequestPayload{RunID: "run_empty", Input: proto.TextInput("")}, out, opencodeHelperConfig()) if err == nil { t.Fatal("expected error on empty prompt") } diff --git a/apps/parsar-daemon/internal/agent/pi/session.go b/apps/parsar-daemon/internal/agent/pi/session.go index 4abb3a195..89661d33b 100644 --- a/apps/parsar-daemon/internal/agent/pi/session.go +++ b/apps/parsar-daemon/internal/agent/pi/session.go @@ -104,7 +104,11 @@ func newSession(parent context.Context, req proto.PromptRequestPayload, out chan } opts = provOpts - buildRes, err := BuildArgs(req.RunID, req.Prompt, req.WorkDir, opts, req.AgentSessionID) + prompt, err := req.Input.TextOnly() + if err != nil { + return nil, err + } + buildRes, err := BuildArgs(req.RunID, prompt, req.WorkDir, opts, req.AgentSessionID) if err != nil { return nil, fmt.Errorf("pi: build args: %w", err) } diff --git a/apps/parsar-daemon/internal/agent/pi/session_provider_test.go b/apps/parsar-daemon/internal/agent/pi/session_provider_test.go index 2e2cfd2da..5190c6356 100644 --- a/apps/parsar-daemon/internal/agent/pi/session_provider_test.go +++ b/apps/parsar-daemon/internal/agent/pi/session_provider_test.go @@ -23,7 +23,7 @@ func TestNewSessionMaterialisesPiProviderModelsJSON(t *testing.T) { RunID: "run_prov", ConversationID: "conv-prov", AgentStateKey: "conv-prov/agent-prov/pi", - Prompt: "hello", + Input: proto.TextInput("hello"), AgentOptions: map[string]any{ "model": "parsar/claude-opus-4-6-thinking-max", "pi_provider": map[string]any{ diff --git a/apps/parsar-daemon/internal/agent/pi/session_skills_test.go b/apps/parsar-daemon/internal/agent/pi/session_skills_test.go index ed7d03181..30009eb84 100644 --- a/apps/parsar-daemon/internal/agent/pi/session_skills_test.go +++ b/apps/parsar-daemon/internal/agent/pi/session_skills_test.go @@ -26,7 +26,7 @@ func TestNewSessionInstallsSkillsAndInjectsSkillFlag(t *testing.T) { req := proto.PromptRequestPayload{ RunID: "run_skill", ConversationID: "conv-skill", - Prompt: "hello", + Input: proto.TextInput("hello"), AgentOptions: map[string]any{ "skills": []any{ map[string]any{ diff --git a/apps/parsar-daemon/internal/agent/pi/session_test.go b/apps/parsar-daemon/internal/agent/pi/session_test.go index 4260c45df..60b83f7cf 100644 --- a/apps/parsar-daemon/internal/agent/pi/session_test.go +++ b/apps/parsar-daemon/internal/agent/pi/session_test.go @@ -119,8 +119,8 @@ func piHelperConfig() pi.SessionConfigForTest { func piHelperReq(runID, prompt, role string) proto.PromptRequestPayload { return proto.PromptRequestPayload{ - RunID: runID, - Prompt: prompt, + RunID: runID, + Input: proto.TextInput(prompt), AgentOptions: map[string]any{ "env": map[string]any{ piHelperEnvKey: role, @@ -297,7 +297,7 @@ func TestSessionRejectsNilOut(t *testing.T) { func TestSessionRejectsEmptyPrompt(t *testing.T) { out := make(chan proto.Envelope, 4) _, err := pi.NewSessionForTest(context.Background(), - proto.PromptRequestPayload{RunID: "run_empty", Prompt: ""}, out, piHelperConfig()) + proto.PromptRequestPayload{RunID: "run_empty", Input: proto.TextInput("")}, out, piHelperConfig()) if err == nil { t.Fatal("expected error on empty prompt") } diff --git a/apps/parsar-daemon/internal/agent/preparation.go b/apps/parsar-daemon/internal/agent/preparation.go index e02d45e28..bd303d464 100644 --- a/apps/parsar-daemon/internal/agent/preparation.go +++ b/apps/parsar-daemon/internal/agent/preparation.go @@ -14,7 +14,7 @@ type Prepared interface { // Start transfers output ownership only when it returns a non-nil Session. // A nil Session leaves the caller as the sole owner of closing out, and the // implementation must not retain or write to it after Start returns. - Start(context.Context, string, string, chan<- proto.Envelope) (Session, error) + Start(context.Context, string, proto.MessageInput, chan<- proto.Envelope) (Session, error) // Close retains unused ownership on error; callers may retry settlement. Close() error } diff --git a/apps/parsar-daemon/internal/cli/agent_discovery.go b/apps/parsar-daemon/internal/cli/agent_discovery.go index f8f4b7427..ee32b3e7d 100644 --- a/apps/parsar-daemon/internal/cli/agent_discovery.go +++ b/apps/parsar-daemon/internal/cli/agent_discovery.go @@ -88,6 +88,7 @@ func discoverAgentCLIs(rc *runContext, profile string, checks agentCLIChecks) (a Steering: true, DurableTurns: true, DurableInputReceipts: true, + MessageImages: true, FunctionTools: true, MCPHTTPTools: true, MCPHTTPBearerAuth: true, diff --git a/apps/parsar-daemon/internal/cli/claude_sdk.go b/apps/parsar-daemon/internal/cli/claude_sdk.go index 8587d23d3..e568a7a79 100644 --- a/apps/parsar-daemon/internal/cli/claude_sdk.go +++ b/apps/parsar-daemon/internal/cli/claude_sdk.go @@ -97,6 +97,7 @@ func discoverClaudeSDK(rc *runContext, profile string, check func(context.Contex caps.WorkspaceReadPreparation, caps.NativeSessionRecovery = true, true } out.Info.Available, out.Info.Version = true, info.SDK + out.Info.Capabilities.MessageImages = info.SupportsMessageImages() out.Info.Capabilities.StructuredOutput = out.Config.Workspace == nil && info.SupportsStructuredOutput() out.Info.Capabilities.SubagentObservations = info.SupportsSubagents() out.Info.Capabilities.MCPHTTPTools = info.SupportsHTTPMCP() diff --git a/apps/parsar-daemon/internal/cli/claude_sdk_live_linux_test.go b/apps/parsar-daemon/internal/cli/claude_sdk_live_linux_test.go index 94b90bc67..175f139d9 100644 --- a/apps/parsar-daemon/internal/cli/claude_sdk_live_linux_test.go +++ b/apps/parsar-daemon/internal/cli/claude_sdk_live_linux_test.go @@ -89,7 +89,7 @@ func TestLiveRegisteredClaudeSDK(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 120*time.Second) defer cancel() id := uuid.NewString() - request := proto.PromptRequestPayload{RunID: id, AgentKind: "claude_sdk", Prompt: prompt, AgentStateKey: "registered-acceptance", AgentSessionID: resume, StrictResume: true, ReleaseOnCompletion: true, ObserveMessages: true, ObserveToolObservations: true, DisableExecutionEnvironment: true, DisableSubagents: true, ExecutionControls: &proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "medium"}, AgentOptions: map[string]any{"model": "MiniMax-M3", "system_prompt": nil}} + request := proto.PromptRequestPayload{RunID: id, AgentKind: "claude_sdk", Input: proto.TextInput(prompt), AgentStateKey: "registered-acceptance", AgentSessionID: resume, StrictResume: true, ReleaseOnCompletion: true, ObserveMessages: true, ObserveToolObservations: true, DisableExecutionEnvironment: true, DisableSubagents: true, ExecutionControls: &proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "medium"}, AgentOptions: map[string]any{"model": "MiniMax-M3", "system_prompt": nil}} if callFunction { request.FunctionTools = []proto.FunctionTool{{Name: "lookup", Description: "Return a verification value.", Parameters: json.RawMessage(`{"type":"object","properties":{"id":{"type":"string"}},"required":["id"],"additionalProperties":false}`)}} } @@ -133,7 +133,7 @@ func TestLiveRegisteredClaudeSDK(t *testing.T) { if !callFunction || proof.FunctionCalls != 1 || call.Name != "lookup" { t.Fatal("unexpected registered function call") } - handle(proto.TypeFunctionResult, proto.FunctionResultPayload{CallID: call.CallID, DeliveryID: "result", Success: true, Content: []proto.FunctionResultContent{{Type: "input_text", Text: &nonce}}}) + handle(proto.TypeFunctionResult, proto.FunctionResultPayload{CallID: call.CallID, DeliveryID: "result", Success: true, Content: []proto.InputContent{{Type: "input_text", Text: &nonce}}}) case proto.TypeInteractionDecisionAck: var ack proto.InteractionDecisionAckPayload if err := event.DecodePayload(&ack); err != nil { diff --git a/apps/parsar-daemon/internal/cli/claude_sdk_test.go b/apps/parsar-daemon/internal/cli/claude_sdk_test.go index a73da6a67..030d6c80f 100644 --- a/apps/parsar-daemon/internal/cli/claude_sdk_test.go +++ b/apps/parsar-daemon/internal/cli/claude_sdk_test.go @@ -94,7 +94,7 @@ func TestClaudeSDKDiscoveryAndRegistration(t *testing.T) { t.Fatalf("incorrect SDK capability scope: %+v", caps) } // Even a ready SDK must not acquire product write access through the wrapper. - _, err = factory(t.Context(), proto.PromptRequestPayload{RunID: "sdk", Prompt: "hello", WorkspaceAuthoring: true}, make(chan proto.Envelope, 1)) + _, err = factory(t.Context(), proto.PromptRequestPayload{RunID: "sdk", Input: proto.TextInput("hello"), WorkspaceAuthoring: true}, make(chan proto.Envelope, 1)) if err == nil || (!tc.ready && !strings.Contains(err.Error(), "runtime is unavailable")) { t.Fatalf("SDK request did not fail closed: %v", err) } diff --git a/apps/parsar-daemon/internal/dispatch/functions_native_test.go b/apps/parsar-daemon/internal/dispatch/functions_native_test.go index d391a3194..b98e7858c 100644 --- a/apps/parsar-daemon/internal/dispatch/functions_native_test.go +++ b/apps/parsar-daemon/internal/dispatch/functions_native_test.go @@ -75,7 +75,7 @@ func TestNativeFunctionBridge(t *testing.T) { continue } found = true - var parts []proto.FunctionResultContent + var parts []proto.InputContent if err := json.Unmarshal(entry.Output, &parts); err != nil { t.Error(err) continue @@ -132,7 +132,7 @@ func TestNativeFunctionBridge(t *testing.T) { nativeID := "" for index := 0; index < 3; index++ { run := fmt.Sprintf("run-%d", index) - request := proto.PromptRequestPayload{AgentKind: "codex", Prompt: "Look up ticket 42.", RunID: run, AgentStateKey: "native-functions", AgentSessionID: nativeID, StrictResume: true, ReleaseOnCompletion: true, DisableExecutionEnvironment: true, ObserveTools: true, + request := proto.PromptRequestPayload{AgentKind: "codex", Input: proto.TextInput("Look up ticket 42."), RunID: run, AgentStateKey: "native-functions", AgentSessionID: nativeID, StrictResume: true, ReleaseOnCompletion: true, DisableExecutionEnvironment: true, ObserveTools: true, FunctionTools: []proto.FunctionTool{{Name: "lookup_ticket", Description: "Read a synthetic ticket", Parameters: json.RawMessage(`{"type":"object","properties":{"ticket":{"type":"string"}},"required":["ticket"],"additionalProperties":false}`)}}, AgentOptions: map[string]any{"model": "gpt-5.5", "codex_provider": map[string]any{"base_url": model.URL + "/v1", "bearer_token": "synthetic-local-token"}}} env, _ := proto.NewEnvelope(proto.TypePromptRequest, run, request) diff --git a/apps/parsar-daemon/internal/dispatch/functions_test.go b/apps/parsar-daemon/internal/dispatch/functions_test.go index ef840965a..e8af2f966 100644 --- a/apps/parsar-daemon/internal/dispatch/functions_test.go +++ b/apps/parsar-daemon/internal/dispatch/functions_test.go @@ -46,7 +46,7 @@ func TestFunctionReceiptsScopeRetriesAndConflicts(t *testing.T) { } defer router.Shutdown(context.Background()) for _, id := range []string{"one", "two"} { - env, _ := proto.NewEnvelope(proto.TypePromptRequest, id, proto.PromptRequestPayload{AgentKind: "function-test", Prompt: "lookup", FunctionTools: []proto.FunctionTool{{Name: "lookup", Parameters: json.RawMessage(`{}`)}}}) + env, _ := proto.NewEnvelope(proto.TypePromptRequest, id, proto.PromptRequestPayload{AgentKind: "function-test", Input: proto.TextInput("lookup"), FunctionTools: []proto.FunctionTool{{Name: "lookup", Parameters: json.RawMessage(`{}`)}}}) if err := router.Handle(t.Context(), env); err != nil { t.Fatal(err) } @@ -72,7 +72,7 @@ func TestFunctionReceiptsScopeRetriesAndConflicts(t *testing.T) { t.Fatal(a) } - invalid, _ := proto.NewEnvelope(proto.TypeFunctionResult, "one", proto.FunctionResultPayload{CallID: "call", DeliveryID: "invalid", Content: []proto.FunctionResultContent{{Type: "input_audio"}}}) + invalid, _ := proto.NewEnvelope(proto.TypeFunctionResult, "one", proto.FunctionResultPayload{CallID: "call", DeliveryID: "invalid", Content: []proto.InputContent{{Type: "input_audio"}}}) if err := router.Handle(t.Context(), invalid); err != nil { t.Fatal(err) } @@ -148,7 +148,7 @@ func TestFunctionToolsRequireAdvertisedSupport(t *testing.T) { } } -func functionResultContent(text string) []proto.FunctionResultContent { +func functionResultContent(text string) []proto.InputContent { picture := image.NewRGBA(image.Rect(0, 0, 1, 1)) picture.Set(0, 0, color.RGBA{R: 255, A: 255}) var encoded bytes.Buffer @@ -157,5 +157,5 @@ func functionResultContent(text string) []proto.FunctionResultContent { } imageURL := "data:image/png;base64," + base64.StdEncoding.EncodeToString(encoded.Bytes()) after := "AFTER-IMAGE" - return []proto.FunctionResultContent{{Type: "input_text", Text: &text}, {Type: "input_image", ImageURL: &imageURL}, {Type: "input_text", Text: &after}} + return []proto.InputContent{{Type: "input_text", Text: &text}, {Type: "input_image", ImageURL: &imageURL}, {Type: "input_text", Text: &after}} } diff --git a/apps/parsar-daemon/internal/dispatch/local_directory.go b/apps/parsar-daemon/internal/dispatch/local_directory.go index d9abde09b..fe61bdac7 100644 --- a/apps/parsar-daemon/internal/dispatch/local_directory.go +++ b/apps/parsar-daemon/internal/dispatch/local_directory.go @@ -13,7 +13,7 @@ func prepareLocalDirectory(context.Context, proto.PromptRequestPayload) (agent.P return localDirectoryPreparation{}, nil } -func (localDirectoryPreparation) Start(context.Context, string, string, chan<- proto.Envelope) (agent.Session, error) { +func (localDirectoryPreparation) Start(context.Context, string, proto.MessageInput, chan<- proto.Envelope) (agent.Session, error) { return nil, agent.ErrWorkspaceReadUnsupported } diff --git a/apps/parsar-daemon/internal/dispatch/local_directory_test.go b/apps/parsar-daemon/internal/dispatch/local_directory_test.go index dcc118460..dbe992d78 100644 --- a/apps/parsar-daemon/internal/dispatch/local_directory_test.go +++ b/apps/parsar-daemon/internal/dispatch/local_directory_test.go @@ -57,7 +57,7 @@ func TestLocalDirectoryPreparationNeedsNoHarnessAndRejectsOtherOwners(t *testing if got := waitWorkspaceRead(t, sender, "foreign"); got.Outcome != "rejected" { t.Fatal("foreign directory accepted", got) } - if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "idle", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "forbidden", Prompt: "work"})); err == nil { + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "idle", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "forbidden", Input: proto.TextInput("work")})); err == nil { t.Fatal("read preparation admitted execution") } bad := request diff --git a/apps/parsar-daemon/internal/dispatch/preparation.go b/apps/parsar-daemon/internal/dispatch/preparation.go index fa34fd04a..f5d892feb 100644 --- a/apps/parsar-daemon/internal/dispatch/preparation.go +++ b/apps/parsar-daemon/internal/dispatch/preparation.go @@ -55,7 +55,7 @@ func (r *Router) handleExecutionPrepare(ctx context.Context, env proto.Envelope) if req, err = r.localWorkspace.Configure(req); err != nil { return r.rejectPreparation(env, "invalid_configuration") } - if req.RunID != "" || req.Prompt != "" || req.ConversationID != "" || req.WorkspaceAuthoring || len(req.Attachments) != 0 || req.EnvironmentID() == "" || strings.TrimSpace(req.AgentStateKey) == "" || !req.StrictResume || !req.ReleaseOnCompletion { + if req.RunID != "" || len(req.Input) != 0 || req.ConversationID != "" || req.WorkspaceAuthoring || req.EnvironmentID() == "" || strings.TrimSpace(req.AgentStateKey) == "" || !req.StrictResume || !req.ReleaseOnCompletion { return r.rejectPreparation(env, "invalid_configuration") } if validateExecutionEnvironment(req, caps) != nil || (len(req.FunctionTools) > 0 && !caps.FunctionTools) { diff --git a/apps/parsar-daemon/internal/dispatch/preparation_cancel_test.go b/apps/parsar-daemon/internal/dispatch/preparation_cancel_test.go index e9e6f6271..d5fd98ec6 100644 --- a/apps/parsar-daemon/internal/dispatch/preparation_cancel_test.go +++ b/apps/parsar-daemon/internal/dispatch/preparation_cancel_test.go @@ -34,7 +34,7 @@ type nonCancellablePreparation struct { once sync.Once } -func (*nonCancellablePreparation) Start(context.Context, string, string, chan<- proto.Envelope) (agent.Session, error) { +func (*nonCancellablePreparation) Start(context.Context, string, proto.MessageInput, chan<- proto.Envelope) (agent.Session, error) { return nil, errors.New("must not start") } @@ -49,7 +49,7 @@ func startCancellationPreparation(t *testing.T, r *dispatch.Router, sender *recS t.Fatal(err) } ready := waitPreparationStatus(t, sender, "request", "ready", "") - if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "run", Prompt: "input"})); err != nil { + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "run", Input: proto.TextInput("input")})); err != nil { t.Fatal(err) } return ready @@ -97,7 +97,7 @@ func TestPreparedCancellationWaitsForOutputAndCleanup(t *testing.T) { Content: "observed", Usage: proto.Usage{Tokens: &proto.TokenUsage{InputTokens: 7, OutputTokens: 3, TotalTokens: 10}}, Metadata: map[string]any{proto.DoneMetaAgentSessionID: "observed-native"}, }} var session *fakeSession - p.start = func(_ context.Context, id, _ string, out chan<- proto.Envelope) (agent.Session, error) { + p.start = func(_ context.Context, id string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { session = &fakeSession{out: out, closeOutOnCancel: true, postCancelEnvelopes: []proto.Envelope{mustEnv(t, proto.TypeDone, id, p.outcome)}} out <- mustEnv(t, proto.TypeDelta, id, proto.DeltaPayload{Delta: "observed"}) @@ -177,7 +177,7 @@ func TestPreparedCancellationBeforeTransferPreservesUnknownOutcome(t *testing.T) cancelOnce.Do(func() { close(cancelled) }) return p.Close() } - p.start = func(_ context.Context, _, _ string, _ chan<- proto.Envelope) (agent.Session, error) { + p.start = func(_ context.Context, _ string, _ proto.MessageInput, _ chan<- proto.Envelope) (agent.Session, error) { close(entered) <-cancelled <-allowReturn @@ -231,7 +231,7 @@ func TestPreparedCancellationFailuresRemainConservative(t *testing.T) { entered, allowReturn := make(chan struct{}), make(chan struct{}) var session *fakeSession p := &cancellationPreparation{controlledPreparation: &controlledPreparation{closed: make(chan struct{})}} - p.start = func(_ context.Context, id, _ string, out chan<- proto.Envelope) (agent.Session, error) { + p.start = func(_ context.Context, id string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { session = &fakeSession{out: out, closeOutOnCancel: true} out <- mustEnv(t, proto.TypeUsage, id, proto.UsagePayload{Usage: proto.Usage{InputTokens: 3}}) close(entered) @@ -296,7 +296,7 @@ func TestPreparedCancellationTimeoutKeepsCapacityUntilStartReturns(t *testing.T) entered, allowReturn := make(chan struct{}), make(chan struct{}) p := &cancellationPreparation{controlledPreparation: &controlledPreparation{closed: make(chan struct{})}} p.cancel = func(context.Context) error { return p.Close() } - p.start = func(ctx context.Context, _, _ string, _ chan<- proto.Envelope) (agent.Session, error) { + p.start = func(ctx context.Context, _ string, _ proto.MessageInput, _ chan<- proto.Envelope) (agent.Session, error) { close(entered) <-allowReturn return nil, ctx.Err() diff --git a/apps/parsar-daemon/internal/dispatch/preparation_cleanup_test.go b/apps/parsar-daemon/internal/dispatch/preparation_cleanup_test.go index 515bbc197..7c08293d6 100644 --- a/apps/parsar-daemon/internal/dispatch/preparation_cleanup_test.go +++ b/apps/parsar-daemon/internal/dispatch/preparation_cleanup_test.go @@ -37,7 +37,7 @@ func TestPreparedCancellationDoesNotAcknowledgeFailedCleanup(t *testing.T) { close(cancelled) return nil }}} - p.start = func(_ context.Context, _, _ string, _ chan<- proto.Envelope) (agent.Session, error) { + p.start = func(_ context.Context, _ string, _ proto.MessageInput, _ chan<- proto.Envelope) (agent.Session, error) { close(entered) <-cancelled return nil, context.Canceled @@ -64,7 +64,7 @@ func TestShutdownRetriesFailedPreparedCancellationOnSameTarget(t *testing.T) { sender := &recSender{} var session *fakeSession p := &cancellationPreparation{controlledPreparation: &controlledPreparation{closed: make(chan struct{})}} - p.start = func(_ context.Context, _, _ string, out chan<- proto.Envelope) (agent.Session, error) { + p.start = func(_ context.Context, _ string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { session = &fakeSession{out: out, closeOutOnCancel: true} return session, nil } @@ -138,7 +138,7 @@ func TestPreparationCloseFailureRetainsCapacityAndRetries(t *testing.T) { if err := r.Handle(t.Context(), replacement); err == nil { t.Fatal("failed cleanup released capacity") } - start := proto.ExecutionStartPayload{Handle: handle, RunID: "run", Prompt: "do not execute"} + start := proto.ExecutionStartPayload{Handle: handle, RunID: "run", Input: proto.TextInput("do not execute")} if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "0", start)); err == nil { t.Fatal("failed cleanup allowed Start") } @@ -234,7 +234,7 @@ func TestPublishedPreparedRunRetainsRetryAfterHandleRetirement(t *testing.T) { sender := &recSender{} session := &fakeSession{closeOutOnCancel: true} p := &cancellationPreparation{controlledPreparation: &controlledPreparation{closed: make(chan struct{})}} - p.start = func(_ context.Context, _, _ string, out chan<- proto.Envelope) (agent.Session, error) { + p.start = func(_ context.Context, _ string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { session.out = out return session, nil } diff --git a/apps/parsar-daemon/internal/dispatch/preparation_start.go b/apps/parsar-daemon/internal/dispatch/preparation_start.go index 95c05204f..0607d407f 100644 --- a/apps/parsar-daemon/internal/dispatch/preparation_start.go +++ b/apps/parsar-daemon/internal/dispatch/preparation_start.go @@ -14,7 +14,7 @@ import ( func (r *Router) handleExecutionStart(_ context.Context, env proto.Envelope) error { var input proto.ExecutionStartPayload - if env.DecodePayload(&input) != nil || input.Handle == "" || strings.TrimSpace(input.RunID) == "" || strings.TrimSpace(input.Prompt) == "" { + if env.DecodePayload(&input) != nil || input.Handle == "" || strings.TrimSpace(input.RunID) == "" || input.Input.Validate() != nil { return r.rejectPreparation(env, "invalid_start") } encoded, _ := json.Marshal(input) @@ -119,7 +119,7 @@ func (r *Router) startPreparedExecution(p *preparationState, state *sessionState if blocked { startErr = context.Canceled } else { - session, startErr = handoff.target.Start(p.ctx, input.RunID, input.Prompt, state.out) + session, startErr = handoff.target.Start(p.ctx, input.RunID, input.Input, state.out) } r.mu.Lock() diff --git a/apps/parsar-daemon/internal/dispatch/preparation_test.go b/apps/parsar-daemon/internal/dispatch/preparation_test.go index 3d02fdfb6..a9cd0e864 100644 --- a/apps/parsar-daemon/internal/dispatch/preparation_test.go +++ b/apps/parsar-daemon/internal/dispatch/preparation_test.go @@ -23,7 +23,7 @@ type controlledPreparation struct { mu sync.Mutex session agent.Session starts atomic.Int32 - start func(context.Context, string, string, chan<- proto.Envelope) (agent.Session, error) + start func(context.Context, string, proto.MessageInput, chan<- proto.Envelope) (agent.Session, error) closeHook func() } @@ -36,7 +36,7 @@ func (p *controlledPreparation) Close() error { }) return nil } -func (p *controlledPreparation) Start(ctx context.Context, id, prompt string, out chan<- proto.Envelope) (agent.Session, error) { +func (p *controlledPreparation) Start(ctx context.Context, id string, prompt proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { p.starts.Add(1) session, err := p.start(ctx, id, prompt, out) if session != nil { @@ -165,7 +165,7 @@ func TestPreparationReleaseDuringBlockedFactory(t *testing.T) { p := &controlledPreparation{closed: make(chan struct{})} entered, allowReturn := make(chan context.Context, 1), make(chan struct{}) r := preparationRouter(t, sender, time.Minute, func(ctx context.Context, req proto.PromptRequestPayload) (agent.Prepared, error) { - if req.RunID != "" || req.Prompt != "" { + if req.RunID != "" || len(req.Input) != 0 { t.Error("run input reached preparation") } entered <- ctx @@ -208,8 +208,8 @@ func TestPreparationSingleTransferAndReleaseDoesNotCancelRun(t *testing.T) { sender := &recSender{} gotSession := make(chan *fakeSession, 1) p := &controlledPreparation{closed: make(chan struct{})} - p.start = func(ctx context.Context, id, prompt string, out chan<- proto.Envelope) (agent.Session, error) { - if id != "real-run" || prompt != "actual input" { + p.start = func(ctx context.Context, id string, prompt proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { + if id != "real-run" || *prompt[0].Content[0].Text != "actual input" { t.Error("start identity or prompt changed") } s := &fakeSession{ctx: ctx, out: out, closeOutOnCancel: true} @@ -225,7 +225,7 @@ func TestPreparationSingleTransferAndReleaseDoesNotCancelRun(t *testing.T) { if err := r.Handle(t.Context(), prepare); err != nil { t.Fatal(err) } - start := mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "real-run", Prompt: "actual input"}) + start := mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "real-run", Input: proto.TextInput("actual input")}) if err := r.Handle(t.Context(), start); err != nil { t.Fatal(err) } @@ -262,7 +262,7 @@ func TestPreparationCancelDuringStartClosesLateSession(t *testing.T) { entered, cancelEntered, allowReturn := make(chan struct{}), make(chan struct{}), make(chan struct{}) lateSession := make(chan *fakeSession, 1) p := &cancellationPreparation{controlledPreparation: &controlledPreparation{closed: make(chan struct{})}} - p.start = func(_ context.Context, _, _ string, out chan<- proto.Envelope) (agent.Session, error) { + p.start = func(_ context.Context, _ string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { close(entered) <-allowReturn s := &fakeSession{out: out, closeOutOnCancel: true} @@ -276,7 +276,7 @@ func TestPreparationCancelDuringStartClosesLateSession(t *testing.T) { r := preparationRouter(t, sender, time.Minute, func(context.Context, proto.PromptRequestPayload) (agent.Prepared, error) { return p, nil }) _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, "request", preparationRequest())) ready := waitPreparationStatus(t, sender, "request", "ready", "") - _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "real-run", Prompt: "input"})) + _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "real-run", Input: proto.TextInput("input")})) <-entered if err := r.Handle(t.Context(), mustEnv(t, proto.TypePromptCancel, "real-run", proto.PromptCancelPayload{DeliveryID: "cancel"})); err != nil { t.Fatal(err) @@ -333,7 +333,7 @@ func TestPreparationCapacityAndConnectionOwnership(t *testing.T) { t.Error("unexpected new native resource") return nil, errors.New("unexpected") }) - if err := other.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "0", proto.ExecutionStartPayload{Handle: handles[0], RunID: "run", Prompt: "input"})); err == nil { + if err := other.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "0", proto.ExecutionStartPayload{Handle: handles[0], RunID: "run", Input: proto.TextInput("input")})); err == nil { t.Fatal("another connection consumed handle") } } @@ -356,7 +356,7 @@ func TestPreparationExpiryAndOldHandleCannotStartReplacement(t *testing.T) { if next.Handle == old.Handle || next.ExpiresAt <= old.ExpiresAt { t.Fatal("replacement reused expired identity") } - if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: old.Handle, RunID: "late", Prompt: "late"})); err == nil { + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: old.Handle, RunID: "late", Input: proto.TextInput("late")})); err == nil { t.Fatal("old handle started replacement") } } @@ -428,11 +428,13 @@ func TestPreparationCapacityIncludesClosingResources(t *testing.T) { func TestPreparationRejectsInputAndProductConfiguration(t *testing.T) { for name, change := range map[string]func(*proto.PromptRequestPayload){ - "run": func(p *proto.PromptRequestPayload) { p.RunID = "run" }, - "input": func(p *proto.PromptRequestPayload) { p.Prompt = "input" }, - "conversation": func(p *proto.PromptRequestPayload) { p.ConversationID = "product" }, - "authoring": func(p *proto.PromptRequestPayload) { p.WorkspaceAuthoring = true }, - "attachment": func(p *proto.PromptRequestPayload) { p.Attachments = []proto.PromptAttachment{{Kind: "image"}} }, + "run": func(p *proto.PromptRequestPayload) { p.RunID = "run" }, + "input": func(p *proto.PromptRequestPayload) { p.Input = proto.TextInput("input") }, + "conversation": func(p *proto.PromptRequestPayload) { p.ConversationID = "product" }, + "authoring": func(p *proto.PromptRequestPayload) { p.WorkspaceAuthoring = true }, + "attachment": func(p *proto.PromptRequestPayload) { + p.Input = proto.MessageInput{{Content: []proto.InputContent{{Type: "input_image"}}}} + }, "missing environment": func(p *proto.PromptRequestPayload) { p.LocalEnvironment = nil }, "resume": func(p *proto.PromptRequestPayload) { p.StrictResume = false }, "release": func(p *proto.PromptRequestPayload) { p.ReleaseOnCompletion = false }, diff --git a/apps/parsar-daemon/internal/dispatch/prepared_handoff_mutation_test.go b/apps/parsar-daemon/internal/dispatch/prepared_handoff_mutation_test.go index 4529c3d95..fa0717b4f 100644 --- a/apps/parsar-daemon/internal/dispatch/prepared_handoff_mutation_test.go +++ b/apps/parsar-daemon/internal/dispatch/prepared_handoff_mutation_test.go @@ -89,7 +89,7 @@ func TestPreparedHandoffReleaseWaitsForMutationReceipt(t *testing.T) { } session := &preparedMutationSession{fakeSession: &fakeSession{closeOutOnCancel: true}, cancelEntered: make(chan struct{})} p := &controlledPreparation{closed: make(chan struct{})} - p.start = func(_ context.Context, _, _ string, out chan<- proto.Envelope) (agent.Session, error) { + p.start = func(_ context.Context, _ string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { session.out = out return session, nil } @@ -110,7 +110,7 @@ func TestPreparedHandoffReleaseWaitsForMutationReceipt(t *testing.T) { waitFor(t, func() bool { return hasFrame(sender.recSender, proto.TypePromptForUserChoice, "run") }, "choice request") mutation = mustEnv(t, proto.TypePromptForUserChoiceDecision, "ask", proto.PromptForUserChoiceDecisionPayload{DeliveryID: sender.deliveryID, Answers: []string{"yes"}}) case "steering": - mutation = mustEnv(t, proto.TypePromptSteer, "run", proto.PromptSteerPayload{InputID: sender.inputID, Text: "continue"}) + mutation = mustEnv(t, proto.TypePromptSteer, "run", proto.PromptSteerPayload{InputID: sender.inputID, Input: proto.TextInput("continue")}) } returned := make(chan error, 1) @@ -152,7 +152,7 @@ func TestPreparedHandoffReleaseWaitsForMutationReceipt(t *testing.T) { } assertDecisionAck(t, sender.recSender, "new-choice", false, "not_pending") case "steering": - late := mustEnv(t, proto.TypePromptSteer, "run", proto.PromptSteerPayload{InputID: "late-steering", Text: "late"}) + late := mustEnv(t, proto.TypePromptSteer, "run", proto.PromptSteerPayload{InputID: "late-steering", Input: proto.TextInput("late")}) if err := r.Handle(t.Context(), late); err != nil { t.Fatal(err) } @@ -242,7 +242,7 @@ func TestPreparedHandoffRouterShutdownWaitsForReceiptAttempt(t *testing.T) { var cancelBeforeReceipt atomic.Bool session := &preparedMutationSession{fakeSession: &fakeSession{closeOutOnCancel: true}, cancelEntered: make(chan struct{})} p := &controlledPreparation{closed: make(chan struct{})} - p.start = func(_ context.Context, _, _ string, out chan<- proto.Envelope) (agent.Session, error) { + p.start = func(_ context.Context, _ string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { session.out = out return session, nil } @@ -270,7 +270,7 @@ func TestPreparedHandoffRouterShutdownWaitsForReceiptAttempt(t *testing.T) { waitFor(t, func() bool { return hasFrame(sender.recSender, proto.TypePromptForUserChoice, "run") }, "choice request") mutation = mustEnv(t, proto.TypePromptForUserChoiceDecision, "ask", proto.PromptForUserChoiceDecisionPayload{DeliveryID: sender.deliveryID, Answers: []string{"yes"}}) case "steering": - mutation = mustEnv(t, proto.TypePromptSteer, "run", proto.PromptSteerPayload{InputID: sender.inputID, Text: "continue"}) + mutation = mustEnv(t, proto.TypePromptSteer, "run", proto.PromptSteerPayload{InputID: sender.inputID, Input: proto.TextInput("continue")}) } go func() { _ = r.Handle(t.Context(), mutation) }() select { @@ -295,7 +295,7 @@ func TestPreparedHandoffEarlyDonePublishesAfterStarted(t *testing.T) { emitted, allowReturn := make(chan struct{}), make(chan struct{}) session := &fakeSession{closeOutOnCancel: true} p := &controlledPreparation{closed: make(chan struct{})} - p.start = func(ctx context.Context, _, _ string, out chan<- proto.Envelope) (agent.Session, error) { + p.start = func(ctx context.Context, _ string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { session.out = out out <- mustEnv(t, proto.TypeDone, "run", proto.DonePayload{Content: "complete"}) close(emitted) diff --git a/apps/parsar-daemon/internal/dispatch/prepared_handoff_test.go b/apps/parsar-daemon/internal/dispatch/prepared_handoff_test.go index d7c135190..d12c8246e 100644 --- a/apps/parsar-daemon/internal/dispatch/prepared_handoff_test.go +++ b/apps/parsar-daemon/internal/dispatch/prepared_handoff_test.go @@ -19,7 +19,7 @@ func TestPreparedHandoffDrainsBurstBeforeStartReturns(t *testing.T) { sent, allowReturn := make(chan struct{}), make(chan struct{}) session := &fakeSession{closeOutOnCancel: true} p := &controlledPreparation{closed: make(chan struct{})} - p.start = func(ctx context.Context, _, _ string, out chan<- proto.Envelope) (agent.Session, error) { + p.start = func(ctx context.Context, _ string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { session.out = out for sequence := uint64(1); sequence <= preparedBurstFrames; sequence++ { select { @@ -113,7 +113,7 @@ func (s *blockingStartingSender) Send(ctx context.Context, env proto.Envelope) e func TestPreparedHandoffAbortBeforeStartAdmissionSkipsNativeStart(t *testing.T) { sender := &blockingStartingSender{recSender: &recSender{}, entered: make(chan struct{}), release: make(chan struct{})} p := &cancellationPreparation{controlledPreparation: &controlledPreparation{closed: make(chan struct{})}} - p.start = func(context.Context, string, string, chan<- proto.Envelope) (agent.Session, error) { + p.start = func(context.Context, string, proto.MessageInput, chan<- proto.Envelope) (agent.Session, error) { t.Fatal("abort that won admission called native Start") return nil, errors.New("unexpected Start") } @@ -159,7 +159,7 @@ func TestPreparedHandoffDuplicateStartDoesNotReexecuteDuringPublication(t *testi sender := &blockingStartedSender{recSender: &recSender{}, entered: make(chan struct{}), release: make(chan struct{})} session := &preparedMutationSession{fakeSession: &fakeSession{closeOutOnCancel: true}, cancelEntered: make(chan struct{})} p := &controlledPreparation{closed: make(chan struct{})} - p.start = func(_ context.Context, _, _ string, out chan<- proto.Envelope) (agent.Session, error) { + p.start = func(_ context.Context, _ string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { session.out = out out <- mustEnv(t, proto.TypePermissionRequest, "run", proto.PermissionRequestPayload{RequestID: "publication-permission"}) out <- mustEnv(t, proto.TypePromptForUserChoice, "run", proto.PromptForUserChoicePayload{AskID: "publication-choice"}) @@ -187,7 +187,7 @@ func TestPreparedHandoffDuplicateStartDoesNotReexecuteDuringPublication(t *testi assertDecisionAck(t, sender.recSender, "publication-function", false, "not_ready") assertDecisionAck(t, sender.recSender, "publication-permission", false, "not_ready") assertDecisionAck(t, sender.recSender, "publication-choice", false, "not_ready") - if err := r.Handle(t.Context(), mustEnv(t, proto.TypePromptSteer, "run", proto.PromptSteerPayload{InputID: "publication-steering", Text: "continue"})); err != nil { + if err := r.Handle(t.Context(), mustEnv(t, proto.TypePromptSteer, "run", proto.PromptSteerPayload{InputID: "publication-steering", Input: proto.TextInput("continue")})); err != nil { t.Fatal(err) } if ack := lastSteeringAck(t, sender.recSender, "run", "publication-steering"); ack.ErrorCode != "not_ready" { @@ -208,7 +208,7 @@ func TestPreparedHandoffDuplicateStartDoesNotReexecuteDuringPublication(t *testi if session.functions.Load() != 0 || session.steers.Load() != 0 || session.reads.Load() != 0 || len(session.submissions()) != 0 || askCalls != 0 { t.Fatal("private Session accepted work before started publication") } - duplicate := mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "run", Prompt: "input"}) + duplicate := mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "run", Input: proto.TextInput("input")}) if err := r.Handle(t.Context(), duplicate); err != nil { t.Fatal(err) } @@ -225,7 +225,7 @@ func TestPreparedHandoffUnsupportedFunctionReleasesOperationBarrier(t *testing.T sender := &recSender{} session := &fakeSession{closeOutOnCancel: true} p := &controlledPreparation{closed: make(chan struct{})} - p.start = func(_ context.Context, _, _ string, out chan<- proto.Envelope) (agent.Session, error) { + p.start = func(_ context.Context, _ string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { session.out = out return session, nil } @@ -250,7 +250,7 @@ func TestPreparedHandoffEarlyDoneStillAllowsExplicitAbort(t *testing.T) { var once sync.Once unblock := func() { once.Do(func() { close(cancelled) }) } p := &cancellationPreparation{controlledPreparation: &controlledPreparation{closed: make(chan struct{})}} - p.start = func(_ context.Context, _, _ string, out chan<- proto.Envelope) (agent.Session, error) { + p.start = func(_ context.Context, _ string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { out <- mustEnv(t, proto.TypeDone, "run", proto.DonePayload{}) <-cancelled return nil, context.Canceled @@ -276,7 +276,7 @@ func TestPreparedHandoffExpiresDuringStartedPublication(t *testing.T) { sender := &blockingStartedSender{recSender: &recSender{}, entered: make(chan struct{}), release: make(chan struct{})} session := &fakeSession{closeOutOnCancel: true} p := &controlledPreparation{closed: make(chan struct{})} - p.start = func(_ context.Context, _, _ string, out chan<- proto.Envelope) (agent.Session, error) { + p.start = func(_ context.Context, _ string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { session.out = out return session, nil } @@ -308,7 +308,7 @@ func TestPreparedHandoffEarlyDoneDetachesPublishedPreparation(t *testing.T) { defer unblock() session := &fakeSession{closeOutOnCancel: true} p := &cancellationPreparation{controlledPreparation: &controlledPreparation{closed: make(chan struct{})}} - p.start = func(_ context.Context, _, _ string, out chan<- proto.Envelope) (agent.Session, error) { + p.start = func(_ context.Context, _ string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { session.out = out out <- mustEnv(t, proto.TypeDone, "run", proto.DonePayload{}) <-allowReturn diff --git a/apps/parsar-daemon/internal/dispatch/prompt.go b/apps/parsar-daemon/internal/dispatch/prompt.go index 58e2d4dcd..f92d972ca 100644 --- a/apps/parsar-daemon/internal/dispatch/prompt.go +++ b/apps/parsar-daemon/internal/dispatch/prompt.go @@ -51,7 +51,7 @@ func (r *Router) handlePromptRequest(callerCtx context.Context, env proto.Envelo } r.log.InfoContext(callerCtx, "handlePromptRequest: decoded", "run_id", runID, "agent_kind", req.AgentKind, - "work_dir", req.WorkDir, "prompt_len", len(req.Prompt), + "work_dir", req.WorkDir, "message_count", len(req.Input), "has_agent_options", req.AgentOptions != nil, "agent_session_id", req.AgentSessionID, "agent_state_key", req.AgentStateKey) diff --git a/apps/parsar-daemon/internal/dispatch/receipt_order_test.go b/apps/parsar-daemon/internal/dispatch/receipt_order_test.go index 8c0801d13..58b086186 100644 --- a/apps/parsar-daemon/internal/dispatch/receipt_order_test.go +++ b/apps/parsar-daemon/internal/dispatch/receipt_order_test.go @@ -73,7 +73,7 @@ func TestDurableCompletionWaitsForSteeringReceiptSend(t *testing.T) { } } handle(proto.TypePromptRequest, proto.PromptRequestPayload{AgentKind: "codex", AgentStateKey: "stable", ReleaseOnCompletion: true}) - input := proto.PromptSteerPayload{InputID: "input-1", Text: "original"} + input := proto.PromptSteerPayload{InputID: "input-1", Input: proto.TextInput("original")} handle(proto.TypePromptSteer, input) <-sender.entered session.out <- mustEnv(t, proto.TypeDone, "ordered", proto.DonePayload{Content: "finished"}) @@ -91,7 +91,7 @@ func TestDurableCompletionWaitsForSteeringReceiptSend(t *testing.T) { if mode == "unknown" && ack.ErrorCode != "outcome_unknown" { t.Fatal("uncertainty lost") } - input.Text = "changed" + input.Input = proto.TextInput("changed") handle(proto.TypePromptSteer, input) input.InputID = "new" handle(proto.TypePromptSteer, input) @@ -142,7 +142,7 @@ func TestShutdownReleasesSteeringWorkerAndCompletionBarrier(t *testing.T) { if err = router.Handle(context.Background(), mustEnv(t, proto.TypePromptRequest, "shutdown", proto.PromptRequestPayload{AgentKind: "codex", ReleaseOnCompletion: true})); err != nil { t.Fatal(err) } - if err = router.Handle(context.Background(), mustEnv(t, proto.TypePromptSteer, "shutdown", proto.PromptSteerPayload{InputID: "one", Text: "text"})); err != nil { + if err = router.Handle(context.Background(), mustEnv(t, proto.TypePromptSteer, "shutdown", proto.PromptSteerPayload{InputID: "one", Input: proto.TextInput("text")})); err != nil { t.Fatal(err) } <-entered diff --git a/apps/parsar-daemon/internal/dispatch/receipt_shutdown_test.go b/apps/parsar-daemon/internal/dispatch/receipt_shutdown_test.go index 9379b6954..81d9a847e 100644 --- a/apps/parsar-daemon/internal/dispatch/receipt_shutdown_test.go +++ b/apps/parsar-daemon/internal/dispatch/receipt_shutdown_test.go @@ -55,7 +55,7 @@ func TestShutdownCancelsCompletionErrorSend(t *testing.T) { if err = router.Handle(context.Background(), mustEnv(t, proto.TypePromptRequest, "shutdown-terminal", proto.PromptRequestPayload{AgentKind: "codex", ReleaseOnCompletion: true})); err != nil { t.Fatal(err) } - if err = router.Handle(context.Background(), mustEnv(t, proto.TypePromptSteer, "shutdown-terminal", proto.PromptSteerPayload{InputID: "one", Text: "text"})); err != nil { + if err = router.Handle(context.Background(), mustEnv(t, proto.TypePromptSteer, "shutdown-terminal", proto.PromptSteerPayload{InputID: "one", Input: proto.TextInput("text")})); err != nil { t.Fatal(err) } <-sender.entered diff --git a/apps/parsar-daemon/internal/dispatch/router_test.go b/apps/parsar-daemon/internal/dispatch/router_test.go index 0dd44ef35..028d6dbc2 100644 --- a/apps/parsar-daemon/internal/dispatch/router_test.go +++ b/apps/parsar-daemon/internal/dispatch/router_test.go @@ -237,14 +237,14 @@ func TestHandlePromptRequestInvokesFactoryAndForwardsOutput(t *testing.T) { defer h.router.Shutdown(context.Background()) env := mustEnv(t, proto.TypePromptRequest, "run_1", proto.PromptRequestPayload{ - AgentKind: "claude_code", Prompt: "hi", ConversationID: "c1", + AgentKind: "claude_code", Input: proto.TextInput("hi"), ConversationID: "c1", }) if err := h.router.Handle(context.Background(), env); err != nil { t.Fatalf("Handle prompt_request: %v", err) } req := <-h.gotReq - if req.RunID != "run_1" || req.AgentKind != "claude_code" || req.Prompt != "hi" { + if req.RunID != "run_1" || req.AgentKind != "claude_code" || *req.Input[0].Content[0].Text != "hi" { t.Errorf("factory got %+v, want run_1/claude_code/hi", req) } sess := <-h.gotSess diff --git a/apps/parsar-daemon/internal/dispatch/steering.go b/apps/parsar-daemon/internal/dispatch/steering.go index 2b5667e07..00265cb91 100644 --- a/apps/parsar-daemon/internal/dispatch/steering.go +++ b/apps/parsar-daemon/internal/dispatch/steering.go @@ -3,6 +3,7 @@ package dispatch import ( "context" "crypto/sha256" + "encoding/json" "errors" "strings" "time" @@ -32,7 +33,7 @@ func (r *Router) handlePromptSteer(ctx context.Context, env proto.Envelope) erro ack.ErrorCode, ack.Error = "invalid_input", "Invalid steering payload." } else { ack.InputID = input.InputID - if env.ID == "" || strings.TrimSpace(input.InputID) == "" || len(input.InputID) > 256 || strings.TrimSpace(input.Text) == "" { + if env.ID == "" || strings.TrimSpace(input.InputID) == "" || len(input.InputID) > 256 || input.Input.Validate() != nil { ack.ErrorCode, ack.Error = "invalid_input", "Run ID, input ID (up to 256 bytes), and non-empty text are required." } else { pending := r.queueSteering(ctx, env, input) @@ -62,7 +63,8 @@ func (r *Router) queueSteering(ctx context.Context, env proto.Envelope, input pr ack.ErrorCode, ack.Error = "run_inactive", "The run is no longer active." return &ack } - fingerprint := sha256.Sum256([]byte(input.Text)) + encoded, _ := json.Marshal(input.Input) + fingerprint := sha256.Sum256(encoded) if previous, ok := state.steering[input.InputID]; ok { if previous.fingerprint != fingerprint || previous.durable != input.DurableReceipt { ack.ErrorCode, ack.Error = "input_conflict", "This input ID was already used with different text." diff --git a/apps/parsar-daemon/internal/dispatch/steering_lifetime_test.go b/apps/parsar-daemon/internal/dispatch/steering_lifetime_test.go index 24efe1841..55cb239ad 100644 --- a/apps/parsar-daemon/internal/dispatch/steering_lifetime_test.go +++ b/apps/parsar-daemon/internal/dispatch/steering_lifetime_test.go @@ -42,7 +42,7 @@ func TestDurableSteeringWaitsBeyondTransportDeadline(t *testing.T) { if err := h.router.Handle(ctx, mustEnv(t, proto.TypePromptRequest, "durable", proto.PromptRequestPayload{AgentKind: "codex", ReleaseOnCompletion: true})); err != nil { t.Fatal(err) } - input := proto.PromptSteerPayload{InputID: "extra", Text: "additional", DurableReceipt: true} + input := proto.PromptSteerPayload{InputID: "extra", Input: proto.TextInput("additional"), DurableReceipt: true} env := mustEnv(t, proto.TypePromptSteer, "durable", input) if err := handleSteeringAndWait(t, h, env); err != nil { t.Fatal(err) @@ -93,7 +93,7 @@ func TestDurableSteeringTransportTimeoutAndShutdown(t *testing.T) { if err := h.router.Handle(ctx, mustEnv(t, proto.TypePromptRequest, "run", proto.PromptRequestPayload{AgentKind: "codex", ReleaseOnCompletion: true})); err != nil { t.Fatal(err) } - if err := h.router.Handle(ctx, mustEnv(t, proto.TypePromptSteer, "run", proto.PromptSteerPayload{InputID: "one", Text: "text", DurableReceipt: true})); err != nil { + if err := h.router.Handle(ctx, mustEnv(t, proto.TypePromptSteer, "run", proto.PromptSteerPayload{InputID: "one", Input: proto.TextInput("text"), DurableReceipt: true})); err != nil { t.Fatal(err) } if phase == "blocked-write" { @@ -142,7 +142,7 @@ func TestDurableSteeringRequiresOptInAndAdapter(t *testing.T) { if err := h.router.Handle(ctx, mustEnv(t, proto.TypePromptRequest, "run", proto.PromptRequestPayload{AgentKind: "codex", ReleaseOnCompletion: !supported})); err != nil { t.Fatal(err) } - if err := handleSteeringAndWait(t, h, mustEnv(t, proto.TypePromptSteer, "run", proto.PromptSteerPayload{InputID: "one", Text: "text", DurableReceipt: true})); err != nil { + if err := handleSteeringAndWait(t, h, mustEnv(t, proto.TypePromptSteer, "run", proto.PromptSteerPayload{InputID: "one", Input: proto.TextInput("text"), DurableReceipt: true})); err != nil { t.Fatal(err) } if ack := lastSteeringAck(t, h.sender, "run", "one"); ack.ErrorCode != "unsupported" { diff --git a/apps/parsar-daemon/internal/dispatch/steering_test.go b/apps/parsar-daemon/internal/dispatch/steering_test.go index d7c01da8d..677a6c298 100644 --- a/apps/parsar-daemon/internal/dispatch/steering_test.go +++ b/apps/parsar-daemon/internal/dispatch/steering_test.go @@ -39,7 +39,7 @@ func TestSteeringReceiptsAndRetries(t *testing.T) { if _, ok := ctx.Deadline(); !ok { t.Error("native request has no deadline") } - if input.InputID != "input-1" || input.Text != "additional text" { + if input.InputID != "input-1" || *input.Input[0].Content[0].Text != "additional text" { t.Errorf("input lost: %+v", input) } if len(h.sender.snapshot()) != 0 { @@ -53,7 +53,7 @@ func TestSteeringReceiptsAndRetries(t *testing.T) { if err := h.router.Handle(ctx, mustEnv(t, proto.TypePromptRequest, "run-1", proto.PromptRequestPayload{AgentKind: "codex"})); err != nil { t.Fatal(err) } - input := proto.PromptSteerPayload{InputID: "input-1", Text: "additional text"} + input := proto.PromptSteerPayload{InputID: "input-1", Input: proto.TextInput("additional text")} env := mustEnv(t, proto.TypePromptSteer, "run-1", input) // An ack transport failure must not cause another native invocation. h.sender.failNow = true @@ -73,7 +73,7 @@ func TestSteeringReceiptsAndRetries(t *testing.T) { t.Fatalf("uncertainty lost: %+v", ack) } } - input.Text = "changed text" + input.Input = proto.TextInput("changed text") if err := handleSteeringAndWait(t, h, mustEnv(t, proto.TypePromptSteer, "run-1", input)); err != nil { t.Fatal(err) } @@ -107,7 +107,7 @@ func TestSteeringReadinessAndUnsupportedRuns(t *testing.T) { if err := h.router.Handle(ctx, mustEnv(t, proto.TypePromptRequest, "run-1", proto.PromptRequestPayload{AgentKind: "codex"})); err != nil { t.Fatal(err) } - input := proto.PromptSteerPayload{InputID: "input-1", Text: "extra"} + input := proto.PromptSteerPayload{InputID: "input-1", Input: proto.TextInput("extra")} env := mustEnv(t, proto.TypePromptSteer, "run-1", input) for _, expected := range []string{"not_ready", ""} { if err := handleSteeringAndWait(t, h, env); err != nil { @@ -117,7 +117,7 @@ func TestSteeringReadinessAndUnsupportedRuns(t *testing.T) { t.Fatalf("expected %q: %+v", expected, ack) } if expected == "not_ready" { - changed := proto.PromptSteerPayload{InputID: "input-1", Text: "different during startup"} + changed := proto.PromptSteerPayload{InputID: "input-1", Input: proto.TextInput("different during startup")} if err := handleSteeringAndWait(t, h, mustEnv(t, proto.TypePromptSteer, "run-1", changed)); err != nil { t.Fatal(err) } @@ -147,7 +147,7 @@ func TestSteeringReadinessAndUnsupportedRuns(t *testing.T) { if ack := lastSteeringAck(t, h.sender, "run-2", "input-1"); ack.ErrorCode != "unsupported" { t.Fatalf("unsupported: %+v", ack) } - input.Text = " \n " + input.Input = proto.TextInput(" \n ") if err := handleSteeringAndWait(t, h, mustEnv(t, proto.TypePromptSteer, "run-2", input)); err != nil { t.Fatal(err) } @@ -181,7 +181,7 @@ func TestSteeringDoesNotBlockOtherRunCancellation(t *testing.T) { other.closeOutOnCancel = true returned := make(chan error, 1) go func() { - returned <- h.router.Handle(ctx, mustEnv(t, proto.TypePromptSteer, "run-1", proto.PromptSteerPayload{InputID: "slow", Text: "extra"})) + returned <- h.router.Handle(ctx, mustEnv(t, proto.TypePromptSteer, "run-1", proto.PromptSteerPayload{InputID: "slow", Input: proto.TextInput("extra")})) }() select { case err := <-returned: @@ -238,7 +238,7 @@ func TestSteeringCapacityPreservesExistingReceipts(t *testing.T) { t.Fatal(err) } for i := range 257 { - input := proto.PromptSteerPayload{InputID: fmt.Sprintf("input-%d", i), Text: "extra"} + input := proto.PromptSteerPayload{InputID: fmt.Sprintf("input-%d", i), Input: proto.TextInput("extra")} if err := handleSteeringAndWait(t, h, mustEnv(t, proto.TypePromptSteer, "run-1", input)); err != nil { t.Fatal(err) } @@ -247,7 +247,7 @@ func TestSteeringCapacityPreservesExistingReceipts(t *testing.T) { t.Fatalf("input %d: %+v", i, ack) } } - if err := handleSteeringAndWait(t, h, mustEnv(t, proto.TypePromptSteer, "run-1", proto.PromptSteerPayload{InputID: "input-0", Text: "extra"})); err != nil { + if err := handleSteeringAndWait(t, h, mustEnv(t, proto.TypePromptSteer, "run-1", proto.PromptSteerPayload{InputID: "input-0", Input: proto.TextInput("extra")})); err != nil { t.Fatal(err) } if ack := lastSteeringAck(t, h.sender, "run-1", "input-0"); !ack.Accepted || calls != 256 { diff --git a/apps/parsar-daemon/internal/dispatch/workspace_directory_test.go b/apps/parsar-daemon/internal/dispatch/workspace_directory_test.go index e31816922..49835c073 100644 --- a/apps/parsar-daemon/internal/dispatch/workspace_directory_test.go +++ b/apps/parsar-daemon/internal/dispatch/workspace_directory_test.go @@ -13,7 +13,7 @@ import ( func TestWorkspaceDirectoryRetainsEnvironmentAndTransferredOwner(t *testing.T) { sender := &recSender{} p := &controlledPreparation{closed: make(chan struct{})} - p.start = func(ctx context.Context, _ string, _ string, out chan<- proto.Envelope) (agent.Session, error) { + p.start = func(ctx context.Context, _ string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { return &fakeSession{out: out, ctx: ctx, closeOutOnCancel: true}, nil } r := preparationRouter(t, sender, time.Minute, func(context.Context, proto.PromptRequestPayload) (agent.Prepared, error) { return p, nil }) @@ -33,7 +33,7 @@ func TestWorkspaceDirectoryRetainsEnvironmentAndTransferredOwner(t *testing.T) { t.Fatal(got) } if phase == "idle" { - _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "prepare", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "run", Prompt: "start"})) + _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "prepare", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "run", Input: proto.TextInput("start")})) waitPreparationStatus(t, sender, "prepare", "started", "") _ = r.Handle(t.Context(), mustEnv(t, proto.TypeWorkspaceRead, "stale", request)) if got := waitWorkspaceRead(t, sender, "stale"); got.Outcome != "rejected" { diff --git a/apps/parsar-daemon/internal/localworkspace/binding.go b/apps/parsar-daemon/internal/localworkspace/binding.go index 3ed9a6f43..6c24974e8 100644 --- a/apps/parsar-daemon/internal/localworkspace/binding.go +++ b/apps/parsar-daemon/internal/localworkspace/binding.go @@ -90,7 +90,7 @@ func (b *Binding) Configure(r proto.PromptRequestPayload) (proto.PromptRequestPa } if b == nil || r.LocalEnvironment == nil || r.LocalEnvironment.ID != b.environment || r.AgentStateKey != b.stateKey || r.DisableExecutionEnvironment || r.WorkDir != "" || - r.ConversationID != "" || r.WorkspaceAuthoring || len(r.Attachments) != 0 || !r.StrictResume || !r.ReleaseOnCompletion { + r.ConversationID != "" || r.WorkspaceAuthoring || !r.StrictResume || !r.ReleaseOnCompletion { return r, errors.New("request does not match the dedicated local Environment") } if !r.WorkspaceReadOnly || r.LocalEnvironment.NetworkAccess != "" || len(r.LocalEnvironment.AllowedDomains) > 0 { diff --git a/apps/parsar-daemon/testdata/onboarding/main.go b/apps/parsar-daemon/testdata/onboarding/main.go index b804d027d..67a29f92b 100644 --- a/apps/parsar-daemon/testdata/onboarding/main.go +++ b/apps/parsar-daemon/testdata/onboarding/main.go @@ -31,6 +31,9 @@ func (s *sender) Send(_ context.Context, e proto.Envelope) error { type harness struct{ history map[string]string } func (h *harness) start(_ context.Context, req proto.PromptRequestPayload, out chan<- proto.Envelope) (agent.Session, error) { + if _, err := req.Input.TextOnly(); err != nil { + return nil, err + } if !req.StrictResume || !req.ReleaseOnCompletion || !req.DisableExecutionEnvironment || !req.DisableSubagents || len(req.FunctionTools) > 0 || req.MCPHTTPServers != nil { return nil, errors.New("unsupported fixture operation") } @@ -75,14 +78,18 @@ func (s *session) Steer(ctx context.Context, p proto.PromptSteerPayload) error { return s.SteerWithReceipt(ctx, p, func() {}) } func (s *session) SteerWithReceipt(_ context.Context, p proto.PromptSteerPayload, written func()) error { + text, err := p.Input.TextOnly() + if err != nil { + return err + } s.mu.Lock() defer s.mu.Unlock() if s.closed { return agent.ErrSteeringInactive } written() - s.emit(proto.TypeDelta, proto.DeltaPayload{Delta: p.Text, Sequence: 2}) - s.emit(proto.TypeDone, proto.DonePayload{Content: "ready" + p.Text, Metadata: map[string]any{proto.DoneMetaAgentSessionID: s.native}}) + s.emit(proto.TypeDelta, proto.DeltaPayload{Delta: text, Sequence: 2}) + s.emit(proto.TypeDone, proto.DonePayload{Content: "ready" + text, Metadata: map[string]any{proto.DoneMetaAgentSessionID: s.native}}) s.closed = true close(s.out) return nil diff --git a/contracts/agents-api/README.md b/contracts/agents-api/README.md index c331ddb01..9824f00eb 100644 --- a/contracts/agents-api/README.md +++ b/contracts/agents-api/README.md @@ -452,8 +452,8 @@ errors, not successful placeholder resources. Add any provider or engine-specifi extension separately from upstream fields and document it here when implemented. `openapi.yaml` is our generated supported surface; it is not the full upstream -specification. The shared Go wire types are in `v1`. Physical Session cleanup, non-text -message input, broader structured-output combinations, broader options/tools, remaining Vault lifecycle, +specification. The shared Go wire types are in `v1`. Physical Session cleanup, broader image +message profiles, broader structured-output combinations, broader options/tools, remaining Vault lifecycle, Subagents and environment/provider resources remain incomplete. Reject unsupported requests explicitly; persisted saved configuration is not execution admission. @@ -539,7 +539,7 @@ historical native transport evidence. ### Public execution admission `POST /v1/agents/sessions/{session_id}/events` accepts `agent.session.input.message` -with user `input_text` content, `agent.session.input.cancel` and +with ordered user `input_text` content and [qualified image content](message-input.md), `agent.session.input.cancel` and `agent.session.input.tool_result`. Successful atomic admission returns 204, as consumed by the official `events.create` method. A retry key identifies the entire ordered request; conflict does not partially admit it. @@ -695,10 +695,11 @@ tool invocation. Omitted, null and explicit medium reused the same creation identity. The tool data was synthetic; model responses were live. This does not establish non-default verbosity, tool-set enforcement or full protocol conformance. -### Initial text at Session creation +### Initial input at Session creation Session creation accepts the pinned string and user-message-array input -forms. It shares text validation and admission with the events endpoint. The +forms. It shares message validation and admission with the events endpoint, +including the [qualified image profile](message-input.md). The Session and its initial work commit atomically; an identical creation retry never re-admits the input, including after later or terminal Turns. With `none`, this includes the first Turn and input Items. With `self_hosted`, it @@ -710,7 +711,7 @@ configured engine must support admission before any initial work is persisted. Fixed SDK/raw HTTP and PostgreSQL tests cover the accepted forms, saved and inline configuration, ordering, tenant isolation, retries, rollback and persistence. -Non-text input remains a gap. Empty arrays and blank text +Image support is bounded as documented above. Empty arrays and blank text currently fail the shared message validator; exact upstream handling of these cases, local size limits and error details remains unverified. Swagger 2 cannot express the string/array union, so input is unconstrained with a type description. diff --git a/contracts/agents-api/harness-onboarding.md b/contracts/agents-api/harness-onboarding.md index b0586a70f..ec521b8cf 100644 --- a/contracts/agents-api/harness-onboarding.md +++ b/contracts/agents-api/harness-onboarding.md @@ -90,6 +90,13 @@ confirmed native output through the existing Message contract. Register public qualification separately from the Runtime capability; see the [structured-output boundary](structured-output.md). No Core engine-name branch is required. +Initial requests, `Prepared.Start` and steering consume the same ordered +`proto.MessageInput`. Text-only adapters use `TextOnly()` to reject images without +discarding content. Image adapters translate each part natively and acknowledge +an active batch only after all its messages are applied. Register +`MessageImages` and qualify `MessageImagePlacements` separately; see the +[message-input contract and real acceptance](message-input.md). + Hosted workspace execution additionally requires verified preparation, workspace reads/output export, network behavior and credential/history isolation. Reuse the same dedicated Runtime binding and shared Files helpers. A native Bash sandbox diff --git a/contracts/agents-api/message-input.md b/contracts/agents-api/message-input.md new file mode 100644 index 000000000..5bc149d3c --- /dev/null +++ b/contracts/agents-api/message-input.md @@ -0,0 +1,84 @@ +# Ordered message input + +The pinned SDK defines user messages as ordered `input_text` and `input_image` +parts. Core retains message boundaries, content order and the supplied image +reference in input persistence and user Items. Session creation and subsequent +events share validation and atomic admission. + +## Supported profile + +Codex and Claude SDK support inline PNG/JPEG data URIs on `environment:none`, for +initial and active input and subsequent Turns. Use a real vision-capable model. +The existing 1 MiB HTTP and 512 KiB durable input limits still apply. A successful +events response acknowledges persistence, not native consumption. Active input +advances its durable receipt only after the adapter confirms application. + +```python +import base64 +from pathlib import Path + +image_url = "data:image/png;base64," + base64.b64encode( + Path("example.png").read_bytes() +).decode() +session = client.beta.agents.sessions.create( + agent={"model": model}, + environment={"type": "none"}, + input=[{"role": "user", "content": [ + {"type": "input_text", "text": "Describe this image."}, + {"type": "input_image", "image_url": image_url}, + ]}], +) +``` + +Query Session/Turn/Items to recover results. SSE remains live-only; reconnecting +does not replay inputs or recreate completed Turns. The request contains no +image `detail` or file-ID extension. + +## Runtime boundary + +Private wire 0.4.0 uses `MessageInput` for initial requests, prepared start and +active steering. Each message contains ordered `InputContent` parts, also reused +by function-result content. Core does not download, transcode or repair media. +Adapters own native encoding and native application-receipt mapping. Existing +text-only adapters reject image parts instead of dropping them. + +Codex converts content to its native flat input list and inserts blank-line +separators between public messages. Public message boundaries remain durable; +independent native message boundaries are not claimed. Claude uses native image +blocks and a UUID for each native user message; one public active input is applied +only after every message in its batch is consumed. Its native 64-message bound is +checked before any part of a batch enters the iterator. + +Public profile qualification and Runtime advertisement are separate. Admission +checks the registered image profile for this placement; device selection and +delivery check actual image support. Text-only operations retain existing offline +admission. Adding an adapter must implement the shared contract and qualify the +public operation, without adding engine-name branches to Core. + +## Validation and remaining gaps + +`TestNativeMessageImagePublicExecution` runs the pinned SDK and raw HTTP against +Core, a dedicated PostgreSQL database, the real daemon and a native harness. +Set `PARSAR_MESSAGE_IMAGE_ENGINE` to `codex` or `claude_sdk`, provide private real +provider options via `PARSAR_MESSAGE_IMAGE_REAL_OPTIONS`, and use the existing +`PARSAR_NATIVE_DAEMON_BIN`, `PARSAR_NATIVE_PROOF_DIR` and +`PARSAR_OFFICIAL_SDK_PYTHON` fixture settings. The test never supplies model responses. + +Real Kimi K3 acceptance on 2026-09-22 used randomized four-color PNGs whose answers +were absent from the input text. Both adapters passed initial ordered input, +active replacement with a different image, retry deduplication, exact retained +user Items, native receipts, cold daemon history continuation, cancellation and +ordinary text continuation, malformed-batch atomic rejection and tenant isolation. +Evidence is under `zju_a100_2:~/.parsar/remediation/20260922/message-image-input/`: +`public-codex-final.log` / `message-image-public-2718944397/public.json` and +`public-claude_sdk-final.log` / `message-image-public-700202073/public.json`. +Native-only probes are feasibility evidence, not public qualification. + +Workspace image workflows, MiniMax Code image input, remote HTTP(S) image URLs, +other media types and full upstream error/default semantics remain unqualified. +MiniMax's fixed ACP advertises `image:false`; its adapter rejects images. These are +implementation gaps, not changes to the official protocol. JPEG parsing/conversion +has deterministic coverage; the recorded real visual fixtures are PNG. Empty +messages, local payload limits and native batch-size parity need upstream evidence. +Function-result image support is unchanged by this batch. No full protocol +compatibility or support for arbitrary vision-model/provider combinations is claimed. diff --git a/contracts/agents-api/openapi.yaml b/contracts/agents-api/openapi.yaml index 3578bf4ff..ef65eeb9b 100644 --- a/contracts/agents-api/openapi.yaml +++ b/contracts/agents-api/openapi.yaml @@ -656,14 +656,16 @@ definitions: type: object v1.InputContent: properties: + image_url: + type: string text: type: string type: enum: - input_text + - input_image type: string required: - - text - type type: object v1.InputMessage: @@ -2679,35 +2681,37 @@ paths: keys or failed credential lookup/decryption never fall back to anonymous execution. Omitted stream defaults to false; stream and agent_id cannot be null. Metadata may be null, but its values must be strings. Initial input accepts a string - or user-message array containing text. None initial input atomically starts - a Turn; self_hosted initial input is reserved while returning its Environment - connection target, with execution deferred to native readiness and Session - failure on initial timeout. Omitted or null input creates an idle Session. - With stream=true, returns live Session events starting at creation; disconnect - does not cancel execution. New Sessions retain their authenticated creator; - all creation retries require the same typed subject, including across key - rotation. Saved-Agent retries and inline requests using Vault attachments - or credential references retain caller intent independently of later resource - changes; unrelated inline retries preserve resolved/default equivalences. - Unknown historical creators reject retries; known creators without recorded - intent retain resolved-snapshot retry rules. These conflict policies are local - and not verified hosted parity. Creation retries observe future events without - replay; retry with stream=false to retrieve the Session. Claude SDK environment:none - supports qualified object-root json_schema output with medium verbosity, single-Agent - execution and ordinary functions; other combinations remain unsupported. Non-text - initial input remains unsupported. Basic Codex and Claude SDK openai_hosted - creation requires an explicitly configured managed provider. The Claude workspace - profile supports non-deferred function tools with text results alongside native - workspace tools; HTTP MCP remains unsupported. Idle Sessions provision automatically; - initial provisioning has no caller connection action. Network defaults to - enabled; disabled and restricted exact ASCII hostnames are supported. Restricted - policy requires 1–100 allowed domains. Unsupported hostname forms and startup - installations are rejected. Confidential env, system/npm/Python packages and - ordered setup commands use the shared initialization lifecycle; requested - network applies after setup. Initial inline and tenant-owned file_id files - freeze encrypted bytes before provisioning, then install through the common - Core lifecycle before native execution or live Files access. Referenced files/env/packages/setup - overrides are rejected pending semantic verification. Tenant-owned environment_template_id + or ordered user-message array. Codex and Claude SDK on none also accept inline + PNG/JPEG image content; other image combinations and remote URLs are unsupported. + None initial input atomically starts a Turn; self_hosted initial input is + reserved while returning its Environment connection target, with execution + deferred to native readiness and Session failure on initial timeout. Omitted + or null input creates an idle Session. With stream=true, returns live Session + events starting at creation; disconnect does not cancel execution. New Sessions + retain their authenticated creator; all creation retries require the same + typed subject, including across key rotation. Saved-Agent retries and inline + requests using Vault attachments or credential references retain caller intent + independently of later resource changes; unrelated inline retries preserve + resolved/default equivalences. Unknown historical creators reject retries; + known creators without recorded intent retain resolved-snapshot retry rules. + These conflict policies are local and not verified hosted parity. Creation + retries observe future events without replay; retry with stream=false to retrieve + the Session. Claude SDK environment:none supports qualified object-root json_schema + output with medium verbosity, single-Agent execution and ordinary functions; + other combinations remain unsupported. Non-text initial input remains unsupported. + Basic Codex and Claude SDK openai_hosted creation requires an explicitly configured + managed provider. The Claude workspace profile supports non-deferred function + tools with text results alongside native workspace tools; HTTP MCP remains + unsupported. Idle Sessions provision automatically; initial provisioning has + no caller connection action. Network defaults to enabled; disabled and restricted + exact ASCII hostnames are supported. Restricted policy requires 1–100 allowed + domains. Unsupported hostname forms and startup installations are rejected. + Confidential env, system/npm/Python packages and ordered setup commands use + the shared initialization lifecycle; requested network applies after setup. + Initial inline and tenant-owned file_id files freeze encrypted bytes before + provisioning, then install through the common Core lifecycle before native + execution or live Files access. Referenced files/env/packages/setup overrides + are rejected pending semantic verification. Tenant-owned environment_template_id references inherit omitted network and allow only narrowing overrides. Referenced network:null is explicitly unsupported pending semantic verification. Core freezes effective configuration; template updates/deletion do not alter Session @@ -3226,8 +3230,10 @@ paths: executor URL settings. Disconnecting the waiting HTTP request does not cancel retained work or restart its deadline. Retry keys identify the whole ordered batch. Function output accepts text or ordered text/image parts subject to - engine support; Claude SDK currently accepts text results only. Message images - are not supported yet. + engine support; Claude SDK currently accepts text results only. Codex and + Claude SDK on none accept ordered inline PNG/JPEG image messages. Workspace + profiles and other engines remain text-only; remote image URLs are unsupported. + Image references are retained unchanged without service-side downloads. parameters: - description: agents=v1 in: header diff --git a/contracts/agents-api/v1/inputs.go b/contracts/agents-api/v1/inputs.go index d7a81c011..2fd8600c5 100644 --- a/contracts/agents-api/v1/inputs.go +++ b/contracts/agents-api/v1/inputs.go @@ -20,8 +20,9 @@ type InputMessage struct { } type InputContent struct { - Type string `json:"type" enums:"input_text" binding:"required"` - Text string `json:"text" binding:"required"` + Type string `json:"type" enums:"input_text,input_image" binding:"required"` + Text *string `json:"text,omitempty"` + ImageURL *string `json:"image_url,omitempty"` } type CreateEventsRequest struct { diff --git a/internal/agentdaemon/device/state.go b/internal/agentdaemon/device/state.go index 7c8c1c8bb..9fe2d9a12 100644 --- a/internal/agentdaemon/device/state.go +++ b/internal/agentdaemon/device/state.go @@ -81,6 +81,7 @@ type KindCapabilities struct { ExecutionControls bool `json:"execution_controls,omitempty"` TextVerbosity bool `json:"text_verbosity,omitempty"` StructuredOutput bool `json:"structured_output,omitempty"` + MessageImages bool `json:"message_images,omitempty"` SubagentControl bool `json:"subagent_control,omitempty"` FunctionTools bool `json:"function_tools,omitempty"` MCPHTTPTools bool `json:"mcp_http_tools,omitempty"` diff --git a/internal/agentdaemon/gateway/mcp_bearer_live_linux_test.go b/internal/agentdaemon/gateway/mcp_bearer_live_linux_test.go index 1920a9079..d0fcb62ae 100644 --- a/internal/agentdaemon/gateway/mcp_bearer_live_linux_test.go +++ b/internal/agentdaemon/gateway/mcp_bearer_live_linux_test.go @@ -87,7 +87,7 @@ func TestLiveMCPBearerGatewayColdContinuation(t *testing.T) { turn := &mcpBearerTurn{} turns = append(turns, turn) runID := uuid.NewString() - request := proto.PromptRequestPayload{AgentKind: "codex", ConversationID: "mcp-bearer-acceptance", RunID: runID, Prompt: prompt, AgentStateKey: "mcp-bearer-acceptance", AgentSessionID: resume, StrictResume: true, ReleaseOnCompletion: true, ObserveMessages: true, ObserveTools: true, ObserveToolObservations: true, DisableExecutionEnvironment: true, DisableSubagents: true, MCPHTTPServers: &servers, AgentOptions: map[string]any{"model": "MiniMax-M3"}, ExecutionControls: &proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "medium"}} + request := proto.PromptRequestPayload{AgentKind: "codex", ConversationID: "mcp-bearer-acceptance", RunID: runID, Input: proto.TextInput(prompt), AgentStateKey: "mcp-bearer-acceptance", AgentSessionID: resume, StrictResume: true, ReleaseOnCompletion: true, ObserveMessages: true, ObserveTools: true, ObserveToolObservations: true, DisableExecutionEnvironment: true, DisableSubagents: true, MCPHTTPServers: &servers, AgentOptions: map[string]any{"model": "MiniMax-M3"}, ExecutionControls: &proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "medium"}} sub, err := peer.SubscribeDurable(runID) if err != nil { t.Fatal("cannot subscribe before real daemon dispatch") diff --git a/internal/agentdaemon/gateway/session.go b/internal/agentdaemon/gateway/session.go index 6adcedf83..e30fdb16d 100644 --- a/internal/agentdaemon/gateway/session.go +++ b/internal/agentdaemon/gateway/session.go @@ -561,6 +561,7 @@ func deviceKindsFromHeartbeat(p proto.HeartbeatPayload) []device.SupportedAgentK WebSearchControl: info.Capabilities.WebSearchControl, TextVerbosity: info.Capabilities.TextVerbosity, StructuredOutput: info.Capabilities.StructuredOutput, + MessageImages: info.Capabilities.MessageImages, ExecutionControls: info.Capabilities.ExecutionControls, SubagentControl: info.Capabilities.SubagentControl, SubagentObservations: info.Capabilities.SubagentObservations, diff --git a/internal/agentdaemon/gateway/session_test.go b/internal/agentdaemon/gateway/session_test.go index f363b2609..778a8eb15 100644 --- a/internal/agentdaemon/gateway/session_test.go +++ b/internal/agentdaemon/gateway/session_test.go @@ -352,7 +352,7 @@ func TestSession_SendWritesToWire(t *testing.T) { env, _ := proto.NewEnvelope(proto.TypePromptRequest, "run-1", proto.PromptRequestPayload{ AgentKind: "claude_code", RunID: "run-1", - Prompt: "hello", + Input: proto.TextInput("hello"), }) if err := sess.Send(context.Background(), env); err != nil { t.Fatalf("Send: %v", err) diff --git a/internal/agentdaemon/proto/envelope_test.go b/internal/agentdaemon/proto/envelope_test.go index 744f18a67..41bdf8cf8 100644 --- a/internal/agentdaemon/proto/envelope_test.go +++ b/internal/agentdaemon/proto/envelope_test.go @@ -88,7 +88,7 @@ func TestVersionCompatible(t *testing.T) { ok bool }{ {Version, true}, // exact match - {"0.3.99", true}, // patch drift OK + {"0.4.99", true}, // patch drift OK {"0.2.99", false}, // minor drift NOT OK {"1.0.0", false}, // major drift NOT OK {"", false}, // missing diff --git a/internal/agentdaemon/proto/functions.go b/internal/agentdaemon/proto/functions.go index c282c0f6e..0fd337959 100644 --- a/internal/agentdaemon/proto/functions.go +++ b/internal/agentdaemon/proto/functions.go @@ -24,17 +24,10 @@ type FunctionCallPayload struct { } type FunctionResultPayload struct { - DeliveryID string `json:"delivery_id"` - CallID string `json:"call_id"` - Success bool `json:"success"` - Content []FunctionResultContent `json:"content"` -} - -// FunctionResultContent is one ordered text or image part of a function result. -type FunctionResultContent struct { - Type string `json:"type"` - Text *string `json:"text,omitempty"` - ImageURL *string `json:"image_url,omitempty"` + DeliveryID string `json:"delivery_id"` + CallID string `json:"call_id"` + Success bool `json:"success"` + Content []InputContent `json:"content"` } func (r FunctionResultPayload) ValidateContent() error { @@ -42,17 +35,9 @@ func (r FunctionResultPayload) ValidateContent() error { return errors.New("function result requires a content array") } for _, part := range r.Content { - switch part.Type { - case "input_text": - if part.Text != nil && part.ImageURL == nil { - continue - } - case "input_image": - if part.ImageURL != nil && part.Text == nil { - continue - } + if err := part.Validate(); err != nil { + return err } - return errors.New("function result requires text or image content") } return nil } diff --git a/internal/agentdaemon/proto/functions_test.go b/internal/agentdaemon/proto/functions_test.go index a34d83dfd..1782136f4 100644 --- a/internal/agentdaemon/proto/functions_test.go +++ b/internal/agentdaemon/proto/functions_test.go @@ -5,7 +5,7 @@ import ( "testing" ) -func TestFunctionResultContentWire(t *testing.T) { +func TestInputContentWire(t *testing.T) { for _, content := range []string{ `[]`, `[{"type":"input_text","text":""}]`, diff --git a/internal/agentdaemon/proto/inbound.go b/internal/agentdaemon/proto/inbound.go index 1ea1623a8..fc36e529c 100644 --- a/internal/agentdaemon/proto/inbound.go +++ b/internal/agentdaemon/proto/inbound.go @@ -273,6 +273,7 @@ type AgentKindCapabilities struct { ExecutionControls bool `json:"execution_controls,omitempty"` TextVerbosity bool `json:"text_verbosity,omitempty"` StructuredOutput bool `json:"structured_output,omitempty"` + MessageImages bool `json:"message_images,omitempty"` SubagentControl bool `json:"subagent_control,omitempty"` DurableInputReceipts bool `json:"durable_input_receipts,omitempty"` // DurableTurns includes strict resume, completion release and cancellation snapshots. diff --git a/internal/agentdaemon/proto/message_images.go b/internal/agentdaemon/proto/message_images.go new file mode 100644 index 000000000..8d440dc36 --- /dev/null +++ b/internal/agentdaemon/proto/message_images.go @@ -0,0 +1,39 @@ +package proto + +import ( + "bytes" + "encoding/base64" + "errors" + "image" + _ "image/jpeg" + _ "image/png" + "strings" +) + +// ValidateInlineImages checks the bounded user-message image profile. It does +// not download references, rewrite bytes or change function-result support. +func (m MessageInput) ValidateInlineImages() error { + for _, message := range m { + for _, part := range message.Content { + if part.Type != "input_image" { + continue + } + if part.ImageURL == nil { + return errors.New("image input requires a reference") + } + header, encoded, ok := strings.Cut(*part.ImageURL, ",") + if !ok || (header != "data:image/png;base64" && header != "data:image/jpeg;base64") { + return errors.New("user image requires inline PNG or JPEG") + } + data, err := base64.StdEncoding.Strict().DecodeString(encoded) + if err != nil || base64.StdEncoding.EncodeToString(data) != encoded { + return errors.New("user image requires valid base64") + } + _, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || header != "data:image/"+format+";base64" { + return errors.New("user image format does not match its media type") + } + } + } + return nil +} diff --git a/internal/agentdaemon/proto/message_images_test.go b/internal/agentdaemon/proto/message_images_test.go new file mode 100644 index 000000000..4da4eb9a3 --- /dev/null +++ b/internal/agentdaemon/proto/message_images_test.go @@ -0,0 +1,45 @@ +package proto + +import ( + "bytes" + "encoding/base64" + "image" + "image/jpeg" + "image/png" + "testing" +) + +func TestInlineMessageImages(t *testing.T) { + for _, format := range []string{"png", "jpeg"} { + var encoded bytes.Buffer + picture := image.NewRGBA(image.Rect(0, 0, 2, 2)) + if format == "png" { + _ = png.Encode(&encoded, picture) + } else { + _ = jpeg.Encode(&encoded, picture, nil) + } + url := "data:image/" + format + ";base64," + base64.StdEncoding.EncodeToString(encoded.Bytes()) + input := MessageInput{{Content: []InputContent{{Type: "input_image", ImageURL: &url}}}} + if err := input.ValidateInlineImages(); err != nil { + t.Fatal(format, err) + } + wrongType := "data:image/gif;base64," + base64.StdEncoding.EncodeToString(encoded.Bytes()) + input[0].Content[0].ImageURL = &wrongType + if input.ValidateInlineImages() == nil { + t.Fatal("accepted unsupported image type") + } + } + for _, url := range []string{"https://example.test/image.png", "file:///private/image.png", "data:image/png;base64,AA==", "data:image/png;base64,!", "data:image/png;base64,"} { + input := MessageInput{{Content: []InputContent{{Type: "input_image", ImageURL: &url}}}} + if input.ValidateInlineImages() == nil { + t.Fatalf("accepted %q", url) + } + // User-message qualification must not narrow the function-output union. + if input.Validate() != nil { + t.Fatalf("inline profile leaked into the shared content union: %q", url) + } + } + if err := TextInput("ordinary text").ValidateInlineImages(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/agentdaemon/proto/message_input.go b/internal/agentdaemon/proto/message_input.go new file mode 100644 index 000000000..b2e01c79e --- /dev/null +++ b/internal/agentdaemon/proto/message_input.go @@ -0,0 +1,99 @@ +package proto + +import ( + "errors" + "strings" +) + +// InputContent is an ordered text or image part, shared by user and tool input. +// ImageURL is passed unchanged; native encoding belongs to the adapter. +type InputContent struct { + Type string `json:"type"` + Text *string `json:"text,omitempty"` + ImageURL *string `json:"image_url,omitempty"` +} + +func (p InputContent) Validate() error { + switch p.Type { + case "input_text": + if p.Text != nil && p.ImageURL == nil { + return nil + } + case "input_image": + if p.ImageURL != nil && p.Text == nil { + return nil + } + } + return errors.New("input requires text or image content") +} + +// MessageInput preserves user-message boundaries and content order on every +// Runtime input path. Adapters map these messages to their native input format. +type MessageInput []InputMessage + +type InputMessage struct { + Content []InputContent `json:"content"` +} + +// TextInput constructs one message without normalizing its text. +func TextInput(text string) MessageInput { + return MessageInput{{Content: []InputContent{{Type: "input_text", Text: &text}}}} +} + +func (m MessageInput) Validate() error { + if len(m) == 0 { + return errors.New("user input requires messages") + } + for _, message := range m { + meaningful := false + for _, part := range message.Content { + if err := part.Validate(); err != nil { + return err + } + if part.Type == "input_image" { + if strings.TrimSpace(*part.ImageURL) == "" { + return errors.New("image input requires a reference") + } + meaningful = true + } else if strings.TrimSpace(*part.Text) != "" { + meaningful = true + } + } + if !meaningful { + return errors.New("user message requires content") + } + } + return nil +} + +func (m MessageInput) HasImages() bool { + for _, message := range m { + for _, part := range message.Content { + if part.Type == "input_image" { + return true + } + } + } + return false +} + +// TextOnly is for text-only native transports. It rejects images rather than +// dropping them, and retains the existing blank-line boundary between messages. +func (m MessageInput) TextOnly() (string, error) { + if err := m.Validate(); err != nil { + return "", err + } + var text strings.Builder + for i, message := range m { + if i > 0 { + text.WriteString("\n\n") + } + for _, part := range message.Content { + if part.Type != "input_text" { + return "", errors.New("native transport does not support image input") + } + text.WriteString(*part.Text) + } + } + return text.String(), nil +} diff --git a/internal/agentdaemon/proto/message_input_test.go b/internal/agentdaemon/proto/message_input_test.go new file mode 100644 index 000000000..dc20d1873 --- /dev/null +++ b/internal/agentdaemon/proto/message_input_test.go @@ -0,0 +1,54 @@ +package proto + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestMessageInputOrderAndTextOnlyRejection(t *testing.T) { + raw := []byte(`[{"content":[{"type":"input_text","text":" before "},{"type":"input_image","image_url":"data:image/png;base64,aW1hZ2U="},{"type":"input_text","text":" after "}]},{"content":[{"type":"input_text","text":"next"}]}]`) + var input MessageInput + if err := json.Unmarshal(raw, &input); err != nil { + t.Fatal(err) + } + if err := input.Validate(); err != nil { + t.Fatal(err) + } + if !input.HasImages() { + t.Fatal("image lost") + } + if _, err := input.TextOnly(); err == nil { + t.Fatal("text-only adapter silently accepted image") + } + encoded, err := json.Marshal(input) + if err != nil || string(encoded) != string(raw) { + t.Fatalf("content changed: %s, %v", encoded, err) + } + text := append(TextInput(" before "), TextInput(" after ")...) + actual, err := text.TextOnly() + if err != nil || actual != " before \n\n after " { + t.Fatalf("text changed: %q %v", actual, err) + } + for _, payload := range []any{PromptRequestPayload{Input: input}, PromptSteerPayload{Input: input}, ExecutionStartPayload{Input: input}} { + encoded, err := json.Marshal(payload) + if err != nil { + t.Fatal(err) + } + var wire struct { + Input MessageInput `json:"input"` + } + if err := json.Unmarshal(encoded, &wire); err != nil || !reflect.DeepEqual(input, wire.Input) { + t.Fatalf("wire loses input: %s", encoded) + } + } +} + +func TestMessageInputInvalidUnion(t *testing.T) { + for _, raw := range []string{`[]`, `[{"content":[]}]`, `[{"content":[{"type":"input_text","text":null}]}]`, `[{"content":[{"type":"input_image","image_url":""}]}]`, `[{"content":[{"type":"input_image","image_url":"image","text":"unexpected"}]}]`} { + var input MessageInput + if err := json.Unmarshal([]byte(raw), &input); err == nil && input.Validate() == nil { + t.Fatalf("accepted %s", raw) + } + } +} diff --git a/internal/agentdaemon/proto/outbound.go b/internal/agentdaemon/proto/outbound.go index a3edfeb72..b1e1a6e95 100644 --- a/internal/agentdaemon/proto/outbound.go +++ b/internal/agentdaemon/proto/outbound.go @@ -50,16 +50,8 @@ type PromptRequestPayload struct { // upstream frame via Envelope.ID. RunID string `json:"run_id"` - // Prompt is the user-facing message that drives this turn. - Prompt string `json:"prompt"` - - // Attachments carries non-text payloads (images from inbound - // messages) alongside Prompt. The daemon-side agent decides how - // to fold them in: claude_code re-encodes them into Anthropic - // image content blocks on the stdin-driven JSON input loop. - // Silently ignored when the agent doesn't understand multimodal - // input — Prompt alone still drives the run. - Attachments []PromptAttachment `json:"attachments,omitempty"` + // Input preserves ordered user messages and content. + Input MessageInput `json:"input,omitempty"` // WorkDir is the cwd for the agent subprocess. Local mode: user's // chosen project root. Sandbox mode: empty — the daemon falls @@ -102,21 +94,6 @@ type PromptRequestPayload struct { DisableSubagents bool `json:"disable_subagents,omitempty"` } -// PromptAttachment is one piece of non-text user input the daemon-side -// agent should fold into the turn alongside Prompt. The field set is -// forward-compatible with file/audio so a wire-schema bump isn't -// required when those land. -// -// DataBase64 is standard-base64 raw bytes; the daemon decodes once -// before forwarding to its agent adapter (claude_code re-wraps as an -// Anthropic image content block on stdin). MIME is forwarded verbatim -// so the agent picks the right block shape (image/png vs image/jpeg). -type PromptAttachment struct { - Kind string `json:"kind"` - MIME string `json:"mime"` - DataBase64 string `json:"data_base64"` -} - // PromptCancelPayload optionally requests an application receipt; identity is on Envelope.ID. type PromptCancelPayload struct { DeliveryID string `json:"delivery_id,omitempty"` diff --git a/internal/agentdaemon/proto/preparation.go b/internal/agentdaemon/proto/preparation.go index eccdf9271..f26c9eb7e 100644 --- a/internal/agentdaemon/proto/preparation.go +++ b/internal/agentdaemon/proto/preparation.go @@ -17,9 +17,9 @@ type ExecutionPreparePayload struct { } type ExecutionStartPayload struct { - Handle string `json:"handle"` - RunID string `json:"run_id"` - Prompt string `json:"prompt"` + Handle string `json:"handle"` + RunID string `json:"run_id"` + Input MessageInput `json:"input"` } type ExecutionReleasePayload struct { diff --git a/internal/agentdaemon/proto/steering.go b/internal/agentdaemon/proto/steering.go index 0c12c0765..6658752cb 100644 --- a/internal/agentdaemon/proto/steering.go +++ b/internal/agentdaemon/proto/steering.go @@ -1,16 +1,16 @@ package proto -// TypePromptSteer appends text to an active run; Envelope.ID is the run ID. +// TypePromptSteer appends messages to an active run; Envelope.ID is the run ID. const TypePromptSteer = "prompt_steer" // TypePromptSteerAck reports input receipt phases on the originating run ID. const TypePromptSteerAck = "prompt_steer_ack" -// PromptSteerPayload identifies one text input within an active run. +// PromptSteerPayload identifies one input batch within an active run. type PromptSteerPayload struct { - InputID string `json:"input_id"` - Text string `json:"text"` - DurableReceipt bool `json:"durable_receipt,omitempty"` + InputID string `json:"input_id"` + Input MessageInput `json:"input"` + DurableReceipt bool `json:"durable_receipt,omitempty"` } // PromptSteerAckPayload distinguishes a completed write from native acceptance. diff --git a/internal/agentdaemon/proto/tool_observations.go b/internal/agentdaemon/proto/tool_observations.go index 6562d55a0..46989a4c7 100644 --- a/internal/agentdaemon/proto/tool_observations.go +++ b/internal/agentdaemon/proto/tool_observations.go @@ -4,19 +4,19 @@ import "encoding/json" // ToolObservation is an engine-neutral execution snapshot, not a public API Item. type ToolObservation struct { - Kind string `json:"kind"` - Status string `json:"status"` - Name string `json:"name,omitempty"` - Command string `json:"command,omitempty"` - Cwd *string `json:"cwd,omitempty"` - ExitCode *int64 `json:"exit_code,omitempty"` - DurationMS *int64 `json:"duration_ms,omitempty"` - Server string `json:"server,omitempty"` - Arguments json.RawMessage `json:"arguments,omitempty"` - Output json.RawMessage `json:"output,omitempty"` - Error json.RawMessage `json:"error,omitempty"` - Content *[]FunctionResultContent `json:"content,omitempty"` - Action *ToolWebSearchAction `json:"action,omitempty"` + Kind string `json:"kind"` + Status string `json:"status"` + Name string `json:"name,omitempty"` + Command string `json:"command,omitempty"` + Cwd *string `json:"cwd,omitempty"` + ExitCode *int64 `json:"exit_code,omitempty"` + DurationMS *int64 `json:"duration_ms,omitempty"` + Server string `json:"server,omitempty"` + Arguments json.RawMessage `json:"arguments,omitempty"` + Output json.RawMessage `json:"output,omitempty"` + Error json.RawMessage `json:"error,omitempty"` + Content *[]InputContent `json:"content,omitempty"` + Action *ToolWebSearchAction `json:"action,omitempty"` } type ToolWebSearchAction struct { diff --git a/internal/agentdaemon/proto/version.go b/internal/agentdaemon/proto/version.go index fe0f84363..641f511bf 100644 --- a/internal/agentdaemon/proto/version.go +++ b/internal/agentdaemon/proto/version.go @@ -5,7 +5,7 @@ package proto // upgrade query (`version=`) and in the bootstrap HTTP // response; mismatches fail closed at WS upgrade. const ( - Version = "0.3.0" + Version = "0.4.0" ) // VersionCompatible returns true when clientVersion's "X.Y" prefix diff --git a/internal/agentdaemon/proto/workspace_read_preparation.go b/internal/agentdaemon/proto/workspace_read_preparation.go index a76ba47e7..a451ff7b5 100644 --- a/internal/agentdaemon/proto/workspace_read_preparation.go +++ b/internal/agentdaemon/proto/workspace_read_preparation.go @@ -4,9 +4,9 @@ package proto // The native adapter supplies temporary state; this request cannot resume or start. func ValidWorkspaceReadPreparation(r PromptRequestPayload) bool { return r.WorkspaceReadOnly && r.LocalEnvironment != nil && r.AgentStateKey != "" && - r.StrictResume && r.ReleaseOnCompletion && r.RunID == "" && r.Prompt == "" && + r.StrictResume && r.ReleaseOnCompletion && r.RunID == "" && len(r.Input) == 0 && r.ConversationID == "" && r.AgentSessionID == "" && r.WorkDir == "" && - !r.RequireExistingNativeSession && !r.WorkspaceAuthoring && !r.DisableExecutionEnvironment && len(r.Attachments) == 0 && + !r.RequireExistingNativeSession && !r.WorkspaceAuthoring && !r.DisableExecutionEnvironment && len(r.AgentOptions) == 0 && r.ExecutionControls == nil && r.MCPHTTPServers == nil && len(r.FunctionTools) == 0 && !r.ObserveMessages && !r.ObserveTools && !r.ObserveToolObservations && !r.ObserveSubagentIdentities diff --git a/packages/claude-sdk-adapter/src/adapter.ts b/packages/claude-sdk-adapter/src/adapter.ts index f3229f3ef..8579673fb 100644 --- a/packages/claude-sdk-adapter/src/adapter.ts +++ b/packages/claude-sdk-adapter/src/adapter.ts @@ -14,7 +14,7 @@ import { MCPProfile } from "./mcp.js"; import { MCPObserver, type MCPEvent } from "./mcp_observer.js"; import { CommandObserver, type CommandEvent } from "./command_observer.js"; import { WorkspaceProfile } from "./workspace.js"; -import { immediatePrompt, type Prepare, type Start } from "./request.js"; +import { immediateInput, type Prepare, type Start } from "./request.js"; import { recoverSession } from "./recovery.js"; export { parseStart, type Start } from "./request.js"; @@ -33,7 +33,7 @@ export type Event = | { type: "result"; session_id: string; text: string } | { type: "error"; code: "invalid_request" | "history_unavailable" | "execution_failed" | "cancelled" }; -export async function execute(request: Start | Prepare, emit: (event: Event) => Promise, abort: AbortController, functions = new FunctionBridge(emit), inputs = new Inputs(immediatePrompt(request)), reads = new WorkspaceReads(emit, abort), directories = new WorkspaceDirectories(emit, abort)): Promise { +export async function execute(request: Start | Prepare, emit: (event: Event) => Promise, abort: AbortController, functions = new FunctionBridge(emit), inputs = new Inputs(immediateInput(request)), reads = new WorkspaceReads(emit, abort), directories = new WorkspaceDirectories(emit, abort)): Promise { const definitions = (request.functions ?? []).map(tool => ({ name: tool.name, description: tool.description, inputSchema: tool.parameters })); const names = definitions.map(tool => `mcp__functions__${tool.name}`); const declarations = request.workspace?.mcp ?? request.mcp_http_servers; @@ -127,7 +127,7 @@ export async function execute(request: Start | Prepare, emit: (event: Event) => if (initialized.hooks_applied !== true || children.length !== 1) throw new Error("MCP initialization unavailable"); if (request.mcp_http_servers?.some(server => server.required)) profile.verifyRequired(await stream.mcpServerStatus()); if (abort.signal.aborted || !nativeAlive) throw new Error("MCP initialization interrupted"); - inputs.release(request.prompt); + inputs.release(request.input); } else stream = query({ prompt: inputs, options }); for await (const message of stream) { subagents?.consume(message); diff --git a/packages/claude-sdk-adapter/src/inputs.ts b/packages/claude-sdk-adapter/src/inputs.ts index 2e7853d60..4c4d1a1b5 100644 --- a/packages/claude-sdk-adapter/src/inputs.ts +++ b/packages/claude-sdk-adapter/src/inputs.ts @@ -1,10 +1,12 @@ +import { parseMessageInput, nativeContent, type MessageInput } from "./message_input.js"; import type { SDKMessage, SDKUserMessage } from "@anthropic-ai/claude-agent-sdk"; import { randomUUID } from "node:crypto"; export type InputEvent = | { type: "input_ready" | "input_closed"; session_id: string } | { type: "input_applied" | "input_rejected"; input_id: string }; -type Input = { id?: string; applied: boolean; completed: boolean }; +type Batch = { id?: string; unapplied: number }; +type Input = { batch: Batch; applied: boolean; completed: boolean }; // This is an SDK input iterator and receipt ledger, never a model/tool loop. export class Inputs implements AsyncIterable { @@ -15,11 +17,11 @@ export class Inputs implements AsyncIterable { private ended = false; private sessionID = ""; - constructor(prompt?: string) { if (prompt !== undefined) this.release(prompt); } + constructor(prompt?: MessageInput) { if (prompt !== undefined) this.release(prompt); } - release(prompt: string): void { - if (this.ended || this.submitted.size || !prompt.trim()) throw new Error("Invalid initial input."); - this.enqueue(prompt); + release(prompt: MessageInput): void { + if (this.ended || this.submitted.size || prompt.length > 64) throw new Error("Invalid initial input."); + this.enqueue(parseMessageInput(prompt)); } start(sessionID: string): InputEvent[] { @@ -33,23 +35,28 @@ export class Inputs implements AsyncIterable { submit(value: unknown): InputEvent[] { if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Invalid input."); const input = value as Record; - if (input.type !== "steer" || Object.keys(input).some(key => !["type", "input_id", "text"].includes(key)) || - typeof input.input_id !== "string" || !input.input_id.trim() || input.input_id.length > 256 || - typeof input.text !== "string" || !input.text.trim()) throw new Error("Invalid input."); + if (input.type !== "steer" || Object.keys(input).some(key => !["type", "input_id", "input"].includes(key)) || + typeof input.input_id !== "string" || !input.input_id.trim() || input.input_id.length > 256) throw new Error("Invalid input."); + const messages = parseMessageInput(input.input); // The native consumed-UUID list has 64 slots, including the opening prompt. - if (this.ended || !this.sessionID || this.submitted.size >= 64 || this.ids.has(input.input_id)) { + if (this.ended || !this.sessionID || this.submitted.size + messages.length > 64 || this.ids.has(input.input_id)) { return [{ type: "input_rejected", input_id: input.input_id }]; } this.ids.add(input.input_id); - this.enqueue(input.text, input.input_id); + this.enqueue(messages, input.input_id); return []; } - private enqueue(text: string, id?: string): void { - const uuid = randomUUID(); - this.submitted.set(uuid, { id, applied: false, completed: false }); - this.queue.push({ type: "user", uuid, session_id: this.sessionID, parent_tool_use_id: null, - message: { role: "user", content: text } }); + private enqueue(messages: MessageInput, id?: string): void { + // Convert the entire batch before admitting any native input. + const contents = messages.map(nativeContent); + const batch: Batch = { id, unapplied: contents.length }; + for (const content of contents) { + const uuid = randomUUID(); + this.submitted.set(uuid, { batch, applied: false, completed: false }); + this.queue.push({ type: "user", uuid, session_id: this.sessionID, parent_tool_use_id: null, + message: { role: "user", content } }); + } this.wake?.(); } @@ -63,7 +70,9 @@ export class Inputs implements AsyncIterable { if (message.type === "result" && !consumed.length) throw new Error("Unattributed native result."); const receipts: InputEvent[] = []; for (const input of consumed) { - if (!input.applied && input.id) receipts.push({ type: "input_applied", input_id: input.id }); + if (!input.applied && --input.batch.unapplied === 0 && input.batch.id) { + receipts.push({ type: "input_applied", input_id: input.batch.id }); + } input.applied = true; if (message.type === "result") input.completed = true; } diff --git a/packages/claude-sdk-adapter/src/main.ts b/packages/claude-sdk-adapter/src/main.ts index 21b0e69f3..496c9ab0a 100644 --- a/packages/claude-sdk-adapter/src/main.ts +++ b/packages/claude-sdk-adapter/src/main.ts @@ -4,7 +4,7 @@ import { Inputs } from "./inputs.js"; import { FunctionBridge } from "./function_bridge.js"; import { createInterface } from "node:readline"; import { execute, type Event } from "./adapter.js"; -import { immediatePrompt, parseRequest, preparedPrompt } from "./request.js"; +import { immediateInput, parseRequest, preparedInput } from "./request.js"; const abort = new AbortController(); const lines = createInterface({ input: process.stdin, crlfDelay: Infinity }); @@ -29,7 +29,7 @@ try { const functions = new FunctionBridge(output); const reads = new WorkspaceReads(output, abort); const directories = new WorkspaceDirectories(output, abort); - const prompts = new Inputs(immediatePrompt(request)); + const prompts = new Inputs(immediateInput(request)); const incoming = (async () => { try { for await (const line of { [Symbol.asyncIterator]: () => input }) { @@ -41,7 +41,7 @@ try { directories.submit(value as Record); } else if (request.type === "prepare" && phase !== "running") { if (phase !== "prepared" || abort.signal.aborted) throw new Error("invalid_request"); - prompts.release(preparedPrompt(value)); + prompts.release(preparedInput(value)); phase = "running"; } else if (value && typeof value === "object" && "type" in value && value.type === "steer") { for (const event of prompts.submit(value)) await output(event); diff --git a/packages/claude-sdk-adapter/src/message_input.ts b/packages/claude-sdk-adapter/src/message_input.ts new file mode 100644 index 000000000..41fa19b16 --- /dev/null +++ b/packages/claude-sdk-adapter/src/message_input.ts @@ -0,0 +1,40 @@ +import type { SDKUserMessage } from "@anthropic-ai/claude-agent-sdk"; + +export type InputContent = { type: "input_text"; text: string } | { type: "input_image"; image_url: string }; +export type MessageInput = { content: InputContent[] }[]; + +// This is the common Runtime representation; native image blocks stay here. +export function parseMessageInput(value: unknown): MessageInput { + if (!Array.isArray(value) || !value.length) throw new Error("Invalid user messages."); + for (const message of value) { + if (!message || typeof message !== "object" || Object.keys(message).some(key => key !== "content") || + !Array.isArray(message.content) || !message.content.length) throw new Error("Invalid user message."); + let meaningful = false; + for (const part of message.content) { + if (!part || typeof part !== "object") throw new Error("Invalid user content."); + if (part.type === "input_text" && typeof part.text === "string" && Object.keys(part).every(key => key === "type" || key === "text")) { + meaningful ||= Boolean(part.text.trim()); + } else if (part.type === "input_image" && typeof part.image_url === "string" && + Object.keys(part).every(key => key === "type" || key === "image_url")) { + imageSource(part.image_url); + meaningful = true; + } else throw new Error("Invalid user content."); + } + if (!meaningful) throw new Error("Empty user message."); + } + return value as MessageInput; +} + +function imageSource(url: string): { type: "base64"; media_type: "image/png" | "image/jpeg"; data: string } { + const match = /^data:(image\/png|image\/jpeg);base64,([A-Za-z0-9+/]+={0,2})$/.exec(url); + if (!match || Buffer.from(match[2]!, "base64").toString("base64") !== match[2]) throw new Error("Unsupported image reference."); + return { type: "base64", media_type: match[1] as "image/png" | "image/jpeg", data: match[2]! }; +} + +export function nativeContent(message: MessageInput[number]): SDKUserMessage["message"]["content"] { + return message.content.map(part => part.type === "input_text" + ? { type: "text" as const, text: part.text } + : { type: "image" as const, source: imageSource(part.image_url) }); +} + +export function textInput(text: string): MessageInput { return [{ content: [{ type: "input_text", text }] }]; } diff --git a/packages/claude-sdk-adapter/src/request.ts b/packages/claude-sdk-adapter/src/request.ts index 3636db135..9ea894c67 100644 --- a/packages/claude-sdk-adapter/src/request.ts +++ b/packages/claude-sdk-adapter/src/request.ts @@ -1,3 +1,4 @@ +import { parseMessageInput, type MessageInput } from "./message_input.js"; import type { Tool } from "@modelcontextprotocol/sdk/types.js"; import { isAbsolute } from "node:path"; import { parseHTTPServers, type HTTPServer } from "./mcp.js"; @@ -6,7 +7,7 @@ import { parseWorkspace, type Workspace } from "./workspace.js"; export type Start = { type: "start"; output_format?: { type: "json_schema"; schema: Record }; - prompt: string; + input: MessageInput; model: string; system_prompt: string; cwd: string; @@ -18,21 +19,21 @@ export type Start = { mcp_http_servers?: HTTPServer[]; workspace?: Workspace; }; -export type Prepare = Omit & { type: "prepare"; workspace: Workspace }; +export type Prepare = Omit & { type: "prepare"; workspace: Workspace }; // MCP startup confirms its hooks before the native input iterator yields. -export function immediatePrompt(request: Start | Prepare): string | undefined { - return request.type === "start" && request.mcp_http_servers === undefined && !request.workspace?.mcp?.length ? request.prompt : undefined; +export function immediateInput(request: Start | Prepare): MessageInput | undefined { + return request.type === "start" && request.mcp_http_servers === undefined && !request.workspace?.mcp?.length ? request.input : undefined; } export function parseRequest(line: string): Start | Prepare { const value: unknown = JSON.parse(line); if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("invalid_request"); const request = value as Record; - const allowed = new Set(["type", "prompt", "model", "system_prompt", "cwd", "resume", "require_history", "observe_messages", "output_format", "subagents", "functions", "mcp_http_servers", "workspace"]); + const allowed = new Set(["type", "input", "model", "system_prompt", "cwd", "resume", "require_history", "observe_messages", "output_format", "subagents", "functions", "mcp_http_servers", "workspace"]); if (Object.keys(request).some(key => !allowed.has(key)) || (request.type !== "start" && request.type !== "prepare") || - (request.type === "start" ? typeof request.prompt !== "string" || !request.prompt.trim() : "prompt" in request) || + (request.type === "start" ? !Array.isArray(request.input) : "input" in request) || typeof request.model !== "string" || !request.model.trim() || typeof request.system_prompt !== "string" || typeof request.cwd !== "string" || !isAbsolute(request.cwd) || @@ -53,6 +54,7 @@ export function parseRequest(line: string): Start | Prepare { !format.schema || format.schema.type !== "object" || !request.observe_messages || request.subagents || request.workspace || request.mcp_http_servers !== undefined) throw new Error("invalid_request"); } + if (request.type === "start") requestInput(request.input); parseHTTPServers(request.mcp_http_servers); const workspace = parseWorkspace(request.workspace, request.cwd); if (request.subagents && workspace?.mcp?.length) throw new Error("invalid_request"); @@ -68,10 +70,14 @@ export function parseStart(line: string): Start { return request; } -export function preparedPrompt(value: unknown): string { +export function preparedInput(value: unknown): MessageInput { if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("invalid_request"); const request = value as Record; - if (request.type !== "start" || Object.keys(request).some(key => key !== "type" && key !== "prompt") || - typeof request.prompt !== "string" || !request.prompt.trim()) throw new Error("invalid_request"); - return request.prompt; + if (request.type !== "start" || Object.keys(request).some(key => key !== "type" && key !== "input") || + !Array.isArray(request.input)) throw new Error("invalid_request"); + return requestInput(request.input); +} + +function requestInput(value: unknown): MessageInput { + try { return parseMessageInput(value); } catch { throw new Error("invalid_request"); } } diff --git a/packages/claude-sdk-adapter/src/runtime_check.ts b/packages/claude-sdk-adapter/src/runtime_check.ts index 6f969facc..6ad0618ce 100644 --- a/packages/claude-sdk-adapter/src/runtime_check.ts +++ b/packages/claude-sdk-adapter/src/runtime_check.ts @@ -45,7 +45,7 @@ try { assert.equal(smoke.error, undefined, "bridge_unavailable"); assert.equal(smoke.status, 0, "bridge_unavailable"); assert.deepEqual(JSON.parse(smoke.stdout), { type: "error", code: "invalid_request" }); - process.stdout.write(JSON.stringify({ type: "runtime_ready", protocol: 1, features: [...(process.platform === "linux" ? ["workspace_directory", "local_runtime_v1", "workspace_functions"] : []), "structured_output", "subagent_resources", "mcp_http_tools", "mcp_http_bearer_auth", "mcp_http_required", "workspace_tools", "workspace_prepare", "workspace_read", "workspace_command_observations"], node: process.versions.node, sdk: sdk.version, mcp: mcp.version, native: nativeVersion }) + "\n"); + process.stdout.write(JSON.stringify({ type: "runtime_ready", protocol: 1, features: [...(process.platform === "linux" ? ["workspace_directory", "local_runtime_v1", "workspace_functions"] : []), "message_images", "structured_output", "subagent_resources", "mcp_http_tools", "mcp_http_bearer_auth", "mcp_http_required", "workspace_tools", "workspace_prepare", "workspace_read", "workspace_command_observations"], node: process.versions.node, sdk: sdk.version, mcp: mcp.version, native: nativeVersion }) + "\n"); } catch { // Native diagnostics can include operator environment; never forward them. process.stdout.write(JSON.stringify({ type: "runtime_unavailable" }) + "\n"); diff --git a/packages/claude-sdk-adapter/tests/execution.test.mjs b/packages/claude-sdk-adapter/tests/execution.test.mjs index 232512a5e..7662be132 100644 --- a/packages/claude-sdk-adapter/tests/execution.test.mjs +++ b/packages/claude-sdk-adapter/tests/execution.test.mjs @@ -17,13 +17,13 @@ const { Inputs } = await import(${JSON.stringify(new URL("../dist/inputs.js", im const { FunctionBridge } = await import(${JSON.stringify(new URL("../dist/function_bridge.js", import.meta.url).href)}); const mode = process.argv[1]; const events = []; -const inputs = new Inputs("opening text"); +const inputs = new Inputs([{ content: [{ type: "input_text", text: "opening text" }] }]); const abort = new AbortController(); let invocation; let functions; const emit = async event => { events.push(event); - if (event.type === "input_ready") inputs.submit({type:"steer",input_id:"extra",text:"later text"}); + if (event.type === "input_ready") inputs.submit({type:"steer",input_id:"extra",input:[{content:[{type:"input_text",text:"later text"}]}]}); if (mode === "later-function" && event.type === "usage" && !invocation) { // The SDK dispatches MCP controls independently while the output consumer is // yielding the previous native turn's result. This call belongs to the next turn. @@ -56,7 +56,7 @@ globalThis.queryFixture = ({prompt,options}) => { }, }; }; -await execute({type:"start",prompt:"opening text",model:"fixture",system_prompt:"",cwd:process.cwd(), +await execute({type:"start",input: [{ content: [{ type: "input_text", text: "opening text" }] }],model:"fixture",system_prompt:"",cwd:process.cwd(), ...(mode === "later-function" ? {functions:[{name:"lookup",description:"fixture",parameters:{type:"object",properties:{}}}]} : {})},emit,abort,functions,inputs); assert.equal(inputs.complete,true); assert.equal(events.filter(e=>e.type === "input_applied" && e.input_id === "extra").length,1); diff --git a/packages/claude-sdk-adapter/tests/inputs.test.mjs b/packages/claude-sdk-adapter/tests/inputs.test.mjs index aea52251d..f228cc7e7 100644 --- a/packages/claude-sdk-adapter/tests/inputs.test.mjs +++ b/packages/claude-sdk-adapter/tests/inputs.test.mjs @@ -3,10 +3,10 @@ import test from "node:test"; import { Inputs } from "../dist/inputs.js"; const result = (ids, session_id = "native") => ({ type: "result", session_id, user_message_uuids: ids }); -const steer = (input_id, text = "additional text") => ({ type: "steer", input_id, text }); +const steer = (input_id, text = "additional text") => ({ type: "steer", input_id, input: [{ content: [{ type: "input_text", text }] }] }); test("queued input survives the first result and completes only with its own native result", async () => { - const inputs = new Inputs("opening text"); + const inputs = new Inputs([{ content: [{ type: "input_text", text: "opening text" }] }]); const stream = inputs[Symbol.asyncIterator](); const first = (await stream.next()).value; assert.deepEqual(inputs.start("native"), [{ type: "input_ready", session_id: "native" }]); @@ -18,7 +18,7 @@ test("queued input survives the first result and completes only with its own nat assert.throws(() => inputs.start("different"), /identity/); const second = (await stream.next()).value; assert.equal(second.session_id, "native"); - assert.equal(second.message.content, "additional text"); + assert.equal(second.message.content[0].text, "additional text"); assert.notEqual(second.uuid, first.uuid); assert.deepEqual(inputs.consume({ type: "assistant", parent_tool_use_id: null, session_id: "native", user_message_uuid: second.uuid }), [{ type: "input_applied", input_id: "second" }]); assert.equal(inputs.complete, false); @@ -29,7 +29,7 @@ test("queued input survives the first result and completes only with its own nat }); test("native folds confirm all consumed UUIDs, never queue/user echoes or unrelated frames", async () => { - const inputs = new Inputs("first"); + const inputs = new Inputs([{ content: [{ type: "input_text", text: "first" }] }]); const stream = inputs[Symbol.asyncIterator](); const first = (await stream.next()).value; inputs.start("native"); inputs.submit(steer("fold")); @@ -51,7 +51,7 @@ test("native folds confirm all consumed UUIDs, never queue/user echoes or unrela }); test("reject before readiness, repeated identities and native receipt capacity without enqueueing", async () => { - const inputs = new Inputs("first"); + const inputs = new Inputs([{ content: [{ type: "input_text", text: "first" }] }]); assert.deepEqual(inputs.submit(steer("early")), [{ type: "input_rejected", input_id: "early" }]); inputs.start("native"); for (let i = 0; i < 63; i++) assert.deepEqual(inputs.submit(steer(String(i))), []); @@ -63,3 +63,52 @@ test("reject before readiness, repeated identities and native receipt capacity w assert.equal(count, 64); assert.equal(inputs.complete, false); }); + +test("ordered image batches keep message boundaries and aggregate native consumption", async () => { + const { textInput } = await import("../dist/message_input.js"); + const inputs = new Inputs(textInput("opening")); + const stream = inputs[Symbol.asyncIterator](); + const first = (await stream.next()).value; + inputs.start("native"); + const input = [{ content: [ + { type: "input_text", text: " before " }, + { type: "input_image", image_url: "data:image/png;base64,aW1hZ2U=" }, + { type: "input_text", text: " after " }, + ] }, ...textInput("second message")]; + assert.deepEqual(inputs.submit({ type: "steer", input_id: "batch", input }), []); + const a = (await stream.next()).value, b = (await stream.next()).value; + assert.deepEqual(a.message.content, [ + { type: "text", text: " before " }, + { type: "image", source: { type: "base64", media_type: "image/png", data: "aW1hZ2U=" } }, + { type: "text", text: " after " }, + ]); + assert.deepEqual(b.message.content, [{ type: "text", text: "second message" }]); + assert.deepEqual(inputs.consume(result([first.uuid, a.uuid])), []); + assert.equal(inputs.complete, false); + assert.deepEqual(inputs.consume({ type: "assistant", parent_tool_use_id: null, session_id: "native", user_message_uuids: [a.uuid] }), []); + assert.deepEqual(inputs.consume(result([b.uuid])), [ + { type: "input_closed", session_id: "native" }, { type: "input_applied", input_id: "batch" }, + ]); +}); + +test("native UUID capacity rejects a whole batch and partial consumption does not apply it", async () => { + const { textInput } = await import("../dist/message_input.js"); + const inputs = new Inputs(Array.from({ length: 63 }, () => textInput("opening")[0])); + inputs.start("native"); + const pair = [...textInput("a"), ...textInput("b")]; + assert.deepEqual(inputs.submit({ type: "steer", input_id: "overflow", input: pair }), [{ type: "input_rejected", input_id: "overflow" }]); + assert.deepEqual(inputs.submit(steer("last")), []); + inputs.close(); + const queued = []; for await (const message of inputs) queued.push(message); + assert.equal(queued.length, 64); + assert.throws(() => new Inputs([...pair, ...Array.from({ length: 63 }, () => textInput("x")[0])]), /Invalid/); + + const partial = new Inputs(textInput("opening")); + const stream = partial[Symbol.asyncIterator](); + await stream.next(); partial.start("native"); + partial.submit({ type: "steer", input_id: "partial", input: pair }); + const first = (await stream.next()).value; + assert.deepEqual(partial.consume(result([first.uuid])), []); + partial.close(); + assert.equal(partial.complete, false); +}); diff --git a/packages/claude-sdk-adapter/tests/mcp.test.mjs b/packages/claude-sdk-adapter/tests/mcp.test.mjs index 279b189d5..70bc102e2 100644 --- a/packages/claude-sdk-adapter/tests/mcp.test.mjs +++ b/packages/claude-sdk-adapter/tests/mcp.test.mjs @@ -26,7 +26,7 @@ test("native selection composes unrestricted, selected and empty servers with ho }); test("invalid remote declarations and wildcard injection fail at the bridge boundary", () => { - const start = { type: "start", prompt: "hello", model: "model", system_prompt: "", cwd: "/tmp" }; + const start = { type: "start", input: [{ content: [{ type: "input_text", text: "hello" }] }], model: "model", system_prompt: "", cwd: "/tmp" }; for (const value of [null, {}, [declaration(["*"])], [{ ...declaration(null), server_label: "functions" }], [{ ...declaration(null), server_url: "https://user:secret@example.invalid/mcp" }], [{ ...declaration(null), server_url: "https://example.invalid/mcp?" }], diff --git a/packages/claude-sdk-adapter/tests/mcp_bearer.test.mjs b/packages/claude-sdk-adapter/tests/mcp_bearer.test.mjs index e5d65034f..6f32eaf81 100644 --- a/packages/claude-sdk-adapter/tests/mcp_bearer.test.mjs +++ b/packages/claude-sdk-adapter/tests/mcp_bearer.test.mjs @@ -5,7 +5,7 @@ import { MCPProfile } from "../dist/mcp.js"; const reference = "PARSAR_MCP_BEARER_" + "A".repeat(26); const server = { server_label: "private", server_url: "https://example.invalid/mcp", allowed_tools: ["echo.v1"], bearer_token_env_var: reference }; -const start = servers => ({ type: "start", prompt: "hello", model: "fixture", system_prompt: "", cwd: "/tmp", mcp_http_servers: servers }); +const start = servers => ({ type: "start", input: [{ content: [{ type: "input_text", text: "hello" }] }], model: "fixture", system_prompt: "", cwd: "/tmp", mcp_http_servers: servers }); test("bearer references remain literal in native configuration and private status is not retained", t => { const token = "fixture-private-bearer+/=="; diff --git a/packages/claude-sdk-adapter/tests/mcp_required.test.mjs b/packages/claude-sdk-adapter/tests/mcp_required.test.mjs index 8bf2842df..a57fa00f7 100644 --- a/packages/claude-sdk-adapter/tests/mcp_required.test.mjs +++ b/packages/claude-sdk-adapter/tests/mcp_required.test.mjs @@ -36,7 +36,7 @@ function create(options){ }, async *[Symbol.asyncIterator](){ const first=await pending;if(first.done)return; - assert.ok(readiness||mode==="optional");process.send({kind:"input",text:first.value.message.content}); + assert.ok(readiness||mode==="optional");process.send({kind:"input",text:first.value.message.content[0].text}); yield {type:"system",subtype:"init",session_id:mode==="wrong-history"?"foreign":"native",tools:["mcp__fixture__echo"],mcp_servers:[]}; yield {type:"result",uuid:"result",session_id:"native",user_message_uuids:[first.value.uuid],subtype:"success",is_error:false,result:"done",usage:{input_tokens:1,output_tokens:1},modelUsage:{}}; } @@ -61,7 +61,7 @@ for (const mode of ["connected", "pending", "failed", "missing", "duplicate", "m const closed = new Promise(resolve => child.once("close", (code, signal) => resolve({ code, signal }))); const timer = setTimeout(() => child.kill("SIGKILL"), 8000); try { - child.stdin.write(JSON.stringify({ type: "start", model: "fixed", prompt: "one input", system_prompt: "", cwd, + child.stdin.write(JSON.stringify({ type: "start", model: "fixed", input: [{ content: [{ type: "input_text", text: "one input" }] }], system_prompt: "", cwd, ...(mode.includes("history") || mode === "resume" ? { resume: "native" } : {}), mcp_http_servers: [{ server_label: "fixture", server_url: "https://example.invalid/mcp", allowed_tools: ["echo"], required: mode !== "optional" }, { server_label: "optional", server_url: "https://optional.invalid/mcp", allowed_tools: [], required: false }] }) + "\n"); diff --git a/packages/claude-sdk-adapter/tests/mcp_workspace.test.mjs b/packages/claude-sdk-adapter/tests/mcp_workspace.test.mjs index 6a23d60dd..03eaac824 100644 --- a/packages/claude-sdk-adapter/tests/mcp_workspace.test.mjs +++ b/packages/claude-sdk-adapter/tests/mcp_workspace.test.mjs @@ -5,7 +5,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { MCPProfile } from "../dist/mcp.js"; import { parseEnvironmentMCP } from "../dist/mcp_environment.js"; -import { immediatePrompt, parseStart } from "../dist/request.js"; +import { immediateInput, parseStart } from "../dist/request.js"; import { WorkspaceProfile } from "../dist/workspace.js"; const stdio = { server_label: "installed", command: "/usr/bin/python3", allowed_tools: null, @@ -24,14 +24,14 @@ function fixture(t, declarations = [stdio]) { const previous = process.env; process.env = { HOME: dirs.home, CLAUDE_CONFIG_DIR: dirs.state, ANTHROPIC_AUTH_TOKEN: "model-secret" }; t.after(() => { process.env = previous; rmSync(root, { recursive: true, force: true }); }); - const request = { type: "start", prompt: "fixture", model: "fixture", system_prompt: "", cwd: dirs.work, workspace: config }; + const request = { type: "start", input: [{ content: [{ type: "input_text", text: "fixture" }] }], model: "fixture", system_prompt: "", cwd: dirs.work, workspace: config }; return { dirs, config, request }; } test("installed MCP private projection cannot launch arbitrary unsandboxed commands", t => { const { request } = fixture(t); assert.deepEqual(parseStart(JSON.stringify(request)), request); - assert.equal(immediatePrompt(request), undefined); + assert.equal(immediateInput(request), undefined); assert.deepEqual(parseEnvironmentMCP([stdio]), [stdio]); for (const value of [[stdio, stdio], [{ ...stdio, command: "/bin/sh" }], [{ ...stdio, env: { TOKEN: "secret" } }], [{ ...stdio, args: ["-c", "untrusted"] }], [{ ...stdio, allowed_tools: ["*"] }], diff --git a/packages/claude-sdk-adapter/tests/messages.test.mjs b/packages/claude-sdk-adapter/tests/messages.test.mjs index c8b3fb5b3..35d5aaaf0 100644 --- a/packages/claude-sdk-adapter/tests/messages.test.mjs +++ b/packages/claude-sdk-adapter/tests/messages.test.mjs @@ -93,7 +93,7 @@ test("unmatched text cannot acquire an invented identity", () => { }); test("observation opt-in is an optional boolean", () => { - const request = { type: "start", prompt: "hello", model: "model", system_prompt: "", cwd: "/tmp" }; + const request = { type: "start", input: [{ content: [{ type: "input_text", text: "hello" }] }], model: "model", system_prompt: "", cwd: "/tmp" }; assert.equal(parseStart(JSON.stringify(request)).observe_messages, undefined); assert.equal(parseStart(JSON.stringify({ ...request, observe_messages: true })).observe_messages, true); assert.throws(() => parseStart(JSON.stringify({ ...request, observe_messages: "true" })), /invalid_request/); diff --git a/packages/claude-sdk-adapter/tests/preparation.test.mjs b/packages/claude-sdk-adapter/tests/preparation.test.mjs index 7de7c4847..a22ebb7f2 100644 --- a/packages/claude-sdk-adapter/tests/preparation.test.mjs +++ b/packages/claude-sdk-adapter/tests/preparation.test.mjs @@ -4,7 +4,7 @@ import { mkdtempSync, mkdirSync, realpathSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; -import { parseRequest, preparedPrompt } from "../dist/request.js"; +import { parseRequest, preparedInput } from "../dist/request.js"; // Exercise the packaged entrypoint and real child ownership with a controlled SDK. // Native dependency enforcement and model effects need separate native acceptance. @@ -55,7 +55,7 @@ globalThis.startupFixture = async ({options, initializeTimeoutMs}) => { const iterator=prompt[Symbol.asyncIterator](); const first=await Promise.race([iterator.next(),closed.then(()=>({done:true}))]); if(first.done)return; - process.send({kind:"input",text:first.value.message.content}); + process.send({kind:"input",text:first.value.message.content[0].text}); yield {type:"system",subtype:"init",session_id:mode === "resume-mismatch"?"other":"native",mcp_servers:[], tools:mode === "bad-inventory"?["Bash","Read","Edit","Agent"]:["Bash","Read","Edit"]}; const ids=[first.value.uuid]; @@ -96,14 +96,14 @@ test("prepare validates a workspace-only immutable configuration and prompt-only try { assert.deepEqual(parseRequest(JSON.stringify(request)), request); assert.deepEqual(parseRequest(JSON.stringify({ ...request, functions: [] })), { ...request, functions: [] }); - for (const fields of [{ prompt: "" }, { prompt: "input" }, { workspace: undefined }, + for (const fields of [{ input: [{ content: [{ type: "input_text", text: "" }] }] }, { input: [{ content: [{ type: "input_text", text: "input" }] }] }, { workspace: undefined }, { mcp_http_servers: [] }, { env: {} }, { resume: "" }, { type: "prepared" }]) { assert.throws(() => parseRequest(JSON.stringify({ ...request, ...fields })), /invalid_request/); } - assert.equal(preparedPrompt({ type: "start", prompt: "first" }), "first"); - for (const value of [{ type: "start", prompt: "" }, { type: "start", prompt: "first", model: "other" }, - { type: "start", prompt: "first", resume: "other" }, { type: "start", prompt: "first", workspace: request.workspace }, - { type: "steer", text: "first" }, null]) assert.throws(() => preparedPrompt(value), /invalid_request/); + assert.deepEqual(preparedInput({ type: "start", input: [{ content: [{ type: "input_text", text: "first" }] }] }), [{ content: [{ type: "input_text", text: "first" }] }]); + for (const value of [{ type: "start", input: [{ content: [{ type: "input_text", text: "" }] }] }, { type: "start", input: [{ content: [{ type: "input_text", text: "first" }] }], model: "other" }, + { type: "start", input: [{ content: [{ type: "input_text", text: "first" }] }], resume: "other" }, { type: "start", input: [{ content: [{ type: "input_text", text: "first" }] }], workspace: request.workspace }, + { type: "steer", text: "first" }, null]) assert.throws(() => preparedInput(value), /invalid_request/); } finally { rmSync(root, { recursive: true, force: true }); } }); @@ -162,17 +162,17 @@ for (const mode of ["release", "resume", "steer", "commands", "unused", "owner-c if (mode === "unused") { child.stdin.end(); await finish("cancelled"); } else if (mode === "owner-cancel") { child.kill("SIGTERM"); await finish("cancelled"); } else if (mode === "native-exit") await finish("execution_failed"); - else if (mode === "replacement") { send({ type: "start", prompt: "first", model: "changed" }); await finish("invalid_request"); } - else if (mode === "early-steer") { send({ type: "steer", input_id: "early", text: "first" }); await finish("invalid_request"); } + else if (mode === "replacement") { send({ type: "start", input: [{ content: [{ type: "input_text", text: "first" }] }], model: "changed" }); await finish("invalid_request"); } + else if (mode === "early-steer") { send({ type: "steer", input_id: "early", input: [{ content: [{ type: "input_text", text: "first" }] }] }); await finish("invalid_request"); } else { - send({ type: "start", prompt: "first" }); - if (mode === "duplicate") { send({ type: "start", prompt: "second" }); await finish("invalid_request"); } + send({ type: "start", input: [{ content: [{ type: "input_text", text: "first" }] }] }); + if (mode === "duplicate") { send({ type: "start", input: [{ content: [{ type: "input_text", text: "second" }] }] }); await finish("invalid_request"); } else if (["bad-inventory", "resume-mismatch"].includes(mode)) { await finish("execution_failed"); assert.equal(events.some(event => event.type === "input_ready"), false); } else { await wait(() => events.some(event => event.type === "input_ready")); - if (mode === "steer") send({ type: "steer", input_id: "extra", text: "second" }); + if (mode === "steer") send({ type: "steer", input_id: "extra", input: [{ content: [{ type: "input_text", text: "second" }] }] }); await finish(); assert.deepEqual(observations.filter(value => value.kind === "input"), [{ kind: "input", text: "first" }]); if (mode === "steer") assert.ok(events.some(event => event.type === "input_applied" && event.input_id === "extra")); @@ -197,7 +197,7 @@ for (const mode of ["missing-history", "missing-hooks", "startup-error", "early- test(`preparation rejects before receipt: ${mode}`, { timeout: 10000 }, async t => { const { child, request, events, observations, send, wait, finish } = await launch(t, mode === "cancel-startup" ? "slow-startup" : mode); send(request); - if (mode === "early-start") send({ type: "start", prompt: "too early" }); + if (mode === "early-start") send({ type: "start", input: [{ content: [{ type: "input_text", text: "too early" }] }] }); if (mode === "cancel-startup") { await wait(() => observations.some(value => value.kind === "spawn")); child.kill("SIGTERM"); } await finish(mode === "missing-history" ? "history_unavailable" : mode === "early-start" ? "invalid_request" : mode === "cancel-startup" ? "cancelled" : "execution_failed"); assert.equal(events.some(event => event.type === "prepared"), false); @@ -212,7 +212,7 @@ test("prepared native reader stays on the same query across Start", { timeout: 1 send(request); await wait(() => events.some(event => event.type === "prepared")); send({type:"workspace_read",id:"before",path:"binary",max_bytes:4}); await wait(()=>observations.some(value=>value.kind === "read")); - send({type:"start",prompt:"first"}); + send({type:"start",input: [{ content: [{ type: "input_text", text: "first" }] }]}); await wait(()=>events.some(event=>event.type === "workspace_read")); assert.deepEqual(events.find(event=>event.type === "workspace_read"),{type:"workspace_read",id:"before",data_base64:"AP+AAQ==",truncated:false}); await finish(); diff --git a/packages/claude-sdk-adapter/tests/structured_output.test.mjs b/packages/claude-sdk-adapter/tests/structured_output.test.mjs index 4d6d355af..878c9fcd9 100644 --- a/packages/claude-sdk-adapter/tests/structured_output.test.mjs +++ b/packages/claude-sdk-adapter/tests/structured_output.test.mjs @@ -22,7 +22,7 @@ test('unrelated or failed results cannot publish a candidate',()=>{ } }); test('output configuration is restricted to the qualified profile',()=>{ - const r={type:'start',prompt:'hello',model:'model',system_prompt:'',cwd:'/tmp',observe_messages:true,output_format:{type:'json_schema',schema:{type:'object'}}}; + const r={type:'start',input: [{ content: [{ type: "input_text", text: 'hello' }] }],model:'model',system_prompt:'',cwd:'/tmp',observe_messages:true,output_format:{type:'json_schema',schema:{type:'object'}}}; assert.deepEqual(parseStart(JSON.stringify(r)),r); for(const change of [{observe_messages:false},{subagents:{max_concurrent:1}},{mcp_http_servers:[]},{output_format:{type:'text',schema:{}}}]) assert.throws(()=>parseStart(JSON.stringify({...r,...change}))); }); diff --git a/packages/claude-sdk-adapter/tests/workspace.test.mjs b/packages/claude-sdk-adapter/tests/workspace.test.mjs index 57f6bf7ef..12e68c4cd 100644 --- a/packages/claude-sdk-adapter/tests/workspace.test.mjs +++ b/packages/claude-sdk-adapter/tests/workspace.test.mjs @@ -15,7 +15,7 @@ function fixture(t) { })); const config = { home: dirs.home, state: dirs.state, scratch: dirs.scratch, protected_dirs: [dirs.protected], dependency_path: dirs.deps, env_names: ["ANTHROPIC_API_KEY", "HTTP_PROXY"] }; - const request = { type: "start", prompt: "fixture", model: "fixture", system_prompt: "", cwd: dirs.workspace, workspace: config }; + const request = { type: "start", input: [{ content: [{ type: "input_text", text: "fixture" }] }], model: "fixture", system_prompt: "", cwd: dirs.workspace, workspace: config }; const previous = process.env; process.env = { HOME: dirs.home, CLAUDE_CONFIG_DIR: dirs.state, ANTHROPIC_API_KEY: "fixture-secret", HTTP_PROXY: "http://fixture-proxy", UNSELECTED_CANARY: "must-not-inherit", NODE_OPTIONS: "unsafe", diff --git a/packages/claude-sdk-adapter/tests/workspace_execution.test.mjs b/packages/claude-sdk-adapter/tests/workspace_execution.test.mjs index 329ed7102..1acab9e8a 100644 --- a/packages/claude-sdk-adapter/tests/workspace_execution.test.mjs +++ b/packages/claude-sdk-adapter/tests/workspace_execution.test.mjs @@ -21,7 +21,7 @@ const dirs = Object.fromEntries(["workspace", "home", "state", "scratch", "deps" const mode = process.argv[1]; const workspace = { home: dirs.home, state: dirs.state, scratch: dirs.scratch, protected_dirs: [], dependency_path: dirs.deps, env_names: ["ANTHROPIC_API_KEY"] }; -const request = { type: "start", prompt: "fixture", model: "fixture", system_prompt: "", cwd: dirs.workspace, +const request = { type: "start", input: [{ content: [{ type: "input_text", text: "fixture" }] }], model: "fixture", system_prompt: "", cwd: dirs.workspace, workspace, ...(mode.startsWith("resume") ? { resume: "native" } : {}) }; process.env.HOME = dirs.home; process.env.CLAUDE_CONFIG_DIR = dirs.state; diff --git a/services/agents-api/internal/api/function_inputs.go b/services/agents-api/internal/api/function_inputs.go index e330ea15c..3d2749932 100644 --- a/services/agents-api/internal/api/function_inputs.go +++ b/services/agents-api/internal/api/function_inputs.go @@ -23,6 +23,19 @@ func decodeInputEvent(raw json.RawMessage) (decodedInputEvent, error) { switch event.Type { case "agent.session.input.message": fields = append(fields, "input") + var messages struct { + Input []struct { + Content json.RawMessage `json:"content"` + } `json:"input"` + } + if json.Unmarshal(raw, &messages) != nil { + return event, store.ErrInvalidInput + } + for _, message := range messages.Input { + if err := validateInputContent(message.Content); err != nil { + return event, err + } + } case "agent.session.input.cancel": case "agent.session.input.tool_result": fields = append(fields, "call_id", "turn_id", "success", "error", "output") @@ -83,6 +96,10 @@ func validateFunctionOutput(raw json.RawMessage) error { if json.Unmarshal(raw, &text) == nil { return nil } + return validateInputContent(raw) +} + +func validateInputContent(raw json.RawMessage) error { var parts []json.RawMessage if json.Unmarshal(raw, &parts) != nil { return store.ErrInvalidInput diff --git a/services/agents-api/internal/api/handler.go b/services/agents-api/internal/api/handler.go index 4fa78bdbb..cc1d3577b 100644 --- a/services/agents-api/internal/api/handler.go +++ b/services/agents-api/internal/api/handler.go @@ -124,7 +124,7 @@ func NewHandler(s ResourceStore, auth *Authenticator, engine string, options ... // createSession atomically reserves or admits initial text with the Session. // @Summary Create an execution Session -// @Description Supports inline configuration or a tenant-owned saved agent_id with per-Session field replacements. Execution supports model/instructions, text verbosity, non-deferred function tools, adapter-qualified multi_agent with persisted Subagent reads, implicit reasoning, service tier auto and environment type none, subject to the configured engine. Codex additionally supports HTTP MCP with explicit service origin, native allowed_tools and boolean required defaulting to false. Session vault_ids attach only project-owned Vaults; credential_id selects an attached static bearer credential for the exact HTTPS URL, while null/omission selects a unique match or remains anonymous. Ambiguous selection rejects creation. Frozen private selections never populate an omitted public credential_id; missing decryption configuration fails dispatch without anonymous fallback. Required initialization uses native startup before the first native Turn, including cold resume, and requires a separately advertised capability; exact hosted creation timing and error parity remain unverified. Other MCP origins and OAuth remain unsupported. The self_hosted profile requires Codex, an absolute workspace_directory and empty capability_directories, with optional non-deferred function tools and HTTP MCP using explicit service origin, optionally authenticated by the attached Vault rules. Remote MCP and remote Bearer authentication each require separately advertised combination support; old peers cannot receive unsupported work. Omitted/null capability_directories use the empty-list default; self_hosted requires configured execution plus executor registry. Claude SDK currently requires medium verbosity and object-root function schemas. It supports anonymous or attached static-bearer service-origin HTTP MCP on none with boolean required and separately advertised MCP/bearer/required runtime support. Required servers must be connected before the first native input is released; pending or failed startup rejects execution. The shared Vault selection and immutable binding rules apply; unsupported native labels/tool names reject before persistence. An attached Vault with no matching credential may remain anonymous; missing keys or failed credential lookup/decryption never fall back to anonymous execution. Omitted stream defaults to false; stream and agent_id cannot be null. Metadata may be null, but its values must be strings. Initial input accepts a string or user-message array containing text. None initial input atomically starts a Turn; self_hosted initial input is reserved while returning its Environment connection target, with execution deferred to native readiness and Session failure on initial timeout. Omitted or null input creates an idle Session. With stream=true, returns live Session events starting at creation; disconnect does not cancel execution. New Sessions retain their authenticated creator; all creation retries require the same typed subject, including across key rotation. Saved-Agent retries and inline requests using Vault attachments or credential references retain caller intent independently of later resource changes; unrelated inline retries preserve resolved/default equivalences. Unknown historical creators reject retries; known creators without recorded intent retain resolved-snapshot retry rules. These conflict policies are local and not verified hosted parity. Creation retries observe future events without replay; retry with stream=false to retrieve the Session. Claude SDK environment:none supports qualified object-root json_schema output with medium verbosity, single-Agent execution and ordinary functions; other combinations remain unsupported. Non-text initial input remains unsupported. Basic Codex and Claude SDK openai_hosted creation requires an explicitly configured managed provider. The Claude workspace profile supports non-deferred function tools with text results alongside native workspace tools; HTTP MCP remains unsupported. Idle Sessions provision automatically; initial provisioning has no caller connection action. Network defaults to enabled; disabled and restricted exact ASCII hostnames are supported. Restricted policy requires 1–100 allowed domains. Unsupported hostname forms and startup installations are rejected. Confidential env, system/npm/Python packages and ordered setup commands use the shared initialization lifecycle; requested network applies after setup. Initial inline and tenant-owned file_id files freeze encrypted bytes before provisioning, then install through the common Core lifecycle before native execution or live Files access. Referenced files/env/packages/setup overrides are rejected pending semantic verification. Tenant-owned environment_template_id references inherit omitted network and allow only narrowing overrides. Referenced network:null is explicitly unsupported pending semantic verification. Core freezes effective configuration; template updates/deletion do not alter Session snapshots or same-intent creation retries. Inline or tenant-owned skill_reference Skills share initialization. Templates preserve default/latest/explicit selectors; Session creation freezes concrete metadata and encrypted content atomically. Skill-list omission inherits and a supplied list replaces; null overrides and null version selectors remain unqualified and reject. Source deletion/default updates cannot change committed Session Skill contents. +// @Description Supports inline configuration or a tenant-owned saved agent_id with per-Session field replacements. Execution supports model/instructions, text verbosity, non-deferred function tools, adapter-qualified multi_agent with persisted Subagent reads, implicit reasoning, service tier auto and environment type none, subject to the configured engine. Codex additionally supports HTTP MCP with explicit service origin, native allowed_tools and boolean required defaulting to false. Session vault_ids attach only project-owned Vaults; credential_id selects an attached static bearer credential for the exact HTTPS URL, while null/omission selects a unique match or remains anonymous. Ambiguous selection rejects creation. Frozen private selections never populate an omitted public credential_id; missing decryption configuration fails dispatch without anonymous fallback. Required initialization uses native startup before the first native Turn, including cold resume, and requires a separately advertised capability; exact hosted creation timing and error parity remain unverified. Other MCP origins and OAuth remain unsupported. The self_hosted profile requires Codex, an absolute workspace_directory and empty capability_directories, with optional non-deferred function tools and HTTP MCP using explicit service origin, optionally authenticated by the attached Vault rules. Remote MCP and remote Bearer authentication each require separately advertised combination support; old peers cannot receive unsupported work. Omitted/null capability_directories use the empty-list default; self_hosted requires configured execution plus executor registry. Claude SDK currently requires medium verbosity and object-root function schemas. It supports anonymous or attached static-bearer service-origin HTTP MCP on none with boolean required and separately advertised MCP/bearer/required runtime support. Required servers must be connected before the first native input is released; pending or failed startup rejects execution. The shared Vault selection and immutable binding rules apply; unsupported native labels/tool names reject before persistence. An attached Vault with no matching credential may remain anonymous; missing keys or failed credential lookup/decryption never fall back to anonymous execution. Omitted stream defaults to false; stream and agent_id cannot be null. Metadata may be null, but its values must be strings. Initial input accepts a string or ordered user-message array. Codex and Claude SDK on none also accept inline PNG/JPEG image content; other image combinations and remote URLs are unsupported. None initial input atomically starts a Turn; self_hosted initial input is reserved while returning its Environment connection target, with execution deferred to native readiness and Session failure on initial timeout. Omitted or null input creates an idle Session. With stream=true, returns live Session events starting at creation; disconnect does not cancel execution. New Sessions retain their authenticated creator; all creation retries require the same typed subject, including across key rotation. Saved-Agent retries and inline requests using Vault attachments or credential references retain caller intent independently of later resource changes; unrelated inline retries preserve resolved/default equivalences. Unknown historical creators reject retries; known creators without recorded intent retain resolved-snapshot retry rules. These conflict policies are local and not verified hosted parity. Creation retries observe future events without replay; retry with stream=false to retrieve the Session. Claude SDK environment:none supports qualified object-root json_schema output with medium verbosity, single-Agent execution and ordinary functions; other combinations remain unsupported. Non-text initial input remains unsupported. Basic Codex and Claude SDK openai_hosted creation requires an explicitly configured managed provider. The Claude workspace profile supports non-deferred function tools with text results alongside native workspace tools; HTTP MCP remains unsupported. Idle Sessions provision automatically; initial provisioning has no caller connection action. Network defaults to enabled; disabled and restricted exact ASCII hostnames are supported. Restricted policy requires 1–100 allowed domains. Unsupported hostname forms and startup installations are rejected. Confidential env, system/npm/Python packages and ordered setup commands use the shared initialization lifecycle; requested network applies after setup. Initial inline and tenant-owned file_id files freeze encrypted bytes before provisioning, then install through the common Core lifecycle before native execution or live Files access. Referenced files/env/packages/setup overrides are rejected pending semantic verification. Tenant-owned environment_template_id references inherit omitted network and allow only narrowing overrides. Referenced network:null is explicitly unsupported pending semantic verification. Core freezes effective configuration; template updates/deletion do not alter Session snapshots or same-intent creation retries. Inline or tenant-owned skill_reference Skills share initialization. Templates preserve default/latest/explicit selectors; Session creation freezes concrete metadata and encrypted content atomically. Skill-list omission inherits and a supplied list replaces; null overrides and null version selectors remain unqualified and reject. Source deletion/default updates cannot change committed Session Skill contents. // @Tags Sessions // @Accept json // @Produce json,text/event-stream diff --git a/services/agents-api/internal/api/inputs.go b/services/agents-api/internal/api/inputs.go index 6685f232c..ca4ab3bf9 100644 --- a/services/agents-api/internal/api/inputs.go +++ b/services/agents-api/internal/api/inputs.go @@ -4,9 +4,9 @@ import ( "context" "encoding/json" "errors" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" "io" "net/http" - "strings" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" "github.com/go-chi/chi/v5" @@ -24,7 +24,7 @@ type Option func(*Handler) func WithExecution(s InputSubmitter) Option { return func(h *Handler) { h.inputs = s } } // @Summary Submit Session input events -// @Description For environment none, atomically accepts text messages, cancellation and function results. Messages steer active work or start a queued Turn. The supported self_hosted and openai_hosted profiles accept text-only batches. Under the Session lock, matching retries retain their original target; new active messages append to the current Turn, while idle messages reserve work and wait up to the original five-minute connection/admission deadline. Return 204 only after durable admission, without claiming native application; active messages create no Turn or reservation. Cancellation-only prepared-environment batches use existing durable cancellation admission and return 204 without waiting for native exit; a new cancellation conflicts while a pre-Turn reservation is pending. Homogeneous tool_result-only prepared-environment batches reuse existing scoped result admission and application receipts without creating a Turn or bypassing a pending reservation. Mixed prepared-environment batches remain unsupported. HTTP expiry/cancellation use local 409 environment_input_expired/environment_input_cancelled errors; exact hosted failure mapping is unverified. Losing execution ownership returns 503. The response write deadline accommodates the admission window for either prepared Environment, independently of new-hosted-admission and executor URL settings. Disconnecting the waiting HTTP request does not cancel retained work or restart its deadline. Retry keys identify the whole ordered batch. Function output accepts text or ordered text/image parts subject to engine support; Claude SDK currently accepts text results only. Message images are not supported yet. +// @Description For environment none, atomically accepts text messages, cancellation and function results. Messages steer active work or start a queued Turn. The supported self_hosted and openai_hosted profiles accept text-only batches. Under the Session lock, matching retries retain their original target; new active messages append to the current Turn, while idle messages reserve work and wait up to the original five-minute connection/admission deadline. Return 204 only after durable admission, without claiming native application; active messages create no Turn or reservation. Cancellation-only prepared-environment batches use existing durable cancellation admission and return 204 without waiting for native exit; a new cancellation conflicts while a pre-Turn reservation is pending. Homogeneous tool_result-only prepared-environment batches reuse existing scoped result admission and application receipts without creating a Turn or bypassing a pending reservation. Mixed prepared-environment batches remain unsupported. HTTP expiry/cancellation use local 409 environment_input_expired/environment_input_cancelled errors; exact hosted failure mapping is unverified. Losing execution ownership returns 503. The response write deadline accommodates the admission window for either prepared Environment, independently of new-hosted-admission and executor URL settings. Disconnecting the waiting HTTP request does not cancel retained work or restart its deadline. Retry keys identify the whole ordered batch. Function output accepts text or ordered text/image parts subject to engine support; Claude SDK currently accepts text results only. Codex and Claude SDK on none accept ordered inline PNG/JPEG image messages. Workspace profiles and other engines remain text-only; remote image URLs are unsupported. Image references are retained unchanged without service-side downloads. // @Tags Sessions // @Accept json // @Security BearerAuth @@ -114,14 +114,11 @@ func executionInputs(events []json.RawMessage) ([]store.Input, error) { if message.Role != "user" || (message.Type != "" && message.Type != "message") || len(message.Content) == 0 { return nil, store.ErrInvalidInput } - var text strings.Builder + converted := proto.MessageInput{{}} for _, content := range message.Content { - if content.Type != "input_text" { - return nil, store.ErrInvalidInput - } - text.WriteString(content.Text) + converted[0].Content = append(converted[0].Content, proto.InputContent{Type: content.Type, Text: content.Text, ImageURL: content.ImageURL}) } - if strings.TrimSpace(text.String()) == "" { + if converted.Validate() != nil || converted.ValidateInlineImages() != nil { return nil, store.ErrInvalidInput } } diff --git a/services/agents-api/internal/api/session_initial_input.go b/services/agents-api/internal/api/session_initial_input.go index de70a65c4..5f5af4cd6 100644 --- a/services/agents-api/internal/api/session_initial_input.go +++ b/services/agents-api/internal/api/session_initial_input.go @@ -18,7 +18,7 @@ func initialSessionInputs(raw json.RawMessage) ([]store.Input, error) { if err := json.Unmarshal(raw, &text); err != nil { return nil, store.ErrInvalidInput } - raw, _ = json.Marshal([]v1.InputMessage{{Role: "user", Content: []v1.InputContent{{Type: "input_text", Text: text}}}}) + raw, _ = json.Marshal([]v1.InputMessage{{Role: "user", Content: []v1.InputContent{{Type: "input_text", Text: &text}}}}) } // Preserve the array's original fields for the shared strict message decoder. event, err := json.Marshal(struct { diff --git a/services/agents-api/internal/engine/claude.go b/services/agents-api/internal/engine/claude.go index e6421b2ef..7143c5d87 100644 --- a/services/agents-api/internal/engine/claude.go +++ b/services/agents-api/internal/engine/claude.go @@ -11,11 +11,12 @@ import ( func claudeProfile() Profile { return Profile{ - StructuredOutput: true, - Placements: []string{"none", "openai_hosted", "self_hosted"}, MCPBearer: true, + StructuredOutput: true, + MessageImagePlacements: []string{"none"}, + Placements: []string{"none", "openai_hosted", "self_hosted"}, MCPBearer: true, ValidateConfiguration: validateClaudeConfiguration, ValidateTools: validateClaudeTools, - ValidateFunctionResult: func(content []proto.FunctionResultContent) error { + ValidateFunctionResult: func(content []proto.InputContent) error { for _, part := range content { if part.Type != "input_text" { return ErrInvalidInput diff --git a/services/agents-api/internal/engine/codex.go b/services/agents-api/internal/engine/codex.go index 24728ec2e..48a1e7212 100644 --- a/services/agents-api/internal/engine/codex.go +++ b/services/agents-api/internal/engine/codex.go @@ -9,8 +9,9 @@ import ( func codexProfile() Profile { return Profile{ - Placements: []string{"none", "self_hosted", "openai_hosted"}, - WebSearchControl: true, TextVerbosity: true, MCPBearer: true, + Placements: []string{"none", "self_hosted", "openai_hosted"}, + MessageImagePlacements: []string{"none"}, + WebSearchControl: true, TextVerbosity: true, MCPBearer: true, ValidateConfiguration: func(agent v1.Agent, _ *v1.Environment, _ bool) error { return rejectSubagentTools(agent, "function", "mcp") }, diff --git a/services/agents-api/internal/engine/profile.go b/services/agents-api/internal/engine/profile.go index 12f4a80ab..8f19929aa 100644 --- a/services/agents-api/internal/engine/profile.go +++ b/services/agents-api/internal/engine/profile.go @@ -15,9 +15,10 @@ type Profile struct { Placements []string WebSearchControl, TextVerbosity, MCPBearer bool StructuredOutput bool + MessageImagePlacements []string ValidateConfiguration func(agent v1.Agent, environment *v1.Environment, hasDaemon bool) error ValidateTools func(environment *v1.Environment, hasDaemon bool, functions []proto.FunctionTool, mcp []proto.MCPHTTPServer) error - ValidateFunctionResult func([]proto.FunctionResultContent) error + ValidateFunctionResult func([]proto.InputContent) error } func (p Profile) Accepts(placement string) bool { @@ -34,6 +35,7 @@ func NewCatalog(profiles map[string]Profile) Catalog { c := Catalog{profiles: make(map[string]Profile, len(profiles))} for kind, profile := range profiles { profile.Placements = slices.Clone(profile.Placements) + profile.MessageImagePlacements = slices.Clone(profile.MessageImagePlacements) c.profiles[kind] = profile } return c @@ -45,6 +47,7 @@ func (c Catalog) Lookup(kind string) (Profile, bool) { } profile, ok := c.profiles[kind] profile.Placements = slices.Clone(profile.Placements) + profile.MessageImagePlacements = slices.Clone(profile.MessageImagePlacements) return profile, ok } diff --git a/services/agents-api/internal/execution/delivery.go b/services/agents-api/internal/execution/delivery.go index 9dde76a62..27e7ea111 100644 --- a/services/agents-api/internal/execution/delivery.go +++ b/services/agents-api/internal/execution/delivery.go @@ -13,7 +13,7 @@ import ( type pendingInput struct { sequence int64 - text string + input proto.MessageInput started time.Time waiting bool written bool @@ -268,19 +268,23 @@ func (d *Dispatcher) deliver(ctx context.Context, tenantID, sessionID string, pe if inputs[0].Kind == "cancel" { continue } - text, err := messageText(inputs[0].Payload) + text, err := messageInput(inputs[0].Payload) if err != nil || inputs[0].Kind != "message" { result.ErrorCode = "invalid_input" return } - pending = &pendingInput{sequence: inputs[0].Sequence, text: text, started: time.Now()} + pending = &pendingInput{sequence: inputs[0].Sequence, input: text, started: time.Now()} } if !pending.written && time.Since(pending.started) > 30*time.Second { result.ErrorCode = "input_outcome_unknown" return } if !pending.waiting && !pending.written { - if send(ctx, peer, proto.TypePromptSteer, request.RunID, proto.PromptSteerPayload{InputID: strconv.FormatInt(pending.sequence, 10), Text: pending.text, DurableReceipt: true}) != nil { + if requireMessageImages(peer, request.AgentKind, pending.input) != nil { + result.ErrorCode = "message_input_unsupported" + return + } + if send(ctx, peer, proto.TypePromptSteer, request.RunID, proto.PromptSteerPayload{InputID: strconv.FormatInt(pending.sequence, 10), Input: pending.input, DurableReceipt: true}) != nil { result.ErrorCode = "input_outcome_unknown" return } diff --git a/services/agents-api/internal/execution/dispatcher.go b/services/agents-api/internal/execution/dispatcher.go index 8b13673c2..e2427eee6 100644 --- a/services/agents-api/internal/execution/dispatcher.go +++ b/services/agents-api/internal/execution/dispatcher.go @@ -75,6 +75,9 @@ func (d *Dispatcher) Run(ctx context.Context, tenantID, sessionID, turnID string if err != nil { return store.Turn{}, err } + if err := d.messageInputSupport(peer, session.Engine, snapshot, text); err != nil { + return store.Turn{}, err + } req, err := d.executionRequest(ctx, session, snapshot, caps, bound) if err != nil { return store.Turn{}, err @@ -82,7 +85,7 @@ func (d *Dispatcher) Run(ctx context.Context, tenantID, sessionID, turnID string if _, err := d.Store.TransitionTurn(ctx, tenantID, sessionID, turnID, store.TurnTransition{ExpectedStatus: store.TurnQueued, Status: store.TurnInProgress}); err != nil { return store.Turn{}, err } - req.ConversationID, req.RunID, req.Prompt = sessionID, turnID, text + req.ConversationID, req.RunID, req.Input = sessionID, turnID, text req.WorkDir, req.DisableExecutionEnvironment = workDir, noEnvironment result, status := d.deliver(ctx, tenantID, sessionID, peer, req, through, nil) return d.finishRun(tenantID, sessionID, turnID, snapshot.Agent.Model, result, status) diff --git a/services/agents-api/internal/execution/engine_profile.go b/services/agents-api/internal/execution/engine_profile.go index a9c07b13e..604301a1f 100644 --- a/services/agents-api/internal/execution/engine_profile.go +++ b/services/agents-api/internal/execution/engine_profile.go @@ -37,12 +37,19 @@ func validateProfileConfiguration(profile engine.Profile, snapshot Snapshot) err return err } -func validateProfileInputs(profile engine.Profile, inputs []store.Input) error { - if profile.ValidateFunctionResult == nil { - return nil - } +func validateProfileInputs(profile engine.Profile, placement string, inputs []store.Input) error { for _, input := range inputs { - if input.Kind != "tool_result" { + if input.Kind == "message" { + messages, err := messageInput(input.Payload) + if err != nil { + return err + } + if err := validateMessageImageProfile(profile, placement, messages); err != nil { + return err + } + continue + } + if input.Kind != "tool_result" || profile.ValidateFunctionResult == nil { continue } var value store.FunctionResultInput diff --git a/services/agents-api/internal/execution/engine_profile_test.go b/services/agents-api/internal/execution/engine_profile_test.go index 7416907d3..139cb840d 100644 --- a/services/agents-api/internal/execution/engine_profile_test.go +++ b/services/agents-api/internal/execution/engine_profile_test.go @@ -45,7 +45,7 @@ func TestAdditionalProfileUsesCommonAdmission(t *testing.T) { } return nil }, - ValidateFunctionResult: func(content []proto.FunctionResultContent) error { + ValidateFunctionResult: func(content []proto.InputContent) error { resultChecked = true if len(content) != 1 || content[0].Text == nil || *content[0].Text != "response" { t.Fatal("common result decoding did not reach profile") @@ -66,7 +66,7 @@ func TestAdditionalProfileUsesCommonAdmission(t *testing.T) { t.Fatal("profile configuration lost public error mapping", err) } inputs := []store.Input{{Kind: "tool_result", Payload: json.RawMessage(`{"call_id":"call","result":{"success":true,"output":"response"}}`)}} - if err := validateProfileInputs(profile, inputs); !errors.Is(err, store.ErrInvalidInput) || !resultChecked { + if err := validateProfileInputs(profile, "none", inputs); !errors.Is(err, store.ErrInvalidInput) || !resultChecked { t.Fatal("profile result lost public error mapping", err) } } diff --git a/services/agents-api/internal/execution/environment_admission.go b/services/agents-api/internal/execution/environment_admission.go index 5196ba936..f818c3551 100644 --- a/services/agents-api/internal/execution/environment_admission.go +++ b/services/agents-api/internal/execution/environment_admission.go @@ -43,6 +43,9 @@ func (w *Worker) validateEnvironmentAdmission(engine string, configuration json. } func (w *Worker) validateCreation(ctx context.Context, input store.CreateSessionInput) error { + if err := w.dispatcher.validateEngineInputs(input.Engine, input.Configuration, input.InitialInputs); err != nil { + return err + } if preparedEnvironmentConfiguration(input.Configuration) { if err := w.validateEnvironmentAdmission(input.Engine, input.Configuration); err != nil { return err diff --git a/services/agents-api/internal/execution/functions.go b/services/agents-api/internal/execution/functions.go index ca0e0baa5..8c8e5dbb8 100644 --- a/services/agents-api/internal/execution/functions.go +++ b/services/agents-api/internal/execution/functions.go @@ -160,7 +160,7 @@ func functionResult(call store.FunctionCall) (proto.FunctionResultPayload, error if value.Success == nil { return proto.FunctionResultPayload{}, errors.New("function result requires success") } - result := proto.FunctionResultPayload{DeliveryID: "function:" + call.CallID, CallID: call.ExecutorCallID, Success: *value.Success, Content: []proto.FunctionResultContent{}} + result := proto.FunctionResultPayload{DeliveryID: "function:" + call.CallID, CallID: call.ExecutorCallID, Success: *value.Success, Content: []proto.InputContent{}} output := bytes.TrimSpace(value.Output) if len(output) > 0 && !bytes.Equal(output, []byte("null")) { if output[0] == '"' { @@ -168,13 +168,13 @@ func functionResult(call store.FunctionCall) (proto.FunctionResultPayload, error if err := json.Unmarshal(output, &text); err != nil { return result, err } - result.Content = append(result.Content, proto.FunctionResultContent{Type: "input_text", Text: &text}) + result.Content = append(result.Content, proto.InputContent{Type: "input_text", Text: &text}) } else if err := json.Unmarshal(output, &result.Content); err != nil { return result, err } } if value.Error != nil { - result.Content = append(result.Content, proto.FunctionResultContent{Type: "input_text", Text: value.Error}) + result.Content = append(result.Content, proto.InputContent{Type: "input_text", Text: value.Error}) } return result, result.ValidateContent() } diff --git a/services/agents-api/internal/execution/functions_test.go b/services/agents-api/internal/execution/functions_test.go index fa15b17a1..f16805ffc 100644 --- a/services/agents-api/internal/execution/functions_test.go +++ b/services/agents-api/internal/execution/functions_test.go @@ -26,19 +26,19 @@ func TestFunctionDefinitionsRejectUnsupportedConfiguration(t *testing.T) { } func TestFunctionResultPreservesCompleteContent(t *testing.T) { - text := func(value string) proto.FunctionResultContent { - return proto.FunctionResultContent{Type: "input_text", Text: &value} + text := func(value string) proto.InputContent { + return proto.InputContent{Type: "input_text", Text: &value} } imageURL := "data:image/png;base64,test" for _, test := range []struct { raw string success bool - content []proto.FunctionResultContent + content []proto.InputContent }{ - {`{"success":true,"output":""}`, true, []proto.FunctionResultContent{text("")}}, - {`{"success":true,"output":null,"error":null}`, true, []proto.FunctionResultContent{}}, - {`{"success":false,"error":"failed"}`, false, []proto.FunctionResultContent{text("failed")}}, - {`{"success":false,"output":[{"type":"input_text","text":"before"},{"type":"input_image","image_url":"data:image/png;base64,test"},{"type":"input_text","text":""}],"error":"failed"}`, false, []proto.FunctionResultContent{text("before"), {Type: "input_image", ImageURL: &imageURL}, text(""), text("failed")}}, + {`{"success":true,"output":""}`, true, []proto.InputContent{text("")}}, + {`{"success":true,"output":null,"error":null}`, true, []proto.InputContent{}}, + {`{"success":false,"error":"failed"}`, false, []proto.InputContent{text("failed")}}, + {`{"success":false,"output":[{"type":"input_text","text":"before"},{"type":"input_image","image_url":"data:image/png;base64,test"},{"type":"input_text","text":""}],"error":"failed"}`, false, []proto.InputContent{text("before"), {Type: "input_image", ImageURL: &imageURL}, text(""), text("failed")}}, } { result, err := functionResult(store.FunctionCall{CallID: "public", ExecutorCallID: "native", Result: json.RawMessage(test.raw)}) if err != nil || result.CallID != "native" || result.DeliveryID != "function:public" || result.Success != test.success || !reflect.DeepEqual(result.Content, test.content) { diff --git a/services/agents-api/internal/execution/input_text.go b/services/agents-api/internal/execution/input_text.go deleted file mode 100644 index 51b58e5a3..000000000 --- a/services/agents-api/internal/execution/input_text.go +++ /dev/null @@ -1,73 +0,0 @@ -package execution - -import ( - "context" - "encoding/json" - "strings" - - v1 "github.com/MiniMax-AI-Dev/parsar/contracts/agents-api/v1" - "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" -) - -func messageText(raw json.RawMessage) (string, error) { - var input struct { - Text string `json:"text"` - Input []v1.InputMessage `json:"input"` - } - if json.Unmarshal(raw, &input) != nil { - return "", store.ErrInvalidInput - } - if len(input.Input) == 0 { - if strings.TrimSpace(input.Text) == "" { - return "", store.ErrInvalidInput - } - return input.Text, nil - } - messages := make([]string, 0, len(input.Input)) - for _, message := range input.Input { - if message.Role != "user" { - return "", store.ErrInvalidInput - } - var text strings.Builder - for _, content := range message.Content { - if content.Type != "input_text" { - return "", store.ErrInvalidInput - } - text.WriteString(content.Text) - } - if strings.TrimSpace(text.String()) == "" { - return "", store.ErrInvalidInput - } - messages = append(messages, text.String()) - } - return strings.Join(messages, "\n\n"), nil -} - -func (d *Dispatcher) initialInput(ctx context.Context, tenant, session, turn string) (string, int64, error) { - inputs, err := d.Store.ListTurnInputs(ctx, tenant, session, turn, 0, 100) - if err != nil { - return "", 0, err - } - var texts []string - var through int64 - size := 0 - for _, input := range inputs { - if input.Kind != "message" { - return "", 0, store.ErrInvalidInput - } - text, err := messageText(input.Payload) - if err != nil { - return "", 0, err - } - if size+len(text) > 512*1024 && len(texts) > 0 { - break - } - texts = append(texts, text) - size += len(text) - through = input.Sequence - } - if len(texts) == 0 { - return "", 0, store.ErrInvalidInput - } - return strings.Join(texts, "\n\n"), through, nil -} diff --git a/services/agents-api/internal/execution/message_input.go b/services/agents-api/internal/execution/message_input.go new file mode 100644 index 000000000..4f3e746b2 --- /dev/null +++ b/services/agents-api/internal/execution/message_input.go @@ -0,0 +1,66 @@ +package execution + +import ( + "context" + "encoding/json" + v1 "github.com/MiniMax-AI-Dev/parsar/contracts/agents-api/v1" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" +) + +func messageInput(raw json.RawMessage) (proto.MessageInput, error) { + var input struct { + Text *string `json:"text"` + Input []v1.InputMessage `json:"input"` + } + if json.Unmarshal(raw, &input) != nil { + return nil, store.ErrInvalidInput + } + var messages proto.MessageInput + if len(input.Input) == 0 && input.Text != nil { + messages = proto.TextInput(*input.Text) + } + for _, message := range input.Input { + if message.Role != "user" { + return nil, store.ErrInvalidInput + } + converted := proto.InputMessage{} + for _, part := range message.Content { + converted.Content = append(converted.Content, proto.InputContent{Type: part.Type, Text: part.Text, ImageURL: part.ImageURL}) + } + messages = append(messages, converted) + } + if messages.Validate() != nil { + return nil, store.ErrInvalidInput + } + return messages, nil +} + +func (d *Dispatcher) initialInput(ctx context.Context, tenant, session, turn string) (proto.MessageInput, int64, error) { + inputs, err := d.Store.ListTurnInputs(ctx, tenant, session, turn, 0, 100) + if err != nil { + return nil, 0, err + } + var messages proto.MessageInput + var through int64 + size := 0 + for _, input := range inputs { + if input.Kind != "message" { + return nil, 0, store.ErrInvalidInput + } + batch, err := messageInput(input.Payload) + if err != nil { + return nil, 0, err + } + if size+len(input.Payload) > 512*1024 && len(messages) > 0 { + break + } + messages = append(messages, batch...) + size += len(input.Payload) + through = input.Sequence + } + if len(messages) == 0 { + return nil, 0, store.ErrInvalidInput + } + return messages, through, nil +} diff --git a/services/agents-api/internal/execution/message_input_test.go b/services/agents-api/internal/execution/message_input_test.go new file mode 100644 index 000000000..b7c130d47 --- /dev/null +++ b/services/agents-api/internal/execution/message_input_test.go @@ -0,0 +1,16 @@ +package execution + +import ( + "encoding/json" + "testing" +) + +func TestMessageInputRetainsPublicBoundariesAndImages(t *testing.T) { + input, err := messageInput(json.RawMessage(`{"input":[{"role":"user","content":[{"type":"input_text","text":" before "},{"type":"input_image","image_url":"data:image/png;base64,aW1hZ2U="},{"type":"input_text","text":" after "}]},{"role":"user","content":[{"type":"input_text","text":"next"}]}]}`)) + if err != nil { + t.Fatal(err) + } + if len(input) != 2 || len(input[0].Content) != 3 || *input[0].Content[0].Text != " before " || *input[0].Content[1].ImageURL != "data:image/png;base64,aW1hZ2U=" || *input[0].Content[2].Text != " after " || *input[1].Content[0].Text != "next" { + t.Fatalf("message input changed: %+v", input) + } +} diff --git a/services/agents-api/internal/execution/message_support.go b/services/agents-api/internal/execution/message_support.go new file mode 100644 index 000000000..62c999b17 --- /dev/null +++ b/services/agents-api/internal/execution/message_support.go @@ -0,0 +1,51 @@ +package execution + +import ( + "errors" + "slices" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/engine" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" +) + +func validateMessageImageProfile(profile engine.Profile, placement string, input proto.MessageInput) error { + if !input.HasImages() { + return nil + } + if !slices.Contains(profile.MessageImagePlacements, placement) || input.ValidateInlineImages() != nil { + return store.ErrInvalidInput + } + return nil +} + +// Image support is checked only for the operation that actually carries images. +func (p Policy) messageInputSupport(peer *gateway.Session, kind string, snapshot Snapshot, input proto.MessageInput) error { + if !input.HasImages() { + return nil + } + profile, ok := p.Engines.Lookup(kind) + if !ok { + return store.ErrInvalidInput + } + placement := "" + if snapshot.Environment != nil { + placement = snapshot.Environment.Type + } + if err := validateMessageImageProfile(profile, placement, input); err != nil { + return err + } + return requireMessageImages(peer, kind, input) +} + +func requireMessageImages(peer *gateway.Session, kind string, input proto.MessageInput) error { + if !input.HasImages() { + return nil + } + info, found, known := peer.AgentKindStatus(kind) + if !found || !known || !info.Available || !info.Capabilities.MessageImages { + return errors.New("Runtime does not support message images") + } + return nil +} diff --git a/services/agents-api/internal/execution/message_support_test.go b/services/agents-api/internal/execution/message_support_test.go new file mode 100644 index 000000000..882f7c4b8 --- /dev/null +++ b/services/agents-api/internal/execution/message_support_test.go @@ -0,0 +1,42 @@ +package execution + +import ( + "encoding/json" + "errors" + "testing" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/engine" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" +) + +func TestMessageImageQualificationIsOperationSpecific(t *testing.T) { + url := "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aXioAAAAASUVORK5CYII=" + input := proto.MessageInput{{Content: []proto.InputContent{{Type: "input_image", ImageURL: &url}}}} + profile := engine.Profile{MessageImagePlacements: []string{"none"}} + if err := validateMessageImageProfile(profile, "none", input); err != nil { + t.Fatal(err) + } + if err := validateMessageImageProfile(profile, "self_hosted", input); !errors.Is(err, store.ErrInvalidInput) { + t.Fatal("unqualified placement accepted", err) + } + if err := validateMessageImageProfile(engine.Profile{}, "none", input); !errors.Is(err, store.ErrInvalidInput) { + t.Fatal("unqualified profile accepted", err) + } + // Text admission and dispatch must not gain an online/image requirement. + if err := (Policy{}).messageInputSupport(nil, "unknown", Snapshot{}, proto.TextInput("queued text")); err != nil { + t.Fatal(err) + } + if err := requireMessageImages(nil, "unknown", proto.TextInput("active text")); err != nil { + t.Fatal(err) + } + // Message validation applies even when no function-result validator exists. + raw, _ := json.Marshal(map[string]any{"input": []any{map[string]any{"role": "user", "content": input[0].Content}}}) + batch := []store.Input{{Kind: "message", Payload: json.RawMessage(`{"text":"valid first"}`)}, {Kind: "message", Payload: raw}} + if err := validateProfileInputs(engine.Profile{}, "none", batch); !errors.Is(err, store.ErrInvalidInput) { + t.Fatal("image escaped profile validation", err) + } + if err := validateProfileInputs(profile, "none", batch); err != nil { + t.Fatal("qualified image rejected", err) + } +} diff --git a/services/agents-api/internal/execution/preparation.go b/services/agents-api/internal/execution/preparation.go index 0181bb6ef..8f1706d3d 100644 --- a/services/agents-api/internal/execution/preparation.go +++ b/services/agents-api/internal/execution/preparation.go @@ -96,7 +96,7 @@ func (d *Dispatcher) awaitPreparation(ctx context.Context, tenant, session strin } func (p *preparedStart) start(ctx context.Context, request proto.PromptRequestPayload) error { - return send(ctx, p.peer, proto.TypeExecutionStart, p.requestID, proto.ExecutionStartPayload{Handle: p.handle, RunID: request.RunID, Prompt: request.Prompt}) + return send(ctx, p.peer, proto.TypeExecutionStart, p.requestID, proto.ExecutionStartPayload{Handle: p.handle, RunID: request.RunID, Input: request.Input}) } func (p *preparedStart) started(env proto.Envelope, runID string) (bool, error) { diff --git a/services/agents-api/internal/execution/prepared_dispatch.go b/services/agents-api/internal/execution/prepared_dispatch.go index 107f5d524..ed12c8295 100644 --- a/services/agents-api/internal/execution/prepared_dispatch.go +++ b/services/agents-api/internal/execution/prepared_dispatch.go @@ -53,16 +53,19 @@ func (d *Dispatcher) RunEnvironmentInput(ctx context.Context, tenantID, sessionI if err != nil { return run, err } - var messages []string + var messages proto.MessageInput for _, input := range run.Reservation.Inputs { if input.Kind != "message" { return run, store.ErrInvalidInput } - text, err := messageText(input.Payload) + text, err := messageInput(input.Payload) if err != nil { return run, err } - messages = append(messages, text) + messages = append(messages, text...) + } + if err := d.messageInputSupport(peer, session.Engine, snapshot, messages); err != nil { + return run, err } if err := d.configurePreparedEnvironment(session, environment, bound.Device, &req); err != nil { return run, err @@ -79,6 +82,9 @@ func (d *Dispatcher) RunEnvironmentInput(ctx context.Context, tenantID, sessionI if err != nil || run.Reservation.State != store.EnvironmentInputPending { return run, err } + if err := d.messageInputSupport(peer, session.Engine, snapshot, messages); err != nil { + return run, err + } run.Reservation, err = d.Store.PromoteEnvironmentInput(owner, tenantID, sessionID, reservationID) if err != nil || run.Reservation.State != store.EnvironmentInputAdmitted { return run, err @@ -87,7 +93,7 @@ func (d *Dispatcher) RunEnvironmentInput(ctx context.Context, tenantID, sessionI return run, nil } req.RunID = run.Reservation.Receipts[0].TurnID - req.Prompt = strings.Join(messages, "\n\n") + req.Input = messages through := run.Reservation.Receipts[len(run.Reservation.Receipts)-1].Sequence result, status := d.deliver(owner, tenantID, sessionID, peer, req, through, prepared) result, status = d.captureCompletedArtifacts(owner, peer, session, environment, bound.Device, req.RunID, result, status) diff --git a/services/agents-api/internal/execution/support.go b/services/agents-api/internal/execution/support.go index 5c187333d..5338a18d3 100644 --- a/services/agents-api/internal/execution/support.go +++ b/services/agents-api/internal/execution/support.go @@ -51,12 +51,20 @@ func (p Policy) canAdmitInputs(engine string, configuration json.RawMessage) boo return p.ValidateSessionConfiguration(engine, configuration) == nil } -func (p Policy) validateEngineInputs(engine string, inputs []store.Input) error { +func (p Policy) validateEngineInputs(engine string, configuration json.RawMessage, inputs []store.Input) error { profile, ok := p.Engines.Lookup(engine) if !ok { return store.ErrInvalidInput } - return validateProfileInputs(profile, inputs) + var snapshot Snapshot + if json.Unmarshal(configuration, &snapshot) != nil { + return store.ErrInvalidInput + } + placement := "" + if snapshot.Environment != nil { + placement = snapshot.Environment.Type + } + return validateProfileInputs(profile, placement, inputs) } // engineCapabilities is shared by device selection and the final preclaim check. diff --git a/services/agents-api/internal/execution/worker.go b/services/agents-api/internal/execution/worker.go index 98054f7f2..903ecf92e 100644 --- a/services/agents-api/internal/execution/worker.go +++ b/services/agents-api/internal/execution/worker.go @@ -62,15 +62,15 @@ func (w *Worker) SubmitInputs(ctx context.Context, tenant, session, key string, if err != nil { return nil, err } + if err := w.dispatcher.validateEngineInputs(value.Engine, value.Configuration, inputs); err != nil { + return nil, err + } if preparedEnvironmentConfiguration(value.Configuration) { return w.submitEnvironmentInputs(ctx, value, key, inputs) } if !w.dispatcher.canAdmitInputs(value.Engine, value.Configuration) { return nil, store.ErrInvalidInput } - if err := w.dispatcher.validateEngineInputs(value.Engine, inputs); err != nil { - return nil, err - } return w.admission.SubmitInputs(ctx, tenant, session, key, inputs) } diff --git a/services/agents-api/internal/execution/worker_device.go b/services/agents-api/internal/execution/worker_device.go index c71627880..87ede5b5d 100644 --- a/services/agents-api/internal/execution/worker_device.go +++ b/services/agents-api/internal/execution/worker_device.go @@ -4,12 +4,17 @@ import ( "context" "encoding/json" "errors" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" ) func (w *Worker) bind(ctx context.Context, item store.ExecutionWork) (bool, error) { - ready, err := w.bindDevice(ctx, item.TenantID, item.SessionID) + input, _, err := w.dispatcher.initialInput(ctx, item.TenantID, item.SessionID, item.TurnID) + if err != nil { + return false, err + } + ready, err := w.bindDevice(ctx, item.TenantID, item.SessionID, input) if !errors.Is(err, store.ErrDeviceBindingConflict) { return ready, err } @@ -20,7 +25,7 @@ func (w *Worker) bind(ctx context.Context, item store.ExecutionWork) (bool, erro return false, err } -func (w *Worker) bindDevice(ctx context.Context, tenantID, sessionID string) (bool, error) { +func (w *Worker) bindDevice(ctx context.Context, tenantID, sessionID string, input proto.MessageInput) (bool, error) { session, err := w.dispatcher.Store.GetSession(ctx, tenantID, sessionID) if errors.Is(err, store.ErrNotFound) { return false, nil @@ -32,7 +37,16 @@ func (w *Worker) bindDevice(ctx context.Context, tenantID, sessionID string) (bo if err := json.Unmarshal(session.Configuration, &snapshot); err != nil { return false, err } - return w.bindSessionDevice(ctx, session, func(id string) bool { return w.ready(ctx, id, session.Engine, snapshot) }) + return w.bindSessionDevice(ctx, session, func(id string) bool { + if !w.ready(ctx, id, session.Engine, snapshot) { + return false + } + if !input.HasImages() { + return true + } + peer, err := w.dispatcher.authorizedPeer(ctx, id) + return err == nil && w.dispatcher.messageInputSupport(peer, session.Engine, snapshot, input) == nil + }) } func (w *Worker) bindSessionDevice(ctx context.Context, session store.Session, ready func(string) bool) (bool, error) { diff --git a/services/agents-api/internal/execution/worker_schedule.go b/services/agents-api/internal/execution/worker_schedule.go index 05717a432..fce848474 100644 --- a/services/agents-api/internal/execution/worker_schedule.go +++ b/services/agents-api/internal/execution/worker_schedule.go @@ -61,7 +61,7 @@ func (s *workerSchedule) selectWork(ctx context.Context, w *Worker, devices []st if item.reservationID == "" { ready, err = w.bind(ctx, item.ExecutionWork) } else { - ready, err = w.bindDevice(ctx, item.TenantID, item.SessionID) + ready, err = w.bindDevice(ctx, item.TenantID, item.SessionID, nil) if errors.Is(err, store.ErrNotFound) || errors.Is(err, store.ErrDeviceBindingConflict) { continue } diff --git a/services/agents-api/internal/store/dispatch_test.go b/services/agents-api/internal/store/dispatch_test.go index 99480b023..259712a6f 100644 --- a/services/agents-api/internal/store/dispatch_test.go +++ b/services/agents-api/internal/store/dispatch_test.go @@ -178,7 +178,7 @@ func TestExecutionDispatchSteeringAndNativeContinuity(t *testing.T) { request := h.read(proto.TypePromptRequest) var prompt proto.PromptRequestPayload _ = request.DecodePayload(&prompt) - if prompt.Prompt != "Initial input" || prompt.ConversationID != h.session.ID || prompt.AgentOptions["model"] != "test-model" || prompt.AgentOptions["system_prompt"] != "Keep this instruction." { + if inputTextForTest(t, prompt.Input) != "Initial input" || prompt.ConversationID != h.session.ID || prompt.AgentOptions["model"] != "test-model" || prompt.AgentOptions["system_prompt"] != "Keep this instruction." { t.Fatalf("wrong resolved request: %+v", prompt) } if _, err := h.d.Run(ctx, uuid.NewString(), h.session.ID, first.TurnID); !errors.Is(err, store.ErrNotFound) { @@ -191,8 +191,8 @@ func TestExecutionDispatchSteeringAndNativeContinuity(t *testing.T) { steer := h.read(proto.TypePromptSteer) var input proto.PromptSteerPayload _ = steer.DecodePayload(&input) - if input.Text != "Follow-up input" { - t.Fatal(input.Text) + if inputTextForTest(t, input.Input) != "Follow-up input" { + t.Fatal(inputTextForTest(t, input.Input)) } h.write(first.TurnID, proto.TypePromptSteerAck, proto.PromptSteerAckPayload{InputID: input.InputID, ErrorCode: "not_ready"}) retry := h.read(proto.TypePromptSteer) diff --git a/services/agents-api/internal/store/environment_admission_test.go b/services/agents-api/internal/store/environment_admission_test.go index 4ad137bd2..70ae826e2 100644 --- a/services/agents-api/internal/store/environment_admission_test.go +++ b/services/agents-api/internal/store/environment_admission_test.go @@ -147,7 +147,7 @@ func TestEnvironmentAdmissionWaitsForPreparedClaimAndRetainsRetry(t *testing.T) for index, receipt := range steered { frame := h.read(proto.TypePromptSteer) var steer proto.PromptSteerPayload - if err := frame.DecodePayload(&steer); err != nil || steer.InputID != strconv.FormatInt(receipt.Sequence, 10) || steer.Text != []string{"first", "second"}[index] { + if err := frame.DecodePayload(&steer); err != nil || steer.InputID != strconv.FormatInt(receipt.Sequence, 10) || inputTextForTest(t, steer.Input) != []string{"first", "second"}[index] { t.Fatal("active delivery changed order or identity", steer, err) } h.write(start.RunID, proto.TypePromptSteerAck, proto.PromptSteerAckPayload{InputID: steer.InputID, Accepted: true}) diff --git a/services/agents-api/internal/store/environment_worker_test.go b/services/agents-api/internal/store/environment_worker_test.go index 19b0c4baf..7b872a49f 100644 --- a/services/agents-api/internal/store/environment_worker_test.go +++ b/services/agents-api/internal/store/environment_worker_test.go @@ -58,7 +58,7 @@ func TestWorkerEnvironmentSharesCapacityThroughClaimAndCleanup(t *testing.T) { firstRuntime.write(first.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 2, State: "ready"}) frame := nextWorkerFrame(t, frames, proto.TypeExecutionStart) var start proto.ExecutionStartPayload - if frame.ID != first.ID || frame.DecodePayload(&start) != nil || start.Handle != handle || start.Prompt != "first" { + if frame.ID != first.ID || frame.DecodePayload(&start) != nil || start.Handle != handle || inputTextForTest(t, start.Input) != "first" { t.Fatal("worker changed preparation at Start") } firstRuntime.write(first.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 3, State: "started", RunID: start.RunID}) diff --git a/services/agents-api/internal/store/function_input_execution_test.go b/services/agents-api/internal/store/function_input_execution_test.go index 4db5ee718..99e763047 100644 --- a/services/agents-api/internal/store/function_input_execution_test.go +++ b/services/agents-api/internal/store/function_input_execution_test.go @@ -43,7 +43,7 @@ func TestExecutionFunctionInputBatchStillSteersMessages(t *testing.T) { h.write(input.TurnID, proto.TypeInteractionDecisionAck, proto.InteractionDecisionAckPayload{DeliveryID: result.DeliveryID, Applied: true}) case proto.TypePromptSteer: var steer proto.PromptSteerPayload - if env.DecodePayload(&steer) != nil || messageSeen || steer.Text != "Follow up" || steer.InputID != strconv.FormatInt(receipts[1].Sequence, 10) { + if env.DecodePayload(&steer) != nil || messageSeen || inputTextForTest(t, steer.Input) != "Follow up" || steer.InputID != strconv.FormatInt(receipts[1].Sequence, 10) { t.Fatal(steer) } messageSeen = true diff --git a/services/agents-api/internal/store/local_environment_worker_test.go b/services/agents-api/internal/store/local_environment_worker_test.go index 7b9a768bd..dee150d24 100644 --- a/services/agents-api/internal/store/local_environment_worker_test.go +++ b/services/agents-api/internal/store/local_environment_worker_test.go @@ -137,7 +137,7 @@ func TestLocalEnvironmentWorkerSchedulesPreparationWithoutRemoteResolver(t *test h.write(frame.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 2, State: "ready"}) startFrame := h.read(proto.TypeExecutionStart) var start proto.ExecutionStartPayload - if startFrame.DecodePayload(&start) != nil || start.Handle != handle || start.RunID == "" || start.Prompt != "first" { + if startFrame.DecodePayload(&start) != nil || start.Handle != handle || start.RunID == "" || inputTextForTest(t, start.Input) != "first" { t.Fatal("local Start changed reservation identity") } h.write(frame.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 3, State: "started", RunID: start.RunID}) diff --git a/services/agents-api/internal/store/message_image_admission_test.go b/services/agents-api/internal/store/message_image_admission_test.go new file mode 100644 index 000000000..660bdb066 --- /dev/null +++ b/services/agents-api/internal/store/message_image_admission_test.go @@ -0,0 +1,56 @@ +package store_test + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/engine" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" +) + +func imageAdmissionBatch() []store.Input { + return []store.Input{ + {Kind: "message", Payload: json.RawMessage(`{"text":"do not partially admit"}`)}, + {Kind: "message", Payload: json.RawMessage(`{"input":[{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aXioAAAAASUVORK5CYII="}]}]}`)}, + } +} + +func TestUnqualifiedImageAdmissionIsAtomic(t *testing.T) { + for _, placement := range []string{"none", "self_hosted"} { + t.Run(placement, func(t *testing.T) { + h := newDispatchHarness(t) + // A registered text-only profile must stay closed regardless of the + // adapters currently qualified by the built-in catalog. + profile, _ := (engine.Catalog{}).Lookup("codex") + profile.MessageImagePlacements = nil + h.d.Policy = execution.Policy{Engines: engine.NewCatalog(map[string]engine.Profile{"codex": profile})} + worker, err := execution.StartWorker(t.Context(), h.d) + if err != nil { + t.Fatal(err) + } + defer func() { ctx, cancel := context.WithCancel(context.Background()); cancel(); _ = worker.Run(ctx) }() + configuration := json.RawMessage(`{"agent":{"model":"fixture"},"environment":{"type":"` + placement + `","workspace_directory":"/workspace"}}`) + create := store.CreateSessionInput{Creator: store.FixtureCreator(), Engine: "codex", IdempotencyKey: "image-create", Configuration: configuration, InitialInputs: imageAdmissionBatch()} + if _, err := worker.CreateSession(t.Context(), h.tenant, create); !errors.Is(err, store.ErrInvalidInput) { + t.Fatal("creation accepted an unqualified image", err) + } + // Create a Session without starting work to exercise both ordinary and + // prepared admission before their respective persistence paths. + create.InitialInputs = nil + session, err := h.s.CreateSession(t.Context(), h.tenant, create) + if err != nil { + t.Fatal(err) + } + if _, err := worker.SubmitInputs(t.Context(), h.tenant, session.ID, "image-batch", imageAdmissionBatch()); !errors.Is(err, store.ErrInvalidInput) { + t.Fatal("batch accepted an unqualified image", err) + } + session, err = h.s.GetSession(t.Context(), h.tenant, session.ID) + if err != nil || session.LastTurn != nil || session.EnvironmentInputActivity != nil { + t.Fatal("rejected batch persisted execution activity", err) + } + }) + } +} diff --git a/services/agents-api/internal/store/message_images_native_test.go b/services/agents-api/internal/store/message_images_native_test.go new file mode 100644 index 000000000..7389cd5ef --- /dev/null +++ b/services/agents-api/internal/store/message_images_native_test.go @@ -0,0 +1,124 @@ +package store_test + +import ( + "context" + "encoding/json" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/api" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" +) + +func TestNativeMessageImagePublicExecution(t *testing.T) { + python, binary, root, optionsFile := os.Getenv("PARSAR_OFFICIAL_SDK_PYTHON"), os.Getenv("PARSAR_NATIVE_DAEMON_BIN"), os.Getenv("PARSAR_NATIVE_PROOF_DIR"), os.Getenv("PARSAR_MESSAGE_IMAGE_REAL_OPTIONS") + if python == "" || binary == "" || root == "" || optionsFile == "" { + t.Skip("native daemon, fixed SDK, real model options and evidence directory required") + } + raw, err := os.ReadFile(optionsFile) + if err != nil { + t.Fatal(err) + } + var options map[string]any + if json.Unmarshal(raw, &options) != nil { + t.Fatal("invalid private options") + } + model, _ := options["model"].(string) + if model == "" { + t.Fatal("real model required") + } + kind := os.Getenv("PARSAR_MESSAGE_IMAGE_ENGINE") + if kind != "codex" && kind != "claude_sdk" { + t.Fatal("image acceptance requires a specified native engine") + } + h := newDispatchHarness(t) + h.d.Options = func(context.Context, store.Session) (map[string]any, error) { return options, nil } + home, err := os.MkdirTemp(root, "message-image-public-") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute) + defer cancel() + worker, err := execution.StartWorker(ctx, h.d) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- worker.Run(ctx) }() + defer func() { + cancel() + select { + case <-done: + case <-time.After(20 * time.Second): + t.Error("worker did not stop") + } + }() + token, foreign := uuid.NewString(), uuid.NewString() + auth, err := api.NewAuthenticator([]api.APIKey{ + {OrganizationID: "test", ProjectID: h.tenant, SubjectKind: "service_account", SubjectID: "owner", TokenSHA256: device.HashCredential(token), TenantID: h.tenant}, + {OrganizationID: "test", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "other", TokenSHA256: device.HashCredential(foreign), TenantID: uuid.NewString()}, + }) + if err != nil { + t.Fatal(err) + } + handler, err := api.NewHandler(h.s, auth, kind, api.WithExecution(worker), api.WithExecutionPolicy(h.d.Policy)) + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(handler) + defer server.Close() + stop := startNativeEngineDaemon(t, h, home, binary, kind) + defer func() { stop() }() + evidence := filepath.Join(home, "public.json") + run := func(stage string) { + cmd := exec.CommandContext(ctx, python, "../../tests/official_message_images.py", server.URL, token, foreign, model, stage, evidence) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("message images %s: %v %s; evidence %s", stage, err, output, home) + } + } + run("initial") + var proof struct { + Session string `json:"session"` + Turn string `json:"turn"` + Call string `json:"call"` + } + raw, err = os.ReadFile(evidence) + if err != nil || json.Unmarshal(raw, &proof) != nil { + t.Fatal("invalid evidence", err) + } + call, err := h.s.GetFunctionCall(ctx, h.tenant, proof.Session, proof.Turn, proof.Call) + if err != nil || !call.Applied { + t.Fatal("function application receipt missing", err) + } + turn, err := h.s.GetTurn(ctx, h.tenant, proof.Session, proof.Turn) + if err != nil { + t.Fatal(err) + } + var outcome execution.Result + if json.Unmarshal(turn.Outcome, &outcome) != nil || outcome.AppliedThrough < 1 { + t.Fatal("native input receipt missing") + } + inputs, err := h.s.ListTurnInputs(ctx, h.tenant, proof.Session, proof.Turn, 0, 100) + if err != nil || len(inputs) != 3 || inputs[0].Kind != "message" || inputs[1].Kind != "message" || inputs[2].Kind != "tool_result" || outcome.AppliedThrough != inputs[2].Sequence { + t.Fatal("active image batch was not applied exactly once in the same Turn", err) + } + before, err := h.s.GetSessionExecutionBinding(ctx, h.tenant, proof.Session) + if err != nil || before.NativeSessionID == "" { + t.Fatal("native binding missing", err) + } + stop() + stop = startNativeEngineDaemon(t, h, home, binary, kind) + run("resume") + after, err := h.s.GetSessionExecutionBinding(ctx, h.tenant, proof.Session) + if err != nil || before.NativeSessionID != after.NativeSessionID { + t.Fatal("native history changed", err) + } + t.Logf("Real message image SDK/raw HTTP, active receipt, cold daemon recovery and isolation passed: %s", home) +} diff --git a/services/agents-api/internal/store/message_input_helpers_test.go b/services/agents-api/internal/store/message_input_helpers_test.go new file mode 100644 index 000000000..4016ca1c7 --- /dev/null +++ b/services/agents-api/internal/store/message_input_helpers_test.go @@ -0,0 +1,15 @@ +package store_test + +import ( + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "testing" +) + +func inputTextForTest(t *testing.T, input proto.MessageInput) string { + t.Helper() + text, err := input.TextOnly() + if err != nil { + t.Fatal(err) + } + return text +} diff --git a/services/agents-api/internal/store/prepared_dispatch_test.go b/services/agents-api/internal/store/prepared_dispatch_test.go index 984e7833e..b46643c5f 100644 --- a/services/agents-api/internal/store/prepared_dispatch_test.go +++ b/services/agents-api/internal/store/prepared_dispatch_test.go @@ -66,7 +66,7 @@ func readyPreparedDispatch(t *testing.T, h *dispatchHarness, request, handle str h.write(request, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 2, State: "ready", ExpiresAt: time.Now().Add(5 * time.Minute).UnixMilli()}) frame := h.read(proto.TypeExecutionStart) var start proto.ExecutionStartPayload - if frame.ID != request || frame.DecodePayload(&start) != nil || start.Handle != handle || start.RunID == "" || start.Prompt != "first\n\nsecond" { + if frame.ID != request || frame.DecodePayload(&start) != nil || start.Handle != handle || start.RunID == "" || inputTextForTest(t, start.Input) != "first\n\nsecond" { t.Fatal("Start changed preparation or original batch", frame.ID, start) } turn, err := h.s.GetTurn(t.Context(), h.tenant, h.session.ID, start.RunID) @@ -81,7 +81,7 @@ func TestPreparedDispatchPromotesOriginalBatchAndPersistsCompletion(t *testing.T result := runPreparedDispatch(h, t.Context(), pending) frame := h.read(proto.TypeExecutionPrepare) var prepare proto.ExecutionPreparePayload - if frame.DecodePayload(&prepare) != nil || prepare.Configuration.Prompt != "" || prepare.Configuration.RunID != "" || prepare.Configuration.ConversationID != "" || prepare.Configuration.LocalEnvironment == nil || prepare.Configuration.LocalEnvironment.ID != h.device.EnvironmentID || prepare.Configuration.DisableExecutionEnvironment { + if frame.DecodePayload(&prepare) != nil || len(prepare.Configuration.Input) != 0 || prepare.Configuration.RunID != "" || prepare.Configuration.ConversationID != "" || prepare.Configuration.LocalEnvironment == nil || prepare.Configuration.LocalEnvironment.ID != h.device.EnvironmentID || prepare.Configuration.DisableExecutionEnvironment { t.Fatal("invalid preparation configuration", prepare) } session, err := h.s.GetSession(t.Context(), h.tenant, h.session.ID) @@ -98,7 +98,7 @@ func TestPreparedDispatchPromotesOriginalBatchAndPersistsCompletion(t *testing.T h.write(frame.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 3, State: "started", RunID: start.RunID}) steering := h.read(proto.TypePromptSteer) var steer proto.PromptSteerPayload - if steering.ID != start.RunID || steering.DecodePayload(&steer) != nil || steer.Text != "third" || !steer.DurableReceipt { + if steering.ID != start.RunID || steering.DecodePayload(&steer) != nil || inputTextForTest(t, steer.Input) != "third" || !steer.DurableReceipt { t.Fatal("later input bypassed ordinary steering", steer) } h.write(start.RunID, proto.TypePromptSteerAck, proto.PromptSteerAckPayload{InputID: steer.InputID, Accepted: true}) diff --git a/services/agents-api/internal/store/public_execution_test.go b/services/agents-api/internal/store/public_execution_test.go index 543ec7449..c6b00bf61 100644 --- a/services/agents-api/internal/store/public_execution_test.go +++ b/services/agents-api/internal/store/public_execution_test.go @@ -63,7 +63,7 @@ func TestExecutionWorkerAdmissionBindingAndRecovery(t *testing.T) { if err := request.DecodePayload(&prompt); err != nil { t.Fatal(err) } - if prompt.Prompt != "First\n\nSecond" || !prompt.DisableExecutionEnvironment || !prompt.DisableSubagents || prompt.ExecutionControls == nil || *prompt.ExecutionControls != (proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "medium"}) || prompt.AgentOptions["web_search"] != nil || prompt.AgentOptions["model_verbosity"] != nil { + if inputTextForTest(t, prompt.Input) != "First\n\nSecond" || !prompt.DisableExecutionEnvironment || !prompt.DisableSubagents || prompt.ExecutionControls == nil || *prompt.ExecutionControls != (proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "medium"}) || prompt.AgentOptions["web_search"] != nil || prompt.AgentOptions["model_verbosity"] != nil { t.Fatal(prompt) } bound, err := h.s.GetSessionDevice(ctx, h.tenant, h.session.ID) diff --git a/services/agents-api/tests/official_message_images.py b/services/agents-api/tests/official_message_images.py new file mode 100644 index 000000000..05d9d960b --- /dev/null +++ b/services/agents-api/tests/official_message_images.py @@ -0,0 +1,176 @@ +"""Ordered image input through the pinned client, Core and a real native Runtime.""" +import base64 +import importlib.metadata +import json +import secrets +import struct +import sys +import time +import uuid +import zlib +from pathlib import Path + +import httpx2 +from openai import OpenAI + +base, token, foreign, model, stage, evidence = sys.argv[1:] +pin = json.loads((Path(__file__).resolve().parents[3] / "contracts/agents-api/upstream.json").read_text()) +dist = importlib.metadata.distribution("openai") +assert dist.version == pin["sdk_version"] +assert json.loads(dist.read_text("direct_url.json"))["vcs_info"]["commit_id"] == pin["commit"] +http = httpx2.Client(trust_env=False, timeout=180) +client = OpenAI(base_url=base + "/v1", api_key=token, max_retries=0, + _strict_response_validation=True, http_client=http) +sessions = client.beta.agents.sessions +headers = {"Authorization": "Bearer " + token, "OpenAI-Beta": "agents=v1"} +proof = {} if stage == "initial" else json.loads(Path(evidence).read_text()) + + +def save(): + Path(evidence).write_text(json.dumps(proof, indent=2)) + + +def picture(names): + colors = {"red": (255, 0, 0), "blue": (0, 0, 255), "green": (0, 170, 0), "yellow": (255, 255, 0)} + rows = b"".join(b"\0" + b"".join(bytes(colors[n]) * 100 for n in names) for _ in range(140)) + + def chunk(kind, value): + return struct.pack(">I", len(value)) + kind + value + struct.pack(">I", zlib.crc32(kind + value) & 0xffffffff) + + image = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 400, 140, 8, 2, 0, 0, 0)) + chunk(b"IDAT", zlib.compress(rows)) + chunk(b"IEND", b"") + return "data:image/png;base64," + base64.b64encode(image).decode() + + +def text(value): + return {"type": "input_text", "text": value} + + +def messages(parts): + return [{"role": "user", "content": parts}] + + +def image_messages(url): + return messages([text("Read this image. "), {"type": "input_image", "image_url": url}, + text(" Remember this as the latest image.")]) + messages([ + text("Reply only with the four band colors from left to right, separated by commas. Do not use tools.")]) + + +def message_event(value): + return {"type": "agent.session.input.message", "input": value} + + +def answer(sid, expected): + items = sessions.items.list(sid, order="asc", limit=100).data + answers = [i for i in items if i.type == "message" and i.role == "assistant"] + assert answers, "missing native answer" + result = " ".join(c.text for c in answers[-1].content if c.type == "output_text").lower() + positions = [result.find(name) for name in expected] + assert all(p >= 0 for p in positions) and positions == sorted(positions), result + return answers[-1].id + + +def wait_initial(sid): + deadline = time.monotonic() + 150 + while time.monotonic() < deadline: + turns = sessions.turns.list(sid, order="asc", limit=100).data + if turns and turns[-1].status in {"completed", "failed", "cancelled"}: + assert len(turns) == 1 and turns[-1].status == "completed", [t.to_dict() for t in turns] + return turns[-1].id + time.sleep(.25) + raise AssertionError("initial input did not complete") + + +def run(sid, prompt, active=False, cancel=False): + events, handled = [], False + with sessions.events.stream(sid, timeout=180) as stream: + sessions.events.create(sid, events=[message_event(messages([text(prompt)]))]) + for event in stream: + events.append(event.to_dict()) + if event.type == "agent.session.requires_action": + assert not handled and (active or cancel) + handled = True + action = event.session.required_actions[0] + assert action.name == "wait_for_image" + if cancel: + sessions.events.create(sid, events=[{"type": "agent.session.input.cancel"}]) + else: + proof.update(turn=action.turn_id, call=action.call_id) + batch = [message_event(image_messages(proof["second_url"]))] + for _ in range(2): + sessions.events.create(sid, events=batch, idempotency_key="same-active-image") + sessions.events.create(sid, events=[{"type": "agent.session.input.tool_result", "turn_id": action.turn_id, + "call_id": action.call_id, "success": True, "output": "The latest image has been supplied. Read it and give the colors; do not call tools again."}]) + assert event.type not in {"agent.session.failed", "agent.session.turn.failed"}, event.to_dict() + if event.type == "agent.session.idle": + break + else: + raise AssertionError("stream ended without idle") + proof.setdefault("runs", []).append(events) + save() + terminal = "agent.session.turn." + ("cancelled" if cancel else "completed") + types = [e["type"] for e in events] + assert types.index("agent.session.turn.created") < types.index(terminal) < types.index("agent.session.idle") + assert len({e["event_id"] for e in events}) == len(events) + for index, event in enumerate(events): + if event["type"] == "agent.session.turn.item.done" and event["item"].get("role") == "assistant": + item_id = event["item"]["id"] + added = next(i for i, e in enumerate(events) if e["type"] == "agent.session.turn.item.added" and e["item"]["id"] == item_id) + assert added < index < types.index(terminal) + turns = sessions.turns.list(sid, order="asc", limit=100).data + assert turns[-1].status == ("cancelled" if cancel else "completed") + if active or cancel: + assert handled + return events + + +def check_items(sid): + expected = image_messages(proof["first_url"]) + messages([text("Call wait_for_image exactly once, then follow the incoming image instructions.")]) + image_messages(proof["second_url"]) + response = http.get(base + "/v1/agents/sessions/" + sid + "/items", headers=headers, params={"order": "asc", "limit": 100}) + assert response.status_code == 200 + actual = [i for i in response.json()["data"] if i["type"] == "message" and i["role"] == "user"] + assert [i["content"] for i in actual] == [m["content"] for m in expected] + assert [i.content for i in sessions.items.list(sid, order="asc", limit=100).data if i.type == "message" and i.role == "user"] + + +try: + if stage == "initial": + first = ["red", "green", "blue", "yellow"] + secrets.SystemRandom().shuffle(first) + second = first[1:] + first[:1] + proof.update(first=first, second=second, first_url=picture(first), second_url=picture(second)) + session = sessions.create(agent={"model": model, "tools": [{"type": "function", "name": "wait_for_image", + "description": "Wait for the user to supply the next image.", "parameters": {"type": "object", "properties": {}, "additionalProperties": False}}]}, + environment={"type": "none"}, input=image_messages(proof["first_url"])) + proof["session"] = session.id + save() + proof["initial_turn"] = wait_initial(session.id) + proof["initial_answer"] = answer(session.id, first) + run(session.id, "Call wait_for_image exactly once, then follow the incoming image instructions.", active=True) + proof["active_answer"] = answer(session.id, second) + assert len(sessions.turns.list(session.id).data) == 2 + check_items(session.id) + before = [t.id for t in sessions.turns.list(session.id).data] + invalid = [message_event(messages([text("must not be admitted")])), message_event(messages([{"type": "input_image", "image_url": "https://example.test/image.png"}]))] + response = http.post(base + "/v1/agents/sessions/" + session.id + "/events", headers=headers, json={"events": invalid}) + assert response.status_code == 400 + assert [t.id for t in sessions.turns.list(session.id).data] == before + check_items(session.id) + foreign_headers = {**headers, "Authorization": "Bearer " + foreign} + for suffix in ["", "/items", "/turns"]: + assert http.get(base + "/v1/agents/sessions/" + session.id + suffix, headers=foreign_headers).status_code == 404 + assert http.post(base + "/v1/agents/sessions/" + session.id + "/events", headers=foreign_headers, + json={"events": [message_event(image_messages(proof["second_url"]))]}).status_code == 404 + else: + sid = proof["session"] + check_items(sid) + run(sid, "Recall the latest image I supplied, not the first. Reply only with its four band colors in order. Do not call tools.") + proof["resumed_answer"] = answer(sid, proof["second"]) + assert len(sessions.turns.list(sid).data) == 3 + run(sid, "Call wait_for_image exactly once and wait for its result.", cancel=True) + run(sid, "Reply PLAIN_OK only. Do not call tools.") + items = sessions.items.list(sid, order="asc", limit=100).data + assert any(i.type == "message" and i.role == "assistant" and any(c.type == "output_text" and "PLAIN_OK" in c.text for c in i.content) for i in items) + proof["passed"] = True +finally: + save() + client.close() From 29d78c9ffe0080313d9706c6a79f4f8b9f7ece80 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Tue, 22 Sep 2026 12:14:41 +0800 Subject: [PATCH 2/3] Update the packaged Runtime capability assertion --- scripts/check-claude-sdk-runtime.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check-claude-sdk-runtime.mjs b/scripts/check-claude-sdk-runtime.mjs index 8c353f055..45f25c08f 100644 --- a/scripts/check-claude-sdk-runtime.mjs +++ b/scripts/check-claude-sdk-runtime.mjs @@ -23,7 +23,7 @@ assert.equal(probe.status, 0, "Exported runtime is unavailable"); const report = JSON.parse(probe.stdout); assert.equal(report.type, "runtime_ready"); assert.equal(report.protocol, 1); -assert.deepEqual(report.features, [...(process.platform === "linux" ? ["workspace_directory", "local_runtime_v1", "workspace_functions"] : []), "structured_output", "subagent_resources", "mcp_http_tools", "mcp_http_bearer_auth", "mcp_http_required", "workspace_tools", "workspace_prepare", "workspace_read", "workspace_command_observations"]); +assert.deepEqual(report.features, [...(process.platform === "linux" ? ["workspace_directory", "local_runtime_v1", "workspace_functions"] : []), "message_images", "structured_output", "subagent_resources", "mcp_http_tools", "mcp_http_bearer_auth", "mcp_http_required", "workspace_tools", "workspace_prepare", "workspace_read", "workspace_command_observations"]); assert.equal(report.sdk, source.dependencies["@anthropic-ai/claude-agent-sdk"]); assert.equal(report.mcp, source.dependencies["@modelcontextprotocol/sdk"]); console.log(`Verified exported SDK ${report.sdk}, MCP ${report.mcp}, ${report.native}`); From 8b10a2d60485836b4c16e0bd313017fc3077cbc5 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Tue, 22 Sep 2026 12:34:07 +0800 Subject: [PATCH 3/3] Guard input candidate races and require the ordered Claude bridge --- CONTRIBUTING.md | 2 + .../claudesdk/preparation_fixture_test.go | 2 +- .../internal/agent/claudesdk/readiness.go | 2 +- .../agent/claudesdk/readiness_test.go | 6 +- .../agent/claudesdk/workspace_launch_test.go | 2 +- contracts/agents-api/message-input.md | 16 +++ .../claude-sdk-adapter/src/runtime_check.ts | 2 +- scripts/check-claude-sdk-runtime.mjs | 2 +- .../internal/execution/worker_device.go | 20 +++- .../internal/store/worker_input_race_test.go | 109 ++++++++++++++++++ .../agents-api/tests/official_mcode_native.py | 6 +- 11 files changed, 159 insertions(+), 10 deletions(-) create mode 100644 services/agents-api/internal/store/worker_input_race_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 639cd1197..c88868317 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -225,6 +225,8 @@ input list and uses blank-line separators between messages; this does not preser independent native user-message boundaries. No old wire fallback is maintained. Deploy Core and daemon together; the existing major/minor WebSocket check rejects older major/minor peers before dispatch rather than ignoring removed fields. +The independently packaged Claude bridge uses protocol 2 for ordered input; +readiness rejects packages reporting the old string-input protocol. Image-bearing messages require a qualified profile/placement before persistence and image support from the selected Runtime before native delivery. These checks apply to that operation only; ordinary text retains offline queueing. Initial, diff --git a/apps/parsar-daemon/internal/agent/claudesdk/preparation_fixture_test.go b/apps/parsar-daemon/internal/agent/claudesdk/preparation_fixture_test.go index c0d187d5c..3986d6ae2 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/preparation_fixture_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/preparation_fixture_test.go @@ -44,7 +44,7 @@ func runPreparationHelper() { } else if mode == "old-command-runtime" { features = []string{"workspace_tools", "workspace_prepare"} } - _ = json.NewEncoder(os.Stdout).Encode(RuntimeInfo{Type: "runtime_ready", Protocol: 1, Node: "fixture", SDK: "fixture", MCP: "fixture", Native: "fixture", Features: features}) + _ = json.NewEncoder(os.Stdout).Encode(RuntimeInfo{Type: "runtime_ready", Protocol: 2, Node: "fixture", SDK: "fixture", MCP: "fixture", Native: "fixture", Features: features}) return } state := os.Getenv("CLAUDE_CONFIG_DIR") diff --git a/apps/parsar-daemon/internal/agent/claudesdk/readiness.go b/apps/parsar-daemon/internal/agent/claudesdk/readiness.go index d799b2fef..9c08e1a39 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/readiness.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/readiness.go @@ -120,7 +120,7 @@ func CheckRuntime(ctx context.Context, config Config) (RuntimeInfo, error) { return RuntimeInfo{}, fmt.Errorf("claudesdk: runtime check failed") } var info RuntimeInfo - if json.Unmarshal(raw, &info) != nil || info.Type != "runtime_ready" || info.Protocol != 1 || + if json.Unmarshal(raw, &info) != nil || info.Type != "runtime_ready" || info.Protocol != 2 || info.Node == "" || info.SDK == "" || info.MCP == "" || info.Native == "" { return RuntimeInfo{}, fmt.Errorf("claudesdk: invalid runtime readiness report") } diff --git a/apps/parsar-daemon/internal/agent/claudesdk/readiness_test.go b/apps/parsar-daemon/internal/agent/claudesdk/readiness_test.go index 7236ca8ea..3ac5f669c 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/readiness_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/readiness_test.go @@ -15,7 +15,7 @@ import ( "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" ) -const readyReport = `{"type":"runtime_ready","protocol":1,"node":"22.22.2","sdk":"0.3.269","mcp":"1.30.0","native":"2.1.269 (Claude Code)"}` +const readyReport = `{"type":"runtime_ready","protocol":2,"node":"22.22.2","sdk":"0.3.269","mcp":"1.30.0","native":"2.1.269 (Claude Code)"}` func TestRequiredMCPNeedsQualifiedRuntime(t *testing.T) { root := t.TempDir() @@ -128,11 +128,11 @@ func runReadinessHelper() { os.Exit(5) } } - _, _ = fmt.Fprintln(os.Stdout, strings.Replace(readyReport, `"protocol":1`, `"protocol":1,"features":["mcp_http_tools"]`, 1)) + _, _ = fmt.Fprintln(os.Stdout, strings.Replace(readyReport, `"protocol":2`, `"protocol":2,"features":["mcp_http_tools"]`, 1)) case "malformed": _, _ = fmt.Fprintln(os.Stdout, "not-json") case "wrong-protocol": - _, _ = fmt.Fprintln(os.Stdout, strings.Replace(readyReport, `"protocol":1`, `"protocol":2`, 1)) + _, _ = fmt.Fprintln(os.Stdout, strings.Replace(readyReport, `"protocol":2`, `"protocol":1`, 1)) case "missing-version": _, _ = fmt.Fprintln(os.Stdout, strings.Replace(readyReport, `"mcp":"1.30.0"`, `"mcp":""`, 1)) case "multiple": diff --git a/apps/parsar-daemon/internal/agent/claudesdk/workspace_launch_test.go b/apps/parsar-daemon/internal/agent/claudesdk/workspace_launch_test.go index 2d1f59283..095ab2d31 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/workspace_launch_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/workspace_launch_test.go @@ -24,7 +24,7 @@ test "$ANTHROPIC_AUTH_TOKEN" = selected-provider-fixture || exit 23 test "$TMPDIR" != "$CLAUDE_CONFIG_DIR/tmp" || exit 24 case "$1" in */runtime_check.js) - printf '%s\n' '{"type":"runtime_ready","protocol":1,"node":"fixture","sdk":"fixture","mcp":"fixture","native":"fixture","features":["workspace_tools","workspace_prepare"]}' ;; + printf '%s\n' '{"type":"runtime_ready","protocol":2,"node":"fixture","sdk":"fixture","mcp":"fixture","native":"fixture","features":["workspace_tools","workspace_prepare"]}' ;; *) IFS= read -r request printf '%s\n' '{"type":"prepared"}' diff --git a/contracts/agents-api/message-input.md b/contracts/agents-api/message-input.md index 5bc149d3c..4e5b4ae79 100644 --- a/contracts/agents-api/message-input.md +++ b/contracts/agents-api/message-input.md @@ -39,6 +39,7 @@ image `detail` or file-ID extension. Private wire 0.4.0 uses `MessageInput` for initial requests, prepared start and active steering. Each message contains ordered `InputContent` parts, also reused by function-result content. Core does not download, transcode or repair media. +The separate Claude bridge reports protocol 2; daemon readiness rejects protocol 1. Adapters own native encoding and native application-receipt mapping. Existing text-only adapters reject image parts instead of dropping them. @@ -72,6 +73,21 @@ ordinary text continuation, malformed-batch atomic rejection and tenant isolatio Evidence is under `zju_a100_2:~/.parsar/remediation/20260922/message-image-input/`: `public-codex-final.log` / `message-image-public-2718944397/public.json` and `public-claude_sdk-final.log` / `message-image-public-700202073/public.json`. +After final lifecycle and bridge-version changes, both complete public chains +passed again: `public-codex-reviewed.log` (59.416s) and +`public-claude-reviewed.log` (79.525s). MiniMax ordinary text, active steering, +receipt, cancellation and cold continuation passed in +`public-mcode-network-fixed.log` (48.963s). The first MiniMax attempt hit a stale +hosted-fixture assertion; a subsequent attempt failed because its test network +relay was absent. Both failures are retained, and no production model behavior +was changed to obtain the passing result. + +The final required gate was split by host: server `make -o check-web check` +passed, and local Node22/pnpm10.30.3 `make check-web` passed all 73 browser tests. +An earlier complete server `make check` stopped at its missing Chrome executable. +The dedicated PostgreSQL cancellation/deletion interleaving regression passed +five repetitions, and shared input/dispatch race checks passed. These checks do +not qualify workspace images or additional native/provider combinations. Native-only probes are feasibility evidence, not public qualification. Workspace image workflows, MiniMax Code image input, remote HTTP(S) image URLs, diff --git a/packages/claude-sdk-adapter/src/runtime_check.ts b/packages/claude-sdk-adapter/src/runtime_check.ts index 6ad0618ce..51b083e9e 100644 --- a/packages/claude-sdk-adapter/src/runtime_check.ts +++ b/packages/claude-sdk-adapter/src/runtime_check.ts @@ -45,7 +45,7 @@ try { assert.equal(smoke.error, undefined, "bridge_unavailable"); assert.equal(smoke.status, 0, "bridge_unavailable"); assert.deepEqual(JSON.parse(smoke.stdout), { type: "error", code: "invalid_request" }); - process.stdout.write(JSON.stringify({ type: "runtime_ready", protocol: 1, features: [...(process.platform === "linux" ? ["workspace_directory", "local_runtime_v1", "workspace_functions"] : []), "message_images", "structured_output", "subagent_resources", "mcp_http_tools", "mcp_http_bearer_auth", "mcp_http_required", "workspace_tools", "workspace_prepare", "workspace_read", "workspace_command_observations"], node: process.versions.node, sdk: sdk.version, mcp: mcp.version, native: nativeVersion }) + "\n"); + process.stdout.write(JSON.stringify({ type: "runtime_ready", protocol: 2, features: [...(process.platform === "linux" ? ["workspace_directory", "local_runtime_v1", "workspace_functions"] : []), "message_images", "structured_output", "subagent_resources", "mcp_http_tools", "mcp_http_bearer_auth", "mcp_http_required", "workspace_tools", "workspace_prepare", "workspace_read", "workspace_command_observations"], node: process.versions.node, sdk: sdk.version, mcp: mcp.version, native: nativeVersion }) + "\n"); } catch { // Native diagnostics can include operator environment; never forward them. process.stdout.write(JSON.stringify({ type: "runtime_unavailable" }) + "\n"); diff --git a/scripts/check-claude-sdk-runtime.mjs b/scripts/check-claude-sdk-runtime.mjs index 45f25c08f..c3beb0bf9 100644 --- a/scripts/check-claude-sdk-runtime.mjs +++ b/scripts/check-claude-sdk-runtime.mjs @@ -22,7 +22,7 @@ const probe = spawnSync(process.execPath, [join(root, "dist/runtime_check.js"), assert.equal(probe.status, 0, "Exported runtime is unavailable"); const report = JSON.parse(probe.stdout); assert.equal(report.type, "runtime_ready"); -assert.equal(report.protocol, 1); +assert.equal(report.protocol, 2); assert.deepEqual(report.features, [...(process.platform === "linux" ? ["workspace_directory", "local_runtime_v1", "workspace_functions"] : []), "message_images", "structured_output", "subagent_resources", "mcp_http_tools", "mcp_http_bearer_auth", "mcp_http_required", "workspace_tools", "workspace_prepare", "workspace_read", "workspace_command_observations"]); assert.equal(report.sdk, source.dependencies["@anthropic-ai/claude-agent-sdk"]); assert.equal(report.mcp, source.dependencies["@modelcontextprotocol/sdk"]); diff --git a/services/agents-api/internal/execution/worker_device.go b/services/agents-api/internal/execution/worker_device.go index 87ede5b5d..d8f26cada 100644 --- a/services/agents-api/internal/execution/worker_device.go +++ b/services/agents-api/internal/execution/worker_device.go @@ -10,11 +10,29 @@ import ( ) func (w *Worker) bind(ctx context.Context, item store.ExecutionWork) (bool, error) { - input, _, err := w.dispatcher.initialInput(ctx, item.TenantID, item.SessionID, item.TurnID) + input, _, inputErr := w.dispatcher.initialInput(ctx, item.TenantID, item.SessionID, item.TurnID) + if inputErr != nil && !errors.Is(inputErr, store.ErrInvalidInput) && !errors.Is(inputErr, store.ErrNotFound) { + return false, inputErr + } + // Candidate selection is a snapshot. Cancellation can append a control input + // before this read, so recheck eligibility after reading the input history. + turn, err := w.dispatcher.Store.GetTurn(ctx, item.TenantID, item.SessionID, item.TurnID) + if errors.Is(err, store.ErrNotFound) { + return false, nil + } if err != nil { return false, err } + if turn.Status != store.TurnQueued || !turn.CancelRequestedAt.IsZero() { + return false, nil + } + if inputErr != nil { + return false, inputErr + } ready, err := w.bindDevice(ctx, item.TenantID, item.SessionID, input) + if errors.Is(err, store.ErrNotFound) { + return false, nil + } if !errors.Is(err, store.ErrDeviceBindingConflict) { return ready, err } diff --git a/services/agents-api/internal/store/worker_input_race_test.go b/services/agents-api/internal/store/worker_input_race_test.go new file mode 100644 index 000000000..b4ca8c2a3 --- /dev/null +++ b/services/agents-api/internal/store/worker_input_race_test.go @@ -0,0 +1,109 @@ +package store_test + +import ( + "context" + "encoding/json" + "strings" + "sync" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// Intercept the actual database read after candidate selection. The concurrent +// mutation uses a separate pool and the production cancellation/deletion path. +type beforeInputRead struct { + once sync.Once + run func() +} + +func (h *beforeInputRead) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context { + if strings.HasPrefix(data.SQL, "-- name: ListTurnInputs ") { + h.once.Do(h.run) + } + return ctx +} +func (*beforeInputRead) TraceQueryEnd(context.Context, *pgx.Conn, pgx.TraceQueryEndData) {} + +func TestWorkerInputReadSkipsConcurrentlyCancelledCandidate(t *testing.T) { + for _, deleted := range []bool{false, true} { + name := "cancel" + if deleted { + name = "delete" + } + t.Run(name, func(t *testing.T) { + h := newDispatchHarness(t) + candidate := h.message("candidate", "queued") + candidateSession := h.session.ID + _, pool := store.NewTestStore(t) + cfg := pool.Config() + mutated := make(chan error, 1) + cfg.ConnConfig.Tracer = &beforeInputRead{run: func() { + if deleted { + mutated <- h.s.DeleteSession(t.Context(), h.tenant, candidateSession) + return + } + _, err := h.s.SubmitInputs(t.Context(), h.tenant, candidateSession, "cancel", []store.Input{{Kind: "cancel", Payload: json.RawMessage(`{}`)}}) + mutated <- err + }} + instrumented, err := pgxpool.NewWithConfig(t.Context(), cfg) + if err != nil { + t.Fatal(err) + } + defer instrumented.Close() + h.d.Store = store.New(instrumented) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + worker, err := execution.StartWorker(ctx, h.d) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- worker.Run(ctx) }() + defer func() { + cancel() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Error("worker did not stop") + } + }() + select { + case err := <-mutated: + if err != nil { + t.Fatal(err) + } + case <-time.After(10 * time.Second): + t.Fatal("input-read interleaving was not reached") + } + turn, err := h.s.GetTurn(ctx, h.tenant, candidateSession, candidate.TurnID) + if err != nil || turn.Status != store.TurnCancelled { + t.Fatal("candidate was not cancelled", err) + } + // A later Session must still execute through this same Worker. + h.session, err = h.s.CreateSession(ctx, h.tenant, store.CreateSessionInput{Creator: store.FixtureCreator(), Engine: "codex", IdempotencyKey: "healthy", Configuration: h.session.Configuration}) + if err != nil { + t.Fatal(err) + } + healthy := h.message("healthy", "continue") + select { + case err := <-done: + // Keep teardown able to consume the already completed Worker. + done <- err + t.Fatal("a stale candidate terminated the global Worker", err) + case <-time.After(650 * time.Millisecond): + } + frame := h.read(proto.TypePromptRequest) + if frame.ID != healthy.TurnID { + t.Fatal("cancelled candidate reached the Runtime", frame.ID) + } + h.write(healthy.TurnID, proto.TypeDone, proto.DonePayload{Content: "continued"}) + waitTurn(t, h, healthy.TurnID, store.TurnCompleted) + }) + } +} diff --git a/services/agents-api/tests/official_mcode_native.py b/services/agents-api/tests/official_mcode_native.py index dcacd60ba..5ccc960e5 100644 --- a/services/agents-api/tests/official_mcode_native.py +++ b/services/agents-api/tests/official_mcode_native.py @@ -63,9 +63,13 @@ def answer(sid): session = sessions.create(agent={"model": model, "instructions": "Follow user instructions. Remember supplied markers. Do not use tools."}, environment={"type": "none"}) record = {"session": session.id, "marker": marker, "model": model, "checks": []} Path(output).write_text(json.dumps(record, indent=2)) - for agent_patch, environment in [({"tools": [{"type": "function", "name": "f", "parameters": {"type": "object"}}]}, {"type": "none"}), ({"text": {"verbosity": "high"}}, {"type": "none"}), ({}, {"type": "openai_hosted"})]: + for agent_patch, environment in [({"tools": [{"type": "function", "name": "f", "parameters": {"type": "object"}}]}, {"type": "none"}), ({"text": {"verbosity": "high"}}, {"type": "none"})]: r = http.post(base + "/v1/agents/sessions", headers=headers, json={"agent": {"model": model, **agent_patch}, "environment": environment}) assert r.status_code == 400, r.status_code + # This text-only fixture deliberately has no hosted provisioner. + r = http.post(base + "/v1/agents/sessions", headers=headers, json={ + "agent": {"model": model}, "environment": {"type": "openai_hosted"}}) + assert r.status_code == 503 and r.json()["error"]["code"] == "execution_unavailable" record["initial_events"] = execute(session.id, "Remember " + marker + ". Write 120 numbered lines explaining addition, one sentence per line. Start immediately.", steer=True) turns = sessions.turns.list(session.id, order="asc", limit=100).data assert len(turns) == 1 and turns[0].status == "completed", [(t.id, t.status) for t in turns]