diff --git a/backend/internal/application/conversation/model_option_policy_test.go b/backend/internal/application/conversation/model_option_policy_test.go index 27430961..505dbbf1 100644 --- a/backend/internal/application/conversation/model_option_policy_test.go +++ b/backend/internal/application/conversation/model_option_policy_test.go @@ -72,7 +72,16 @@ func TestFilterModelOptionsAllowlistUsesDefaultAndProtocolPaths(t *testing.T) { func TestFilterModelOptionsAllowsGeminiInteractionResponseFormatArray(t *testing.T) { filtered := filterModelOptions(map[string]interface{}{ "response_format": []interface{}{ - map[string]interface{}{"type": "text"}, + map[string]interface{}{ + "type": "text", + "mime_type": "application/json", + "schema": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "answer": map[string]interface{}{"type": "string"}, + }, + }, + }, map[string]interface{}{"type": "image", "image_size": "1K", "delivery": "b64_json"}, }, }, llm.AdapterGeminiInteractions, modelOptionPolicyConfig{ @@ -85,6 +94,11 @@ func TestFilterModelOptionsAllowsGeminiInteractionResponseFormatArray(t *testing if !ok || len(formats) != 2 { t.Fatalf("expected Gemini Interactions response_format array to pass, got %#v", filtered) } + textFormat := formats[0].(map[string]interface{}) + schema, ok := textFormat["schema"].(map[string]interface{}) + if !ok || schema["type"] != "object" { + t.Fatalf("expected whitelisted text schema to pass, got %#v", textFormat) + } imageFormat := formats[1].(map[string]interface{}) if imageFormat["image_size"] != "1K" { t.Fatalf("expected whitelisted image_size to pass, got %#v", imageFormat) @@ -976,7 +990,7 @@ func TestFilterModelOptionsGeminiInteractionsAllowsVideoParams(t *testing.T) { "response_format": map[string]interface{}{ "aspect_ratio": "16:9", "image_size": "1K", - "mime_type": "image/png", + "mime_type": "image/jpeg", "delivery": "b64_json", }, "generation_config": map[string]interface{}{ @@ -997,7 +1011,7 @@ func TestFilterModelOptionsGeminiInteractionsAllowsVideoParams(t *testing.T) { }) responseFormat, ok := filtered["response_format"].(map[string]interface{}) - if !ok || responseFormat["aspect_ratio"] != "16:9" || responseFormat["image_size"] != "1K" || responseFormat["mime_type"] != "image/png" { + if !ok || responseFormat["aspect_ratio"] != "16:9" || responseFormat["image_size"] != "1K" || responseFormat["mime_type"] != "image/jpeg" { t.Fatalf("expected Gemini response_format aspect ratio to pass, got %#v", filtered) } if _, ok := responseFormat["delivery"]; ok { @@ -1133,7 +1147,7 @@ func TestFilterModelOptionsSelectsGeminiToolsForResolvedRouteProtocol(t *testing } } -func TestFilterModelOptionsGeminiInteractionsAllowsCamelCaseVideoConfig(t *testing.T) { +func TestFilterModelOptionsGeminiInteractionsRejectsLegacyCamelCaseConfig(t *testing.T) { filtered := filterModelOptions(map[string]interface{}{ "generationConfig": map[string]interface{}{ "videoConfig": map[string]interface{}{ @@ -1146,13 +1160,8 @@ func TestFilterModelOptionsGeminiInteractionsAllowsCamelCaseVideoConfig(t *testi DeniedPathsJSON: config.DefaultModelOptionDeniedPathsJSON(), }) - generationConfig, ok := filtered["generationConfig"].(map[string]interface{}) - if !ok { - t.Fatalf("expected camelCase Gemini generationConfig to pass, got %#v", filtered) - } - videoConfig, ok := generationConfig["videoConfig"].(map[string]interface{}) - if !ok || videoConfig["task"] != "text_to_video" { - t.Fatalf("expected camelCase Gemini video task to pass, got %#v", filtered) + if len(filtered) != 0 { + t.Fatalf("expected legacy camelCase Interactions options to be rejected, got %#v", filtered) } } diff --git a/backend/internal/application/conversation/service_media_generation.go b/backend/internal/application/conversation/service_media_generation.go index 6c884fe1..5d133a80 100644 --- a/backend/internal/application/conversation/service_media_generation.go +++ b/backend/internal/application/conversation/service_media_generation.go @@ -755,12 +755,6 @@ func withGeminiInteractionResponseType(options map[string]interface{}, responseT format[key] = value } } - if raw, ok := next["responseFormat"].(map[string]interface{}); ok { - for key, value := range raw { - format[key] = value - } - delete(next, "responseFormat") - } format["type"] = strings.TrimSpace(responseType) next["response_format"] = format return next diff --git a/backend/internal/application/settings/service.go b/backend/internal/application/settings/service.go index f78e48bb..141220ff 100644 --- a/backend/internal/application/settings/service.go +++ b/backend/internal/application/settings/service.go @@ -220,10 +220,29 @@ func isLegacyDefaultModelOptionAllowedPaths(value string) bool { previousCombinedDefault["gemini_interactions"], "generation_config.thinking_summaries", ) + legacyInteractionsDefault := cloneStringSliceMap(latestDefault) + legacyInteractionsDefault["gemini_interactions"] = append( + removeStringValue(legacyInteractionsDefault["gemini_interactions"], "response_format.schema"), + "responseFormat.type", + "responseFormat.aspectRatio", + "responseFormat.imageSize", + "responseFormat.mimeType", + "generationConfig.videoConfig.task", + ) + legacyInteractionsWithoutSummaries := cloneStringSliceMap(legacyInteractionsDefault) + legacyInteractionsWithoutSummaries["gemini_interactions"] = removeStringValue( + legacyInteractionsWithoutSummaries["gemini_interactions"], + "generation_config.thinking_summaries", + ) + legacyCombinedDefault := cloneStringSliceMap(legacyInteractionsWithoutSummaries) + legacyCombinedDefault["gemini_generate_content"] = previousGenerateContentDefault["gemini_generate_content"] previousDefaults := []map[string][]string{ previousGenerateContentDefault, previousInteractionsDefault, previousCombinedDefault, + legacyInteractionsDefault, + legacyInteractionsWithoutSummaries, + legacyCombinedDefault, } for _, previousDefault := range previousDefaults { if sameStringSliceMap(current, previousDefault) { diff --git a/backend/internal/application/settings/service_seed_test.go b/backend/internal/application/settings/service_seed_test.go index a68eea0c..cb2941bb 100644 --- a/backend/internal/application/settings/service_seed_test.go +++ b/backend/internal/application/settings/service_seed_test.go @@ -213,6 +213,39 @@ func TestSeedAddsGeminiThinkingSummariesToPreviousDefaultModelOptionAllowedPaths } } +func TestSeedReplacesLegacyGeminiInteractionsOptionPaths(t *testing.T) { + previousDefault := map[string][]string{} + if err := json.Unmarshal([]byte(config.DefaultModelOptionAllowedPathsJSON()), &previousDefault); err != nil { + t.Fatalf("decode current model option defaults: %v", err) + } + previousDefault["gemini_interactions"] = append( + removeStringValue(previousDefault["gemini_interactions"], "response_format.schema"), + "responseFormat.type", + "responseFormat.aspectRatio", + "responseFormat.imageSize", + "responseFormat.mimeType", + "generationConfig.videoConfig.task", + ) + previousJSON, err := json.Marshal(previousDefault) + if err != nil { + t.Fatalf("encode previous model option defaults: %v", err) + } + repo := newSettingsSeedRepo(domainsettings.SystemSetting{ + Namespace: "chat", + Key: "model_option_allowed_paths", + Value: string(previousJSON), + ValueType: "json", + }) + service := NewService(repo, "") + + if err := service.Seed(context.Background(), config.Config{}); err != nil { + t.Fatalf("seed settings: %v", err) + } + if got := repo.items["chat:model_option_allowed_paths"].Value; got != config.DefaultModelOptionAllowedPathsJSON() { + t.Fatalf("expected legacy Gemini Interactions paths to migrate, got %q", got) + } +} + func TestSeedAddsGeminiGenerateContentThinkingPathsToPreviousDefaultModelOptionAllowedPaths(t *testing.T) { previousDefault := map[string][]string{} if err := json.Unmarshal([]byte(config.DefaultModelOptionAllowedPathsJSON()), &previousDefault); err != nil { diff --git a/backend/internal/infra/config/config.go b/backend/internal/infra/config/config.go index 6406fb82..48575dce 100644 --- a/backend/internal/infra/config/config.go +++ b/backend/internal/infra/config/config.go @@ -142,11 +142,7 @@ func DefaultModelOptionAllowedPathsJSON() string { "response_format.aspect_ratio", "response_format.image_size", "response_format.mime_type", - "responseFormat.type", - "responseFormat.aspectRatio", - "responseFormat.imageSize", - "responseFormat.mimeType", - "generationConfig.videoConfig.task", + "response_format.schema", "generation_config.video_config.task" ], "xai_responses": [ diff --git a/backend/internal/infra/llm/gemini_interactions.go b/backend/internal/infra/llm/gemini_interactions.go index 7e01f5b4..3aab83ab 100644 --- a/backend/internal/infra/llm/gemini_interactions.go +++ b/backend/internal/infra/llm/gemini_interactions.go @@ -129,6 +129,11 @@ func (c *Client) generateGeminiInteractionStream( if err = consumeGeminiInteractionStream(streamBody, result, onEvent); err != nil { return nil, MarkRequestAccepted(attachUpstreamDebug(err, upstreamDebugSnapshot(req, payload, resp, streamErrorBody(streamBody, err)))) } + for index := range result.GeneratedImages { + if result.GeneratedImages[index].RevisedPrompt == "" { + result.GeneratedImages[index].RevisedPrompt = result.Text + } + } return result, nil } @@ -352,9 +357,6 @@ func buildGeminiInteractionResponseFormat(route RouteConfig, options map[string] } } raw := modelParamMap(options, "response_format") - if len(raw) == 0 { - raw = modelParamMap(options, "responseFormat") - } return normalizeGeminiInteractionResponseFormat(route, raw) } @@ -378,53 +380,46 @@ func normalizeGeminiInteractionResponseFormat(route RouteConfig, raw map[string] if aspectRatio := geminiInteractionAspectRatio(getString(raw["aspect_ratio"]), responseType); aspectRatio != "" { format["aspect_ratio"] = aspectRatio } - if aspectRatio := geminiInteractionAspectRatio(getString(raw["aspectRatio"]), responseType); aspectRatio != "" { - format["aspect_ratio"] = aspectRatio - } if imageSize := geminiInteractionImageSize(getString(raw["image_size"])); imageSize != "" { format["image_size"] = imageSize } - if imageSize := geminiInteractionImageSize(getString(raw["imageSize"])); imageSize != "" { - format["image_size"] = imageSize - } if mimeType := geminiInteractionMIMEType(getString(raw["mime_type"]), responseType); mimeType != "" { format["mime_type"] = mimeType } - if mimeType := geminiInteractionMIMEType(getString(raw["mimeType"]), responseType); mimeType != "" { - format["mime_type"] = mimeType + if responseType == "text" && format["mime_type"] == "application/json" { + if schema := asMap(raw["schema"]); len(schema) > 0 { + format["schema"] = schema + } } return format } func firstGeminiInteractionResponseFormatList(options map[string]interface{}) ([]map[string]interface{}, bool) { - for _, key := range []string{"response_format", "responseFormat"} { - value, ok := options[key] - if !ok { - continue - } - switch typed := value.(type) { - case []map[string]interface{}: - items := make([]map[string]interface{}, 0, len(typed)) - for _, item := range typed { - if len(item) > 0 { - items = append(items, item) - } + value, ok := options["response_format"] + if !ok { + return nil, false + } + switch typed := value.(type) { + case []map[string]interface{}: + items := make([]map[string]interface{}, 0, len(typed)) + for _, item := range typed { + if len(item) > 0 { + items = append(items, item) } - return items, len(items) > 0 - case []interface{}: - items := make([]map[string]interface{}, 0, len(typed)) - for _, raw := range typed { - item := asMap(raw) - if len(item) > 0 { - items = append(items, item) - } + } + return items, len(items) > 0 + case []interface{}: + items := make([]map[string]interface{}, 0, len(typed)) + for _, raw := range typed { + item := asMap(raw) + if len(item) > 0 { + items = append(items, item) } - return items, len(items) > 0 - default: - return nil, false } + return items, len(items) > 0 + default: + return nil, false } - return nil, false } func geminiInteractionResponseType(value string) string { @@ -443,36 +438,13 @@ func geminiInteractionResponseType(value string) string { func buildGeminiInteractionGenerationConfig(options map[string]interface{}) map[string]interface{} { config := map[string]interface{}{} raw := modelParamMap(options, "generation_config") - if len(raw) == 0 { - raw = modelParamMap(options, "generationConfig") - } for key, value := range raw { - if strings.TrimSpace(key) != "" { - config[camelToSnakeGeminiInteractionKey(key)] = value + if strings.TrimSpace(key) != "" && key != "video_config" { + config[key] = value } } - if maxTokens, ok := firstGeminiIntOption(options, "max_output_tokens", "max_completion_tokens", "maxOutputTokens"); ok && maxTokens > 0 { - config["max_output_tokens"] = maxTokens - } - if value, ok := modelParamFloat(options, "temperature"); ok { - config["temperature"] = value - } - if value, ok := firstGeminiFloatOption(options, "top_p", "topP"); ok { - config["top_p"] = value - } - if topK, ok := firstGeminiIntOption(options, "top_k", "topK"); ok && topK > 0 { - config["top_k"] = topK - } - if stops := firstGeminiStringListOption(options, "stop", "stop_sequences", "stopSequences"); len(stops) > 0 { - config["stop_sequences"] = stops - } - if level := firstGeminiStringOption(options, "thinking_level", "thinkingLevel"); level != "" { - config["thinking_level"] = level - } if videoConfig := buildGeminiInteractionVideoConfig(modelParamMap(raw, "video_config")); len(videoConfig) > 0 { config["video_config"] = videoConfig - } else if videoConfig := buildGeminiInteractionVideoConfig(modelParamMap(raw, "videoConfig")); len(videoConfig) > 0 { - config["video_config"] = videoConfig } return config } @@ -535,38 +507,19 @@ func geminiInteractionMIMEType(value string, responseType string) string { return "" } switch responseType { - case "image": + case "text": switch normalized { - case "image/png", "image/jpeg", "image/webp": + case "application/json", "text/plain": return normalized } - case "video": - if strings.HasPrefix(normalized, "video/") { + case "image": + if normalized == "image/jpeg" { return normalized } } return "" } -func camelToSnakeGeminiInteractionKey(value string) string { - switch strings.TrimSpace(value) { - case "maxOutputTokens": - return "max_output_tokens" - case "topP": - return "top_p" - case "topK": - return "top_k" - case "stopSequences": - return "stop_sequences" - case "videoConfig": - return "video_config" - case "thinkingLevel": - return "thinking_level" - default: - return value - } -} - func buildGeminiInteractionTools(tools []ToolDefinition) []map[string]interface{} { if len(tools) == 0 { return nil @@ -592,27 +545,11 @@ func geminiInteractionsProtectedProviderOptionKeys() []string { "input", "model", "response_format", - "responseFormat", - "thinking_level", - "thinkingLevel", "generation_config", - "generationConfig", - "max_completion_tokens", - "max_output_tokens", - "maxOutputTokens", "previous_interaction_id", - "previousInteractionId", - "stop", - "stopSequences", "stream", "system_instruction", - "systemInstruction", - "temperature", "tools", - "top_k", - "top_p", - "topK", - "topP", } } @@ -670,17 +607,18 @@ func consumeGeminiInteractionStream( return flush() } -type geminiInteractionStreamToolStep struct { - callID string - name string -} - type geminiInteractionStreamState struct { - toolSteps map[int64]geminiInteractionStreamToolStep + toolCallIndexes map[int64]int + argumentDeltaStarted map[int64]bool + serverToolCallIDs map[int64]string } func newGeminiInteractionStreamState() *geminiInteractionStreamState { - return &geminiInteractionStreamState{toolSteps: make(map[int64]geminiInteractionStreamToolStep)} + return &geminiInteractionStreamState{ + toolCallIndexes: make(map[int64]int), + argumentDeltaStarted: make(map[int64]bool), + serverToolCallIDs: make(map[int64]string), + } } // applyGeminiInteractionStreamEvent 将单个官方 event_type 事件归并到统一生成结果并向会话层发送增量。 @@ -697,38 +635,44 @@ func applyGeminiInteractionStreamEvent( if responseID := geminiInteractionStreamResponseID(parsed, eventType); responseID != "" { result.ResponseID = responseID } + if serviceTier := geminiInteractionStreamServiceTier(parsed); serviceTier != "" { + result.Usage.ServiceTier = serviceTier + } if finalPayload := geminiInteractionStreamFinalPayload(parsed, eventType); len(finalPayload) > 0 { return mergeGeminiInteractionStreamFinal(result, finalPayload, onEvent) } - if delta := geminiInteractionStreamTextDelta(parsed); delta != "" { - result.Text += delta + if reasoning := geminiInteractionStreamReasoningDelta(parsed, eventType); reasoning != nil { + mergeReasoningDeltaOutput(&result.Reasoning, reasoning) if onEvent != nil { if err := onEvent(GenerateStreamEvent{ - Delta: delta, + Reasoning: reasoning, ResponseID: result.ResponseID, }); err != nil { return err } } } - if reasoning := parseGeminiInteractionReasoningDelta(parsed); reasoning != nil { - mergeReasoningDeltaOutput(&result.Reasoning, reasoning) + if delta := geminiInteractionStreamText(parsed, eventType); delta != "" { + result.Text += delta if onEvent != nil { if err := onEvent(GenerateStreamEvent{ - Reasoning: reasoning, + Delta: delta, ResponseID: result.ResponseID, }); err != nil { return err } } } - for _, call := range parseGeminiInteractionFunctionCalls(parsed) { - result.ToolCalls = append(result.ToolCalls, call) + if err := applyGeminiInteractionStreamMedia(parsed, eventType, result, onEvent); err != nil { + return err } - result.ToolCalls = dedupeGeminiInteractionToolCalls(result.ToolCalls) - for _, call := range parseGeminiInteractionStreamServerToolCalls(parsed, streamState) { - merged := appendGeminiInteractionServerToolCall(result, call) + updateGeminiInteractionStreamToolCall(result, streamState, parsed, eventType) + if call, ok := updateGeminiInteractionStreamServerToolCall(streamState, parsed, eventType); ok { + appendUniqueToolCall(&result.ServerToolCalls, call) + result.ServerSideToolUsage = geminiInteractionServerToolUsage(result.ServerToolCalls) + result.Citations = geminiInteractionServerToolCitations(result.ServerToolCalls) if onEvent != nil { + merged := geminiInteractionServerToolCall(result.ServerToolCalls, call.ToolCallID) if err := onEvent(GenerateStreamEvent{ ServerToolCall: &merged, ResponseID: result.ResponseID, @@ -737,10 +681,10 @@ func applyGeminiInteractionStreamEvent( } } } - result.ServerSideToolUsage = geminiInteractionServerToolUsage(result.ServerToolCalls) - result.GeneratedImages = dedupeGeminiInteractionImages(append(result.GeneratedImages, extractGeminiInteractionGeneratedImages(parsed)...)) - result.GeneratedVideos = dedupeGeminiInteractionVideos(append(result.GeneratedVideos, extractGeminiInteractionGeneratedVideos(parsed)...)) if usage := parseGeminiInteractionUsage(parsed); usage != (Usage{}) { + if usage.ServiceTier == "" { + usage.ServiceTier = result.Usage.ServiceTier + } result.Usage = usage if onEvent != nil { return onEvent(GenerateStreamEvent{ @@ -752,19 +696,26 @@ func applyGeminiInteractionStreamEvent( return nil } +func geminiInteractionStreamServiceTier(parsed map[string]interface{}) string { + if interaction := asMap(parsed["interaction"]); len(interaction) > 0 { + return strings.TrimSpace(getString(interaction["service_tier"])) + } + return strings.TrimSpace(getString(parsed["service_tier"])) +} + func geminiInteractionStreamResponseID(parsed map[string]interface{}, eventType string) string { if interaction := asMap(parsed["interaction"]); len(interaction) > 0 { - return firstString(interaction, "id", "name") + return strings.TrimSpace(getString(interaction["id"])) } - if strings.HasPrefix(strings.ToLower(eventType), "interaction.") { - return firstString(parsed, "id", "name", "interaction_id", "interactionId") + if strings.EqualFold(strings.TrimSpace(eventType), "interaction.status_update") { + return strings.TrimSpace(getString(parsed["interaction_id"])) } - return firstString(parsed, "interaction_id", "interactionId") + return "" } func geminiInteractionStreamFinalPayload(parsed map[string]interface{}, eventType string) map[string]interface{} { eventType = strings.ToLower(strings.TrimSpace(eventType)) - if eventType != "interaction.completed" && eventType != "completed" { + if eventType != "interaction.completed" { return nil } return parsed @@ -798,79 +749,178 @@ func mergeGeminiInteractionStreamFinal( } } if finalOutput.Usage != (Usage{}) { - result.Usage = finalOutput.Usage + usage := finalOutput.Usage + if usage.ServiceTier == "" { + usage.ServiceTier = result.Usage.ServiceTier + } + result.Usage = usage if onEvent != nil { if err := onEvent(GenerateStreamEvent{ - Usage: finalOutput.Usage, + Usage: usage, ResponseID: result.ResponseID, }); err != nil { return err } } } - result.ToolCalls = dedupeGeminiInteractionToolCalls(append(result.ToolCalls, finalOutput.ToolCalls...)) mergeReasoningOutput(&result.Reasoning, finalOutput.Reasoning) - if len(finalOutput.ServerToolCalls) > 0 { - result.ServerToolCalls = mergeGeminiInteractionFinalServerToolCalls(result.ServerToolCalls, finalOutput.ServerToolCalls) - result.ServerSideToolUsage = geminiInteractionServerToolUsage(result.ServerToolCalls) - if onEvent != nil { - for index := range result.ServerToolCalls { - if err := onEvent(GenerateStreamEvent{ - ServerToolCall: &result.ServerToolCalls[index], - ResponseID: result.ResponseID, - }); err != nil { - return err - } - } - } + for _, call := range finalOutput.ToolCalls { + appendUniqueToolCall(&result.ToolCalls, call) } + result.ServerToolCalls = mergeGeminiInteractionFinalServerToolCalls(result.ServerToolCalls, finalOutput.ServerToolCalls) + result.ServerSideToolUsage = geminiInteractionServerToolUsage(result.ServerToolCalls) + result.Citations = appendUniqueStrings(result.Citations, finalOutput.Citations...) result.GeneratedImages = dedupeGeminiInteractionImages(append(result.GeneratedImages, finalOutput.GeneratedImages...)) result.GeneratedVideos = dedupeGeminiInteractionVideos(append(result.GeneratedVideos, finalOutput.GeneratedVideos...)) return nil } -func geminiInteractionStreamTextDelta(parsed map[string]interface{}) string { - eventType := strings.ToLower(strings.TrimSpace(getString(parsed["event_type"]))) - if strings.Contains(eventType, "text") || strings.Contains(eventType, "output") { - for _, key := range []string{"delta", "text", "output_text"} { - if text := geminiInteractionTextDeltaFromValue(parsed[key]); text != "" { - return text +func applyGeminiInteractionStreamMedia( + parsed map[string]interface{}, + eventType string, + result *GenerateOutput, + onEvent func(GenerateStreamEvent) error, +) error { + if result == nil { + return nil + } + images, videos := geminiInteractionStreamMedia(parsed, eventType) + for _, image := range images { + if geminiInteractionImageExists(result.GeneratedImages, image) { + continue + } + image.RevisedPrompt = result.Text + imageIndex := int64(len(result.GeneratedImages)) + result.GeneratedImages = append(result.GeneratedImages, image) + if onEvent != nil { + if err := onEvent(GenerateStreamEvent{ + GeneratedImage: &image, + GeneratedImageIndex: imageIndex, + GeneratedImagePartial: true, + ResponseID: result.ResponseID, + }); err != nil { + return err } } } - return geminiInteractionTextDeltaFromValue(parsed["delta"]) + result.GeneratedVideos = dedupeGeminiInteractionVideos(append(result.GeneratedVideos, videos...)) + return nil } -func geminiInteractionTextDeltaFromValue(raw interface{}) string { - switch typed := raw.(type) { - case string: - return typed - case []interface{}: - parts := make([]string, 0, len(typed)) - for _, item := range typed { - if text := geminiInteractionTextDeltaFromValue(item); text != "" { - parts = append(parts, text) - } +func geminiInteractionStreamMedia(parsed map[string]interface{}, eventType string) ([]GeneratedImage, []GeneratedVideo) { + var value interface{} + switch strings.ToLower(strings.TrimSpace(eventType)) { + case "step.start": + step := asMap(parsed["step"]) + if strings.ToLower(strings.TrimSpace(getString(step["type"]))) != "model_output" { + return nil, nil } - return strings.Join(parts, "") - case map[string]interface{}: - itemType := strings.ToLower(strings.TrimSpace(getString(typed["type"]))) - switch itemType { - case "text", "output_text", "model_output": - if text := getString(typed["text"]); text != "" { - return text - } - return geminiInteractionTextDeltaFromValue(typed["content"]) - case "thinking", "thought", "thought_summary", "thought_signature", "reasoning", "function_call", "function_result", "image", "video": + value = step["content"] + case "step.delta": + delta := asMap(parsed["delta"]) + switch strings.ToLower(strings.TrimSpace(getString(delta["type"]))) { + case "image", "video": + value = delta + default: + return nil, nil + } + default: + return nil, nil + } + images := make([]GeneratedImage, 0) + videos := make([]GeneratedVideo, 0) + walkGeminiInteractionImages(value, &images) + walkGeminiInteractionVideos(value, &videos) + return dedupeGeminiInteractionImages(images), dedupeGeminiInteractionVideos(videos) +} + +func geminiInteractionImageExists(images []GeneratedImage, candidate GeneratedImage) bool { + key := strings.TrimSpace(candidate.URL) + if key == "" { + key = strings.TrimSpace(candidate.B64JSON) + } + if key == "" { + return true + } + for _, image := range images { + current := strings.TrimSpace(image.URL) + if current == "" { + current = strings.TrimSpace(image.B64JSON) + } + if current == key { + return true + } + } + return false +} + +func geminiInteractionStreamText(parsed map[string]interface{}, eventType string) string { + switch strings.ToLower(strings.TrimSpace(eventType)) { + case "step.start": + step := asMap(parsed["step"]) + if strings.ToLower(strings.TrimSpace(getString(step["type"]))) != "model_output" { return "" } - if text := getString(typed["text"]); text != "" { - return text + var text strings.Builder + for _, rawContent := range asSlice(step["content"]) { + content := asMap(rawContent) + if strings.ToLower(strings.TrimSpace(getString(content["type"]))) == "text" { + text.WriteString(getString(content["text"])) + } + } + return text.String() + case "step.delta": + delta := asMap(parsed["delta"]) + if strings.ToLower(strings.TrimSpace(getString(delta["type"]))) == "text" { + return getString(delta["text"]) + } + } + return "" +} + +func geminiInteractionStreamReasoningDelta(parsed map[string]interface{}, eventType string) *ReasoningDelta { + result := &ReasoningDelta{ + EventType: eventType, + ItemID: fmt.Sprintf("%v", parsed["index"]), + Status: "streaming", + } + switch strings.ToLower(strings.TrimSpace(eventType)) { + case "step.start": + step := asMap(parsed["step"]) + if strings.ToLower(strings.TrimSpace(getString(step["type"]))) != "thought" { + return nil + } + result.Kind = "summary_text" + result.Text = geminiInteractionSummaryText(step["summary"]) + result.Signature = strings.TrimSpace(getString(step["signature"])) + if result.Text == "" && result.Signature == "" { + return nil + } + case "step.delta": + delta := asMap(parsed["delta"]) + switch strings.ToLower(strings.TrimSpace(getString(delta["type"]))) { + case "thought_summary": + content := asMap(delta["content"]) + if strings.ToLower(strings.TrimSpace(getString(content["type"]))) != "text" { + return nil + } + result.Kind = "summary_text" + result.Text = getString(content["text"]) + if result.Text == "" { + return nil + } + case "thought_signature": + result.Signature = strings.TrimSpace(getString(delta["signature"])) + if result.Signature == "" { + return nil + } + default: + return nil } - return geminiInteractionTextDeltaFromValue(typed["content"]) default: - return "" + return nil } + return result } func parseGeminiInteractionOutput(body []byte) (*GenerateOutput, error) { @@ -891,8 +941,8 @@ func parseGeminiInteractionPayload(parsed map[string]interface{}) *GenerateOutpu } usage := parseGeminiInteractionUsage(parsed) output := &GenerateOutput{ - ResponseID: firstString(payload, "id", "name"), - Text: strings.TrimSpace(firstString(payload, "text", "output_text")), + ResponseID: strings.TrimSpace(getString(payload["id"])), + Text: geminiInteractionTextFromSteps(payload["steps"]), Reasoning: parseGeminiInteractionReasoning(payload), Usage: usage, ToolCalls: parseGeminiInteractionFunctionCalls(payload), @@ -901,12 +951,7 @@ func parseGeminiInteractionPayload(parsed map[string]interface{}) *GenerateOutpu GeneratedVideos: extractGeminiInteractionGeneratedVideos(payload), } output.ServerSideToolUsage = geminiInteractionServerToolUsage(output.ServerToolCalls) - if output.Text == "" { - output.Text = geminiInteractionTextFromOutput(payload["output"]) - } - if output.Text == "" { - output.Text = geminiInteractionTextFromSteps(payload["steps"]) - } + output.Citations = geminiInteractionServerToolCitations(output.ServerToolCalls) for i := range output.GeneratedImages { if output.GeneratedImages[i].RevisedPrompt == "" { output.GeneratedImages[i].RevisedPrompt = output.Text @@ -940,63 +985,40 @@ func parseGeminiInteractionUsage(parsed map[string]interface{}) Usage { } } -// parseGeminiInteractionReasoning 提取完整响应中的 thought 摘要和校验签名。 func parseGeminiInteractionReasoning(parsed map[string]interface{}) *ReasoningOutput { result := &ReasoningOutput{} - for _, raw := range asSlice(parsed["steps"]) { - step := asMap(raw) - if strings.TrimSpace(strings.ToLower(getString(step["type"]))) != "thought" { + summaryParts := make([]string, 0) + for _, rawStep := range asSlice(parsed["steps"]) { + step := asMap(rawStep) + if strings.ToLower(strings.TrimSpace(getString(step["type"]))) != "thought" { continue } - result.Summary += extractReasoningDeltaText(step["summary"]) + if summary := geminiInteractionSummaryText(step["summary"]); summary != "" { + summaryParts = append(summaryParts, summary) + } if signature := strings.TrimSpace(getString(step["signature"])); signature != "" { result.Signature = signature } } - if strings.TrimSpace(result.Summary) == "" && strings.TrimSpace(result.Signature) == "" { + result.Summary = strings.Join(summaryParts, "\n\n") + if result.Summary == "" && result.Signature == "" { return nil } return result } -// parseGeminiInteractionReasoningDelta 映射官方 thought_summary 与 thought_signature 流式增量。 -func parseGeminiInteractionReasoningDelta(parsed map[string]interface{}) *ReasoningDelta { - eventType := strings.ToLower(strings.TrimSpace(getString(parsed["event_type"]))) - switch eventType { - case "step.start": - step := asMap(parsed["step"]) - if strings.TrimSpace(strings.ToLower(getString(step["type"]))) != "thought" { - return nil +func geminiInteractionSummaryText(raw interface{}) string { + parts := make([]string, 0) + for _, rawContent := range asSlice(raw) { + content := asMap(rawContent) + if strings.ToLower(strings.TrimSpace(getString(content["type"]))) != "text" { + continue } - return geminiInteractionReasoningDelta(eventType, step) - case "step.delta": - delta := asMap(parsed["delta"]) - switch strings.TrimSpace(strings.ToLower(getString(delta["type"]))) { - case "thought_summary", "thought_signature": - return geminiInteractionReasoningDelta(eventType, delta) + if text := strings.TrimSpace(getString(content["text"])); text != "" { + parts = append(parts, text) } } - return nil -} - -func geminiInteractionReasoningDelta(eventType string, payload map[string]interface{}) *ReasoningDelta { - payloadType := strings.TrimSpace(strings.ToLower(getString(payload["type"]))) - text := "" - if payloadType == "thought" { - text = extractReasoningDeltaText(payload["summary"]) - } else if payloadType == "thought_summary" { - text = extractReasoningDeltaText(payload["content"]) - } - signature := strings.TrimSpace(getString(payload["signature"])) - if text == "" && signature == "" { - return nil - } - return &ReasoningDelta{ - EventType: eventType, - Kind: "summary_text", - Text: text, - Signature: signature, - } + return strings.Join(parts, "\n\n") } func rawJSONFromValue(value interface{}) string { @@ -1012,162 +1034,108 @@ func rawJSONFromValue(value interface{}) string { func parseGeminiInteractionFunctionCalls(parsed map[string]interface{}) []ToolCall { calls := make([]ToolCall, 0) - walkGeminiInteractionFunctionCalls(parsed["steps"], &calls) - walkGeminiInteractionFunctionCalls(parsed["output"], &calls) + for _, rawStep := range asSlice(parsed["steps"]) { + if call, ok := geminiInteractionToolCallFromMap(asMap(rawStep)); ok { + calls = append(calls, call) + } + } return dedupeGeminiInteractionToolCalls(calls) } -// parseGeminiInteractionServerToolCalls 将完整响应中的调用步骤与结果步骤合并为稳定的服务端工具轨迹。 func parseGeminiInteractionServerToolCalls(parsed map[string]interface{}) []ToolCall { calls := make([]ToolCall, 0) - for index, raw := range asSlice(parsed["steps"]) { - call, isResult, ok := geminiInteractionServerToolCallFromStep(asMap(raw)) - if ok { - if !isResult && call.ToolCallID == "" { - call.ToolCallID = geminiInteractionStreamToolCallID(call.ToolName, int64(index)) - } + for _, rawStep := range asSlice(parsed["steps"]) { + if call, ok := parseGeminiInteractionServerToolCall(asMap(rawStep), false); ok { appendUniqueToolCall(&calls, call) } } return calls } -// parseGeminiInteractionStreamServerToolCalls 按步骤索引关联不携带调用 ID 的增量事件，避免同一次调用被拆成多条轨迹。 -func parseGeminiInteractionStreamServerToolCalls( - parsed map[string]interface{}, +func updateGeminiInteractionStreamServerToolCall( state *geminiInteractionStreamState, -) []ToolCall { + parsed map[string]interface{}, + eventType string, +) (ToolCall, bool) { if state == nil { - state = newGeminiInteractionStreamState() + return ToolCall{}, false } - index := toInt64(parsed["index"]) - switch strings.ToLower(strings.TrimSpace(getString(parsed["event_type"]))) { + index, ok := geminiInteractionStreamStepIndex(parsed) + if !ok { + return ToolCall{}, false + } + var payload map[string]interface{} + switch strings.ToLower(strings.TrimSpace(eventType)) { case "step.start": - step := asMap(parsed["step"]) - call, isResult, ok := geminiInteractionServerToolCallFromStep(step) - if !ok { - return nil - } - callID := call.ToolCallID - if !isResult && callID == "" { - callID = geminiInteractionStreamToolCallID(call.ToolName, index) - call.ToolCallID = callID - } - state.toolSteps[index] = geminiInteractionStreamToolStep{ - callID: callID, - name: call.ToolName, - } - if isResult && call.OutputJSON == "" && call.ErrorJSON == "" { - return nil - } - return []ToolCall{call} + payload = asMap(parsed["step"]) case "step.delta": - delta := asMap(parsed["delta"]) - call, _, ok := geminiInteractionServerToolCallFromStep(delta) - if !ok { - return nil - } - if step, exists := state.toolSteps[index]; exists { - if call.ToolCallID == "" { - call.ToolCallID = step.callID - } - if call.ToolName == "" { - call.ToolName = step.name - call.ToolType = step.name - } - } - return []ToolCall{call} + payload = asMap(parsed["delta"]) default: - return nil + return ToolCall{}, false + } + call, ok := parseGeminiInteractionServerToolCall(payload, true) + if !ok { + return ToolCall{}, false } + if call.ToolCallID == "" { + call.ToolCallID = state.serverToolCallIDs[index] + } + if call.ToolCallID == "" { + _, isResult := geminiInteractionServerToolName(getString(payload["type"])) + if isResult { + return ToolCall{}, false + } + call.ToolCallID = geminiInteractionStreamToolCallID(call.ToolName, index) + } + if state.serverToolCallIDs == nil { + state.serverToolCallIDs = make(map[int64]string) + } + state.serverToolCallIDs[index] = call.ToolCallID + return call, true } -// geminiInteractionServerToolCallFromStep 只解析 Interactions 原生托管工具步骤，并标记当前步骤是否为结果。 -func geminiInteractionServerToolCallFromStep(step map[string]interface{}) (ToolCall, bool, bool) { - name, isResult, ok := geminiInteractionServerToolStepType(getString(step["type"])) - if !ok { - return ToolCall{}, false, false +func parseGeminiInteractionServerToolCall(item map[string]interface{}, streaming bool) (ToolCall, bool) { + itemType := strings.ToLower(strings.TrimSpace(getString(item["type"]))) + toolName, isResult := geminiInteractionServerToolName(itemType) + if toolName == "" { + return ToolCall{}, false } + callID := strings.TrimSpace(getString(item["id"])) if isResult { - result, hasResult := step["result"] - if !hasResult { - return ToolCall{ - ToolCallID: strings.TrimSpace(getString(step["call_id"])), - ToolType: name, - ToolName: name, - }, true, true - } - call := ToolCall{ - ToolCallID: strings.TrimSpace(getString(step["call_id"])), - ToolType: name, - ToolName: name, - ThoughtSignature: strings.TrimSpace(getString(step["signature"])), - Status: "completed", - OutputJSON: rawJSONFromValue(result), - } - if failed, _ := step["is_error"].(bool); failed { - call.Status = "error" - call.ErrorJSON = call.OutputJSON - } - return call, true, true + callID = strings.TrimSpace(getString(item["call_id"])) + } + status := "in_progress" + outputJSON := "" + errorJSON := "" + if isResult { + _, hasResult := item["result"] + if !streaming || hasResult { + status = "completed" + } + outputJSON = rawJSONFromValue(item["result"]) + if isError, _ := item["is_error"].(bool); isError { + status = "error" + errorJSON = outputJSON + } } return ToolCall{ - ToolCallID: strings.TrimSpace(getString(step["id"])), - ToolType: name, - ToolName: name, - ArgumentsJSON: rawJSONFromValue(step["arguments"]), - ThoughtSignature: strings.TrimSpace(getString(step["signature"])), - Status: "in_progress", - }, false, true + ToolCallID: callID, + ToolType: toolName, + ToolName: toolName, + ArgumentsJSON: rawJSONFromValue(item["arguments"]), + ThoughtSignature: strings.TrimSpace(getString(item["signature"])), + Status: status, + OutputJSON: outputJSON, + ErrorJSON: errorJSON, + }, true } func geminiInteractionStreamToolCallID(name string, index int64) string { return fmt.Sprintf("gemini_interaction_%s_%d", name, index) } -func geminiInteractionServerToolStepType(stepType string) (name string, isResult bool, ok bool) { - switch strings.ToLower(strings.TrimSpace(stepType)) { - case "google_search_call": - return "google_search", false, true - case "google_search_result": - return "google_search", true, true - case "code_execution_call": - return "code_execution", false, true - case "code_execution_result": - return "code_execution", true, true - case "url_context_call": - return "url_context", false, true - case "url_context_result": - return "url_context", true, true - default: - return "", false, false - } -} - -// appendGeminiInteractionServerToolCall 将流式调用与结果合并，并优先复用上游提供的调用 ID。 -func appendGeminiInteractionServerToolCall(result *GenerateOutput, call ToolCall) ToolCall { - if result == nil { - return call - } - if call.ToolCallID == "" && (call.OutputJSON != "" || call.ErrorJSON != "") { - for index := len(result.ServerToolCalls) - 1; index >= 0; index-- { - current := result.ServerToolCalls[index] - if current.ToolName == call.ToolName && current.OutputJSON == "" && current.ErrorJSON == "" { - call.ToolCallID = current.ToolCallID - break - } - } - } - appendUniqueToolCall(&result.ServerToolCalls, call) - for _, current := range result.ServerToolCalls { - if shouldMergeToolCall(current, call) { - return current - } - } - return call -} - -// mergeGeminiInteractionFinalServerToolCalls 用完成事件的完整轨迹补齐增量状态，同时保留已对外发送的稳定 ID。 +// mergeGeminiInteractionFinalServerToolCalls uses the completed interaction to fill streamed +// native-tool traces while preserving any stable ID already emitted to conversation consumers. func mergeGeminiInteractionFinalServerToolCalls(current []ToolCall, final []ToolCall) []ToolCall { if len(current) == 0 { return final @@ -1207,35 +1175,172 @@ func mergeGeminiInteractionFinalServerToolCalls(current []ToolCall, final []Tool return merged } +func geminiInteractionServerToolName(itemType string) (string, bool) { + switch strings.ToLower(strings.TrimSpace(itemType)) { + case "google_search_call": + return "google_search", false + case "google_search_result": + return "google_search", true + case "code_execution_call": + return "code_execution", false + case "code_execution_result": + return "code_execution", true + case "url_context_call": + return "url_context", false + case "url_context_result": + return "url_context", true + default: + return "", false + } +} + +func geminiInteractionServerToolCall(calls []ToolCall, callID string) ToolCall { + for _, call := range calls { + if strings.TrimSpace(call.ToolCallID) == strings.TrimSpace(callID) { + return call + } + } + return ToolCall{ToolCallID: strings.TrimSpace(callID)} +} + func geminiInteractionServerToolUsage(calls []ToolCall) map[string]int64 { if len(calls) == 0 { return nil } usage := make(map[string]int64) for _, call := range calls { - if name := strings.TrimSpace(call.ToolName); name != "" { + name := strings.TrimSpace(call.ToolName) + if name == "" { + name = strings.TrimSpace(call.ToolType) + } + if name != "" { usage[name]++ } } + if len(usage) == 0 { + return nil + } return usage } -func walkGeminiInteractionFunctionCalls(value interface{}, calls *[]ToolCall) { +func geminiInteractionServerToolCitations(calls []ToolCall) []string { + citations := make([]string, 0) + for _, call := range calls { + if call.ToolName != "google_search" && call.ToolName != "url_context" { + continue + } + var output interface{} + if err := json.Unmarshal([]byte(call.OutputJSON), &output); err != nil { + continue + } + walkGeminiInteractionCitationURLs(output, &citations) + } + return appendUniqueStrings(nil, citations...) +} + +func walkGeminiInteractionCitationURLs(value interface{}, citations *[]string) { switch typed := value.(type) { case map[string]interface{}: - if call, ok := geminiInteractionToolCallFromMap(typed); ok { - *calls = append(*calls, call) - } - for _, child := range typed { - walkGeminiInteractionFunctionCalls(child, calls) + for key, child := range typed { + if key == "url" || key == "uri" { + if citation := strings.TrimSpace(getString(child)); citation != "" { + *citations = append(*citations, citation) + } + } + walkGeminiInteractionCitationURLs(child, citations) } case []interface{}: for _, child := range typed { - walkGeminiInteractionFunctionCalls(child, calls) + walkGeminiInteractionCitationURLs(child, citations) } } } +func updateGeminiInteractionStreamToolCall( + result *GenerateOutput, + state *geminiInteractionStreamState, + parsed map[string]interface{}, + eventType string, +) { + if result == nil || state == nil { + return + } + index, ok := geminiInteractionStreamStepIndex(parsed) + if !ok { + return + } + switch strings.ToLower(strings.TrimSpace(eventType)) { + case "step.start": + step := asMap(parsed["step"]) + if strings.ToLower(strings.TrimSpace(getString(step["type"]))) != "function_call" { + return + } + name := strings.TrimSpace(getString(step["name"])) + if name == "" { + return + } + if state.toolCallIndexes == nil { + state.toolCallIndexes = make(map[int64]int) + } + arguments := normalizeJSONString(step["arguments"]) + result.ToolCalls = append(result.ToolCalls, ToolCall{ + ToolCallID: strings.TrimSpace(getString(step["id"])), + ToolType: "function", + ToolName: name, + ArgumentsJSON: arguments, + Status: "requested", + }) + state.toolCallIndexes[index] = len(result.ToolCalls) - 1 + case "step.delta": + delta := asMap(parsed["delta"]) + if strings.ToLower(strings.TrimSpace(getString(delta["type"]))) != "arguments_delta" { + return + } + partial := getString(delta["arguments"]) + if partial == "" { + return + } + callIndex, ok := geminiInteractionStreamToolCallIndex(result, state, index) + if !ok { + return + } + if !state.argumentDeltaStarted[index] { + if state.argumentDeltaStarted == nil { + state.argumentDeltaStarted = make(map[int64]bool) + } + result.ToolCalls[callIndex].ArgumentsJSON = "" + state.argumentDeltaStarted[index] = true + } + result.ToolCalls[callIndex].ArgumentsJSON += partial + case "step.stop": + callIndex, ok := geminiInteractionStreamToolCallIndex(result, state, index) + if !ok || strings.TrimSpace(result.ToolCalls[callIndex].ArgumentsJSON) != "" { + return + } + result.ToolCalls[callIndex].ArgumentsJSON = "{}" + } +} + +func geminiInteractionStreamStepIndex(parsed map[string]interface{}) (int64, bool) { + rawIndex, ok := parsed["index"] + if !ok { + return 0, false + } + index := toInt64(rawIndex) + return index, index >= 0 +} + +func geminiInteractionStreamToolCallIndex(result *GenerateOutput, state *geminiInteractionStreamState, index int64) (int, bool) { + if result == nil || state == nil { + return 0, false + } + callIndex, ok := state.toolCallIndexes[index] + if !ok || callIndex >= len(result.ToolCalls) { + return 0, false + } + return callIndex, true +} + func geminiInteractionToolCallFromMap(item map[string]interface{}) (ToolCall, bool) { if strings.TrimSpace(strings.ToLower(getString(item["type"]))) != "function_call" { return ToolCall{}, false @@ -1245,14 +1350,11 @@ func geminiInteractionToolCallFromMap(item map[string]interface{}) (ToolCall, bo return ToolCall{}, false } arguments := normalizeJSONString(item["arguments"]) - if arguments == "" { - arguments = normalizeJSONString(item["args"]) - } if arguments == "" { arguments = "{}" } return ToolCall{ - ToolCallID: firstString(item, "id", "call_id", "tool_call_id"), + ToolCallID: strings.TrimSpace(getString(item["id"])), ToolType: "function", ToolName: name, ArgumentsJSON: arguments, @@ -1282,7 +1384,6 @@ func dedupeGeminiInteractionToolCalls(calls []ToolCall) []ToolCall { func extractGeminiInteractionGeneratedImages(parsed map[string]interface{}) []GeneratedImage { images := make([]GeneratedImage, 0) - walkGeminiInteractionImages(parsed["output"], &images) walkGeminiInteractionModelOutputContent(parsed["steps"], func(content interface{}) { walkGeminiInteractionImages(content, &images) }) @@ -1318,14 +1419,10 @@ func walkGeminiInteractionModelOutputContent(raw interface{}, walk func(interfac } for _, rawStep := range asSlice(raw) { step := asMap(rawStep) - if stepType := strings.TrimSpace(strings.ToLower(getString(step["type"]))); stepType != "" && stepType != "model_output" { + if strings.TrimSpace(strings.ToLower(getString(step["type"]))) != "model_output" { continue } - if content, ok := step["content"]; ok { - walk(content) - continue - } - walk(step) + walk(step["content"]) } } @@ -1335,71 +1432,28 @@ func walkGeminiInteractionImages(value interface{}, images *[]GeneratedImage) { if image, ok := geminiImageFromInteractionMap(typed); ok { *images = append(*images, image) } - for _, child := range typed { - walkGeminiInteractionImages(child, images) - } case []interface{}: for _, child := range typed { - walkGeminiInteractionImages(child, images) + if image, ok := geminiImageFromInteractionMap(asMap(child)); ok { + *images = append(*images, image) + } } } } func geminiImageFromInteractionMap(item map[string]interface{}) (GeneratedImage, bool) { - mimeType := strings.TrimSpace(firstString(item, "mime_type", "mimeType")) - if mimeType == "" { - if inlineData := asMap(item["inlineData"]); len(inlineData) > 0 { - mimeType = strings.TrimSpace(firstString(inlineData, "mimeType", "mime_type")) - } + if strings.TrimSpace(strings.ToLower(getString(item["type"]))) != "image" { + return GeneratedImage{}, false } + mimeType := strings.TrimSpace(getString(item["mime_type"])) if mimeType != "" && !strings.HasPrefix(strings.ToLower(mimeType), "image/") { return GeneratedImage{}, false } - - url := strings.TrimSpace(firstString(item, "uri", "url", "file_uri", "fileUri")) - b64 := strings.TrimSpace(firstString(item, "b64_json", "b64Json", "data")) - if fileData := asMap(item["fileData"]); len(fileData) > 0 { - if url == "" { - url = strings.TrimSpace(firstString(fileData, "fileUri", "file_uri", "uri", "url")) - } - if mimeType == "" { - mimeType = strings.TrimSpace(firstString(fileData, "mimeType", "mime_type")) - } - } - if fileData := asMap(item["file_data"]); len(fileData) > 0 { - if url == "" { - url = strings.TrimSpace(firstString(fileData, "fileUri", "file_uri", "uri", "url")) - } - if mimeType == "" { - mimeType = strings.TrimSpace(firstString(fileData, "mimeType", "mime_type")) - } - } - if inlineData := asMap(item["inlineData"]); len(inlineData) > 0 { - if b64 == "" { - b64 = strings.TrimSpace(firstString(inlineData, "data", "b64_json", "b64Json")) - } - if mimeType == "" { - mimeType = strings.TrimSpace(firstString(inlineData, "mimeType", "mime_type")) - } - } - if inlineData := asMap(item["inline_data"]); len(inlineData) > 0 { - if b64 == "" { - b64 = strings.TrimSpace(firstString(inlineData, "data", "b64_json", "b64Json")) - } - if mimeType == "" { - mimeType = strings.TrimSpace(firstString(inlineData, "mimeType", "mime_type")) - } - } - if nested := asMap(item["image"]); len(nested) > 0 { - image, ok := geminiImageFromInteractionMap(nested) - if ok { - return image, true - } - } - itemType := strings.TrimSpace(strings.ToLower(firstString(item, "type"))) - if mimeType == "" && itemType == "image" { + if mimeType == "" { mimeType = "image/png" } + url := strings.TrimSpace(getString(item["uri"])) + b64 := strings.TrimSpace(getString(item["data"])) if !strings.HasPrefix(strings.ToLower(mimeType), "image/") || (url == "" && b64 == "") { return GeneratedImage{}, false } @@ -1412,7 +1466,6 @@ func geminiImageFromInteractionMap(item map[string]interface{}) (GeneratedImage, func extractGeminiInteractionGeneratedVideos(parsed map[string]interface{}) []GeneratedVideo { videos := make([]GeneratedVideo, 0) - walkGeminiInteractionVideos(parsed["output"], &videos) walkGeminiInteractionModelOutputContent(parsed["steps"], func(content interface{}) { walkGeminiInteractionVideos(content, &videos) }) @@ -1448,53 +1501,25 @@ func walkGeminiInteractionVideos(value interface{}, videos *[]GeneratedVideo) { if video, ok := geminiVideoFromMap(typed); ok { *videos = append(*videos, video) } - for _, child := range typed { - walkGeminiInteractionVideos(child, videos) - } case []interface{}: for _, child := range typed { - walkGeminiInteractionVideos(child, videos) + if video, ok := geminiVideoFromMap(asMap(child)); ok { + *videos = append(*videos, video) + } } } } func geminiVideoFromMap(item map[string]interface{}) (GeneratedVideo, bool) { - fileData := asMap(item["fileData"]) - if len(fileData) == 0 { - fileData = asMap(item["file_data"]) - } - inlineData := asMap(item["inlineData"]) - if len(inlineData) == 0 { - inlineData = asMap(item["inline_data"]) - } - mimeType := strings.TrimSpace(firstString(item, "mime_type", "mimeType")) - if mimeType == "" { - if len(inlineData) > 0 { - mimeType = strings.TrimSpace(firstString(inlineData, "mimeType", "mime_type")) - } + if strings.TrimSpace(strings.ToLower(getString(item["type"]))) != "video" { + return GeneratedVideo{}, false } + mimeType := strings.TrimSpace(getString(item["mime_type"])) if mimeType != "" && !strings.HasPrefix(strings.ToLower(mimeType), "video/") { return GeneratedVideo{}, false } - - url := strings.TrimSpace(firstString(item, "uri", "url", "file_uri", "fileUri")) - b64 := strings.TrimSpace(firstString(item, "b64_json", "b64Json", "data")) - if len(fileData) > 0 { - if url == "" { - url = strings.TrimSpace(firstString(fileData, "fileUri", "file_uri", "uri", "url")) - } - if mimeType == "" { - mimeType = strings.TrimSpace(firstString(fileData, "mimeType", "mime_type")) - } - } - if len(inlineData) > 0 { - if b64 == "" { - b64 = strings.TrimSpace(firstString(inlineData, "data", "b64_json", "b64Json")) - } - if mimeType == "" { - mimeType = strings.TrimSpace(firstString(inlineData, "mimeType", "mime_type")) - } - } + url := strings.TrimSpace(getString(item["uri"])) + b64 := strings.TrimSpace(getString(item["data"])) if mimeType == "" { mimeType = "video/mp4" } @@ -1505,67 +1530,25 @@ func geminiVideoFromMap(item map[string]interface{}) (GeneratedVideo, bool) { URL: url, B64JSON: b64, MIMEType: mimeType, - FileName: strings.TrimSpace(firstString(item, "file_name", "fileName", "name")), - DurationSeconds: generatedMediaDurationSeconds( - item["duration_seconds"], - item["durationSeconds"], - item["duration"], - fileData["duration_seconds"], - fileData["durationSeconds"], - ), }, true } -func geminiInteractionTextFromOutput(raw interface{}) string { - items, ok := raw.([]interface{}) - if !ok { - return "" - } - parts := make([]string, 0, len(items)) - for _, item := range items { - if text := strings.TrimSpace(getString(asMap(item)["text"])); text != "" { - parts = append(parts, text) - } - } - return strings.Join(parts, "\n\n") -} - func geminiInteractionTextFromSteps(raw interface{}) string { - steps, ok := raw.([]interface{}) - if !ok { - return "" - } - parts := make([]string, 0, len(steps)) - for _, rawStep := range steps { + parts := make([]string, 0) + for _, rawStep := range asSlice(raw) { step := asMap(rawStep) - if stepType := strings.TrimSpace(strings.ToLower(getString(step["type"]))); stepType != "" && stepType != "model_output" { + if strings.TrimSpace(strings.ToLower(getString(step["type"]))) != "model_output" { continue } - parts = appendGeminiInteractionTextParts(parts, step["content"]) - } - return strings.Join(parts, "\n\n") -} - -func appendGeminiInteractionTextParts(parts []string, raw interface{}) []string { - switch typed := raw.(type) { - case string: - if text := strings.TrimSpace(typed); text != "" { - return append(parts, text) - } - case map[string]interface{}: - if itemType := strings.TrimSpace(strings.ToLower(getString(typed["type"]))); itemType != "" && itemType != "text" { - return parts - } - if text := strings.TrimSpace(getString(typed["text"])); text != "" { - return append(parts, text) - } - if content, ok := typed["content"]; ok { - parts = appendGeminiInteractionTextParts(parts, content) - } - case []interface{}: - for _, child := range typed { - parts = appendGeminiInteractionTextParts(parts, child) + for _, rawContent := range asSlice(step["content"]) { + content := asMap(rawContent) + if strings.TrimSpace(strings.ToLower(getString(content["type"]))) != "text" { + continue + } + if text := strings.TrimSpace(getString(content["text"])); text != "" { + parts = append(parts, text) + } } } - return parts + return strings.Join(parts, "\n\n") } diff --git a/backend/internal/infra/llm/gemini_interactions_test.go b/backend/internal/infra/llm/gemini_interactions_test.go index 21838eee..923d8f1c 100644 --- a/backend/internal/infra/llm/gemini_interactions_test.go +++ b/backend/internal/infra/llm/gemini_interactions_test.go @@ -132,20 +132,28 @@ func TestBuildGeminiInteractionRequestBodySupportsUniversalOptionsAndTools(t *te }}, Options: map[string]interface{}{ "response_format": []interface{}{ - map[string]interface{}{"type": "text"}, + map[string]interface{}{ + "type": "text", + "mime_type": "application/json", + "schema": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "summary": map[string]interface{}{"type": "string"}, + }, + }, + }, map[string]interface{}{ "type": "image", "aspect_ratio": "1:1", "image_size": "1K", - "mime_type": "image/png", + "mime_type": "image/jpeg", }, }, - "temperature": 0.4, - "top_p": 0.9, - "max_output_tokens": 512, - "thinking_level": "low", "generation_config": map[string]interface{}{ - "thinkingLevel": "high", + "temperature": 0.4, + "top_p": 0.9, + "max_output_tokens": 512, + "thinking_level": "low", "thinking_summaries": "auto", }, }, @@ -160,8 +168,12 @@ func TestBuildGeminiInteractionRequestBodySupportsUniversalOptionsAndTools(t *te if _, leaked := asMap(payload["response_format"])["_list"]; leaked { t.Fatalf("response_format must not use private _list wrapper: %#v", payload["response_format"]) } + textFormat := asMap(formats[0]) + if textFormat["mime_type"] != "application/json" || asMap(textFormat["schema"])["type"] != "object" { + t.Fatalf("unexpected structured text response_format: %#v", textFormat) + } imageFormat := asMap(formats[1]) - if imageFormat["type"] != "image" || imageFormat["aspect_ratio"] != "1:1" || imageFormat["image_size"] != "1K" || imageFormat["mime_type"] != "image/png" { + if imageFormat["type"] != "image" || imageFormat["aspect_ratio"] != "1:1" || imageFormat["image_size"] != "1K" || imageFormat["mime_type"] != "image/jpeg" { t.Fatalf("unexpected image response_format: %#v", imageFormat) } config, ok := payload["generation_config"].(map[string]interface{}) @@ -377,11 +389,14 @@ func TestParseGeminiInteractionOutputExtractsVideoURIAndInlineData(t *testing.T) inline := base64.StdEncoding.EncodeToString([]byte("video")) body := []byte(`{ "id": "interaction-1", - "output": [ - {"type": "video", "durationSeconds": 5.2, "fileData": {"fileUri": "https://example.com/video.mp4", "mimeType": "video/mp4"}}, - {"type": "video", "file_data": {"file_uri": "https://example.com/video.mp4", "mime_type": "video/mp4"}}, - {"type": "video", "duration_seconds": 3, "inlineData": {"data": "` + inline + `", "mimeType": "video/webm"}} - ], + "steps": [{ + "type": "model_output", + "content": [ + {"type": "video", "uri": "https://example.com/video.mp4", "mime_type": "video/mp4"}, + {"type": "video", "uri": "https://example.com/video.mp4", "mime_type": "video/mp4"}, + {"type": "video", "data": "` + inline + `", "mime_type": "video/webm"} + ] + }], "usage": {"total_input_tokens": 3, "total_output_tokens": 5} }`) output, err := parseGeminiInteractionOutput(body) @@ -394,10 +409,10 @@ func TestParseGeminiInteractionOutputExtractsVideoURIAndInlineData(t *testing.T) if got := len(output.GeneratedVideos); got != 2 { t.Fatalf("expected duplicate URI to be deduped, got %d videos: %#v", got, output.GeneratedVideos) } - if output.GeneratedVideos[0].URL != "https://example.com/video.mp4" || output.GeneratedVideos[0].MIMEType != "video/mp4" || output.GeneratedVideos[0].DurationSeconds != 6 { + if output.GeneratedVideos[0].URL != "https://example.com/video.mp4" || output.GeneratedVideos[0].MIMEType != "video/mp4" { t.Fatalf("unexpected URI video: %#v", output.GeneratedVideos[0]) } - if output.GeneratedVideos[1].B64JSON != inline || output.GeneratedVideos[1].MIMEType != "video/webm" || output.GeneratedVideos[1].DurationSeconds != 3 { + if output.GeneratedVideos[1].B64JSON != inline || output.GeneratedVideos[1].MIMEType != "video/webm" { t.Fatalf("unexpected inline video: %#v", output.GeneratedVideos[1]) } if output.Usage.InputTokens != 3 || output.Usage.OutputTokens != 5 { @@ -405,6 +420,41 @@ func TestParseGeminiInteractionOutputExtractsVideoURIAndInlineData(t *testing.T) } } +func TestParseGeminiInteractionOutputExtractsReasoningAndOfficialUsage(t *testing.T) { + body := []byte(`{ + "id": "interaction-reasoning", + "service_tier": "priority", + "steps": [{ + "type": "thought", + "summary": [ + {"type": "text", "text": "Check the inputs."}, + {"type": "text", "text": " Then answer."} + ], + "signature": "thought-signature" + }], + "usage": { + "total_input_tokens": 10, + "total_cached_tokens": 4, + "total_output_tokens": 6, + "total_thought_tokens": 3, + "total_tool_use_tokens": 2 + } + }`) + output, err := parseGeminiInteractionOutput(body) + if err != nil { + t.Fatalf("parse Gemini interaction output: %v", err) + } + if output.Reasoning == nil || output.Reasoning.Summary != "Check the inputs.\n\nThen answer." || output.Reasoning.Signature != "thought-signature" { + t.Fatalf("unexpected reasoning: %#v", output.Reasoning) + } + if output.Usage.InputTokens != 6 || output.Usage.CacheReadTokens != 4 || output.Usage.OutputTokens != 6 || output.Usage.ReasoningTokens != 3 || output.Usage.ServiceTier != "priority" { + t.Fatalf("unexpected usage: %#v", output.Usage) + } + if !strings.Contains(output.Usage.RawUsageJSON, `"total_tool_use_tokens":2`) { + t.Fatalf("expected raw usage to preserve tool-use tokens, got %q", output.Usage.RawUsageJSON) + } +} + func TestParseGeminiInteractionOutputExtractsTextAndImages(t *testing.T) { inline := base64.StdEncoding.EncodeToString([]byte("png")) inputInline := base64.StdEncoding.EncodeToString([]byte("source")) @@ -412,12 +462,12 @@ func TestParseGeminiInteractionOutputExtractsTextAndImages(t *testing.T) { "id": "interaction-2", "steps": [ {"type": "user_input", "content": [ - {"type": "image", "inlineData": {"data": "` + inputInline + `", "mimeType": "image/png"}} + {"type": "image", "data": "` + inputInline + `", "mime_type": "image/png"} ]}, {"type": "model_output", "content": [ {"type": "text", "text": "A revised prompt"}, - {"type": "image", "inlineData": {"data": "` + inline + `", "mimeType": "image/png"}}, - {"type": "image", "fileData": {"fileUri": "https://example.com/image.png", "mimeType": "image/png"}} + {"type": "image", "data": "` + inline + `", "mime_type": "image/png"}, + {"type": "image", "uri": "https://example.com/image.png", "mime_type": "image/png"} ]} ] }`) @@ -446,11 +496,8 @@ func TestParseGeminiInteractionOutputExtractsFunctionCalls(t *testing.T) { body := []byte(`{ "id": "interaction-tools", "steps": [ - {"type": "model_output", "content": "Let me check."}, - {"type": "function_call", "id": "call_weather", "name": "get_weather", "arguments": {"location": "Paris"}}, - {"type": "model_output", "content": [ - {"type": "function_call", "id": "call_weather", "name": "get_weather", "arguments": {"location": "Paris"}} - ]} + {"type": "model_output", "content": [{"type": "text", "text": "Let me check."}]}, + {"type": "function_call", "id": "call_weather", "name": "get_weather", "arguments": {"location": "Paris"}} ] }`) output, err := parseGeminiInteractionOutput(body) @@ -461,7 +508,7 @@ func TestParseGeminiInteractionOutputExtractsFunctionCalls(t *testing.T) { t.Fatalf("expected text from model_output only, got %q", output.Text) } if len(output.ToolCalls) != 1 { - t.Fatalf("expected deduped function call, got %#v", output.ToolCalls) + t.Fatalf("expected function call, got %#v", output.ToolCalls) } call := output.ToolCalls[0] if call.ToolCallID != "call_weather" || call.ToolType != "function" || call.ToolName != "get_weather" || call.Status != "requested" { @@ -664,7 +711,7 @@ func TestGenerateGeminiInteractionPostsInteractionsRequest(t *testing.T) { t.Fatalf("decode request payload: %v", err) } w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"id":"interaction-1","output":[{"fileData":{"fileUri":"https://example.com/video.mp4","mimeType":"video/mp4"}}]}`)) + _, _ = w.Write([]byte(`{"id":"interaction-1","steps":[{"type":"model_output","content":[{"type":"video","uri":"https://example.com/video.mp4","mime_type":"video/mp4"}]}]}`)) })) defer server.Close() @@ -703,22 +750,55 @@ func TestGenerateGeminiInteractionStreamPostsStreamRequest(t *testing.T) { t.Fatalf("decode request payload: %v", err) } w.Header().Set("Content-Type", "text/event-stream") - _, _ = w.Write([]byte(`data: {"event_type":"interaction.created","interaction":{"id":"interaction-stream-1"}} + _, _ = w.Write([]byte(`data: {"event_type":"interaction.created","interaction":{"id":"interaction-stream-1","service_tier":"standard"}} -data: {"event_type":"step.delta","interaction_id":"interaction-stream-1","delta":{"type":"text","text":"Hello"}} +data: {"event_type":"step.start","index":0,"step":{"type":"thought","summary":[{"type":"text","text":"I should check"}],"signature":""}} -data: {"event_type":"step.delta","interaction_id":"interaction-stream-1","delta":{"type":"text","text":" world"}} +data: {"event_type":"step.delta","index":0,"delta":{"type":"thought_summary","content":{"type":"text","text":" the weather."}}} -data: {"event_type":"interaction.completed","interaction":{"id":"interaction-stream-1","steps":[{"type":"model_output","content":[{"type":"text","text":"Hello world"}]}],"usage":{"total_input_tokens":4,"total_output_tokens":2}}} +data: {"event_type":"step.delta","index":0,"delta":{"type":"thought_signature","signature":"stream-signature"}} -data: {"event_type":"done"} +data: {"event_type":"step.start","index":1,"step":{"type":"function_call","id":"call_weather","name":"get_weather","arguments":{"stale":true}}} + +data: {"event_type":"step.delta","index":1,"delta":{"type":"arguments_delta","arguments":"{\"location\":\""}} + +data: {"event_type":"step.delta","index":1,"delta":{"type":"arguments_delta","arguments":"Paris\"}"}} + +data: {"event_type":"step.stop","index":1,"status":"waiting"} + +data: {"event_type":"step.start","index":2,"step":{"type":"model_output","content":[{"type":"text","text":"Hello"}]}} + +data: {"event_type":"step.delta","index":2,"delta":{"type":"text","text":" world"},"metadata":{"total_usage":{"total_input_tokens":4,"total_cached_tokens":1,"total_output_tokens":2,"total_thought_tokens":3,"total_tool_use_tokens":1}}} + +data: {"event_type":"step.start","index":3,"step":{"type":"google_search_call","id":"search_call_1","arguments":{}}} + +data: {"event_type":"step.delta","index":3,"delta":{"type":"google_search_call","arguments":{"queries":["Gemini streaming"]},"signature":"search-signature"}} + +data: {"event_type":"step.start","index":4,"step":{"type":"google_search_result","call_id":"search_call_1"}} + +data: {"event_type":"step.delta","index":4,"delta":{"type":"google_search_result","result":[{"title":"Gemini","url":"https://ai.google.dev/gemini-api/docs/streaming"}],"signature":"result-signature"}} + +data: {"event_type":"step.start","index":5,"step":{"type":"model_output"}} + +data: {"event_type":"step.delta","index":5,"delta":{"type":"image","mime_type":"image/jpeg","data":"aW1hZ2U="}} + +data: {"event_type":"step.start","index":6,"step":{"type":"model_output"}} + +data: {"event_type":"step.delta","index":6,"delta":{"type":"video","mime_type":"video/mp4","uri":"https://example.com/video.mp4"}} + +data: {"event_type":"interaction.completed","interaction":{"id":"interaction-stream-1","status":"completed","usage":{"total_input_tokens":4,"total_cached_tokens":1,"total_output_tokens":2,"total_thought_tokens":3,"total_tool_use_tokens":1}}} + +data: [DONE] `)) })) defer server.Close() var deltas []string + var reasoningDeltas []ReasoningDelta var usageEvents []Usage + var serverToolEvents []ToolCall + var imageEvents []GenerateStreamEvent output, err := newTestClient().GenerateStream(context.Background(), RouteConfig{ Protocol: AdapterGeminiInteractions, BaseURL: server.URL, @@ -730,9 +810,18 @@ data: {"event_type":"done"} if event.Delta != "" { deltas = append(deltas, event.Delta) } + if event.Reasoning != nil { + reasoningDeltas = append(reasoningDeltas, *event.Reasoning) + } if event.Usage != (Usage{}) { usageEvents = append(usageEvents, event.Usage) } + if event.ServerToolCall != nil { + serverToolEvents = append(serverToolEvents, *event.ServerToolCall) + } + if event.GeneratedImage != nil { + imageEvents = append(imageEvents, event) + } return nil }) if err != nil { @@ -747,11 +836,131 @@ data: {"event_type":"done"} if strings.Join(deltas, "") != "Hello world" { t.Fatalf("unexpected stream deltas: %#v", deltas) } - if len(usageEvents) != 1 || output.Usage.InputTokens != 4 || output.Usage.OutputTokens != 2 { + if output.Reasoning == nil || output.Reasoning.Summary != "I should check the weather." || output.Reasoning.Signature != "stream-signature" { + t.Fatalf("unexpected stream reasoning: %#v", output.Reasoning) + } + if len(reasoningDeltas) != 3 || reasoningDeltas[0].Kind != "summary_text" || reasoningDeltas[0].Text != "I should check" || reasoningDeltas[1].Text != " the weather." || reasoningDeltas[2].Signature != "stream-signature" { + t.Fatalf("unexpected reasoning deltas: %#v", reasoningDeltas) + } + if len(output.ToolCalls) != 1 || output.ToolCalls[0].ToolCallID != "call_weather" || output.ToolCalls[0].ToolName != "get_weather" || output.ToolCalls[0].ArgumentsJSON != `{"location":"Paris"}` { + t.Fatalf("unexpected stream tool calls: %#v", output.ToolCalls) + } + if len(output.ServerToolCalls) != 1 || output.ServerToolCalls[0].ToolCallID != "search_call_1" || output.ServerToolCalls[0].ToolName != "google_search" || output.ServerToolCalls[0].Status != "completed" { + t.Fatalf("unexpected stream server tool calls: %#v", output.ServerToolCalls) + } + if output.ServerToolCalls[0].ArgumentsJSON != `{"queries":["Gemini streaming"]}` || !strings.Contains(output.ServerToolCalls[0].OutputJSON, `"https://ai.google.dev/gemini-api/docs/streaming"`) { + t.Fatalf("unexpected stream server tool payload: %#v", output.ServerToolCalls[0]) + } + if len(serverToolEvents) != 4 || serverToolEvents[len(serverToolEvents)-1].Status != "completed" || output.ServerSideToolUsage["google_search"] != 1 { + t.Fatalf("unexpected server tool events=%#v usage=%#v", serverToolEvents, output.ServerSideToolUsage) + } + if len(output.Citations) != 1 || output.Citations[0] != "https://ai.google.dev/gemini-api/docs/streaming" { + t.Fatalf("unexpected citations: %#v", output.Citations) + } + if len(output.GeneratedImages) != 1 || output.GeneratedImages[0].B64JSON != "aW1hZ2U=" || output.GeneratedImages[0].MIMEType != "image/jpeg" || output.GeneratedImages[0].RevisedPrompt != "Hello world" { + t.Fatalf("unexpected streamed images: %#v", output.GeneratedImages) + } + if len(imageEvents) != 1 || !imageEvents[0].GeneratedImagePartial || imageEvents[0].GeneratedImageIndex != 0 { + t.Fatalf("unexpected streamed image events: %#v", imageEvents) + } + if len(output.GeneratedVideos) != 1 || output.GeneratedVideos[0].URL != "https://example.com/video.mp4" || output.GeneratedVideos[0].MIMEType != "video/mp4" { + t.Fatalf("unexpected streamed videos: %#v", output.GeneratedVideos) + } + if len(usageEvents) != 2 || output.Usage.InputTokens != 3 || output.Usage.CacheReadTokens != 1 || output.Usage.OutputTokens != 2 || output.Usage.ReasoningTokens != 3 || output.Usage.ServiceTier != "standard" { t.Fatalf("unexpected stream usage events=%#v output=%#v", usageEvents, output.Usage) } } +func TestParseGeminiInteractionUsageReadsOfficialStepShapes(t *testing.T) { + tests := []struct { + name string + payload map[string]interface{} + }{ + { + name: "accumulated step usage", + payload: map[string]interface{}{ + "usage": map[string]interface{}{ + "total_input_tokens": float64(8), + "total_cached_tokens": float64(3), + "total_output_tokens": float64(5), + }, + }, + }, + { + name: "delta metadata total usage", + payload: map[string]interface{}{ + "metadata": map[string]interface{}{ + "total_usage": map[string]interface{}{ + "total_input_tokens": float64(8), + "total_cached_tokens": float64(3), + "total_output_tokens": float64(5), + "total_thought_tokens": float64(2), + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + usage := parseGeminiInteractionUsage(tt.payload) + if usage.InputTokens != 5 || usage.CacheReadTokens != 3 || usage.OutputTokens != 5 { + t.Fatalf("unexpected usage: %#v", usage) + } + }) + } +} + +func TestParseGeminiInteractionOutputExtractsOfficialServerToolSteps(t *testing.T) { + body := []byte(`{ + "id":"interaction-native-tools", + "steps":[ + {"type":"code_execution_call","id":"code_call_1","arguments":{"code":"print(42)","language":"python"}}, + {"type":"code_execution_result","call_id":"code_call_1","result":"42\n"}, + {"type":"url_context_call","id":"url_call_1","arguments":{"urls":["https://example.com"]}}, + {"type":"url_context_result","call_id":"url_call_1","result":[{"url":"https://example.com","status":"success"}]} + ] + }`) + output, err := parseGeminiInteractionOutput(body) + if err != nil { + t.Fatalf("parse Gemini interaction output: %v", err) + } + if len(output.ServerToolCalls) != 2 { + t.Fatalf("unexpected server tool calls: %#v", output.ServerToolCalls) + } + if output.ServerToolCalls[0].ToolName != "code_execution" || output.ServerToolCalls[0].Status != "completed" || output.ServerToolCalls[0].ArgumentsJSON != `{"code":"print(42)","language":"python"}` || output.ServerToolCalls[0].OutputJSON != `"42\n"` { + t.Fatalf("unexpected code execution call: %#v", output.ServerToolCalls[0]) + } + if output.ServerToolCalls[1].ToolName != "url_context" || output.ServerToolCalls[1].Status != "completed" || !strings.Contains(output.ServerToolCalls[1].OutputJSON, `"https://example.com"`) { + t.Fatalf("unexpected URL context call: %#v", output.ServerToolCalls[1]) + } + if output.ServerSideToolUsage["code_execution"] != 1 || output.ServerSideToolUsage["url_context"] != 1 { + t.Fatalf("unexpected server-side tool usage: %#v", output.ServerSideToolUsage) + } + if len(output.Citations) != 1 || output.Citations[0] != "https://example.com" { + t.Fatalf("unexpected citations: %#v", output.Citations) + } +} + +func TestGeminiInteractionStreamToolCallKeepsStepStartArgumentsWithoutDeltas(t *testing.T) { + result := &GenerateOutput{} + state := &geminiInteractionStreamState{} + updateGeminiInteractionStreamToolCall(result, state, map[string]interface{}{ + "index": float64(0), + "step": map[string]interface{}{ + "type": "function_call", + "id": "call-1", + "name": "lookup", + "arguments": map[string]interface{}{"query": "weather"}, + }, + }, "step.start") + updateGeminiInteractionStreamToolCall(result, state, map[string]interface{}{ + "index": float64(0), + }, "step.stop") + if len(result.ToolCalls) != 1 || result.ToolCalls[0].ArgumentsJSON != `{"query":"weather"}` { + t.Fatalf("unexpected tool calls: %#v", result.ToolCalls) + } +} + func TestNewGeminiRequestUsesOnlyGoogleAPIKeyForOfficialHost(t *testing.T) { req, err := newTestClient().newGeminiRequest(context.Background(), http.MethodPost, "https://generativelanguage.googleapis.com/v1beta/interactions", nil, RouteConfig{ APIKey: "test-key", diff --git a/frontend/features/admin/components/sections/conversation/admin-conversation.tsx b/frontend/features/admin/components/sections/conversation/admin-conversation.tsx index 29908055..d899e621 100644 --- a/frontend/features/admin/components/sections/conversation/admin-conversation.tsx +++ b/frontend/features/admin/components/sections/conversation/admin-conversation.tsx @@ -263,15 +263,12 @@ generationConfig.safetySettings.threshold`} "generation_config.top_p", "generation_config.max_output_tokens", "generation_config.thinking_level", + "generation_config.thinking_summaries", "response_format.type", "response_format.aspect_ratio", "response_format.image_size", "response_format.mime_type", - "responseFormat.type", - "responseFormat.aspectRatio", - "responseFormat.imageSize", - "responseFormat.mimeType", - "generationConfig.videoConfig.task", + "response_format.schema", "generation_config.video_config.task" ], "xai_image": [ diff --git a/frontend/features/admin/model/conversation-settings.ts b/frontend/features/admin/model/conversation-settings.ts index 88e65fb3..0b318c35 100644 --- a/frontend/features/admin/model/conversation-settings.ts +++ b/frontend/features/admin/model/conversation-settings.ts @@ -150,11 +150,7 @@ export const DEFAULT_MODEL_OPTION_ALLOWED_PATHS = `{ "response_format.aspect_ratio", "response_format.image_size", "response_format.mime_type", - "responseFormat.type", - "responseFormat.aspectRatio", - "responseFormat.imageSize", - "responseFormat.mimeType", - "generationConfig.videoConfig.task", + "response_format.schema", "generation_config.video_config.task" ], "xai_responses": [ diff --git a/frontend/features/chat/components/app-chat-area.tsx b/frontend/features/chat/components/app-chat-area.tsx index 682fbd35..fb9d8151 100644 --- a/frontend/features/chat/components/app-chat-area.tsx +++ b/frontend/features/chat/components/app-chat-area.tsx @@ -212,7 +212,11 @@ export function AppChatArea() { router.push(projectID ? `/chat?project_id=${encodeURIComponent(projectID)}` : "/chat"); }, [requestNewConversation, routeProjectID, router]); const activeGenerationRunsRef = React.useRef>(new Set()); - const failedGenerationRunsRef = React.useRef>(new Set()); + // Set 的原地增删不会触发 effect，revision 用于同步断流恢复判断。 + const [activeGenerationRunsRevision, setActiveGenerationRunsRevision] = React.useState(0); + const onActiveGenerationRunsChange = React.useCallback(() => { + setActiveGenerationRunsRevision((current) => current + 1); + }, []); const { autoGenerateLabels, deleteFilesByDefault, @@ -245,7 +249,7 @@ export function AppChatArea() { resumingRunID, } = useChatData(conversationID, { activeGenerationRunsRef, - failedGenerationRunsRef, + activeGenerationRunsRevision, }); const { greetingTitle } = useChatViewerProfile(); const [manualConversationTitle, setManualConversationTitle] = React.useState(""); @@ -641,7 +645,8 @@ export function AppChatArea() { setAttachments, releaseAttachments, activeGenerationRunsRef, - failedGenerationRunsRef, + activeGenerationRunsRevision, + onActiveGenerationRunsChange, resumingRunID, }); const generating = sending; diff --git a/frontend/features/chat/hooks/use-chat-data.ts b/frontend/features/chat/hooks/use-chat-data.ts index 55b535d5..4d68d30d 100644 --- a/frontend/features/chat/hooks/use-chat-data.ts +++ b/frontend/features/chat/hooks/use-chat-data.ts @@ -74,10 +74,10 @@ export function useChatData( conversationID: string | null, { activeGenerationRunsRef, - failedGenerationRunsRef, + activeGenerationRunsRevision = 0, }: { activeGenerationRunsRef?: React.RefObject>; - failedGenerationRunsRef?: React.RefObject>; + activeGenerationRunsRevision?: number; } = {}, ) { const t = useTranslations("chat.data"); @@ -320,6 +320,11 @@ export function useChatData( }, [state.messages]); const pendingRunID = pendingAssistant?.runID?.trim() || ""; + // revision 仅用于重新读取可变 Set；effect 只依赖当前 pending run 的实际活动状态。 + const pendingRunIsActive = React.useMemo( + () => Boolean(pendingRunID && activeGenerationRunsRef?.current.has(pendingRunID)), + [activeGenerationRunsRef, activeGenerationRunsRevision, pendingRunID], + ); React.useEffect(() => { pendingAssistantContentRef.current = pendingAssistant?.content ?? ""; @@ -329,8 +334,7 @@ export function useChatData( if ( !conversationID || !pendingRunID || - activeGenerationRunsRef?.current.has(pendingRunID) || - failedGenerationRunsRef?.current.has(pendingRunID) + pendingRunIsActive ) { setResumingRunID(""); return; @@ -510,11 +514,10 @@ export function useChatData( } }; }, [ - activeGenerationRunsRef, clearResumeCheckpoint, conversationID, - failedGenerationRunsRef, pendingRunID, + pendingRunIsActive, reload, tSubmit, ]); @@ -523,8 +526,7 @@ export function useChatData( if ( !conversationID || !pendingAssistant || - activeGenerationRunsRef?.current.has(pendingRunID) || - failedGenerationRunsRef?.current.has(pendingRunID) || + pendingRunIsActive || (pendingRunID && pendingRunID === resumingRunID) ) { return; @@ -535,7 +537,7 @@ export function useChatData( return () => { window.clearTimeout(timer); }; - }, [activeGenerationRunsRef, conversationID, failedGenerationRunsRef, pendingAssistant, pendingRunID, reload, resumingRunID]); + }, [conversationID, pendingAssistant, pendingRunID, pendingRunIsActive, reload, resumingRunID]); return { ...state, diff --git a/frontend/features/chat/hooks/use-chat-message-submit.ts b/frontend/features/chat/hooks/use-chat-message-submit.ts index c63395dd..e7e78b14 100644 --- a/frontend/features/chat/hooks/use-chat-message-submit.ts +++ b/frontend/features/chat/hooks/use-chat-message-submit.ts @@ -481,7 +481,8 @@ export function useChatMessageSubmit({ resetStreamBuffer, startStream, activeGenerationRunsRef, - failedGenerationRunsRef, + activeGenerationRunsRevision, + onActiveGenerationRunsChange, resumeGenerationActive = false, }: { conversationID: string | null; @@ -525,11 +526,11 @@ export function useChatMessageSubmit({ resetStreamBuffer: (exchangeKey?: string) => void; startStream: (exchangeKey: string, runID?: string) => void; activeGenerationRunsRef?: React.RefObject>; - failedGenerationRunsRef?: React.RefObject>; + activeGenerationRunsRevision: number; + onActiveGenerationRunsChange?: () => void; resumeGenerationActive?: boolean; }) { const t = useTranslations("chat.submit"); - const [activeRunRevision, setActiveRunRevision] = React.useState(0); const activeStreamsRef = React.useRef(new Map()); const conversationIDRef = React.useRef(conversationID); const conversationScopeKeyRef = React.useRef(conversationScopeKey); @@ -570,12 +571,12 @@ export function useChatMessageSubmit({ visibleMessages, ), ), - [activeRunRevision, conversationScopeKey, visibleBranchScopePath, visibleMessages], + [activeGenerationRunsRevision, conversationScopeKey, visibleBranchScopePath, visibleMessages], ); const syncActiveRuns = React.useCallback(() => { - setActiveRunRevision((current) => current + 1); - }, []); + onActiveGenerationRunsChange?.(); + }, [onActiveGenerationRunsChange]); const updatePendingExchange = React.useCallback( (exchangeKey: string, update: (current: PendingExchange) => PendingExchange) => { @@ -1182,7 +1183,6 @@ export function useChatMessageSubmit({ : await streamImageEdit(token, targetConversationID, mediaPayload, streamOptions); } - failedGenerationRunsRef?.current.delete(clientRunID); sentSuccessfully = true; flushStreamTextNow(exchangeKey); flushUpstreamThinkNow(exchangeKey); @@ -1415,7 +1415,6 @@ export function useChatMessageSubmit({ const errorMessage = resolveErrorMessage(error, t("retryLater")); const errorDetails = resolveErrorDetails(error); const errorSummary = resolveErrorSummary(error, t("retryLater")); - failedGenerationRunsRef?.current.add(clientRunID); shouldKeepConversationLayout = true; if ( resetComposer && @@ -1494,7 +1493,6 @@ export function useChatMessageSubmit({ [ activeGenerationRunsRef, autoGenerateLabels, - failedGenerationRunsRef, enqueueUpstreamThinkDelta, enqueueStreamText, flushStreamTextNow, @@ -1929,7 +1927,7 @@ export function useChatMessageSubmit({ dispatchingQueuedSubmissionIDsRef.current.delete(queuedSubmission.id); }); }, [ - activeRunRevision, + activeGenerationRunsRevision, combinedMessages, conversationScopeKey, currentLeafMessage?.publicID, diff --git a/frontend/features/chat/hooks/use-chat-runtime.ts b/frontend/features/chat/hooks/use-chat-runtime.ts index f099646e..bee83692 100644 --- a/frontend/features/chat/hooks/use-chat-runtime.ts +++ b/frontend/features/chat/hooks/use-chat-runtime.ts @@ -112,7 +112,8 @@ export function useChatRuntime({ setAttachments, releaseAttachments, activeGenerationRunsRef, - failedGenerationRunsRef, + activeGenerationRunsRevision, + onActiveGenerationRunsChange, resumingRunID = "", }: { conversationID: string | null; @@ -140,7 +141,8 @@ export function useChatRuntime({ setAttachments: React.Dispatch>; releaseAttachments: (items: PendingAttachment[]) => void; activeGenerationRunsRef?: React.RefObject>; - failedGenerationRunsRef?: React.RefObject>; + activeGenerationRunsRevision: number; + onActiveGenerationRunsChange?: () => void; resumingRunID?: string; }) { const [showConversationLayout, setShowConversationLayout] = React.useState(false); @@ -215,7 +217,8 @@ export function useChatRuntime({ combinedMessages: branchState.combinedMessages, serverMessagePublicIDs: branchState.serverMessagePublicIDs, activeGenerationRunsRef, - failedGenerationRunsRef, + activeGenerationRunsRevision, + onActiveGenerationRunsChange, resumeGenerationActive: visibleResumeGenerationActive, }); diff --git a/frontend/features/chat/hooks/use-chat-submit-stream.ts b/frontend/features/chat/hooks/use-chat-submit-stream.ts index 52ae1717..ede35a16 100644 --- a/frontend/features/chat/hooks/use-chat-submit-stream.ts +++ b/frontend/features/chat/hooks/use-chat-submit-stream.ts @@ -53,7 +53,8 @@ export function useChatSubmitStream({ combinedMessages, serverMessagePublicIDs, activeGenerationRunsRef, - failedGenerationRunsRef, + activeGenerationRunsRevision, + onActiveGenerationRunsChange, resumeGenerationActive, }: { conversationID: string | null; @@ -91,7 +92,8 @@ export function useChatSubmitStream({ combinedMessages: ChatAreaMessage[]; serverMessagePublicIDs: Set; activeGenerationRunsRef?: React.RefObject>; - failedGenerationRunsRef?: React.RefObject>; + activeGenerationRunsRevision: number; + onActiveGenerationRunsChange?: () => void; resumeGenerationActive?: boolean; }) { const streamBuffer = useChatStreamBuffer({ @@ -140,7 +142,8 @@ export function useChatSubmitStream({ resetStreamBuffer: streamBuffer.resetStreamBuffer, startStream: streamBuffer.startStream, activeGenerationRunsRef, - failedGenerationRunsRef, + activeGenerationRunsRevision, + onActiveGenerationRunsChange, resumeGenerationActive, }); diff --git a/frontend/features/chat/model/chat-task.ts b/frontend/features/chat/model/chat-task.ts index 19b66959..cf48c56b 100644 --- a/frontend/features/chat/model/chat-task.ts +++ b/frontend/features/chat/model/chat-task.ts @@ -64,7 +64,7 @@ function requestedResponseType(options?: ConversationOptions): "image" | "video" if (!options) { return ""; } - return responseFormatType(options.response_format ?? options.responseFormat); + return responseFormatType(options.response_format); } export function resolveChatSubmitDecision(