diff --git a/cmd/gomodel/docs/docs.go b/cmd/gomodel/docs/docs.go index e120c13e..969ca1bb 100644 --- a/cmd/gomodel/docs/docs.go +++ b/cmd/gomodel/docs/docs.go @@ -4173,6 +4173,93 @@ const docTemplate = `{ ] } }, + "/v1/audio/translations": { + "post": { + "consumes": [ + "multipart/form-data" + ], + "produces": [ + "application/json", + "text/plain" + ], + "tags": [ + "audio" + ], + "summary": "Translate audio into English", + "parameters": [ + { + "type": "file", + "description": "Audio file to translate", + "name": "file", + "in": "formData", + "required": true + }, + { + "type": "string", + "description": "Model ID", + "name": "model", + "in": "formData", + "required": true + }, + { + "type": "string", + "description": "Optional English text to guide the model", + "name": "prompt", + "in": "formData" + }, + { + "type": "string", + "description": "json, text, srt, verbose_json, or vtt", + "name": "response_format", + "in": "formData" + }, + { + "type": "number", + "description": "Sampling temperature (0-1)", + "name": "temperature", + "in": "formData" + } + ], + "responses": { + "200": { + "description": "English translation in the requested response_format", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/core.OpenAIErrorEnvelope" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/core.OpenAIErrorEnvelope" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/core.OpenAIErrorEnvelope" + } + }, + "502": { + "description": "Bad Gateway", + "schema": { + "$ref": "#/definitions/core.OpenAIErrorEnvelope" + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ] + } + }, "/v1/batches": { "get": { "produces": [ diff --git a/docs/openapi.json b/docs/openapi.json index 846cd4a3..69c52734 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -6252,6 +6252,140 @@ } } }, + "/v1/audio/translations": { + "post": { + "tags": [ + "audio" + ], + "summary": "Translate audio into English", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "properties": { + "file": { + "description": "Audio file to translate", + "type": "string", + "format": "binary" + }, + "model": { + "description": "Model ID", + "type": "string" + }, + "prompt": { + "description": "Optional English text to guide the model", + "type": "string" + }, + "response_format": { + "description": "json, text, srt, verbose_json, or vtt", + "type": "string" + }, + "temperature": { + "description": "Sampling temperature (0-1)", + "type": "number" + } + }, + "required": [ + "file", + "model" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "English translation in the requested response_format", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "text/plain": { + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/core.OpenAIErrorEnvelope" + } + }, + "text/plain": { + "schema": { + "$ref": "#/components/schemas/core.OpenAIErrorEnvelope" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/core.OpenAIErrorEnvelope" + } + }, + "text/plain": { + "schema": { + "$ref": "#/components/schemas/core.OpenAIErrorEnvelope" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/core.OpenAIErrorEnvelope" + } + }, + "text/plain": { + "schema": { + "$ref": "#/components/schemas/core.OpenAIErrorEnvelope" + } + } + } + }, + "502": { + "description": "Bad Gateway", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/core.OpenAIErrorEnvelope" + } + }, + "text/plain": { + "schema": { + "$ref": "#/components/schemas/core.OpenAIErrorEnvelope" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "x-mint": { + "metadata": { + "sidebarTitle": "/v1/audio/translations" + } + } + } + }, "/v1/batches": { "get": { "tags": [ diff --git a/internal/auditlog/auditlog_test.go b/internal/auditlog/auditlog_test.go index c3dedb98..8dcdc116 100644 --- a/internal/auditlog/auditlog_test.go +++ b/internal/auditlog/auditlog_test.go @@ -1007,6 +1007,7 @@ func TestIsModelInteractionPath(t *testing.T) { {"batches prefix overmatch", "/v1/batcheship", false}, {"audio speech", "/v1/audio/speech", true}, {"audio transcriptions", "/v1/audio/transcriptions", true}, + {"audio translations", "/v1/audio/translations", true}, {"models", "/v1/models", false}, {"models with subpath", "/v1/models/gpt-4", false}, {"health", "/health", false}, diff --git a/internal/core/endpoints.go b/internal/core/endpoints.go index 13595a62..96abd1b2 100644 --- a/internal/core/endpoints.go +++ b/internal/core/endpoints.go @@ -27,6 +27,7 @@ const ( OperationFiles Operation = "files" OperationAudioSpeech Operation = "audio_speech" OperationAudioTranscriptions Operation = "audio_transcriptions" + OperationAudioTranslations Operation = "audio_translations" OperationRealtime Operation = "realtime" OperationProviderPassthrough Operation = "provider_passthrough" OperationMCP Operation = "mcp" @@ -142,6 +143,12 @@ func describeEndpointPath(path string) EndpointDescriptor { Dialect: "openai_compat", Operation: OperationAudioTranscriptions, } + case path == "/v1/audio/translations": + return EndpointDescriptor{ + ModelInteraction: true, + Dialect: "openai_compat", + Operation: OperationAudioTranslations, + } case path == "/v1/realtime" || path == "/v1/realtime/calls" || path == "/v1/realtime/client_secrets": // The realtime endpoints relay the provider's schema verbatim: /v1/realtime // upgrades to a websocket, /v1/realtime/calls exchanges WebRTC SDP, and @@ -212,7 +219,7 @@ func bodyModeForEndpoint(method, path string, operation Operation) BodyMode { return BodyModeNone case OperationAudioSpeech: return BodyModeJSON - case OperationAudioTranscriptions: + case OperationAudioTranscriptions, OperationAudioTranslations: return BodyModeMultipart case OperationRealtime: if method == http.MethodPost && path == "/v1/realtime/client_secrets" { diff --git a/internal/core/endpoints_test.go b/internal/core/endpoints_test.go index 34c6df5c..5629081b 100644 --- a/internal/core/endpoints_test.go +++ b/internal/core/endpoints_test.go @@ -29,6 +29,7 @@ func TestDescribeEndpointPath(t *testing.T) { {path: "/v1/files/file_1", managed: true, dialect: "openai_compat", operation: OperationFiles, bodyMode: BodyModeNone, interaction: true}, {path: "/v1/audio/speech", managed: false, dialect: "openai_compat", operation: OperationAudioSpeech, bodyMode: BodyModeJSON, interaction: true}, {path: "/v1/audio/transcriptions", managed: false, dialect: "openai_compat", operation: OperationAudioTranscriptions, bodyMode: BodyModeMultipart, interaction: true}, + {path: "/v1/audio/translations", managed: false, dialect: "openai_compat", operation: OperationAudioTranslations, bodyMode: BodyModeMultipart, interaction: true}, {path: "/mcp", managed: false, dialect: "mcp", operation: OperationMCP, bodyMode: BodyModeNone, interaction: true}, {path: "/mcp/linear", managed: false, dialect: "mcp", operation: OperationMCP, bodyMode: BodyModeNone, interaction: true}, {path: "/p/openai/responses", managed: true, dialect: "provider_passthrough", operation: OperationProviderPassthrough, bodyMode: BodyModeOpaque, interaction: true}, @@ -86,6 +87,7 @@ func TestDescribeEndpoint_UsesMethodForBodyMode(t *testing.T) { {method: http.MethodGet, path: "/v1/files/file_1", bodyMode: BodyModeNone}, {method: http.MethodPost, path: "/v1/audio/speech", bodyMode: BodyModeJSON}, {method: http.MethodPost, path: "/v1/audio/transcriptions", bodyMode: BodyModeMultipart}, + {method: http.MethodPost, path: "/v1/audio/translations", bodyMode: BodyModeMultipart}, {method: http.MethodPost, path: "/v1/batches/batch_1/cancel", bodyMode: BodyModeNone}, {method: http.MethodPost, path: "/mcp", bodyMode: BodyModeJSON}, {method: http.MethodGet, path: "/mcp", bodyMode: BodyModeNone}, diff --git a/internal/core/interfaces.go b/internal/core/interfaces.go index f0b1c627..78662348 100644 --- a/internal/core/interfaces.go +++ b/internal/core/interfaces.go @@ -36,6 +36,14 @@ type AudioProvider interface { CreateTranscription(ctx context.Context, req *AudioTranscriptionRequest) (*AudioResponse, error) } +// AudioTranslationProvider is implemented by audio providers that support +// translating spoken audio into English through POST /v1/audio/translations. +// It is separate from AudioProvider because transcription support does not imply +// translation support for every upstream provider. +type AudioTranslationProvider interface { + CreateTranslation(ctx context.Context, req *AudioTranscriptionRequest) (*AudioResponse, error) +} + // NativeBatchProvider is implemented by providers that support native discounted batching. // This is intentionally separate from Provider so unsupported providers can still implement // regular synchronous APIs without batch capabilities. diff --git a/internal/providers/groq/groq.go b/internal/providers/groq/groq.go index 1c8f300f..e714ae36 100644 --- a/internal/providers/groq/groq.go +++ b/internal/providers/groq/groq.go @@ -119,3 +119,9 @@ func (p *Provider) CreateSpeech(ctx context.Context, req *core.AudioSpeechReques func (p *Provider) CreateTranscription(ctx context.Context, req *core.AudioTranscriptionRequest) (*core.AudioResponse, error) { return p.compat.CreateTranscription(ctx, req) } + +// CreateTranslation translates audio through Groq's OpenAI-compatible +// /audio/translations API (whisper models). +func (p *Provider) CreateTranslation(ctx context.Context, req *core.AudioTranscriptionRequest) (*core.AudioResponse, error) { + return p.compat.CreateTranslation(ctx, req) +} diff --git a/internal/providers/openai/audio.go b/internal/providers/openai/audio.go index 6353a8a2..e4ba1054 100644 --- a/internal/providers/openai/audio.go +++ b/internal/providers/openai/audio.go @@ -59,8 +59,25 @@ func speechResponseContentType(raw *llmclient.Response, format string) string { // The request is multipart/form-data; the response (JSON or text per response_format) // is proxied verbatim. func (p *CompatibleProvider) CreateTranscription(ctx context.Context, req *core.AudioTranscriptionRequest) (*core.AudioResponse, error) { + return p.createAudioTranscription(ctx, req, "/audio/transcriptions", true, "audio transcription request is required") +} + +// CreateTranslation implements OpenAI speech translation (POST /audio/translations). +// Translation accepts the transcription upload fields except language and timestamp +// granularities, and always returns English text. +func (p *CompatibleProvider) CreateTranslation(ctx context.Context, req *core.AudioTranscriptionRequest) (*core.AudioResponse, error) { + return p.createAudioTranscription(ctx, req, "/audio/translations", false, "audio translation request is required") +} + +func (p *CompatibleProvider) createAudioTranscription( + ctx context.Context, + req *core.AudioTranscriptionRequest, + endpoint string, + includeTranscriptionFields bool, + nilRequestMessage string, +) (*core.AudioResponse, error) { if req == nil { - return nil, core.NewInvalidRequestError("audio transcription request is required", nil) + return nil, core.NewInvalidRequestError(nilRequestMessage, nil) } content := req.FileReader if content == nil && len(req.File) > 0 { @@ -70,10 +87,10 @@ func (p *CompatibleProvider) CreateTranscription(ctx context.Context, req *core. return nil, core.NewInvalidRequestError("file is required", nil) } - body, contentType := transcriptionMultipart(req, content) + body, contentType := audioTranscriptionMultipart(req, content, includeTranscriptionFields) raw, err := p.client.DoRaw(ctx, p.prepareRequest(llmclient.Request{ Method: http.MethodPost, - Endpoint: "/audio/transcriptions", + Endpoint: endpoint, RawBodyReader: body, Headers: http.Header{"Content-Type": {contentType}}, })) @@ -86,11 +103,10 @@ func (p *CompatibleProvider) CreateTranscription(ctx context.Context, req *core. }, nil } -// transcriptionMultipart streams a multipart/form-data body for a transcription -// request and returns the reader plus its Content-Type. It mirrors the file-upload -// adapter: the body is produced on a goroutine through an io.Pipe so large audio -// files are never buffered whole. -func transcriptionMultipart(req *core.AudioTranscriptionRequest, content io.Reader) (io.Reader, string) { +// audioTranscriptionMultipart streams a multipart/form-data body for a +// transcription or translation request. Translation omits fields that OpenAI's +// translations endpoint does not accept. +func audioTranscriptionMultipart(req *core.AudioTranscriptionRequest, content io.Reader, includeTranscriptionFields bool) (io.Reader, string) { filename := strings.TrimSpace(req.Filename) if filename == "" { filename = "audio" @@ -101,13 +117,15 @@ func transcriptionMultipart(req *core.AudioTranscriptionRequest, content io.Read go func() { defer func() { _ = pw.Close() }() - fields := [...][2]string{ - {"model", req.Model}, - {"language", req.Language}, - {"prompt", req.Prompt}, - {"response_format", req.ResponseFormat}, - {"temperature", req.Temperature}, + fields := [][2]string{{"model", req.Model}} + if includeTranscriptionFields { + fields = append(fields, [2]string{"language", req.Language}) } + fields = append(fields, + [2]string{"prompt", req.Prompt}, + [2]string{"response_format", req.ResponseFormat}, + [2]string{"temperature", req.Temperature}, + ) for _, field := range fields { if strings.TrimSpace(field[1]) == "" { continue @@ -117,13 +135,15 @@ func transcriptionMultipart(req *core.AudioTranscriptionRequest, content io.Read return } } - for _, granularity := range req.TimestampGranularities { - if strings.TrimSpace(granularity) == "" { - continue - } - if err := writer.WriteField("timestamp_granularities[]", granularity); err != nil { - _ = pw.CloseWithError(core.NewInvalidRequestError("failed to write timestamp_granularities field", err)) - return + if includeTranscriptionFields { + for _, granularity := range req.TimestampGranularities { + if strings.TrimSpace(granularity) == "" { + continue + } + if err := writer.WriteField("timestamp_granularities[]", granularity); err != nil { + _ = pw.CloseWithError(core.NewInvalidRequestError("failed to write timestamp_granularities field", err)) + return + } } } diff --git a/internal/providers/openai/audio_test.go b/internal/providers/openai/audio_test.go index dae4bada..81aa1930 100644 --- a/internal/providers/openai/audio_test.go +++ b/internal/providers/openai/audio_test.go @@ -1,7 +1,10 @@ package openai import ( + "bytes" "context" + "errors" + "io" "net/http" "net/http/httptest" "testing" @@ -62,3 +65,88 @@ func TestCreateSpeech_PreservesUpstreamContentType(t *testing.T) { }) } } + +func TestCreateTranslation_UsesTranslationMultipartShape(t *testing.T) { + provider := newSpeechTestProvider(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/audio/translations" { + t.Errorf("path = %q, want /audio/translations", r.URL.Path) + } + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("ParseMultipartForm: %v", err) + } + for field, want := range map[string]string{ + "model": "whisper-1", "prompt": "Use product names", "response_format": "text", "temperature": "0.2", + } { + if got := r.FormValue(field); got != want { + t.Errorf("%s = %q, want %q", field, got, want) + } + } + if _, ok := r.MultipartForm.Value["language"]; ok { + t.Error("translation request must not forward language") + } + if _, ok := r.MultipartForm.Value["timestamp_granularities[]"]; ok { + t.Error("translation request must not forward timestamp granularities") + } + + file, header, err := r.FormFile("file") + if err != nil { + t.Fatalf("FormFile: %v", err) + } + defer func() { _ = file.Close() }() + data, err := io.ReadAll(file) + if err != nil { + t.Fatalf("ReadAll(file): %v", err) + } + if header.Filename != "speech.wav" || !bytes.Equal(data, []byte("wave-bytes")) { + t.Errorf("file = %q %q, want speech.wav wave-bytes", header.Filename, data) + } + + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte("Hello from GoModel.")) + }) + + resp, err := provider.CreateTranslation(context.Background(), &core.AudioTranscriptionRequest{ + Model: "whisper-1", + Filename: "speech.wav", + File: []byte("wave-bytes"), + Language: "de", + Prompt: "Use product names", + ResponseFormat: "text", + Temperature: "0.2", + TimestampGranularities: []string{"word"}, + }) + if err != nil { + t.Fatalf("CreateTranslation() error = %v", err) + } + if resp.ContentType != "text/plain; charset=utf-8" { + t.Errorf("ContentType = %q, want text/plain; charset=utf-8", resp.ContentType) + } + if string(resp.Data) != "Hello from GoModel." { + t.Errorf("Data = %q, want translated text", resp.Data) + } +} + +func TestCreateTranslation_RejectsInvalidRequests(t *testing.T) { + tests := []struct { + name string + req *core.AudioTranscriptionRequest + wantMessage string + }{ + {name: "nil request", wantMessage: "audio translation request is required"}, + {name: "missing file", req: &core.AudioTranscriptionRequest{Model: "whisper-1"}, wantMessage: "file is required"}, + } + + provider := &CompatibleProvider{} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := provider.CreateTranslation(context.Background(), tt.req) + var gatewayErr *core.GatewayError + if !errors.As(err, &gatewayErr) { + t.Fatalf("CreateTranslation() error = %v, want GatewayError", err) + } + if gatewayErr.Message != tt.wantMessage { + t.Fatalf("CreateTranslation() message = %q, want %q", gatewayErr.Message, tt.wantMessage) + } + }) + } +} diff --git a/internal/providers/router.go b/internal/providers/router.go index bc20616d..128901cc 100644 --- a/internal/providers/router.go +++ b/internal/providers/router.go @@ -750,6 +750,28 @@ func (r *Router) CreateTranscription(ctx context.Context, req *core.AudioTranscr ) } +// CreateTranslation routes a speech translation request to a provider that +// explicitly supports the OpenAI-compatible translations endpoint. +func (r *Router) CreateTranslation(ctx context.Context, req *core.AudioTranscriptionRequest) (*core.AudioResponse, error) { + if req == nil { + return nil, core.NewInvalidRequestError("audio translation request is required", nil) + } + resp, _, err := routeResolvedModelCall( + r, ctx, req.Model, req.Provider, + func(selector core.ModelSelector) *core.AudioTranscriptionRequest { + return forwardAudioTranscriptionRequest(req, selector) + }, + func(ctx context.Context, provider core.Provider, forwardReq *core.AudioTranscriptionRequest) (*core.AudioResponse, error) { + translator, ok := provider.(core.AudioTranslationProvider) + if !ok { + return nil, core.NewInvalidRequestError(fmt.Sprintf("model %q does not support audio translations", req.Model), nil) + } + return translator.CreateTranslation(ctx, forwardReq) + }, + ) + return resp, err +} + // routeAudioCall resolves the model, requires the target provider to implement // core.AudioProvider, and invokes call. It mirrors routeNative*Call but for the // optional audio capability. diff --git a/internal/providers/router_test.go b/internal/providers/router_test.go index 0883fe80..5d11351a 100644 --- a/internal/providers/router_test.go +++ b/internal/providers/router_test.go @@ -126,6 +126,20 @@ type mockProvider struct { passthroughResp *core.PassthroughResponse } +type mockAudioTranslationProvider struct { + *mockProvider + translationResponse *core.AudioResponse + lastTranslationReq *core.AudioTranscriptionRequest +} + +func (m *mockAudioTranslationProvider) CreateTranslation(_ context.Context, req *core.AudioTranscriptionRequest) (*core.AudioResponse, error) { + m.lastTranslationReq = req + if m.err != nil { + return nil, m.err + } + return m.translationResponse, nil +} + func readAndCloseBody(t *testing.T, body io.ReadCloser) string { t.Helper() if body == nil { @@ -367,6 +381,90 @@ func TestNewRouter(t *testing.T) { }) } +func TestRouterCreateTranslation(t *testing.T) { + translator := &mockAudioTranslationProvider{ + mockProvider: &mockProvider{name: "openai"}, + translationResponse: &core.AudioResponse{ContentType: "application/json", Data: []byte(`{"text":"hello"}`)}, + } + lookup := newMockLookup() + lookup.addModel("openai/whisper-1", translator, "openai") + router, _ := NewRouter(lookup) + + resp, err := router.CreateTranslation(context.Background(), &core.AudioTranscriptionRequest{ + Model: "whisper-1", Provider: "openai", File: []byte("audio"), Prompt: "names", + }) + if err != nil { + t.Fatalf("CreateTranslation() error = %v", err) + } + if string(resp.Data) != `{"text":"hello"}` { + t.Errorf("response = %s", resp.Data) + } + if translator.lastTranslationReq == nil { + t.Fatal("translation provider was not called") + } + if translator.lastTranslationReq.Model != "whisper-1" || translator.lastTranslationReq.Provider != "" { + t.Errorf("forwarded selector = %q/%q, want provider metadata stripped", translator.lastTranslationReq.Provider, translator.lastTranslationReq.Model) + } +} + +func TestRouterCreateTranslation_Errors(t *testing.T) { + providerErr := errors.New("translation provider failed") + tests := []struct { + name string + model string + provider core.Provider + providerType string + req *core.AudioTranscriptionRequest + wantError string + wantIs error + }{ + { + name: "unsupported provider", + model: "transcribe-only", + provider: &mockProvider{name: "cohere"}, + providerType: "cohere", + req: &core.AudioTranscriptionRequest{Model: "transcribe-only", File: []byte("audio")}, + wantError: "does not support audio translations", + }, + { + name: "provider failure", + model: "openai/whisper-1", + provider: &mockAudioTranslationProvider{mockProvider: &mockProvider{name: "openai", err: providerErr}}, + providerType: "openai", + req: &core.AudioTranscriptionRequest{Model: "whisper-1", Provider: "openai", File: []byte("audio")}, + wantIs: providerErr, + }, + { + name: "nil request", + wantError: "audio translation request is required", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lookup := newMockLookup() + if tt.provider != nil { + lookup.addModel(tt.model, tt.provider, tt.providerType) + } + router, err := NewRouter(lookup) + if err != nil { + t.Fatalf("NewRouter() error = %v", err) + } + + _, err = router.CreateTranslation(context.Background(), tt.req) + if tt.wantIs != nil { + if !errors.Is(err, tt.wantIs) { + t.Fatalf("CreateTranslation() error = %v, want %v", err, tt.wantIs) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantError) { + t.Fatalf("CreateTranslation() error = %v, want message containing %q", err, tt.wantError) + } + }) + } +} + func TestRouterEmptyLookup(t *testing.T) { lookup := newMockLookup() // Empty - no models router, _ := NewRouter(lookup) diff --git a/internal/server/audio_service.go b/internal/server/audio_service.go index 1207c3c8..d70c971b 100644 --- a/internal/server/audio_service.go +++ b/internal/server/audio_service.go @@ -46,6 +46,14 @@ func (s *audioService) router() (core.AudioProvider, error) { return router, nil } +func (s *audioService) translationRouter() (core.AudioTranslationProvider, error) { + router, ok := s.provider.(core.AudioTranslationProvider) + if !ok { + return nil, core.NewInvalidRequestError("audio translations are not supported by the current provider router", nil) + } + return router, nil +} + // CreateSpeech handles POST /v1/audio/speech. func (s *audioService) CreateSpeech(c *echo.Context) error { router, err := s.router() @@ -111,12 +119,31 @@ func speechResponseFormat(req *core.AudioSpeechRequest, resp *core.AudioResponse // CreateTranscription handles POST /v1/audio/transcriptions. func (s *audioService) CreateTranscription(c *echo.Context) error { - router, err := s.router() - if err != nil { - return handleError(c, err) + return s.createAudioTranscription(c, false) +} + +// CreateTranslation handles POST /v1/audio/translations. +func (s *audioService) CreateTranslation(c *echo.Context) error { + return s.createAudioTranscription(c, true) +} + +func (s *audioService) createAudioTranscription(c *echo.Context, translation bool) error { + var call func(context.Context, *core.AudioTranscriptionRequest) (*core.AudioResponse, error) + if translation { + router, err := s.translationRouter() + if err != nil { + return handleError(c, err) + } + call = router.CreateTranslation + } else { + router, err := s.router() + if err != nil { + return handleError(c, err) + } + call = router.CreateTranscription } - req, err := transcriptionRequestFromForm(c) + req, err := audioTranscriptionRequestFromForm(c, !translation) if err != nil { return handleError(c, err) } @@ -140,7 +167,7 @@ func (s *audioService) CreateTranscription(c *echo.Context) error { return handleError(c, err) } defer release() - resp, err := router.CreateTranscription(ctx, req) + resp, err := call(ctx, req) if err != nil { return handleError(c, err) } @@ -148,6 +175,9 @@ func (s *audioService) CreateTranscription(c *echo.Context) error { return s.respondAudio(c, resp) // emits the 502 guard before resp.Data is read } s.logUsage(ctx, route, func(pricing *core.ModelPricing) *usage.UsageEntry { + if translation { + return usage.ExtractFromTranslationResponse(resp.Data, route.requestID, route.model, route.providerType, pricing) + } return usage.ExtractFromTranscriptionResponse(resp.Data, route.requestID, route.model, route.providerType, pricing) }) return s.respondAudio(c, resp) @@ -236,12 +266,24 @@ func (s *audioService) logUsage(ctx context.Context, route audioRoute, extract f s.usageLogger.Write(entry) } -func transcriptionRequestFromForm(c *echo.Context) (*core.AudioTranscriptionRequest, error) { +func audioTranscriptionRequestFromForm(c *echo.Context, includeTranscriptionFields bool) (*core.AudioTranscriptionRequest, error) { + form, err := c.MultipartForm() + if err != nil { + return nil, core.NewInvalidRequestError("invalid multipart form", err) + } model := strings.TrimSpace(c.FormValue("model")) if model == "" { return nil, core.NewInvalidRequestError("model is required", nil) } + if !includeTranscriptionFields && form != nil { + for _, field := range []string{"language", "timestamp_granularities", "timestamp_granularities[]"} { + if _, present := form.Value[field]; present { + return nil, core.NewInvalidRequestError(field+" is not supported for audio translations", nil) + } + } + } + fileHeader, err := c.FormFile("file") if err != nil { return nil, core.NewInvalidRequestError("file is required", err) @@ -256,13 +298,17 @@ func transcriptionRequestFromForm(c *echo.Context) (*core.AudioTranscriptionRequ return nil, core.NewInvalidRequestError("failed to read uploaded file", err) } - // Accept both the canonical bracketed key and the unbracketed variant some - // clients send; the adapter always forwards the bracketed form upstream. var granularities []string - if form, err := c.MultipartForm(); err == nil && form != nil { - granularities = form.Value["timestamp_granularities[]"] - if len(granularities) == 0 { - granularities = form.Value["timestamp_granularities"] + var language string + if includeTranscriptionFields { + language = strings.TrimSpace(c.FormValue("language")) + // Accept both the canonical bracketed key and the unbracketed variant some + // clients send; the adapter always forwards the bracketed form upstream. + if form != nil { + granularities = form.Value["timestamp_granularities[]"] + if len(granularities) == 0 { + granularities = form.Value["timestamp_granularities"] + } } } @@ -271,7 +317,7 @@ func transcriptionRequestFromForm(c *echo.Context) (*core.AudioTranscriptionRequ Filename: fileHeader.Filename, FileContentType: fileHeader.Header.Get("Content-Type"), File: data, - Language: strings.TrimSpace(c.FormValue("language")), + Language: language, Prompt: c.FormValue("prompt"), ResponseFormat: strings.TrimSpace(c.FormValue("response_format")), Temperature: strings.TrimSpace(c.FormValue("temperature")), diff --git a/internal/server/audio_service_test.go b/internal/server/audio_service_test.go index 63ea5ccc..f37e86fc 100644 --- a/internal/server/audio_service_test.go +++ b/internal/server/audio_service_test.go @@ -4,9 +4,11 @@ import ( "bytes" "context" "encoding/base64" + "encoding/json" "mime/multipart" "net/http" "net/http/httptest" + "os" "strings" "testing" @@ -23,10 +25,12 @@ type audioMockProvider struct { *mockProvider speechResp *core.AudioResponse transcriptionResp *core.AudioResponse + translationResp *core.AudioResponse audioErr error resolved *core.ModelSelector capturedSpeech *core.AudioSpeechRequest capturedTranscription *core.AudioTranscriptionRequest + capturedTranslation *core.AudioTranscriptionRequest } // ResolveModel lets the fake stand in for the Router so the service can authorize @@ -56,6 +60,14 @@ func (m *audioMockProvider) CreateTranscription(_ context.Context, req *core.Aud return m.transcriptionResp, nil } +func (m *audioMockProvider) CreateTranslation(_ context.Context, req *core.AudioTranscriptionRequest) (*core.AudioResponse, error) { + m.capturedTranslation = req + if m.audioErr != nil { + return nil, m.audioErr + } + return m.translationResp, nil +} + func TestAudioSpeech_HappyPath(t *testing.T) { mock := &audioMockProvider{ mockProvider: &mockProvider{supportedModels: []string{"gpt-4o-mini-tts"}}, @@ -230,6 +242,184 @@ func TestAudioTranscription_HappyPath(t *testing.T) { } } +func TestAudioTranscription_UsesConfiguredMultipartMemoryLimit(t *testing.T) { + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + _ = w.WriteField("model", "gpt-4o-transcribe") + part, err := w.CreateFormFile("file", "speech.mp3") + if err != nil { + t.Fatalf("CreateFormFile: %v", err) + } + audio := bytes.Repeat([]byte("a"), 1024) + if _, err := part.Write(audio); err != nil { + t.Fatalf("write audio: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("close multipart writer: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/v1/audio/transcriptions", &buf) + req.Header.Set("Content-Type", w.FormDataContentType()) + rec := httptest.NewRecorder() + e := echo.NewWithConfig(echo.Config{FormParseMaxMemory: 1}) + c := e.NewContext(req, rec) + + parsed, err := audioTranscriptionRequestFromForm(c, true) + if err != nil { + t.Fatalf("audioTranscriptionRequestFromForm returned error: %v", err) + } + if !bytes.Equal(parsed.File, audio) { + t.Fatalf("parsed file length = %d, want %d", len(parsed.File), len(audio)) + } + if req.MultipartForm == nil { + t.Fatal("multipart form was not parsed") + } + t.Cleanup(func() { _ = req.MultipartForm.RemoveAll() }) + files := req.MultipartForm.File["file"] + if len(files) != 1 { + t.Fatalf("parsed uploads = %d, want 1", len(files)) + } + + upload, err := files[0].Open() + if err != nil { + t.Fatalf("open parsed upload: %v", err) + } + defer func() { _ = upload.Close() }() + if _, ok := upload.(*os.File); !ok { + t.Fatalf("uploaded file type = %T, want disk-backed *os.File", upload) + } +} + +func TestAudioTranslation_HappyPath(t *testing.T) { + mock := &audioMockProvider{ + mockProvider: &mockProvider{supportedModels: []string{"whisper-1"}}, + translationResp: &core.AudioResponse{ContentType: "application/json", Data: []byte(`{"text":"hello"}`)}, + } + srv := New(mock, nil) + + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + _ = w.WriteField("model", "whisper-1") + _ = w.WriteField("prompt", "Use product names") + _ = w.WriteField("response_format", "json") + _ = w.WriteField("temperature", "0.2") + part, err := w.CreateFormFile("file", "speech.wav") + if err != nil { + t.Fatalf("CreateFormFile: %v", err) + } + _, _ = part.Write([]byte("audio-bytes")) + if err := w.Close(); err != nil { + t.Fatalf("close multipart writer: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/v1/audio/translations", &buf) + req.Header.Set("Content-Type", w.FormDataContentType()) + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + if rec.Code != http.StatusOK || rec.Body.String() != `{"text":"hello"}` { + t.Fatalf("status/body = %d %q, want 200 translation response", rec.Code, rec.Body.String()) + } + captured := mock.capturedTranslation + if captured == nil || captured.Model != "whisper-1" || captured.Filename != "speech.wav" { + t.Fatalf("captured translation request mismatch: %+v", captured) + } + if captured.Language != "" || len(captured.TimestampGranularities) != 0 { + t.Errorf("translation accepted transcription-only fields: %+v", captured) + } + if captured.Prompt != "Use product names" || captured.ResponseFormat != "json" || captured.Temperature != "0.2" { + t.Errorf("translation fields were not preserved: %+v", captured) + } +} + +func TestAudioTranslation_ErrorResponses(t *testing.T) { + tests := []struct { + name string + fields map[string][]string + providerError *core.GatewayError + wantStatus int + wantType core.ErrorType + wantMessage string + wantProvider bool + }{ + { + name: "rejects language", + fields: map[string][]string{"language": {"de"}}, + wantStatus: http.StatusBadRequest, + wantType: core.ErrorTypeInvalidRequest, + wantMessage: "language is not supported for audio translations", + }, + { + name: "rejects unbracketed timestamp granularities", + fields: map[string][]string{"timestamp_granularities": {"word"}}, + wantStatus: http.StatusBadRequest, + wantType: core.ErrorTypeInvalidRequest, + wantMessage: "timestamp_granularities is not supported for audio translations", + }, + { + name: "rejects bracketed timestamp granularities", + fields: map[string][]string{"timestamp_granularities[]": {"word"}}, + wantStatus: http.StatusBadRequest, + wantType: core.ErrorTypeInvalidRequest, + wantMessage: "timestamp_granularities[] is not supported for audio translations", + }, + { + name: "propagates provider error", + providerError: core.NewProviderError("openai", http.StatusBadGateway, "translation provider failed", nil), + wantStatus: http.StatusBadGateway, + wantType: core.ErrorTypeProvider, + wantMessage: "translation provider failed", + wantProvider: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mock := &audioMockProvider{ + mockProvider: &mockProvider{supportedModels: []string{"whisper-1"}}, + translationResp: &core.AudioResponse{ContentType: "application/json", Data: []byte(`{"text":"hello"}`)}, + audioErr: tt.providerError, + } + srv := New(mock, nil) + + var buf bytes.Buffer + writer := multipart.NewWriter(&buf) + _ = writer.WriteField("model", "whisper-1") + for field, values := range tt.fields { + for _, value := range values { + _ = writer.WriteField(field, value) + } + } + part, err := writer.CreateFormFile("file", "speech.wav") + if err != nil { + t.Fatalf("CreateFormFile: %v", err) + } + _, _ = part.Write([]byte("audio-bytes")) + if err := writer.Close(); err != nil { + t.Fatalf("close multipart writer: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/v1/audio/translations", &buf) + req.Header.Set("Content-Type", writer.FormDataContentType()) + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + if rec.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d (body: %s)", rec.Code, tt.wantStatus, rec.Body.String()) + } + var envelope core.OpenAIErrorEnvelope + if err := json.Unmarshal(rec.Body.Bytes(), &envelope); err != nil { + t.Fatalf("decode error response: %v", err) + } + if envelope.Error.Type != tt.wantType || envelope.Error.Message != tt.wantMessage { + t.Fatalf("error = %+v, want type %q and message %q", envelope.Error, tt.wantType, tt.wantMessage) + } + if (mock.capturedTranslation != nil) != tt.wantProvider { + t.Fatalf("provider called = %v, want %v", mock.capturedTranslation != nil, tt.wantProvider) + } + }) + } +} + // newTranscriptionRequestWithAuditEntry builds a multipart /v1/audio/transcriptions // request carrying the given audio bytes and seeds an empty audit entry. func newTranscriptionRequestWithAuditEntry(filename string, audio []byte) (*echo.Context, *httptest.ResponseRecorder, *auditlog.LogEntry) { diff --git a/internal/server/handlers.go b/internal/server/handlers.go index 5f646bc7..c8821219 100644 --- a/internal/server/handlers.go +++ b/internal/server/handlers.go @@ -692,6 +692,29 @@ func (h *Handler) AudioTranscriptions(c *echo.Context) error { return h.audio().CreateTranscription(c) } +// AudioTranslations handles POST /v1/audio/translations. +// +// @Summary Translate audio into English +// @Tags audio +// @Accept mpfd +// @Produce json +// @Produce plain +// @Security BearerAuth +// @Param file formData file true "Audio file to translate" +// @Param model formData string true "Model ID" +// @Param prompt formData string false "Optional English text to guide the model" +// @Param response_format formData string false "json, text, srt, verbose_json, or vtt" +// @Param temperature formData number false "Sampling temperature (0-1)" +// @Success 200 {object} map[string]interface{} "English translation in the requested response_format" +// @Failure 400 {object} core.OpenAIErrorEnvelope +// @Failure 401 {object} core.OpenAIErrorEnvelope +// @Failure 404 {object} core.OpenAIErrorEnvelope +// @Failure 502 {object} core.OpenAIErrorEnvelope +// @Router /v1/audio/translations [post] +func (h *Handler) AudioTranslations(c *echo.Context) error { + return h.audio().CreateTranslation(c) +} + // Responses handles POST /v1/responses // // @Summary Create a model response (Responses API) diff --git a/internal/server/http.go b/internal/server/http.go index 818427fd..8762e231 100644 --- a/internal/server/http.go +++ b/internal/server/http.go @@ -433,6 +433,7 @@ func New(provider core.RoutableProvider, cfg *Config) *Server { e.POST("/v1/embeddings", handler.Embeddings) e.POST("/v1/audio/speech", handler.AudioSpeech) e.POST("/v1/audio/transcriptions", handler.AudioTranscriptions) + e.POST("/v1/audio/translations", handler.AudioTranslations) if cfg == nil || cfg.RealtimeEnabled { e.GET("/v1/realtime", handler.Realtime) e.POST("/v1/realtime/calls", handler.RealtimeCalls) diff --git a/internal/server/http_start_test.go b/internal/server/http_start_test.go index f59cf79e..ae58c942 100644 --- a/internal/server/http_start_test.go +++ b/internal/server/http_start_test.go @@ -73,23 +73,27 @@ func TestNewGatewayStartConfig_ConfiguresGracefulDrain(t *testing.T) { } func TestModelInteractionWriteDeadlineMiddleware_ClearsDeadlineForModelRoutes(t *testing.T) { - e := echo.New() - writer := &deadlineTrackingWriter{ResponseRecorder: httptest.NewRecorder()} - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil) - c := e.NewContext(req, writer) - - handler := modelInteractionWriteDeadlineMiddleware()(func(c *echo.Context) error { - return c.String(http.StatusOK, "ok") - }) - - if err := handler(c); err != nil { - t.Fatalf("handler() error = %v", err) - } - if len(writer.deadlines) != 1 { - t.Fatalf("deadline calls = %d, want 1", len(writer.deadlines)) - } - if !writer.deadlines[0].IsZero() { - t.Fatalf("deadline = %v, want zero time", writer.deadlines[0]) + for _, path := range []string{"/v1/chat/completions", "/v1/audio/translations"} { + t.Run(path, func(t *testing.T) { + e := echo.New() + writer := &deadlineTrackingWriter{ResponseRecorder: httptest.NewRecorder()} + req := httptest.NewRequest(http.MethodPost, path, nil) + c := e.NewContext(req, writer) + + handler := modelInteractionWriteDeadlineMiddleware()(func(c *echo.Context) error { + return c.String(http.StatusOK, "ok") + }) + + if err := handler(c); err != nil { + t.Fatalf("handler() error = %v", err) + } + if len(writer.deadlines) != 1 { + t.Fatalf("deadline calls = %d, want 1", len(writer.deadlines)) + } + if !writer.deadlines[0].IsZero() { + t.Fatalf("deadline = %v, want zero time", writer.deadlines[0]) + } + }) } } diff --git a/internal/usage/audio.go b/internal/usage/audio.go index 9ca20acd..47547c94 100644 --- a/internal/usage/audio.go +++ b/internal/usage/audio.go @@ -13,6 +13,7 @@ import ( const ( endpointAudioSpeech = "/v1/audio/speech" endpointAudioTranscriptions = "/v1/audio/transcriptions" + endpointAudioTranslations = "/v1/audio/translations" // rawKeyInputCharacters and rawKeyAudioSeconds are the RawData keys that carry // the non-token billable units audio providers do not report as tokens: input @@ -89,13 +90,23 @@ type transcriptionUsage struct { // interaction stays observable even when the provider reports no usage (whisper, // or non-JSON response formats such as text/srt/vtt). func ExtractFromTranscriptionResponse(body []byte, requestID, model, provider string, pricing ...*core.ModelPricing) *UsageEntry { + return extractFromAudioTextResponse(body, requestID, model, provider, endpointAudioTranscriptions, pricing...) +} + +// ExtractFromTranslationResponse builds a usage entry for an audio translation +// request while preserving the translations endpoint in usage records. +func ExtractFromTranslationResponse(body []byte, requestID, model, provider string, pricing ...*core.ModelPricing) *UsageEntry { + return extractFromAudioTextResponse(body, requestID, model, provider, endpointAudioTranslations, pricing...) +} + +func extractFromAudioTextResponse(body []byte, requestID, model, provider, endpoint string, pricing ...*core.ModelPricing) *UsageEntry { entry := &UsageEntry{ ID: uuid.New().String(), RequestID: requestID, Timestamp: time.Now().UTC(), Model: model, Provider: provider, - Endpoint: endpointAudioTranscriptions, + Endpoint: endpoint, } var parsed struct { @@ -114,7 +125,7 @@ func ExtractFromTranscriptionResponse(body []byte, requestID, model, provider st } } - applyUsageCosts(entry, provider, endpointAudioTranscriptions, pricing...) + applyUsageCosts(entry, provider, endpoint, pricing...) return entry } diff --git a/internal/usage/audio_test.go b/internal/usage/audio_test.go index d8e43262..435516b3 100644 --- a/internal/usage/audio_test.go +++ b/internal/usage/audio_test.go @@ -92,14 +92,43 @@ func TestExtractFromTranscriptionResponse_TokenUsage(t *testing.T) { if entry == nil { t.Fatal("expected a usage entry") } - if entry.Endpoint != endpointAudioTranscriptions { - t.Errorf("endpoint = %q, want %q", entry.Endpoint, endpointAudioTranscriptions) - } if entry.InputTokens != 14 || entry.OutputTokens != 45 || entry.TotalTokens != 59 { t.Errorf("token counts mismatch: %+v", entry) } } +func TestExtractFromAudioTextResponse_UsesEndpoint(t *testing.T) { + tests := []struct { + name string + extract func() *UsageEntry + endpoint string + }{ + { + name: "transcription", + extract: func() *UsageEntry { + return ExtractFromTranscriptionResponse([]byte(`{"text":"hello"}`), "req", "whisper-1", "openai") + }, + endpoint: endpointAudioTranscriptions, + }, + { + name: "translation", + extract: func() *UsageEntry { + return ExtractFromTranslationResponse([]byte(`{"text":"hello"}`), "req", "whisper-1", "openai") + }, + endpoint: endpointAudioTranslations, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + entry := tt.extract() + if entry.Endpoint != tt.endpoint { + t.Errorf("endpoint = %q, want %q", entry.Endpoint, tt.endpoint) + } + }) + } +} + func TestExtractFromTranscriptionResponse_TotalTokensDerived(t *testing.T) { body := []byte(`{"usage":{"input_tokens":10,"output_tokens":20}}`)