diff --git a/core/http/endpoints/openai/realtime.go b/core/http/endpoints/openai/realtime.go index 17592e3bd85c..e2a032a9f131 100644 --- a/core/http/endpoints/openai/realtime.go +++ b/core/http/endpoints/openai/realtime.go @@ -12,6 +12,7 @@ import ( "maps" "math" "os" + "regexp" "strconv" "strings" "sync" @@ -2017,6 +2018,19 @@ func commitUtterance(ctx context.Context, utt []byte, session *Session, conv *Co commitUtteranceWithTranscript(ctx, utt, nil, nil, "", session, conv, t, slot) } +// eouLangTagRe matches the end-of-utterance language tags that Nemotron-3.5 +// ASR appends to transcripts (BCP-47 style: "", "", ...). +var eouLangTagRe = regexp.MustCompile(`\s*<[a-z]{2,3}(?:-[A-Za-z]{2,3})?>\s*`) + +// stripEOULanguageTags removes the STT EOU language tags from a transcript +// (see the call site in commitUtteranceWithTranscript). +func stripEOULanguageTags(s string) string { + if !strings.Contains(s, "<") { + return s + } + return strings.Join(strings.Fields(eouLangTagRe.ReplaceAllString(s, " ")), " ") +} + // commitUtteranceWithTranscript commits one user turn. live carries the // transcript semantic_vad's live stream already produced (its caption deltas // were streamed to the client during the turn, so only the completed event @@ -2184,6 +2198,15 @@ func commitUtteranceWithTranscript(ctx context.Context, utt []byte, live *liveUt return } + // EOU language tags (e.g. "") are STT metadata: they stay in the + // transcription event emitted above (debugging, client display), but are + // stripped from the LLM context — the per-utterance tag is unreliable for + // short/ambiguous utterances (a German "Hallo." tagged ), and the + // LLM treats it as a language signal (the agent's language rule then + // switches the reply language; see CallcenterAiAgent agent.py, + // 17.10.2026). + transcript = stripEOULanguageTags(transcript) + // Sound-event detection is additive to transcription: classify the same // committed window and emit its scored AudioSet tags as a separate event. // A failure here is logged but must never abort the turn. diff --git a/core/http/endpoints/openai/realtime_eou_langtag_test.go b/core/http/endpoints/openai/realtime_eou_langtag_test.go new file mode 100644 index 000000000000..067f15000ea1 --- /dev/null +++ b/core/http/endpoints/openai/realtime_eou_langtag_test.go @@ -0,0 +1,105 @@ +package openai + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/backend" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/endpoints/openai/types" +) + +// Regression test: ASR backends like Nemotron-3.5 append end-of-utterance +// language tags to transcripts ("", "", ...). The committed +// transcript feeds the LLM conversation (the user item's Transcript field). +// The per-utterance tag is unreliable for short/ambiguous utterances (a +// German "Hallo." tagged ) and the LLM treats it as a language +// signal, switching the reply language against the session instructions. +// The tags must be stripped from the LLM context while the emitted +// transcription event keeps the raw transcript (debugging / client display). +var _ = Describe("stripEOULanguageTags", func() { + It("removes a trailing EOU tag", func() { + Expect(stripEOULanguageTags("Hallo. ")).To(Equal("Hallo.")) + }) + + It("removes mid-text EOU tags and normalizes whitespace", func() { + Expect(stripEOULanguageTags("My name is Benjamin Zimmer. Ich hatte gerne eine Festnetzrufnummer. ")). + To(Equal("My name is Benjamin Zimmer. Ich hatte gerne eine Festnetzrufnummer.")) + }) + + It("handles 3-letter primary subtags", func() { + Expect(stripEOULanguageTags("Hello. ")).To(Equal("Hello.")) + }) + + It("leaves plain text untouched", func() { + Expect(stripEOULanguageTags("Guten Tag, wie kann ich helfen?")). + To(Equal("Guten Tag, wie kann ich helfen?")) + }) +}) + +var _ = Describe("client commit with an EOU-tagged transcript", func() { + utt := []byte{1, 2, 3, 4} + + newModel := func(text string) *heldTranscribeModel { + return &heldTranscribeModel{ + fakeModel: &fakeModel{ + cfg: &config.ModelConfig{}, + predictTokens: []string{"ok"}, + predictResp: backend.LLMResponse{Response: "ok"}, + ttsStreamChunks: [][]byte{{1}}, + ttsStreamRate: 24000, + }, + texts: []string{text}, + release: make(chan struct{}), + started: make(chan int32, 8), + finished: make(chan int32, 8), + } + } + + newSession := func(m *heldTranscribeModel) *Session { + on := true + session := &Session{ + OutputSampleRate: 24000, + InputAudioTranscription: &types.AudioTranscription{}, + ModelInterface: m, + ModelConfig: &config.ModelConfig{ + Pipeline: config.Pipeline{Streaming: config.PipelineStreaming{LLM: &on, TTS: &on}}, + }, + respSink: newResponseSink(), + } + session.sessionCtx, session.sessionCancel = context.WithCancel(context.Background()) + return session + } + + It("keeps the raw tag in the emitted event but strips it from the LLM context", func() { + m := newModel("Hallo. ") + session := newSession(m) + tr := &fakeTransport{} + conv := &Conversation{} + + session.issueClientCommit(utt, conv, tr) + Eventually(m.started, "2s").Should(Receive(Equal(int32(1)))) + close(m.release) + session.respSink.wait() + + // The emitted transcription event keeps the raw transcript. + var emitted string + for _, ev := range tr.events() { + if c, ok := ev.(types.ConversationItemInputAudioTranscriptionCompletedEvent); ok { + emitted = c.Transcript + } + } + Expect(emitted).To(Equal("Hallo. ")) + + // The LLM context (user message) must be tag-free. + var llmUser string + for _, msg := range m.lastMessages { + if msg.Role == string(types.MessageRoleUser) { + llmUser = msg.StringContent + } + } + Expect(llmUser).To(Equal("Hallo.")) + }) +}) diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index 96f9c9e0b84f..3aa64e566135 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -33,6 +33,11 @@ Make sure all referenced models (`silero-vad-ggml`, `whisper-large-turbo`, `qwen A pipeline stage can name a [failover chain]({{%relref "features/model-failover" %}}); the stage then switches targets without closing the session. +Transcription backends can include end-of-utterance language tags, such as `` or ``. +LocalAI removes these tags from the user transcript sent to the LLM. +The transcription events sent to the client retain the original tags. +This prevents a tag from overriding the intended conversation language in the LLM context. + ### Install the Qwen streaming assistant preset The gallery entry `qwen3-streaming-assistant` configures an English voice