From dff488044cdd28672f69c273899636cb086e6f33 Mon Sep 17 00:00:00 2001 From: "nexxtmobile.de" Date: Sat, 17 Oct 2026 20:44:29 +0000 Subject: [PATCH 1/2] fix(realtime): strip EOU language tags from the LLM context ASR backends like Nemotron-3.5 append end-of-utterance language tags to transcripts ("", "", ...). The committed transcript feeds the LLM conversation (the user item Transcript field), but the per-utterance tag is unreliable for short/ambiguous utterances (a German "Hallo." tagged ) and the LLM treats it as a language signal, switching the reply language against the session instructions. commitUtteranceWithTranscript now strips the EOU tags from the transcript that goes to appendUserItem/generateResponse (the LLM context). The emitted input_audio_transcription event keeps the raw transcript (debugging / client display). Regression tests: stripEOULanguageTags unit specs + a client-commit spec asserting the emitted event keeps the raw tag while the LLM user message is tag-free. Signed-off-by: nexxtmobile.de --- core/http/endpoints/openai/realtime.go | 23 ++++ .../openai/realtime_eou_langtag_test.go | 105 ++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 core/http/endpoints/openai/realtime_eou_langtag_test.go diff --git a/core/http/endpoints/openai/realtime.go b/core/http/endpoints/openai/realtime.go index 17592e3bd85c..e2a032a9f131 100644 --- a/core/http/endpoints/openai/realtime.go +++ b/core/http/endpoints/openai/realtime.go @@ -12,6 +12,7 @@ import ( "maps" "math" "os" + "regexp" "strconv" "strings" "sync" @@ -2017,6 +2018,19 @@ func commitUtterance(ctx context.Context, utt []byte, session *Session, conv *Co commitUtteranceWithTranscript(ctx, utt, nil, nil, "", session, conv, t, slot) } +// eouLangTagRe matches the end-of-utterance language tags that Nemotron-3.5 +// ASR appends to transcripts (BCP-47 style: "", "", ...). +var eouLangTagRe = regexp.MustCompile(`\s*<[a-z]{2,3}(?:-[A-Za-z]{2,3})?>\s*`) + +// stripEOULanguageTags removes the STT EOU language tags from a transcript +// (see the call site in commitUtteranceWithTranscript). +func stripEOULanguageTags(s string) string { + if !strings.Contains(s, "<") { + return s + } + return strings.Join(strings.Fields(eouLangTagRe.ReplaceAllString(s, " ")), " ") +} + // commitUtteranceWithTranscript commits one user turn. live carries the // transcript semantic_vad's live stream already produced (its caption deltas // were streamed to the client during the turn, so only the completed event @@ -2184,6 +2198,15 @@ func commitUtteranceWithTranscript(ctx context.Context, utt []byte, live *liveUt return } + // EOU language tags (e.g. "") are STT metadata: they stay in the + // transcription event emitted above (debugging, client display), but are + // stripped from the LLM context — the per-utterance tag is unreliable for + // short/ambiguous utterances (a German "Hallo." tagged ), and the + // LLM treats it as a language signal (the agent's language rule then + // switches the reply language; see CallcenterAiAgent agent.py, + // 17.10.2026). + transcript = stripEOULanguageTags(transcript) + // Sound-event detection is additive to transcription: classify the same // committed window and emit its scored AudioSet tags as a separate event. // A failure here is logged but must never abort the turn. diff --git a/core/http/endpoints/openai/realtime_eou_langtag_test.go b/core/http/endpoints/openai/realtime_eou_langtag_test.go new file mode 100644 index 000000000000..067f15000ea1 --- /dev/null +++ b/core/http/endpoints/openai/realtime_eou_langtag_test.go @@ -0,0 +1,105 @@ +package openai + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/backend" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/endpoints/openai/types" +) + +// Regression test: ASR backends like Nemotron-3.5 append end-of-utterance +// language tags to transcripts ("", "", ...). The committed +// transcript feeds the LLM conversation (the user item's Transcript field). +// The per-utterance tag is unreliable for short/ambiguous utterances (a +// German "Hallo." tagged ) and the LLM treats it as a language +// signal, switching the reply language against the session instructions. +// The tags must be stripped from the LLM context while the emitted +// transcription event keeps the raw transcript (debugging / client display). +var _ = Describe("stripEOULanguageTags", func() { + It("removes a trailing EOU tag", func() { + Expect(stripEOULanguageTags("Hallo. ")).To(Equal("Hallo.")) + }) + + It("removes mid-text EOU tags and normalizes whitespace", func() { + Expect(stripEOULanguageTags("My name is Benjamin Zimmer. Ich hatte gerne eine Festnetzrufnummer. ")). + To(Equal("My name is Benjamin Zimmer. Ich hatte gerne eine Festnetzrufnummer.")) + }) + + It("handles 3-letter primary subtags", func() { + Expect(stripEOULanguageTags("Hello. ")).To(Equal("Hello.")) + }) + + It("leaves plain text untouched", func() { + Expect(stripEOULanguageTags("Guten Tag, wie kann ich helfen?")). + To(Equal("Guten Tag, wie kann ich helfen?")) + }) +}) + +var _ = Describe("client commit with an EOU-tagged transcript", func() { + utt := []byte{1, 2, 3, 4} + + newModel := func(text string) *heldTranscribeModel { + return &heldTranscribeModel{ + fakeModel: &fakeModel{ + cfg: &config.ModelConfig{}, + predictTokens: []string{"ok"}, + predictResp: backend.LLMResponse{Response: "ok"}, + ttsStreamChunks: [][]byte{{1}}, + ttsStreamRate: 24000, + }, + texts: []string{text}, + release: make(chan struct{}), + started: make(chan int32, 8), + finished: make(chan int32, 8), + } + } + + newSession := func(m *heldTranscribeModel) *Session { + on := true + session := &Session{ + OutputSampleRate: 24000, + InputAudioTranscription: &types.AudioTranscription{}, + ModelInterface: m, + ModelConfig: &config.ModelConfig{ + Pipeline: config.Pipeline{Streaming: config.PipelineStreaming{LLM: &on, TTS: &on}}, + }, + respSink: newResponseSink(), + } + session.sessionCtx, session.sessionCancel = context.WithCancel(context.Background()) + return session + } + + It("keeps the raw tag in the emitted event but strips it from the LLM context", func() { + m := newModel("Hallo. ") + session := newSession(m) + tr := &fakeTransport{} + conv := &Conversation{} + + session.issueClientCommit(utt, conv, tr) + Eventually(m.started, "2s").Should(Receive(Equal(int32(1)))) + close(m.release) + session.respSink.wait() + + // The emitted transcription event keeps the raw transcript. + var emitted string + for _, ev := range tr.events() { + if c, ok := ev.(types.ConversationItemInputAudioTranscriptionCompletedEvent); ok { + emitted = c.Transcript + } + } + Expect(emitted).To(Equal("Hallo. ")) + + // The LLM context (user message) must be tag-free. + var llmUser string + for _, msg := range m.lastMessages { + if msg.Role == string(types.MessageRoleUser) { + llmUser = msg.StringContent + } + } + Expect(llmUser).To(Equal("Hallo.")) + }) +}) From 005a2d4ce8aa99bc6b462c9ee6a714fa22d377c7 Mon Sep 17 00:00:00 2001 From: "nexxtmobile.de" Date: Mon, 19 Oct 2026 13:22:42 +0000 Subject: [PATCH 2/2] docs(realtime): explain transcript language tags Assisted-by: Codex:gpt-6 Signed-off-by: nexxtmobile.de --- docs/content/features/openai-realtime.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index 96f9c9e0b84f..3aa64e566135 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -33,6 +33,11 @@ Make sure all referenced models (`silero-vad-ggml`, `whisper-large-turbo`, `qwen A pipeline stage can name a [failover chain]({{%relref "features/model-failover" %}}); the stage then switches targets without closing the session. +Transcription backends can include end-of-utterance language tags, such as `` or ``. +LocalAI removes these tags from the user transcript sent to the LLM. +The transcription events sent to the client retain the original tags. +This prevents a tag from overriding the intended conversation language in the LLM context. + ### Install the Qwen streaming assistant preset The gallery entry `qwen3-streaming-assistant` configures an English voice