From 78b4443c58d2a16332b3e1e7b77f7a519d8a79de Mon Sep 17 00:00:00 2001 From: "nexxtmobile.de" Date: Sat, 17 Oct 2026 20:30:12 +0000 Subject: [PATCH 1/2] fix(realtime): write committed-utterance WAV header at the session input sample rate The client-commit path (commitUtteranceWithTranscript) and writeUtteranceWAV wrote the utterance WAV with a fixed 16 kHz header (laudio.NewWAVHeader), but the input buffer holds audio at the session input sample rate (e.g. 24 kHz for OpenAI-Realtime clients). The file-based transcription backend then decoded 24 kHz data at 16 kHz (1.5x speed/pitch), degrading transcription to garbage or empty output, which failed the follow-up response with input_not_committed (total silence after the greeting). writeUtteranceWAV now takes the sample rate and both call sites pass session.InputSampleRate (laudio.NewWAVHeaderWithRate already existed). Regression test: the committed WAV header carries the requested rate. Signed-off-by: nexxtmobile.de --- core/http/endpoints/openai/realtime.go | 19 +++++-- .../openai/realtime_commit_wav_rate_test.go | 56 +++++++++++++++++++ 2 files changed, 69 insertions(+), 6 deletions(-) create mode 100644 core/http/endpoints/openai/realtime_commit_wav_rate_test.go diff --git a/core/http/endpoints/openai/realtime.go b/core/http/endpoints/openai/realtime.go index 17592e3bd85c..46df1ca10aac 100644 --- a/core/http/endpoints/openai/realtime.go +++ b/core/http/endpoints/openai/realtime.go @@ -2066,7 +2066,13 @@ func commitUtteranceWithTranscript(ctx context.Context, utt []byte, live *liveUt defer os.Remove(f.Name()) xlog.Debug("Writing to file", "file", f.Name()) - hdr := laudio.NewWAVHeader(uint32(len(utt))) + // The input buffer holds audio at the session's input sample rate + // (e.g. 24 kHz for OpenAI-Realtime clients; WebRTC is resampled to + // 16 kHz). A fixed 16 kHz header makes the backend decode the stream + // at the wrong rate (1.5x speed/pitch for 24 kHz), degrading + // transcription to garbage or empty output — which then fails the + // follow-up response with input_not_committed (client-commit path). + hdr := laudio.NewWAVHeaderWithRate(uint32(len(utt)), uint32(session.InputSampleRate)) if err := hdr.Write(f); err != nil { xlog.Error("Failed to write WAV header", "error", err) return @@ -2361,9 +2367,10 @@ func writeWindowWAV(pcm []byte, sampleRate int) (string, error) { return f.Name(), nil } -// writeUtteranceWAV persists raw 16 kHz mono PCM to a temp WAV for the -// file-based transcription paths. The caller must invoke cleanup. -func writeUtteranceWAV(utt []byte) (string, func(), error) { +// writeUtteranceWAV persists raw mono PCM (at the given sample rate) to a +// temp WAV for the file-based transcription paths. The caller must invoke +// cleanup. +func writeUtteranceWAV(utt []byte, sampleRate int) (string, func(), error) { f, err := os.CreateTemp("", "realtime-audio-chunk-*.wav") if err != nil { return "", nil, err @@ -2374,7 +2381,7 @@ func writeUtteranceWAV(utt []byte) (string, func(), error) { } xlog.Debug("Writing to file", "file", f.Name()) - hdr := laudio.NewWAVHeader(uint32(len(utt))) + hdr := laudio.NewWAVHeaderWithRate(uint32(len(utt)), uint32(sampleRate)) if err := hdr.Write(f); err != nil { cleanup() return "", nil, err @@ -2391,7 +2398,7 @@ func writeUtteranceWAV(utt []byte) (string, func(), error) { // the semantic_vad retranscribe gate. The result's Eou flag reports whether // the batch decode also ended on the end-of-utterance token. func transcribeUtterance(ctx context.Context, utt []byte, session *Session) (*schema.TranscriptionResult, error) { - path, cleanup, err := writeUtteranceWAV(utt) + path, cleanup, err := writeUtteranceWAV(utt, session.InputSampleRate) if err != nil { return nil, err } diff --git a/core/http/endpoints/openai/realtime_commit_wav_rate_test.go b/core/http/endpoints/openai/realtime_commit_wav_rate_test.go new file mode 100644 index 000000000000..e70e17607a22 --- /dev/null +++ b/core/http/endpoints/openai/realtime_commit_wav_rate_test.go @@ -0,0 +1,56 @@ +package openai + +import ( + "bytes" + "encoding/binary" + "fmt" + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// wavSampleRateOf reads the sample rate from a WAV file's fmt chunk +// (RIFF/WAVE, little-endian), without external dependencies. +func wavSampleRateOf(path string) (int, error) { + b, err := os.ReadFile(path) + if err != nil { + return 0, err + } + if len(b) < 44 || !bytes.Equal(b[0:4], []byte("RIFF")) || + !bytes.Equal(b[8:12], []byte("WAVE")) || + !bytes.Equal(b[12:16], []byte("fmt ")) { + return 0, fmt.Errorf("not a RIFF/WAVE file: %s", path) + } + return int(binary.LittleEndian.Uint32(b[24:28])), nil +} + +// Regression test: the committed-utterance WAV must carry the session's +// input sample rate in its header. The client-commit path used to write a +// FIXED 16 kHz header (laudio.NewWAVHeader) while the input buffer holds +// audio at the session input rate (e.g. 24 kHz for OpenAI-Realtime clients). +// The file-based transcription backend then decoded 24 kHz data at 16 kHz +// (1.5x speed/pitch), degrading transcription to garbage or empty output, +// which failed the follow-up response with input_not_committed (total +// silence after the greeting). +var _ = Describe("writeUtteranceWAV", func() { + It("writes the header at the requested sample rate (24 kHz input)", func() { + utt := make([]byte, 24000*2) // 1 s of 16-bit mono PCM + path, cleanup, err := writeUtteranceWAV(utt, 24000) + defer cleanup() + Expect(err).ToNot(HaveOccurred()) + rate, herr := wavSampleRateOf(path) + Expect(herr).ToNot(HaveOccurred()) + Expect(rate).To(Equal(24000)) + }) + + It("writes the header at the requested sample rate (16 kHz input)", func() { + utt := make([]byte, 16000*2) + path, cleanup, err := writeUtteranceWAV(utt, 16000) + defer cleanup() + Expect(err).ToNot(HaveOccurred()) + rate, herr := wavSampleRateOf(path) + Expect(herr).ToNot(HaveOccurred()) + Expect(rate).To(Equal(16000)) + }) +}) From 5859326bffe3ab82f486ece85bf7f865ac26a20c Mon Sep 17 00:00:00 2001 From: "nexxtmobile.de" Date: Mon, 19 Oct 2026 13:22:41 +0000 Subject: [PATCH 2/2] docs(realtime): explain committed audio sample rates Assisted-by: Codex:gpt-6 Signed-off-by: nexxtmobile.de --- docs/content/features/openai-realtime.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index 96f9c9e0b84f..d72f1895232e 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -479,6 +479,10 @@ ws://localhost:8080/v1/realtime?model=gpt-realtime Audio is sent and received as raw PCM in the WebSocket messages, following the OpenAI Realtime API protocol. +LocalAI writes committed audio to the transcription backend at the session input sample rate. +For example, 24 kHz WebSocket input produces a WAV file with a 24 kHz header. +WebRTC input is resampled to 16 kHz before transcription. + ### WebRTC The WebRTC transport enables browser-based voice conversations with lower latency. OpenAI-compatible clients can send a raw SDP offer and select the model with the query parameter: