From 51f97ddd35cf035f32151b2385616a2a0ff3ddab Mon Sep 17 00:00:00 2001 From: faryal-9 Date: Thu, 27 Aug 2026 16:30:13 +0500 Subject: [PATCH] Add DirectPolling microphone capture mode (fixes Quest capture reliability) The default capture path plays the microphone clip through an AudioSource and captures it via AudioProbe's OnAudioFilterRead. That couples two clocks: the microphone driver writes the clip ring at the input device's rate while the AudioSource reads it at the output pipeline's rate. On devices where those clocks drift, the filter's read position slides against the microphone's write position until it consumes stale or not-yet-written regions of the ring, which surfaces as intermittent silence and crackle. We measured this in production on Meta Quest 3 (whose capture pipeline runs at 24 kHz). This adds an opt-in MicrophoneCaptureMode.DirectPolling that reads 10 ms frames straight out of the clip with Microphone.GetPosition/GetData on the main thread and raises AudioRead directly - no AudioSource or AudioProbe involved, so the microphone's own write position is the only clock and drift cannot accumulate. This is the same capture strategy Photon Voice and Mumble use on the same hardware. Details: - Default behavior is unchanged: the existing constructor keeps the filter-based path; the new mode is selected via a constructor overload. - Ring wrap-around is handled modularly; a transient GetPosition() == -1 (observed on some Android device states) is skipped rather than fed into GetData. - After a main-thread hitch (GC, scene load) the backlog beyond a 300 ms catch-up budget is dropped oldest-first: the native audio source ingests 10 ms frames into a bounded queue at real time, so emitting hundreds of ms of stale audio in one tick overflows it, and real-time voice cannot use that audio anyway. - Channel shape is adapted to the source's resolved device format (mono -> N duplication for the common Android mono-clip case, N -> mono averaging, first-channel fallback otherwise). - A throwing AudioRead subscriber is contained (throttled warning): the loop is the sole frame producer, so an escaped exception would leave the microphone permanently silent. - Stop/restart (including the iOS pause/resume path) invalidate the loop via a generation counter. Tested on Meta Quest 3 (Unity 2022.3, Android, 24 kHz mono capture, stereo source): multi-hour sessions with repeated network drops and SDK reconnects show steady 10 ms frame production (~250 packets per 5 s via OutboundRtp counters) with no silence or drift, where the filter-based path previously degraded. Editor (Windows, 48 kHz) verified against a remote peer for parity. --- Runtime/Scripts/Audio/MicrophoneSource.cs | 212 +++++++++++++++++++++- 1 file changed, 211 insertions(+), 1 deletion(-) diff --git a/Runtime/Scripts/Audio/MicrophoneSource.cs b/Runtime/Scripts/Audio/MicrophoneSource.cs index 75bec1f0..31eecd7c 100644 --- a/Runtime/Scripts/Audio/MicrophoneSource.cs +++ b/Runtime/Scripts/Audio/MicrophoneSource.cs @@ -12,15 +12,56 @@ namespace LiveKit /// /// Ensure microphone permissions are granted before calling . /// + /// + /// Strategy used by to move samples from Unity's microphone + /// ring buffer into the audio pipeline. + /// + public enum MicrophoneCaptureMode + { + /// + /// Play the microphone clip through an and capture it via + /// (OnAudioFilterRead). This is the default and matches the + /// SDK's historical behavior. + /// + AudioFilter, + + /// + /// Read samples directly out of the microphone clip with + /// / on the main + /// thread, without creating an AudioSource or AudioProbe. + /// + /// couples two clocks: the microphone driver writes the clip + /// at the input device's rate while the AudioSource reads it at the output pipeline's + /// rate. On devices where those clocks drift (measured on Meta Quest, whose capture + /// pipeline runs at 24 kHz), the read position slides against the write position until + /// the filter consumes stale or not-yet-written regions of the ring, which surfaces as + /// intermittent silence and crackle. Direct polling uses the microphone's own write + /// position as the only clock, so drift cannot accumulate; this is the same capture + /// strategy used by Photon Voice and Mumble on the same hardware. + /// + DirectPolling, + } + sealed public class MicrophoneSource : RtcAudioSource { + // DirectPolling: never emit more than this much backlog in one tick. The native audio + // source ingests 10 ms frames into a bounded (~1 s) queue at real time; after a main + // thread hitch (GC, scene load) Microphone.GetPosition can report hundreds of ms of + // backlog, and pushing it all at once overflows that queue. Real-time voice cannot use + // stale audio anyway, so the loop drops the oldest samples beyond this budget and + // resynchronizes to the newest. + private const int CatchUpBudgetMs = 300; + private readonly GameObject _sourceObject; private readonly string _deviceName; + private readonly MicrophoneCaptureMode _captureMode; public override event Action AudioRead; private bool _disposed = false; private bool _started = false; + // Invalidates an in-flight polling loop when the microphone is stopped or restarted. + private int _pollGeneration = 0; /// /// Creates a new microphone source for the given device. @@ -29,10 +70,28 @@ sealed public class MicrophoneSource : RtcAudioSource /// get the list of available devices. /// The GameObject to attach the AudioSource to. The object must be kept in the scene /// for the duration of the source's lifetime. - public MicrophoneSource(string deviceName, GameObject sourceObject) : base(RtcAudioSourceType.AudioSourceMicrophone) + public MicrophoneSource(string deviceName, GameObject sourceObject) + : this(deviceName, sourceObject, MicrophoneCaptureMode.AudioFilter) + { + } + + /// + /// Creates a new microphone source for the given device using the given capture mode. + /// + /// The name of the device to capture from. Use to + /// get the list of available devices. + /// The GameObject to attach the AudioSource to (unused by + /// , which creates no components). The object must be kept + /// in the scene for the duration of the source's lifetime. + /// How samples are moved out of the microphone ring buffer. Use + /// on devices where the default filter-based capture + /// exhibits drift (e.g. Meta Quest). + public MicrophoneSource(string deviceName, GameObject sourceObject, MicrophoneCaptureMode captureMode) + : base(RtcAudioSourceType.AudioSourceMicrophone) { _deviceName = deviceName; _sourceObject = sourceObject; + _captureMode = captureMode; } /// @@ -98,6 +157,28 @@ private IEnumerator StartMicrophone() yield break; } + if (_captureMode == MicrophoneCaptureMode.DirectPolling) + { + // Wait for the microphone to actually start producing data before polling. + const float pollTimeout = 2f; + float pollElapsed = 0f; + while (Microphone.GetPosition(_deviceName) <= 0 && pollElapsed < pollTimeout) + { + yield return new WaitForSeconds(0.05f); + pollElapsed += 0.05f; + } + if (Microphone.GetPosition(_deviceName) <= 0) + { + Utils.Error($"MicrophoneSource: Microphone did not start producing data after {pollTimeout}s"); + yield break; + } + + int generation = ++_pollGeneration; + MonoBehaviourContext.RunCoroutine(PollMicrophone(clip, generation)); + Utils.Debug($"MicrophoneSource device='{_deviceName}' started successfully (direct polling)"); + yield break; + } + // Ensure no duplicate components exist before adding new ones. // This is important during app resume on iOS where components might not be // fully destroyed yet due to Unity's deferred Destroy(). @@ -153,6 +234,8 @@ public override void Stop() private IEnumerator StopMicrophone() { + _pollGeneration++; // ends an in-flight DirectPolling loop + if (Microphone.IsRecording(_deviceName)) Microphone.End(_deviceName); @@ -175,6 +258,133 @@ private IEnumerator StopMicrophone() yield return null; } + /// + /// DirectPolling capture: reads 10 ms frames straight out of the microphone clip using + /// the microphone's own write position as the only clock. Runs once per rendered frame; + /// exits when the source is stopped, restarted, or the device stops recording. + /// + private IEnumerator PollMicrophone(AudioClip clip, int generation) + { + int sampleRate = clip.frequency; + int clipChannels = clip.channels; + int expectedChannels = (int)_expectedChannels; + int clipLength = clip.samples; // samples per channel + int frameSize = sampleRate / 100; // 10 ms per channel + int maxCatchUpFrames = CatchUpBudgetMs / 10; + + var readBuf = new float[frameSize * clipChannels]; + var sendBuf = clipChannels == expectedChannels ? readBuf : new float[frameSize * expectedChannels]; + + int lastPos = Microphone.GetPosition(_deviceName); + if (lastPos < 0) lastPos = 0; + + int droppedSinceWarn = 0; + float nextWarnTime = 0f; + bool channelMismatchWarned = false; + + while (_started && generation == _pollGeneration && Microphone.IsRecording(_deviceName)) + { + int micPos = Microphone.GetPosition(_deviceName); + // GetPosition can transiently return -1 on some Android device states; never + // feed a negative offset into the modulo below or into GetData. + if (micPos < 0) + { + yield return null; + continue; + } + + int available = micPos - lastPos; + if (available < 0) available += clipLength; // ring wrap + + // A stall longer than the ring can hold makes the modulo untrustworthy (the + // write position may have lapped us); resync to the newest data outright. + if (available >= clipLength - frameSize) + { + lastPos = micPos; + yield return null; + continue; + } + + // Drop the oldest backlog beyond the catch-up budget so a main-thread hitch + // can never flood the native queue in a single tick. + int backlogFrames = available / frameSize; + if (backlogFrames > maxCatchUpFrames) + { + int dropFrames = backlogFrames - maxCatchUpFrames; + lastPos = (lastPos + dropFrames * frameSize) % clipLength; + droppedSinceWarn += dropFrames; + } + + if (droppedSinceWarn > 0 && Time.unscaledTime >= nextWarnTime) + { + Utils.Warning($"MicrophoneSource: dropped {droppedSinceWarn} stale mic frame(s) to avoid flooding the capture queue (main-thread hitch?)"); + droppedSinceWarn = 0; + nextWarnTime = Time.unscaledTime + 1f; + } + + while (_started && generation == _pollGeneration && ((micPos - lastPos + clipLength) % clipLength) >= frameSize) + { + clip.GetData(readBuf, lastPos); + lastPos = (lastPos + frameSize) % clipLength; + + if (clipChannels != expectedChannels) + { + if (clipChannels == 1) + { + // Mono clip, multi-channel source (the common Android case): + // duplicate each sample across the expected channels. + for (int i = 0; i < frameSize; i++) + for (int c = 0; c < expectedChannels; c++) + sendBuf[i * expectedChannels + c] = readBuf[i]; + } + else if (expectedChannels == 1) + { + // Multi-channel clip, mono source: average. + for (int i = 0; i < frameSize; i++) + { + float sum = 0f; + for (int c = 0; c < clipChannels; c++) + sum += readBuf[i * clipChannels + c]; + sendBuf[i] = sum / clipChannels; + } + } + else + { + // Unusual pairing: carry the first clip channel into every output + // channel rather than sending misinterleaved audio. + if (!channelMismatchWarned) + { + channelMismatchWarned = true; + Utils.Warning($"MicrophoneSource: clip has {clipChannels} channels but source expects {expectedChannels}; using first channel"); + } + for (int i = 0; i < frameSize; i++) + for (int c = 0; c < expectedChannels; c++) + sendBuf[i * expectedChannels + c] = readBuf[i * clipChannels]; + } + } + + try + { + AudioRead?.Invoke(sendBuf, expectedChannels, sampleRate); + } + catch (Exception e) + { + // This loop is the sole frame producer: a throwing subscriber must not + // kill it, or the microphone goes permanently silent. + if (Time.unscaledTime >= nextWarnTime) + { + Utils.Warning($"MicrophoneSource: AudioRead subscriber threw: {e.Message}"); + nextWarnTime = Time.unscaledTime + 1f; + } + } + } + + yield return null; + } + + Utils.Debug($"MicrophoneSource device='{_deviceName}' polling loop ended"); + } + private void OnAudioRead(float[] data, int channels, int sampleRate) { AudioRead?.Invoke(data, channels, sampleRate);