From 26546d51cf663cd94ff8a53ebb71f92522136c5d Mon Sep 17 00:00:00 2001 From: lukasIO Date: Thu, 30 Jul 2026 15:30:20 +0200 Subject: [PATCH] consistently clean up audio contexts and graphs --- src/room/Room.ts | 235 +++++++++++++++++------ src/room/participant/LocalParticipant.ts | 30 +-- src/room/participant/Participant.ts | 9 +- src/room/track/LocalAudioTrack.ts | 98 +++++++++- src/room/track/LocalTrack.ts | 9 +- src/room/track/RemoteAudioTrack.ts | 47 +++-- src/room/track/utils.ts | 32 ++- src/room/utils.ts | 61 ++++-- 8 files changed, 400 insertions(+), 121 deletions(-) diff --git a/src/room/Room.ts b/src/room/Room.ts index 47b3087777..d4a233e6f6 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -127,6 +127,7 @@ import { isSafariBased, isWeb, numberToBigInt, + releaseEmptyAudioStreamTrack, sleep, supportsSetSinkId, toHttpUrl, @@ -144,6 +145,8 @@ export enum ConnectionState { const CONNECTION_RECONCILE_FREQUENCY_MS = 4 * 1000; +const DUMMY_AUDIO_ELEMENT_ID = 'livekit-dummy-audio-el'; + /** * In LiveKit, a room is the logical grouping for a list of participants. * Participants in a room can publish tracks, and subscribe to others' tracks. @@ -191,6 +194,24 @@ class Room extends (EventEmitter as new () => TypedEmitter) private audioContext?: AudioContext; + /** + * whether `audioContext` was created by the SDK and therefore has to be closed by it. + * A context handed in through `webAudioMix` is owned by the application instead. + */ + private ownsAudioContext = false; + + /** + * set once the audio context has been released as part of a disconnect. Guards against + * creating a new context outside of a connection lifecycle, where nothing would close it again. + */ + private audioContextReleased = false; + + /** tears down the iOS dummy audio element, set while one is in place. See `startAudio` */ + private releaseDummyAudioElement?: () => void; + + /** silent audio tracks this room acquired, handed back to the shared source on disconnect */ + private acquiredEmptyAudioTracks = new Set(); + /** used for aborting pending connections to a LiveKit server */ private abortController?: AbortController; @@ -1323,6 +1344,79 @@ class Room extends (EventEmitter as new () => TypedEmitter) await this.disconnect(); }; + /** + * iOS blocks audio element playback unless some audio source is already playing, so keep an + * element with a silent track around. Reuses an existing one instead of stacking up listeners. + */ + private acquireDummyAudioElement(): HTMLAudioElement { + const existingEl = document.getElementById(DUMMY_AUDIO_ELEMENT_ID); + if (existingEl instanceof HTMLAudioElement) { + return existingEl; + } + + const dummyAudioEl = document.createElement('audio'); + dummyAudioEl.id = DUMMY_AUDIO_ELEMENT_ID; + dummyAudioEl.autoplay = true; + dummyAudioEl.hidden = true; + const track = this.acquireEmptyAudioStreamTrack(); + track.enabled = true; + const stream = new MediaStream([track]); + dummyAudioEl.srcObject = stream; + + const handleVisibilityChange = () => { + // set the srcObject to null on page hide in order to prevent lock screen controls to show up for it + dummyAudioEl.srcObject = document.hidden ? null : stream; + if (!document.hidden) { + this.log.debug( + 'page visible again, triggering startAudio to resume playback and update playback status', + ); + this.startAudio().catch((error) => + this.log.warn('Could not restart audio on page becoming visible', { + ...this.logContext, + error, + }), + ); + } + }; + document.addEventListener('visibilitychange', handleVisibilityChange); + document.body.append(dummyAudioEl); + + this.releaseDummyAudioElement = () => { + this.releaseDummyAudioElement = undefined; + document.removeEventListener('visibilitychange', handleVisibilityChange); + dummyAudioEl.srcObject = null; + dummyAudioEl.remove(); + this.releaseAcquiredEmptyAudioTrack(track); + }; + + return dummyAudioEl; + } + + /** + * Obtains a silent audio track from the shared source and keeps track of it, so that it can be + * handed back on disconnect and the shared `AudioContext` behind it can be closed again. + */ + private acquireEmptyAudioStreamTrack(): MediaStreamTrack { + const track = getEmptyAudioStreamTrack(); + this.acquiredEmptyAudioTracks.add(track); + return track; + } + + private releaseAcquiredEmptyAudioTrack(track: MediaStreamTrack) { + if (!this.acquiredEmptyAudioTracks.delete(track)) { + return; + } + releaseEmptyAudioStreamTrack(track).catch((error) => + this.log.warn('Could not release empty audio stream track', { ...this.logContext, error }), + ); + } + + private releaseAcquiredEmptyAudioTracks() { + for (const track of this.acquiredEmptyAudioTracks) { + this.releaseAcquiredEmptyAudioTrack(track); + } + } + /** * Browsers have different policies regarding audio playback. Most requiring * some form of user interaction (click/tap/etc). @@ -1334,45 +1428,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) const elements: Array = []; const browser = getBrowser(); if (browser && browser.os === 'iOS') { - /** - * iOS blocks audio element playback if - * - user is not publishing audio themselves and - * - no other audio source is playing - * - * as a workaround, we create an audio element with an empty track, so that - * silent audio is always playing - */ - const audioId = 'livekit-dummy-audio-el'; - let dummyAudioEl = document.getElementById(audioId) as HTMLAudioElement | null; - if (!dummyAudioEl) { - dummyAudioEl = document.createElement('audio'); - dummyAudioEl.id = audioId; - dummyAudioEl.autoplay = true; - dummyAudioEl.hidden = true; - const track = getEmptyAudioStreamTrack(); - track.enabled = true; - const stream = new MediaStream([track]); - dummyAudioEl.srcObject = stream; - document.addEventListener('visibilitychange', () => { - if (!dummyAudioEl) { - return; - } - // set the srcObject to null on page hide in order to prevent lock screen controls to show up for it - dummyAudioEl.srcObject = document.hidden ? null : stream; - if (!document.hidden) { - this.log.debug( - 'page visible again, triggering startAudio to resume playback and update playback status', - ); - this.startAudio(); - } - }); - document.body.append(dummyAudioEl); - this.once(RoomEvent.Disconnected, () => { - dummyAudioEl?.remove(); - dummyAudioEl = null; - }); - } - elements.push(dummyAudioEl); + elements.push(this.acquireDummyAudioElement()); } this.remoteParticipants.forEach((p) => { @@ -1387,7 +1443,9 @@ class Room extends (EventEmitter as new () => TypedEmitter) try { await Promise.all([ - this.acquireAudioContext(), + // once the audio context has been released as part of a disconnect there is nothing left + // to unlock, and creating a new context here would leak it as no disconnect will follow + ...(this.audioContextReleased ? [] : [this.acquireAudioContext()]), ...elements.map((e) => { // when webAudioMix is enabled, attached elements are deliberately kept muted by // RemoteAudioTrack.attach() and audio is routed through the web audio graph instead. @@ -1837,6 +1895,14 @@ class Room extends (EventEmitter as new () => TypedEmitter) } try { + // audio tracks that outlive this room are about to become unreachable through the local + // participant, collect them while we can so they can be detached from the context below + const retainedAudioTracks = shouldStopTracks + ? [] + : Array.from(this.localParticipant.audioTrackPublications.values()) + .map((pub) => pub.track) + .filter(isLocalAudioTrack); + this.remoteParticipants.forEach((p) => { p.trackPublications.forEach((pub) => { p.unpublishTrack(pub.trackSid); @@ -1879,10 +1945,13 @@ class Room extends (EventEmitter as new () => TypedEmitter) this.remoteParticipants.clear(); this.sidToIdentity.clear(); this.activeSpeakers = []; - if (this.audioContext && typeof this.options.webAudioMix === 'boolean') { - this.audioContext.close(); - this.audioContext = undefined; - } + this.releaseDummyAudioElement?.(); + this.releaseAcquiredEmptyAudioTracks(); + // teardown is asynchronous (processors need to release their web audio nodes), but + // `handleDisconnect` is not, so let it finish in the background + this.releaseAudioContext(retainedAudioTracks).catch((error) => + this.log.warn('Could not release audio context', { ...this.logContext, error }), + ); if (isWeb()) { window.removeEventListener('beforeunload', this.onPageLeave); window.removeEventListener('pagehide', this.onPageLeave); @@ -2319,23 +2388,20 @@ class Room extends (EventEmitter as new () => TypedEmitter) }; private async acquireAudioContext() { - if (typeof this.options.webAudioMix !== 'boolean' && this.options.webAudioMix.audioContext) { + this.audioContextReleased = false; + const providedAudioContext = + typeof this.options.webAudioMix !== 'boolean' + ? this.options.webAudioMix.audioContext + : undefined; + if (providedAudioContext) { // override audio context with custom audio context if supplied by user - this.audioContext = this.options.webAudioMix.audioContext; + await this.setRoomAudioContext(providedAudioContext, false); } else if (!this.audioContext || this.audioContext.state === 'closed') { // by using an AudioContext, it reduces lag on audio elements // https://stackoverflow.com/questions/9811429/html5-audio-tag-on-safari-has-a-delay/54119854#54119854 - this.audioContext = getNewAudioContext() ?? undefined; - } - - if (this.options.webAudioMix) { - this.remoteParticipants.forEach((participant) => - participant.setAudioContext(this.audioContext), - ); + await this.setRoomAudioContext(getNewAudioContext() ?? undefined, true); } - this.localParticipant.setAudioContext(this.audioContext); - if (this.audioContext && this.audioContext.state === 'suspended') { // for iOS a newly created AudioContext is always in `suspended` state. // we try our best to resume the context here, if that doesn't work, we just continue with regular processing @@ -2353,6 +2419,57 @@ class Room extends (EventEmitter as new () => TypedEmitter) } } + /** + * Points the room and its participants at `audioContext`, rebuilding the web audio graph of + * every audio track in it. Closes the previous context if the SDK `owned` it. + */ + private async setRoomAudioContext(audioContext: AudioContext | undefined, owned: boolean) { + const ownsAudioContext = !!audioContext && owned; + if (this.audioContext === audioContext) { + this.ownsAudioContext = ownsAudioContext; + return; + } + const contextToClose = this.ownsAudioContext ? this.audioContext : undefined; + this.audioContext = audioContext; + this.ownsAudioContext = ownsAudioContext; + + // every audio track rebuilds its nodes in the new context, which has to happen before the + // previous one is closed so that processors can move off it while it is still alive + if (this.options.webAudioMix) { + await Promise.all( + Array.from(this.remoteParticipants.values()).map((participant) => + participant.setAudioContext(audioContext), + ), + ); + } + await this.localParticipant.setAudioContext(audioContext); + + if (contextToClose && contextToClose.state !== 'closed') { + try { + await contextToClose.close(); + } catch (error) { + this.log.warn('Could not close audio context', { ...this.logContext, error }); + } + } + } + + /** + * Detaches the web audio graph the SDK built from the current audio context and closes that + * context if the SDK created it. A context provided through `webAudioMix` is left untouched. + */ + private async releaseAudioContext(retainedAudioTracks: LocalAudioTrack[] = []) { + this.audioContextReleased = true; + if (!this.ownsAudioContext) { + // a provided context stays open, so tracks that outlive the room stay functional on it. + // The tracks this room detached have released their nodes during track teardown already. + return; + } + // tracks that outlive the room aren't reachable through the local participant anymore, but + // still hold web audio nodes in the context we're about to close + await Promise.all(retainedAudioTracks.map((track) => track.setAudioContext(undefined))); + await this.setRoomAudioContext(undefined, false); + } + private createParticipant(identity: string, info?: ParticipantInfo): RemoteParticipant { let participant: RemoteParticipant; if (info) { @@ -2380,7 +2497,9 @@ class Room extends (EventEmitter as new () => TypedEmitter) ); } if (this.options.webAudioMix) { - participant.setAudioContext(this.audioContext); + participant + .setAudioContext(this.audioContext) + .catch((e) => this.log.warn(`Could not set audio context: ${e.message}`)); } if (this.options.audioOutput?.deviceId) { participant @@ -2829,7 +2948,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) new LocalAudioTrack( publishOptions.useRealTracks && navigator.mediaDevices?.getUserMedia ? (await navigator.mediaDevices.getUserMedia({ audio: true })).getAudioTracks()[0] - : getEmptyAudioStreamTrack(), + : this.acquireEmptyAudioStreamTrack(), undefined, false, this.audioContext, @@ -2872,7 +2991,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) info.tracks = [...info.tracks, videoTrack]; } if (participantOptions.audio) { - const dummyTrack = getEmptyAudioStreamTrack(); + const dummyTrack = this.acquireEmptyAudioStreamTrack(); const audioTrack = new TrackInfo({ source: TrackSource.MICROPHONE, sid: Math.floor(Math.random() * 10_000).toString(), diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index 1993a1d154..0fc060491d 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -686,19 +686,21 @@ export default class LocalParticipant extends Participant { loggerName: this.roomOptions.loggerName, loggerContextCb: () => this.logContext, }); - const localTracks = tracks.map((track) => { - if (isAudioTrack(track)) { - this.microphoneError = undefined; - track.setAudioContext(this.audioContext); - track.source = Track.Source.Microphone; - this.emit(ParticipantEvent.AudioStreamAcquired); - } - if (isVideoTrack(track)) { - this.cameraError = undefined; - track.source = Track.Source.Camera; - } - return track; - }); + const localTracks = await Promise.all( + tracks.map(async (track) => { + if (isAudioTrack(track)) { + this.microphoneError = undefined; + await track.setAudioContext(this.audioContext); + track.source = Track.Source.Microphone; + this.emit(ParticipantEvent.AudioStreamAcquired); + } + if (isVideoTrack(track)) { + this.cameraError = undefined; + track.source = Track.Source.Camera; + } + return track; + }), + ); return localTracks; } catch (err) { if (err instanceof Error) { @@ -814,7 +816,7 @@ export default class LocalParticipant extends Participant { hasRetriedAfterNegotiationError = false, ): Promise { if (isLocalAudioTrack(track)) { - track.setAudioContext(this.audioContext); + await track.setAudioContext(this.audioContext); } await this.reconnectFuture?.promise; diff --git a/src/room/participant/Participant.ts b/src/room/participant/Participant.ts index 300318c166..401028b741 100644 --- a/src/room/participant/Participant.ts +++ b/src/room/participant/Participant.ts @@ -363,10 +363,13 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter /** * @internal */ - setAudioContext(ctx: AudioContext | undefined) { + async setAudioContext(ctx: AudioContext | undefined) { this.audioContext = ctx; - this.audioTrackPublications.forEach( - (track) => isAudioTrack(track.track) && track.track.setAudioContext(ctx), + await Promise.all( + Array.from(this.audioTrackPublications.values()) + .map((publication) => publication.track) + .filter(isAudioTrack) + .map((track) => track.setAudioContext(ctx)), ); } diff --git a/src/room/track/LocalAudioTrack.ts b/src/room/track/LocalAudioTrack.ts index 3898aa6a7b..6e3b779965 100644 --- a/src/room/track/LocalAudioTrack.ts +++ b/src/room/track/LocalAudioTrack.ts @@ -18,6 +18,12 @@ export default class LocalAudioTrack extends LocalTrack { private isKrispNoiseFilterEnabled = false; + /** + * the processed track we currently have krisp feature listeners registered on. Tracked + * separately, as `processor.processedTrack` can be swapped out while we hold the listeners. + */ + private observedProcessedTrack?: MediaStreamTrack; + protected processor?: TrackProcessor | undefined; /** @@ -184,6 +190,44 @@ export default class LocalAudioTrack extends LocalTrack { ); }; + /** + * registers the krisp feature listeners on the given processed track, moving them over from + * a previously observed one. Passing `undefined` just removes the existing listeners. + */ + private observeProcessedTrack(processedTrack: MediaStreamTrack | undefined) { + if (this.observedProcessedTrack === processedTrack) { + return; + } + if (this.observedProcessedTrack) { + this.observedProcessedTrack.removeEventListener( + 'enable-lk-krisp-noise-filter', + this.handleKrispNoiseFilterEnable, + ); + this.observedProcessedTrack.removeEventListener( + 'disable-lk-krisp-noise-filter', + this.handleKrispNoiseFilterDisable, + ); + this.observedProcessedTrack = undefined; + if (this.isKrispNoiseFilterEnabled) { + // we're no longer observing the track that had the filter enabled, so the feature + // can't be active anymore. Make sure the state doesn't get stuck on `true`. + this.handleKrispNoiseFilterDisable(); + } + } + if (!processedTrack) { + return; + } + processedTrack.addEventListener( + 'enable-lk-krisp-noise-filter', + this.handleKrispNoiseFilterEnable, + ); + processedTrack.addEventListener( + 'disable-lk-krisp-noise-filter', + this.handleKrispNoiseFilterDisable, + ); + this.observedProcessedTrack = processedTrack; + } + async setProcessor(processor: TrackProcessor) { const unlock = await this.trackChangeLock.lock(); try { @@ -209,14 +253,7 @@ export default class LocalAudioTrack extends LocalTrack { this.processor = processor; if (this.processor.processedTrack) { await this.sender?.replaceTrack(this.processor.processedTrack); - this.processor.processedTrack.addEventListener( - 'enable-lk-krisp-noise-filter', - this.handleKrispNoiseFilterEnable, - ); - this.processor.processedTrack.addEventListener( - 'disable-lk-krisp-noise-filter', - this.handleKrispNoiseFilterDisable, - ); + this.observeProcessedTrack(this.processor.processedTrack); } this.emit(TrackEvent.TrackProcessorUpdate, this.processor); } finally { @@ -224,12 +261,53 @@ export default class LocalAudioTrack extends LocalTrack { } } + protected async internalStopProcessor(keepElement = true) { + this.observeProcessedTrack(undefined); + await super.internalStopProcessor(keepElement); + } + + stop() { + this.observeProcessedTrack(undefined); + super.stop(); + } + /** * @internal * @experimental + * + * A processor's nodes (`AudioWorkletNode`, ...) can't move between contexts, so the processor + * is rebuilt whenever the context changes and torn down when there is none left to run in. */ - setAudioContext(audioContext: AudioContext | undefined) { - this.audioContext = audioContext; + async setAudioContext(audioContext: AudioContext | undefined) { + if (this.audioContext === audioContext) { + return; + } + const unlock = await this.trackChangeLock.lock(); + try { + this.audioContext = audioContext; + if (!this.processor || isReactNative()) { + return; + } + if (!audioContext) { + this.log.debug('stopping audio processor, audio context has been removed', this.logContext); + await this.internalStopProcessor(); + return; + } + this.log.debug('restarting audio processor on new audio context', this.logContext); + await this.processor.restart({ + kind: this.kind, + track: this._mediaStreamTrack, + audioContext, + localTrack: this, + }); + if (this.processor.processedTrack) { + await this.sender?.replaceTrack(this.processor.processedTrack); + this.observeProcessedTrack(this.processor.processedTrack); + } + this.emit(TrackEvent.TrackProcessorUpdate, this.processor); + } finally { + unlock(); + } } async getSenderStats(): Promise { diff --git a/src/room/track/LocalTrack.ts b/src/room/track/LocalTrack.ts index 275e9ea3d4..f1dbf4bd72 100644 --- a/src/room/track/LocalTrack.ts +++ b/src/room/track/LocalTrack.ts @@ -194,6 +194,9 @@ export default abstract class LocalTrack< track: newTrack, kind: this.kind, element: this.processorElement, + // audio processors need the context they were initialised with in order to rebuild + // their graph, omitting it here would leave them without one + audioContext: this.audioContext, localTrack: this, }); processedTrack = this.processor.processedTrack; @@ -464,7 +467,11 @@ export default abstract class LocalTrack< this._mediaStreamTrack.removeEventListener('ended', this.handleEnded); this._mediaStreamTrack.removeEventListener('mute', this.handleTrackMuteEvent); this._mediaStreamTrack.removeEventListener('unmute', this.handleTrackUnmuteEvent); - this.processor?.destroy(); + // `stop` is synchronous, so we can't await the teardown of the processor here. + // make sure a failing teardown doesn't surface as an unhandled rejection. + this.processor?.destroy().catch((error) => { + this.log.error('failed to destroy processor', { ...this.logContext, error }); + }); this.processor = undefined; } diff --git a/src/room/track/RemoteAudioTrack.ts b/src/room/track/RemoteAudioTrack.ts index 7d48caa7a2..dc20b08d5d 100644 --- a/src/room/track/RemoteAudioTrack.ts +++ b/src/room/track/RemoteAudioTrack.ts @@ -42,10 +42,11 @@ export default class RemoteAudioTrack extends RemoteTrack { * sets the volume for all attached audio elements */ setVolume(volume: number) { - for (const el of this.attachedElements) { - if (this.audioContext) { - this.gainNode?.gain.setTargetAtTime(volume, 0, 0.1); - } else { + if (this.audioContext) { + // playback goes through the web audio graph, the attached elements themselves stay silent + this.gainNode?.gain.setTargetAtTime(volume, 0, 0.1); + } else { + for (const el of this.attachedElements) { el.volume = volume; } } @@ -96,7 +97,9 @@ export default class RemoteAudioTrack extends RemoteTrack { attach(): HTMLMediaElement; attach(element: HTMLMediaElement): HTMLMediaElement; attach(element?: HTMLMediaElement): HTMLMediaElement { - const needsNewWebAudioConnection = this.attachedElements.length === 0; + // the graph is fed by the track rather than by a specific element, so one connection covers + // all attached elements + const needsNewWebAudioConnection = !this.sourceNode; if (!element) { element = super.attach(); } else { @@ -108,11 +111,15 @@ export default class RemoteAudioTrack extends RemoteTrack { this.log.error('Failed to set sink id on remote audio track', e, this.logContext); }); } - if (this.audioContext && needsNewWebAudioConnection) { - this.log.debug('using audio context mapping', this.logContext); - this.connectWebAudio(this.audioContext, element); + if (this.audioContext) { + // audio is routed through the web audio graph, so every attached element has to stay + // silent, no matter whether it is the one the graph was built from element.volume = 0; element.muted = true; + if (needsNewWebAudioConnection) { + this.log.debug('using audio context mapping', this.logContext); + this.connectWebAudio(this.audioContext, element); + } } if (this.elementVolume) { @@ -158,11 +165,26 @@ export default class RemoteAudioTrack extends RemoteTrack { * @experimental */ setAudioContext(audioContext: AudioContext | undefined) { + if (this.audioContext === audioContext) { + return; + } this.audioContext = audioContext; - if (audioContext && this.attachedElements.length > 0) { - this.connectWebAudio(audioContext, this.attachedElements[0]); - } else if (!audioContext) { + if (audioContext) { + // playback moves into the web audio graph, mute the elements to avoid double playback + for (const element of this.attachedElements) { + element.volume = 0; + element.muted = true; + } + if (this.attachedElements.length > 0) { + this.connectWebAudio(audioContext, this.attachedElements[0]); + } + } else { this.disconnectWebAudio(); + // playback moves back into the elements, which `attach` had muted in favour of the graph + for (const element of this.attachedElements) { + element.muted = false; + element.volume = this.elementVolume ?? 1; + } } } @@ -216,6 +238,9 @@ export default class RemoteAudioTrack extends RemoteTrack { private disconnectWebAudio() { this.gainNode?.disconnect(); this.sourceNode?.disconnect(); + // the plugin nodes belong to whoever passed them in, but the connections between them are ours + // to undo, otherwise stale routing survives into the next graph the same nodes are used in + this.webAudioPluginNodes.forEach((node) => node.disconnect()); this.gainNode = undefined; this.sourceNode = undefined; } diff --git a/src/room/track/utils.ts b/src/room/track/utils.ts index 19e99354e4..e8e2555c54 100644 --- a/src/room/track/utils.ts +++ b/src/room/track/utils.ts @@ -107,22 +107,44 @@ export function constraintsForOptions(options: CreateLocalTracksOptions): MediaS */ export async function detectSilence(track: AudioTrack, timeOffset = 200): Promise { const ctx = getNewAudioContext(); - if (ctx) { - const analyser = ctx.createAnalyser(); + if (!ctx) { + return false; + } + let source: MediaStreamAudioSourceNode | undefined; + let analyser: AnalyserNode | undefined; + try { + analyser = ctx.createAnalyser(); analyser.fftSize = 2048; const bufferLength = analyser.frequencyBinCount; const dataArray = new Uint8Array(bufferLength); - const source = ctx.createMediaStreamSource(new MediaStream([track.mediaStreamTrack])); + source = ctx.createMediaStreamSource(new MediaStream([track.mediaStreamTrack])); source.connect(analyser); + // read through a function so that `state` isn't narrowed to its value from before `resume` + const isRunning = () => ctx.state === 'running'; + if (!isRunning()) { + try { + await ctx.resume(); + } catch { + // autoplay policies can keep us from resuming the context + } + if (!isRunning()) { + // a context that isn't running doesn't process any audio, so the analyser would only ever + // read silence. Report the track as not silent rather than raising a false positive. + return false; + } + } await sleep(timeOffset); analyser.getByteTimeDomainData(dataArray); const someNoise = dataArray.some((sample) => sample !== 128 && sample !== 0); - ctx.close(); return !someNoise; + } finally { + source?.disconnect(); + analyser?.disconnect(); + // don't let a failing teardown mask the detection result + await ctx.close().catch(() => {}); } - return false; } /** diff --git a/src/room/utils.ts b/src/room/utils.ts index 3ad2e3798f..52b63911b4 100644 --- a/src/room/utils.ts +++ b/src/room/utils.ts @@ -434,6 +434,15 @@ export function createDummyVideoStreamTrack( let emptyAudioStreamTrack: MediaStreamTrack | undefined; +let emptyAudioStreamContext: AudioContext | undefined; + +/** tracks handed out by `getEmptyAudioStreamTrack` that haven't been released yet */ +const outstandingEmptyAudioStreamTracks = new Set(); + +/** + * Returns a silent audio track, cloned off a shared `AudioContext` created on first use. Hand the + * clone back to `releaseEmptyAudioStreamTrack` rather than stopping it, so the context can close. + */ export function getEmptyAudioStreamTrack() { if (!emptyAudioStreamTrack) { // implementation adapted from https://blog.mozilla.org/webrtc/warm-up-with-replacetrack/ @@ -447,31 +456,36 @@ export function getEmptyAudioStreamTrack() { oscillator.start(); [emptyAudioStreamTrack] = dst.stream.getAudioTracks(); if (!emptyAudioStreamTrack) { + // don't hold on to a context we can't get a track out of + ctx.close().catch(() => {}); throw Error('Could not get empty media stream audio track'); } emptyAudioStreamTrack.enabled = false; + emptyAudioStreamContext = ctx; } - return emptyAudioStreamTrack.clone(); + const track = emptyAudioStreamTrack.clone(); + outstandingEmptyAudioStreamTracks.add(track); + return track; } -export function getStereoAudioStreamTrack() { - const ctx = new AudioContext(); - const oscLeft = ctx.createOscillator(); - const oscRight = ctx.createOscillator(); - oscLeft.frequency.value = 440; - oscRight.frequency.value = 220; - const merger = ctx.createChannelMerger(2); - oscLeft.connect(merger, 0, 0); // left channel - oscRight.connect(merger, 0, 1); // right channel - const dst = ctx.createMediaStreamDestination(); - merger.connect(dst); - oscLeft.start(); - oscRight.start(); - const [stereoTrack] = dst.stream.getAudioTracks(); - if (!stereoTrack) { - throw Error('Could not get stereo media stream audio track'); +/** + * Stops a track obtained from `getEmptyAudioStreamTrack` and closes the shared `AudioContext` + * backing it once every track it handed out has been released. + */ +export async function releaseEmptyAudioStreamTrack(track: MediaStreamTrack) { + track.stop(); + if (!outstandingEmptyAudioStreamTracks.delete(track)) { + // the track wasn't obtained from `getEmptyAudioStreamTrack`, or has been released before + return; } - return stereoTrack; + if (outstandingEmptyAudioStreamTracks.size > 0) { + return; + } + const context = emptyAudioStreamContext; + emptyAudioStreamTrack?.stop(); + emptyAudioStreamTrack = undefined; + emptyAudioStreamContext = undefined; + await context?.close(); } /** An object that represents a serialized version of a `new Promise((resolve, reject) => {})` @@ -583,11 +597,20 @@ export function createAudioAnalyser( return volume; }; + let isCleanedUp = false; + const cleanup = async () => { - await audioContext.close(); + if (isCleanedUp) { + // closing an already closed context would reject, make repeated cleanups a no-op instead + return; + } + isCleanedUp = true; + mediaStreamSource.disconnect(); + analyser.disconnect(); if (opts.cloneTrack) { streamTrack.stop(); } + await audioContext.close(); }; return { calculateVolume, analyser, cleanup };