Skip to content

consistently clean up audio contexts and graphs #2029

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
lukasIO wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

consistently clean up audio contexts and graphs #2029

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
235 changes: 177 additions & 58 deletions src/room/Room.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ import {
isSafariBased,
isWeb,
numberToBigInt,
releaseEmptyAudioStreamTrack,
sleep,
supportsSetSinkId,
toHttpUrl,
Expand All @@ -144,6 +145,8 @@ export enum ConnectionState {

const CONNECTION_RECONCILE_FREQUENCY_MS = 4 * 1000;

const DUMMY_AUDIO_ELEMENT_ID = 'livekit-dummy-audio-el';

/**
* In LiveKit, a room is the logical grouping for a list of participants.
* Participants in a room can publish tracks, and subscribe to others' tracks.
Expand Down Expand Up @@ -191,6 +194,24 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)

private audioContext?: AudioContext;

/**
* whether `audioContext` was created by the SDK and therefore has to be closed by it.
* A context handed in through `webAudioMix` is owned by the application instead.
*/
private ownsAudioContext = false;

/**
* set once the audio context has been released as part of a disconnect. Guards against
* creating a new context outside of a connection lifecycle, where nothing would close it again.
*/
private audioContextReleased = false;

/** tears down the iOS dummy audio element, set while one is in place. See `startAudio` */
private releaseDummyAudioElement?: () => void;

/** silent audio tracks this room acquired, handed back to the shared source on disconnect */
private acquiredEmptyAudioTracks = new Set<MediaStreamTrack>();

/** used for aborting pending connections to a LiveKit server */
private abortController?: AbortController;

Expand Down Expand Up @@ -1323,6 +1344,79 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
await this.disconnect();
};

/**
* iOS blocks audio element playback unless some audio source is already playing, so keep an
* element with a silent track around. Reuses an existing one instead of stacking up listeners.
*/
private acquireDummyAudioElement(): HTMLAudioElement {
const existingEl = document.getElementById(DUMMY_AUDIO_ELEMENT_ID);
if (existingEl instanceof HTMLAudioElement) {
return existingEl;
}

const dummyAudioEl = document.createElement('audio');
dummyAudioEl.id = DUMMY_AUDIO_ELEMENT_ID;
dummyAudioEl.autoplay = true;
dummyAudioEl.hidden = true;
const track = this.acquireEmptyAudioStreamTrack();
track.enabled = true;
const stream = new MediaStream([track]);
dummyAudioEl.srcObject = stream;

const handleVisibilityChange = () => {
// set the srcObject to null on page hide in order to prevent lock screen controls to show up for it
dummyAudioEl.srcObject = document.hidden ? null : stream;
if (!document.hidden) {
this.log.debug(
'page visible again, triggering startAudio to resume playback and update playback status',
);
this.startAudio().catch((error) =>
this.log.warn('Could not restart audio on page becoming visible', {
...this.logContext,
error,
}),
);
}
};
document.addEventListener('visibilitychange', handleVisibilityChange);
document.body.append(dummyAudioEl);

this.releaseDummyAudioElement = () => {
this.releaseDummyAudioElement = undefined;
document.removeEventListener('visibilitychange', handleVisibilityChange);
dummyAudioEl.srcObject = null;
dummyAudioEl.remove();
this.releaseAcquiredEmptyAudioTrack(track);
};

return dummyAudioEl;
}

/**
* Obtains a silent audio track from the shared source and keeps track of it, so that it can be
* handed back on disconnect and the shared `AudioContext` behind it can be closed again.
*/
private acquireEmptyAudioStreamTrack(): MediaStreamTrack {
const track = getEmptyAudioStreamTrack();
this.acquiredEmptyAudioTracks.add(track);
return track;
}

private releaseAcquiredEmptyAudioTrack(track: MediaStreamTrack) {
if (!this.acquiredEmptyAudioTracks.delete(track)) {
return;
}
releaseEmptyAudioStreamTrack(track).catch((error) =>
this.log.warn('Could not release empty audio stream track', { ...this.logContext, error }),
);
}

private releaseAcquiredEmptyAudioTracks() {
for (const track of this.acquiredEmptyAudioTracks) {
this.releaseAcquiredEmptyAudioTrack(track);
}
}

/**
* Browsers have different policies regarding audio playback. Most requiring
* some form of user interaction (click/tap/etc).
Expand All @@ -1334,45 +1428,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
const elements: Array<HTMLMediaElement> = [];
const browser = getBrowser();
if (browser && browser.os === 'iOS') {
/**
* iOS blocks audio element playback if
* - user is not publishing audio themselves and
* - no other audio source is playing
*
* as a workaround, we create an audio element with an empty track, so that
* silent audio is always playing
*/
const audioId = 'livekit-dummy-audio-el';
let dummyAudioEl = document.getElementById(audioId) as HTMLAudioElement | null;
if (!dummyAudioEl) {
dummyAudioEl = document.createElement('audio');
dummyAudioEl.id = audioId;
dummyAudioEl.autoplay = true;
dummyAudioEl.hidden = true;
const track = getEmptyAudioStreamTrack();
track.enabled = true;
const stream = new MediaStream([track]);
dummyAudioEl.srcObject = stream;
document.addEventListener('visibilitychange', () => {
if (!dummyAudioEl) {
return;
}
// set the srcObject to null on page hide in order to prevent lock screen controls to show up for it
dummyAudioEl.srcObject = document.hidden ? null : stream;
if (!document.hidden) {
this.log.debug(
'page visible again, triggering startAudio to resume playback and update playback status',
);
this.startAudio();
}
});
document.body.append(dummyAudioEl);
this.once(RoomEvent.Disconnected, () => {
dummyAudioEl?.remove();
dummyAudioEl = null;
});
}
elements.push(dummyAudioEl);
elements.push(this.acquireDummyAudioElement());
}

this.remoteParticipants.forEach((p) => {
Expand All @@ -1387,7 +1443,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)

try {
await Promise.all([
this.acquireAudioContext(),
// once the audio context has been released as part of a disconnect there is nothing left
// to unlock, and creating a new context here would leak it as no disconnect will follow
...(this.audioContextReleased ? [] : [this.acquireAudioContext()]),
...elements.map((e) => {
// when webAudioMix is enabled, attached elements are deliberately kept muted by
// RemoteAudioTrack.attach() and audio is routed through the web audio graph instead.
Expand Down Expand Up @@ -1837,6 +1895,14 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
}

try {
// audio tracks that outlive this room are about to become unreachable through the local
// participant, collect them while we can so they can be detached from the context below
const retainedAudioTracks = shouldStopTracks
? []
: Array.from(this.localParticipant.audioTrackPublications.values())
.map((pub) => pub.track)
.filter(isLocalAudioTrack);

this.remoteParticipants.forEach((p) => {
p.trackPublications.forEach((pub) => {
p.unpublishTrack(pub.trackSid);
Expand Down Expand Up @@ -1879,10 +1945,13 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
this.remoteParticipants.clear();
this.sidToIdentity.clear();
this.activeSpeakers = [];
if (this.audioContext && typeof this.options.webAudioMix === 'boolean') {
this.audioContext.close();
this.audioContext = undefined;
}
this.releaseDummyAudioElement?.();
this.releaseAcquiredEmptyAudioTracks();
// teardown is asynchronous (processors need to release their web audio nodes), but
// `handleDisconnect` is not, so let it finish in the background
this.releaseAudioContext(retainedAudioTracks).catch((error) =>
this.log.warn('Could not release audio context', { ...this.logContext, error }),
);
if (isWeb()) {
window.removeEventListener('beforeunload', this.onPageLeave);
window.removeEventListener('pagehide', this.onPageLeave);
Expand Down Expand Up @@ -2319,23 +2388,20 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
};

private async acquireAudioContext() {
if (typeof this.options.webAudioMix !== 'boolean' && this.options.webAudioMix.audioContext) {
this.audioContextReleased = false;
const providedAudioContext =
typeof this.options.webAudioMix !== 'boolean'
? this.options.webAudioMix.audioContext
: undefined;
if (providedAudioContext) {
// override audio context with custom audio context if supplied by user
this.audioContext = this.options.webAudioMix.audioContext;
await this.setRoomAudioContext(providedAudioContext, false);
} else if (!this.audioContext || this.audioContext.state === 'closed') {
// by using an AudioContext, it reduces lag on audio elements
// https://stackoverflow.com/questions/9811429/html5-audio-tag-on-safari-has-a-delay/54119854#54119854
this.audioContext = getNewAudioContext() ?? undefined;
}

if (this.options.webAudioMix) {
this.remoteParticipants.forEach((participant) =>
participant.setAudioContext(this.audioContext),
);
await this.setRoomAudioContext(getNewAudioContext() ?? undefined, true);
}

this.localParticipant.setAudioContext(this.audioContext);

if (this.audioContext && this.audioContext.state === 'suspended') {
// for iOS a newly created AudioContext is always in `suspended` state.
// we try our best to resume the context here, if that doesn't work, we just continue with regular processing
Expand All @@ -2353,6 +2419,57 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
}
}

/**
* Points the room and its participants at `audioContext`, rebuilding the web audio graph of
* every audio track in it. Closes the previous context if the SDK `owned` it.
*/
private async setRoomAudioContext(audioContext: AudioContext | undefined, owned: boolean) {
const ownsAudioContext = !!audioContext && owned;
if (this.audioContext === audioContext) {
this.ownsAudioContext = ownsAudioContext;
return;
}
const contextToClose = this.ownsAudioContext ? this.audioContext : undefined;
this.audioContext = audioContext;
this.ownsAudioContext = ownsAudioContext;

// every audio track rebuilds its nodes in the new context, which has to happen before the
// previous one is closed so that processors can move off it while it is still alive
if (this.options.webAudioMix) {
await Promise.all(
Array.from(this.remoteParticipants.values()).map((participant) =>
participant.setAudioContext(audioContext),
),
);
}
await this.localParticipant.setAudioContext(audioContext);

if (contextToClose && contextToClose.state !== 'closed') {
try {
await contextToClose.close();
} catch (error) {
this.log.warn('Could not close audio context', { ...this.logContext, error });
}
}
}

/**
* Detaches the web audio graph the SDK built from the current audio context and closes that
* context if the SDK created it. A context provided through `webAudioMix` is left untouched.
*/
private async releaseAudioContext(retainedAudioTracks: LocalAudioTrack[] = []) {
this.audioContextReleased = true;
if (!this.ownsAudioContext) {
// a provided context stays open, so tracks that outlive the room stay functional on it.
// The tracks this room detached have released their nodes during track teardown already.
return;
}
// tracks that outlive the room aren't reachable through the local participant anymore, but
// still hold web audio nodes in the context we're about to close
await Promise.all(retainedAudioTracks.map((track) => track.setAudioContext(undefined)));
await this.setRoomAudioContext(undefined, false);
}

private createParticipant(identity: string, info?: ParticipantInfo): RemoteParticipant {
let participant: RemoteParticipant;
if (info) {
Expand Down Expand Up @@ -2380,7 +2497,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
);
}
if (this.options.webAudioMix) {
participant.setAudioContext(this.audioContext);
participant
.setAudioContext(this.audioContext)
.catch((e) => this.log.warn(`Could not set audio context: ${e.message}`));
}
if (this.options.audioOutput?.deviceId) {
participant
Expand Down Expand Up @@ -2829,7 +2948,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
new LocalAudioTrack(
publishOptions.useRealTracks && navigator.mediaDevices?.getUserMedia
? (await navigator.mediaDevices.getUserMedia({ audio: true })).getAudioTracks()[0]
: getEmptyAudioStreamTrack(),
: this.acquireEmptyAudioStreamTrack(),
undefined,
false,
this.audioContext,
Expand Down Expand Up @@ -2872,7 +2991,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
info.tracks = [...info.tracks, videoTrack];
}
if (participantOptions.audio) {
const dummyTrack = getEmptyAudioStreamTrack();
const dummyTrack = this.acquireEmptyAudioStreamTrack();
const audioTrack = new TrackInfo({
source: TrackSource.MICROPHONE,
sid: Math.floor(Math.random() * 10_000).toString(),
Expand Down
30 changes: 16 additions & 14 deletions src/room/participant/LocalParticipant.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -686,19 +686,21 @@ export default class LocalParticipant extends Participant {
loggerName: this.roomOptions.loggerName,
loggerContextCb: () => this.logContext,
});
const localTracks = tracks.map((track) => {
if (isAudioTrack(track)) {
this.microphoneError = undefined;
track.setAudioContext(this.audioContext);
track.source = Track.Source.Microphone;
this.emit(ParticipantEvent.AudioStreamAcquired);
}
if (isVideoTrack(track)) {
this.cameraError = undefined;
track.source = Track.Source.Camera;
}
return track;
});
const localTracks = await Promise.all(
tracks.map(async (track) => {
if (isAudioTrack(track)) {
this.microphoneError = undefined;
await track.setAudioContext(this.audioContext);
track.source = Track.Source.Microphone;
this.emit(ParticipantEvent.AudioStreamAcquired);
}
if (isVideoTrack(track)) {
this.cameraError = undefined;
track.source = Track.Source.Camera;
}
return track;
}),
);
return localTracks;
} catch (err) {
if (err instanceof Error) {
Expand Down Expand Up @@ -814,7 +816,7 @@ export default class LocalParticipant extends Participant {
hasRetriedAfterNegotiationError = false,
): Promise<LocalTrackPublication> {
if (isLocalAudioTrack(track)) {
track.setAudioContext(this.audioContext);
await track.setAudioContext(this.audioContext);
}

await this.reconnectFuture?.promise;
Expand Down
9 changes: 6 additions & 3 deletions src/room/participant/Participant.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,10 +363,13 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
/**
* @internal
*/
setAudioContext(ctx: AudioContext | undefined) {
async setAudioContext(ctx: AudioContext | undefined) {
this.audioContext = ctx;
this.audioTrackPublications.forEach(
(track) => isAudioTrack(track.track) && track.track.setAudioContext(ctx),
await Promise.all(
Array.from(this.audioTrackPublications.values())
.map((publication) => publication.track)
.filter(isAudioTrack)
.map((track) => track.setAudioContext(ctx)),
);
}

Expand Down
Loading
Loading