fix: stabilize browser media sessions

app/api/connection-details/route.ts

@@ -1,7 +1,6 @@
 import { NextResponse } from 'next/server';
 import { AccessToken, type AccessTokenOptions, type VideoGrant } from 'livekit-server-sdk';
 import { randomUUID } from 'node:crypto';
-import { RoomConfiguration } from '@livekit/protocol';
 import { deriveLiveKitRoomName, resolveConnectionSessionId } from '@/lib/connection-room-id';
 
 type ConnectionDetails = {
@@ -32,23 +31,19 @@ export async function POST(req: Request) {
       throw new Error('LIVEKIT_API_SECRET is not defined');
     }
 
-    // Parse room configuration from request body
     const body = await req.json();
-    const roomConfig = body?.room_config
-      ? RoomConfiguration.fromJson(body.room_config, { ignoreUnknownFields: true })
-      : new RoomConfiguration();
-    const tokenRoomConfig = buildTokenRoomConfig(roomConfig);
 
     // Generate participant token
     const participantName = 'user';
     const sessionId = resolveConnectionSessionId(body, randomUUID);
     const participantIdentity = `voice_assistant_user_${sessionId}`;
     const roomName = deriveLiveKitRoomName(sessionId);
 
+    // Explicit dispatch is handled by /api/session/dispatch. Omitting roomConfig
+    // also keeps participant tokens compatible with older LiveKit servers.
     const participantToken = await createParticipantToken(
       { identity: participantIdentity, name: participantName },
-      roomName,
-      tokenRoomConfig
+      roomName
     );
 
     // Return connection details
@@ -76,11 +71,7 @@ export async function POST(req: Request) {
   }
 }
 
-function createParticipantToken(
-  userInfo: AccessTokenOptions,
-  roomName: string,
-  roomConfig: RoomConfiguration | undefined
-): Promise<string> {
+function createParticipantToken(userInfo: AccessTokenOptions, roomName: string): Promise<string> {
   const at = new AccessToken(API_KEY, API_SECRET, {
     ...userInfo,
     ttl: '15m',
@@ -94,21 +85,5 @@ function createParticipantToken(
   };
   at.addGrant(grant);
 
-  if (roomConfig) {
-    at.roomConfig = roomConfig;
-  }
-
   return at.toJwt();
 }
-
-function buildTokenRoomConfig(roomConfig: RoomConfiguration) {
-  if (roomConfig.agents.length === 0) {
-    return roomConfig;
-  }
-
-  // Explicit dispatch is handled by /api/session/dispatch; token agents would create duplicate jobs.
-  return new RoomConfiguration({
-    ...roomConfig,
-    agents: [],
-  });
-}

app/api/session/stop/route.ts

@@ -10,6 +10,7 @@ import {
 } from '@/lib/connection-room-id';
 import {
   executeRoomInputStopsSequentially,
+  isLiveKitRoomNotFoundError,
   resolveRoomInputStopUrls as resolveConfiguredRoomInputStopUrls,
   resolveLiveKitHttpUrl,
 } from '@/lib/session-stop';
@@ -252,6 +253,10 @@ async function deleteLiveKitRoom(roomName: string): Promise<StopResult> {
     await roomService.deleteRoom(roomName);
     return { target: 'livekit_room', ok: true };
   } catch (error) {
+    if (isLiveKitRoomNotFoundError(error)) {
+      return { target: 'livekit_room', ok: true, skipped: true, status: 404 };
+    }
+
     return {
       target: 'livekit_room',
       ok: false,

components/livekit/filtered-audio-renderer.tsx

@@ -10,6 +10,7 @@ import {
   Track,
 } from 'livekit-client';
 import { useRoomContext } from '@livekit/components-react';
+import { buildBrowserAudioPlaybackDiagnostics } from '@/lib/browser-audio-capture';
 import { startMediaTrackAudioObserver } from '@/lib/frontend-audio-observer';
 import {
   FRONTEND_EVENTS,
@@ -398,6 +399,15 @@ export function FilteredAudioRenderer({
           if (!playbackSource || playbackObserverStops.has(elementKey)) {
             return;
           }
+          const playbackDiagnostics = buildBrowserAudioPlaybackDiagnostics(
+            participantIdentity,
+            trackName,
+            audioElements.values(),
+            createdAudioElement
+          );
+          const logPlaybackDiagnostics =
+            playbackDiagnostics.activeAudioElementCount === 1 ? console.info : console.warn;
+          logPlaybackDiagnostics('[browser-audio] playback diagnostics', playbackDiagnostics);
           pendingPlayback.delete(elementKey);
           startPlaybackObserver(
             elementKey,

hooks/useBrowserSourceClient.ts

@@ -11,7 +11,13 @@ import {
   createLocalVideoTrack,
 } from 'livekit-client';
 import type { AppConfig } from '@/app-config';
+import {
+  BROWSER_AUDIO_CONSTRAINTS,
+  assertBrowserEchoCancellationActive,
+  inspectBrowserAudioCapture,
+} from '@/lib/browser-audio-capture';
 import { BrowserAudioGateDevice } from '@/lib/browser-audio-gate-device';
+import { awaitBrowserMediaCapture } from '@/lib/browser-media-capture-timeout';
 import {
   detachCurrentRuntime,
   isCurrentRuntime,
@@ -33,12 +39,8 @@ const DEFAULT_BROWSER_MEDIA_STREAM_NAME = 'browser_input';
 const BROWSER_VIDEO_DEFAULT_ENABLED = true;
 const BROWSER_VIDEO_STATS_INTERVAL_MS = 5000;
 const BROWSER_MEDIA_GATE_MAX_OPEN_LEASE_MS = 3000;
-const BROWSER_AUDIO_CONSTRAINTS: MediaTrackConstraints = {
-  echoCancellation: true,
-  noiseSuppression: true,
-  autoGainControl: true,
-};
-
+const BROWSER_VIDEO_CAPTURE_TIMEOUT_MS = 8000;
+const BROWSER_VIDEO_PUBLISH_TIMEOUT_MS = 5000;
 interface BrowserSourceRuntime {
   audioTrack: LocalAudioTrack | null;
   videoTrack: LocalVideoTrack | null;
@@ -163,6 +165,7 @@ export function useBrowserSourceClient(
         );
         recordFrontendObservability(FRONTEND_EVENTS.BROWSER_AUDIO_CAPTURE_FINISHED);
         const captureTrack = audioTrack.mediaStreamTrack;
+        logBrowserAudioCaptureDiagnostics(captureTrack);
         audioTrack.mediaStreamTrack.enabled = false;
 
         try {
@@ -281,15 +284,22 @@ export function useBrowserSourceClient(
       }
 
       recordFrontendObservability(FRONTEND_EVENTS.BROWSER_VIDEO_CAPTURE_STARTED);
-      const videoTrack = await createLocalVideoTrack({
-        facingMode: 'user',
-        frameRate: { ideal: browserVideoFrameRate, max: browserVideoFrameRate },
-        resolution: {
-          width: browserVideoWidth,
-          height: browserVideoHeight,
-          frameRate: browserVideoFrameRate,
-        },
-      });
+      const videoTrack = await awaitBrowserMediaCapture(
+        createLocalVideoTrack({
+          facingMode: 'user',
+          frameRate: { ideal: browserVideoFrameRate, max: browserVideoFrameRate },
+          resolution: {
+            width: browserVideoWidth,
+            height: browserVideoHeight,
+            frameRate: browserVideoFrameRate,
+          },
+        }),
+        {
+          timeoutMs: BROWSER_VIDEO_CAPTURE_TIMEOUT_MS,
+          label: 'camera',
+          disposeLateResult: (track) => track.stop(),
+        }
+      );
       recordFrontendObservability(FRONTEND_EVENTS.BROWSER_VIDEO_CAPTURE_FINISHED);
       videoTrack.mediaStreamTrack.enabled = runtime.videoEnabled;
       if (!isCurrentRuntime(runtimeRef, runtime)) {
@@ -299,17 +309,26 @@ export function useBrowserSourceClient(
 
       try {
         recordFrontendObservability(FRONTEND_EVENTS.BROWSER_VIDEO_PUBLISH_STARTED);
-        const publication = await room.localParticipant.publishTrack(videoTrack, {
-          name: BROWSER_VIDEO_TRACK_NAME,
-          source: Track.Source.Camera,
-          stream: browserMediaStreamName,
-          simulcast: false,
-          degradationPreference: 'maintain-resolution',
-          videoEncoding: {
-            maxBitrate: browserVideoMaxBitrate,
-            maxFramerate: browserVideoFrameRate,
-          },
-        });
+        const publication = await awaitBrowserMediaCapture(
+          room.localParticipant.publishTrack(videoTrack, {
+            name: BROWSER_VIDEO_TRACK_NAME,
+            source: Track.Source.Camera,
+            stream: browserMediaStreamName,
+            simulcast: false,
+            degradationPreference: 'maintain-resolution',
+            videoEncoding: {
+              maxBitrate: browserVideoMaxBitrate,
+              maxFramerate: browserVideoFrameRate,
+            },
+          }),
+          {
+            timeoutMs: BROWSER_VIDEO_PUBLISH_TIMEOUT_MS,
+            label: 'camera publish',
+            disposeLateResult: () => {
+              void room.localParticipant.unpublishTrack(videoTrack, true).catch(() => undefined);
+            },
+          }
+        );
         recordFrontendObservability(FRONTEND_EVENTS.BROWSER_VIDEO_PUBLISH_FINISHED);
         if (!isCurrentRuntime(runtimeRef, runtime)) {
           await room.localParticipant.unpublishTrack(videoTrack, true).catch(() => undefined);
@@ -712,6 +731,15 @@ function buildAudioCaptureOptions(deviceId: string | null) {
   };
 }
 
+function logBrowserAudioCaptureDiagnostics(track: MediaStreamTrack) {
+  const diagnostics = inspectBrowserAudioCapture(
+    track,
+    navigator.mediaDevices.getSupportedConstraints()
+  );
+  console.info('[browser-audio] capture diagnostics', diagnostics);
+  assertBrowserEchoCancellationActive(diagnostics);
+}
+
 function syncTrackEnabled(track: LocalAudioTrack | LocalVideoTrack | null, enabled: boolean) {
   if (!track) return;
 

hooks/useRoom.ts

@@ -259,6 +259,11 @@ export function useRoom(appConfig: AppConfig) {
 
     try {
       await waitForAgentSessionStop();
+      // A Room disconnect can make the welcome view visible without running the
+      // explicit End Call path. Clear the previous capture/gate runtime before
+      // reusing this Room, otherwise start() is a no-op and the new agent is
+      // dispatched into a room that never receives the browser microphone.
+      await browserSourceClient.stop();
       await waitForRoomDisconnected(room);
 
       if (usesManagedRoomInput) {

lib/browser-audio-capture.ts

@@ -0,0 +1,66 @@
+export const BROWSER_AUDIO_CONSTRAINTS: MediaTrackConstraints = {
+  echoCancellation: true,
+  noiseSuppression: true,
+  autoGainControl: true,
+};
+
+type InspectableAudioTrack = Pick<MediaStreamTrack, 'id' | 'getConstraints' | 'getSettings'>;
+
+export interface BrowserAudioCaptureDiagnostics {
+  trackId: string;
+  supported: MediaTrackSupportedConstraints;
+  constraints: MediaTrackConstraints;
+  settings: MediaTrackSettings;
+}
+
+export interface BrowserAudioPlaybackDiagnostics {
+  participantIdentity: string;
+  trackName: string;
+  activeAudioElementCount: number;
+  paused: boolean;
+  readyState: number;
+}
+
+type BrowserAudioElementState = Pick<HTMLAudioElement, 'ended' | 'paused' | 'readyState'>;
+
+export function buildBrowserAudioPlaybackDiagnostics(
+  participantIdentity: string,
+  trackName: string,
+  audioElements: Iterable<BrowserAudioElementState>,
+  currentElement: BrowserAudioElementState
+): BrowserAudioPlaybackDiagnostics {
+  return {
+    participantIdentity,
+    trackName,
+    activeAudioElementCount: Array.from(audioElements).filter(
+      (element) => !element.paused && !element.ended && element.readyState >= 2
+    ).length,
+    paused: currentElement.paused,
+    readyState: currentElement.readyState,
+  };
+}
+
+export function inspectBrowserAudioCapture(
+  track: InspectableAudioTrack,
+  supported: MediaTrackSupportedConstraints
+): BrowserAudioCaptureDiagnostics {
+  const constraints = track.getConstraints();
+  const settings = track.getSettings();
+
+  return {
+    trackId: track.id,
+    supported,
+    constraints,
+    settings,
+  };
+}
+
+export function assertBrowserEchoCancellationActive(
+  diagnostics: BrowserAudioCaptureDiagnostics
+): void {
+  if (diagnostics.supported.echoCancellation && diagnostics.settings.echoCancellation !== true) {
+    throw new Error(
+      'Browser echo cancellation was requested but is not active on the microphone track.'
+    );
+  }
+}

lib/browser-media-capture-timeout.ts

@@ -0,0 +1,40 @@
+export class BrowserMediaCaptureTimeoutError extends Error {
+  constructor(label: string, timeoutMs: number) {
+    super(`${label} capture did not become ready within ${timeoutMs}ms`);
+    this.name = 'BrowserMediaCaptureTimeoutError';
+  }
+}
+
+type BrowserMediaCaptureOptions<T> = {
+  timeoutMs: number;
+  label: string;
+  disposeLateResult: (result: T) => void;
+};
+
+export async function awaitBrowserMediaCapture<T>(
+  capture: Promise<T>,
+  options: BrowserMediaCaptureOptions<T>
+): Promise<T> {
+  let timeoutHandle: ReturnType<typeof setTimeout> | null = null;
+  let timedOut = false;
+
+  void capture.then(
+    (result) => {
+      if (timedOut) options.disposeLateResult(result);
+    },
+    () => undefined
+  );
+
+  const timeout = new Promise<never>((_resolve, reject) => {
+    timeoutHandle = setTimeout(() => {
+      timedOut = true;
+      reject(new BrowserMediaCaptureTimeoutError(options.label, options.timeoutMs));
+    }, options.timeoutMs);
+  });
+
+  try {
+    return await Promise.race([capture, timeout]);
+  } finally {
+    if (timeoutHandle !== null) clearTimeout(timeoutHandle);
+  }
+}

lib/livekit-media-gate.ts

@@ -1,4 +1,5 @@
 import { type RemoteParticipant, type Room, RoomEvent } from 'livekit-client';
+import { awaitBrowserMediaCapture } from './browser-media-capture-timeout';
 import {
   MEDIA_CONTROL_TOPIC,
   MEDIA_STATE_TOPIC,
@@ -8,6 +9,8 @@ import {
   encodeMediaState,
 } from './media-control-protocol';
 
+const MEDIA_STATE_PUBLISH_TIMEOUT_MS = 3000;
+
 export type MediaGateExecutorPort = {
   start(): Promise<void>;
   bindController(controllerIdentity: string): Promise<void>;
@@ -228,11 +231,18 @@ export async function publishLiveKitMediaState(
   throwIfAborted(signal);
   const payload = encodeMediaState(state);
   throwIfAborted(signal);
-  await room.localParticipant.publishData(payload, {
-    reliable: true,
-    destinationIdentities: [controllerIdentity],
-    topic: MEDIA_STATE_TOPIC,
-  });
+  await awaitBrowserMediaCapture(
+    room.localParticipant.publishData(payload, {
+      reliable: true,
+      destinationIdentities: [controllerIdentity],
+      topic: MEDIA_STATE_TOPIC,
+    }),
+    {
+      timeoutMs: MEDIA_STATE_PUBLISH_TIMEOUT_MS,
+      label: 'media gate state',
+      disposeLateResult: () => undefined,
+    }
+  );
 }
 
 function mediaControlErrorCode(error: unknown): string {