From 066d209fad1c2b415005b3f4e30afaf6fd4e72ae Mon Sep 17 00:00:00 2001 From: gino98005 Date: Sun, 18 Oct 2026 02:28:59 +0800 Subject: [PATCH] Let the candidate switch microphones mid-interview Once the interview starts, nothing can move it off the microphone the preflight opened, so a headset plugged in afterwards goes unused while the laptop microphone keeps hearing Jim, and the only way out is to end the interview and start over. Choosing a device before the interview is #206; this covers the interview itself. The switch reuses the one capture path and replaces the published track rather than calling LiveKit's switchActiveDevice, which would reopen the device inside the vendored client and leave the stream the integrity checks read holding a stopped track. Because that trail exists to say whether the microphone is still the one the interview started with, a switch records MICROPHONE_SWITCHED, which the agent now accepts. --- docs/integrity-evidence.md | 35 + scripts/gen-wire-fixtures.mjs | 15 + src/agent/integrity.rs | 6 + tests/browser/devices.test.js | 61 +- tests/browser/mic-switch.test.js | 1426 +++++++++++++++++++++++++++ tests/fixtures/integrity-chain.json | 12 + web/audio-input.js | 4 +- web/devices.js | 43 +- web/integrity.js | 19 +- web/interview.html | 25 + web/interview.js | 230 ++++- web/mic-switch.js | 298 ++++++ web/styles.css | 19 + 13 files changed, 2174 insertions(+), 19 deletions(-) create mode 100644 tests/browser/mic-switch.test.js create mode 100644 web/mic-switch.js diff --git a/docs/integrity-evidence.md b/docs/integrity-evidence.md index 6fba4ba4d..5ad7a26ce 100644 --- a/docs/integrity-evidence.md +++ b/docs/integrity-evidence.md @@ -26,6 +26,41 @@ the camera required, because the recording notice describes a video recording. - `declined`: the candidate skipped a camera with no error, or with an error that names none of the causes above. +## Microphone switches + +A candidate can move the interview to another microphone from the interview +page, and the page does so by itself when the input in use ends. Either way, +once the new input is in use, the browser records a signed `MICROPHONE_SWITCHED` +event, because the heartbeat that follows names a live microphone and would +otherwise read as the one the interview started with. Its detail is +`reason=chosen` or `reason=lost`, then `microphone=` and the new device's label, +last because a label is the browser's text and may contain the separators. A +switch is not a microphone stopping, so it records no `MICROPHONE_STOPPED`; an +input that ended before the page switched away from it still records one, since +it did stop. + +A chosen switch that fails leaves the previous input in use and records nothing, +unless the previous input cannot be handed back to the sender either; the page +then drops it, says the interviewer cannot hear the candidate, and asks for a +microphone again. That drop records `MICROPHONE_STOPPED` with +`reason=switch_failed`, because stopping a working input fires no `ended` and +the state poll skips a missing track, so nothing else would. A move after a lost +input that fails leaves the interview with no working microphone; the page says +so, records no switch, and tries the move again whenever a device is plugged in, +while the `MICROPHONE_STOPPED` for the loss stays the last word on the input +that ended. + +While a switch is under way, LiveKit's own mute disables the input still in +use, so that the replacement is not sent before the Mic button says. For that +stretch the heartbeat and the state poll report the Mic button's state instead +of the input's, since a `MICROPHONE_STATE_CHANGED` to `off` there would record +the switch muting a candidate who never pressed Mic. + +The server drops an integrity event whose type it does not list before it +checks the chain, so that event's sequence number is never accepted and every +later event is refused as out of sequence. A new event type is therefore added +to `sanitize_integrity_event` in the same change that starts sending it. + ## Response windows The replay page lists a *response window* for each turn the interviewer took: diff --git a/scripts/gen-wire-fixtures.mjs b/scripts/gen-wire-fixtures.mjs index 082f2f455..88685fcbb 100755 --- a/scripts/gen-wire-fixtures.mjs +++ b/scripts/gen-wire-fixtures.mjs @@ -24,6 +24,7 @@ const { reportRecoveryLimits } = await import( join(ROOT, "web", "report-recovery.js") ); const lib = await import(join(ROOT, "web", "lib.js")); +const { switchDetail } = await import(join(ROOT, "web", "mic-switch.js")); const { ALL_LANGUAGES } = await import( join(ROOT, "web", "compiler-explorer.js") ); @@ -361,6 +362,20 @@ const INTEGRITY_INPUTS = [ "5555555555555", ], }, + // Last, so the rows above keep their hashes. A type the agent's allowlist + // lacks is refused and stops the chain, which is how a switch the browser + // records would otherwise vanish from every report. The detail comes from the + // producer the page uses, with a label carrying the separators it is placed + // last to survive. + { + type: "MICROPHONE_SWITCHED", + source: "microphone", + severity: "info", + at: "2026-08-18T06:40:30.512Z", + detail: switchDetail("chosen", { + label: "USB Audio; Rear=Mic (0d8c:0014)", + }), + }, ]; async function integrityChain() { diff --git a/src/agent/integrity.rs b/src/agent/integrity.rs index dfee02669..3ab66aa1c 100644 --- a/src/agent/integrity.rs +++ b/src/agent/integrity.rs @@ -276,6 +276,12 @@ pub fn sanitize_integrity_event(payload: &serde_json::Value) -> Option "CAMERA_RELEASED_TO_PRESENTER", "CAMERA_NOT_USED" => "CAMERA_NOT_USED", "MICROPHONE_STATE_CHANGED" => "MICROPHONE_STATE_CHANGED", + + // The candidate moved the interview to another input, or the page did + // after the one in use ended. Without it the heartbeat goes on naming a + // live microphone and nothing says it is no longer the one the + // interview started with. + "MICROPHONE_SWITCHED" => "MICROPHONE_SWITCHED", _ => return None, }; let at = text("at")?; diff --git a/tests/browser/devices.test.js b/tests/browser/devices.test.js index 4da35e698..c0d9cd413 100644 --- a/tests/browser/devices.test.js +++ b/tests/browser/devices.test.js @@ -2,7 +2,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { FakeStream, FakeTrack, settle } from "./fake-media.js"; -import { createDevicePool } from "../../web/devices.js"; +import { createDevicePool, openMicrophone } from "../../web/devices.js"; function poolWith(getUserMedia, options = {}) { const changes = { count: 0 }; @@ -571,3 +571,62 @@ test("the pool's own drop of an ended track also removes before notifying", asyn finished = true; pool.cancelRetry(); }); + +// The switch the candidate asked for names its device exactly: opening some +// other input and calling it theirs would be worse than saying it failed. +test("a mid-interview microphone is asked for by exact device", async () => { + const asked = []; + const track = new FakeTrack("audio"); + const opened = await openMicrophone( + { + getUserMedia: async (constraints) => { + asked.push(constraints); + return new FakeStream([track]); + }, + }, + "headset", + ); + + assert.equal(opened, track); + assert.deepEqual(asked, [ + { + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + deviceId: { exact: "headset" }, + }, + }, + ]); +}); + +test("the default microphone is asked for with no device id", async () => { + const asked = []; + await openMicrophone( + { + getUserMedia: async (constraints) => { + asked.push(constraints); + return new FakeStream([new FakeTrack("audio")]); + }, + }, + "", + ); + + assert.equal("deviceId" in asked[0].audio, false); +}); + +test("a grant with no live input is refused and leaves nothing open", async () => { + const spare = new FakeTrack("video"); + const dead = new FakeTrack("audio"); + dead.readyState = "ended"; + + await assert.rejects( + openMicrophone( + { getUserMedia: async () => new FakeStream([dead, spare]) }, + "headset", + ), + /no active audio track/, + ); + assert.ok(spare.stopped, "a track nothing will read was left running"); + assert.ok(dead.stopped); +}); diff --git a/tests/browser/mic-switch.test.js b/tests/browser/mic-switch.test.js new file mode 100644 index 000000000..a6c27c2fe --- /dev/null +++ b/tests/browser/mic-switch.test.js @@ -0,0 +1,1426 @@ +// Run with: node --test tests/browser/mic-switch.test.js +// +// Switching the microphone is an ordering problem: open the new input, hand it +// to the published track, then retire the old one, and back out cleanly at +// every step that can fail. These drive the real switch against fakes for the +// stream and the LiveKit track, so each step is executed rather than read. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { FakeStream, FakeTrack } from "./fake-media.js"; +import { loadInterview, read } from "./source.js"; + +import { + INPUT_NOTES, + createMicrophoneSwitch, + holdMute, + inputOptions, + liftMute, + switchDetail, +} from "../../web/mic-switch.js"; + +/// A microphone that reports the device it opened, which is what the switch +/// compares a pick against and what the select marks as in use. +class FakeMic extends FakeTrack { + constructor(deviceId, label = deviceId) { + super("audio"); + this.deviceId = deviceId; + this.label = label; + } + getSettings() { + return { deviceId: this.deviceId }; + } +} + +/// Stands in for LiveKit's LocalAudioTrack. Like the real one, it re-enables +/// the track it is handed from its own mute flag, which the page never sets. +class FakePublished { + constructor(track) { + this.sending = track; + this.calls = []; + this.refuse = false; + // Refuses every call after this many, so a test can fail only a later one. + this.refuseAfter = Infinity; + // Set by a test to hold the replace open while it changes the session. + this.pending = null; + // Whether each track was enabled at the moment the sender took it. + this.sentEnabled = []; + // LiveKit's own mute flag, which it raises when a user-provided track ends. + this.isMuted = false; + this.liveAfterReplace = []; + this.unmutes = 0; + this.liveAfterUnmute = []; + this.listeners = {}; + } + async replaceTrack(track, options) { + this.calls.push(options); + this.sentEnabled.push(track.enabled); + // LiveKit's setMediaStreamTrack returns at once for the track it already + // holds, so a hand-back to an input never replaced cannot fail. + if (track === this.sending) return; + if (this.pending) await this.pending; + if (this.refuse || this.calls.length > this.refuseAfter) + throw new Error("sender closed"); + // What LiveKit does when a user-provided track it is listening to ends. + if (this.muteOnEnded && track.readyState === "ended") this.isMuted = true; + // LiveKit hands the track to the sender and then resumes upstream, which + // can still reject with the new track already being sent. + // One-shot, so only the replace under test rejects. + if (this.rejectAfterSend) { + this.rejectAfterSend = false; + this.sending = track; + throw new Error("resume upstream failed"); + } + track.enabled = !this.isMuted; + // What the sender was given to send once LiveKit had set it up, which is + // the window a muted candidate must not be live in. + this.liveAfterReplace.push(track.enabled); + this.sending = track; + } + async mute() { + this.isMuted = true; + this.sending.enabled = false; + } + // LiveKit's setTrackMuted: enable the track, then announce it in the same + // synchronous step. + async unmute() { + if (this.unmutePending) await this.unmutePending; + if (this.refuseUnmute) throw new Error("unmute refused"); + this.unmutes += 1; + this.isMuted = false; + this.sending.enabled = true; + for (const listener of [...(this.listeners.unmuted || [])]) listener(this); + // What is left enabled once LiveKit's own step is over. + this.liveAfterUnmute.push(this.sending.enabled); + } + get mediaStreamTrack() { + return this.sending; + } + on(name, listener) { + (this.listeners[name] ||= []).push(listener); + } + off(name, listener) { + this.listeners[name] = (this.listeners[name] || []).filter( + (each) => each !== listener, + ); + } +} + +function setup({ open, live = true, muted = false, published = true } = {}) { + const old = new FakeMic("laptop"); + const stream = new FakeStream([old]); + const sender = published ? new FakePublished(old) : null; + const state = { + live, + muted, + opened: [], + switched: [], + failed: [], + outcomes: [], + dropped: [], + holding: false, + }; + const microphone = createMicrophoneSwitch({ + open: async (deviceId) => { + state.opened.push(deviceId); + return open ? open(deviceId) : new FakeMic(deviceId); + }, + streamOf: () => stream, + publishedTrack: () => sender, + isLive: () => state.live, + micEnabled: () => !state.muted, + onHold: (holding) => { + state.holding = holding; + }, + onSwitched: (track, reason) => state.switched.push({ track, reason }), + onFailed: (reason, error, outcome, droppedLive) => { + state.failed.push(reason); + state.outcomes.push(outcome); + state.dropped.push(droppedLive); + }, + }); + return { microphone, old, stream, sender, state }; +} + +const audioIn = (stream) => + stream.getTracks().filter((t) => t.kind === "audio"); + +test("a chosen input replaces the sent track and retires the old one", async () => { + const { microphone, old, stream, sender, state } = setup(); + + assert.equal(await microphone.switchTo("headset"), "switched"); + + const [now] = audioIn(stream); + assert.equal(now.deviceId, "headset"); + assert.equal(audioIn(stream).length, 1, "the stream holds exactly one input"); + assert.equal(sender.sending, now, "LiveKit sends the input the stream holds"); + assert.deepEqual(sender.calls, [{ userProvidedTrack: true }]); + assert.ok(old.stopped, "the previous device stays open"); + const switched = state.switched.map((s) => [s.reason, s.track.deviceId]); + assert.deepEqual(switched, [["chosen", "headset"]]); +}); + +// The Mic button mutes by disabling the track. LiveKit's replace re-enables +// whatever it is given, so a muted candidate who switches would be live again. +test("a muted microphone stays muted across a switch", async () => { + const { microphone, stream, sender } = setup({ muted: true }); + + await microphone.switchTo("headset"); + + assert.equal(audioIn(stream)[0].enabled, false); + assert.deepEqual( + sender.sentEnabled, + [false], + "the sender took the new input while it was still enabled", + ); + // LiveKit enables the new track from its own flag inside the replace, so the + // flag has to read muted by then, or the input is live until the page + // disables it again. + assert.deepEqual(sender.liveAfterReplace, [false]); + // And down again afterwards, without the input going live as it comes down: + // a raised flag is a server-side mute the agent hears as no frames at all. + assert.equal(sender.isMuted, false); + assert.deepEqual(sender.liveAfterUnmute, [false]); +}); + +test("an input that will not open leaves the current one in use", async () => { + const { microphone, old, stream, sender, state } = setup({ + open: () => { + throw new Error("NotReadableError"); + }, + }); + + assert.equal(await microphone.switchTo("headset"), "refused"); + + assert.deepEqual(audioIn(stream), [old]); + assert.equal(old.stopped, false); + assert.equal(sender.sending, old); + assert.deepEqual(state.failed, ["chosen"]); + assert.deepEqual(state.switched, []); +}); + +test("a refused replace stops the new device and keeps the old", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + sender.refuse = true; + + assert.equal(await microphone.switchTo("headset"), "refused"); + + assert.ok(opened[0].stopped, "the device opened for the switch was left on"); + assert.deepEqual(audioIn(stream), [old]); + assert.equal(old.stopped, false); + assert.deepEqual(state.failed, ["chosen"]); +}); + +// A permission prompt can stay up past the end of the interview. +test("an input granted after the interview ended is stopped, not swapped in", async () => { + let grant = null; + const pending = new FakeMic("headset"); + const { microphone, old, stream, state } = setup({ + open: () => + new Promise((resolve) => { + grant = () => resolve(pending); + }), + }); + + const result = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + state.live = false; + grant(); + + assert.equal(await result, "idle"); + assert.ok(pending.stopped); + assert.deepEqual(audioIn(stream), [old]); + assert.deepEqual(state.switched, []); +}); + +test("nothing is opened while there is no published track to hand it to", async () => { + const { microphone, state } = setup({ published: false }); + + assert.equal(await microphone.switchTo("headset"), "idle"); + assert.deepEqual(state.opened, []); +}); + +test("picking the input already in use opens nothing", async () => { + const { microphone, state } = setup(); + + assert.equal(await microphone.switchTo("laptop"), "unchanged"); + assert.deepEqual(state.opened, []); +}); + +// Two picks while the first prompt is still up. Unserialized, the first could +// resolve last and leave the stream and LiveKit holding different inputs. +test("rapid picks resolve in the order they were made", async () => { + const grants = []; + const tracks = []; + const { microphone, stream, sender } = setup({ + open: (deviceId) => + new Promise((resolve) => { + const track = new FakeMic(deviceId); + tracks.push(track); + grants.push(() => resolve(track)); + }), + }); + + const first = microphone.switchTo("headset"); + const second = microphone.switchTo("webcam"); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(grants.length, 1, "the second pick waits for the first"); + grants[0](); + assert.equal(await first, "switched"); + await new Promise((resolve) => setTimeout(resolve, 0)); + grants[1](); + assert.equal(await second, "switched"); + + assert.equal(audioIn(stream)[0].deviceId, "webcam"); + assert.equal(sender.sending, audioIn(stream)[0]); + assert.ok(tracks[0].stopped, "the input switched away from stays open"); +}); + +test("an input that ended moves the interview to the default", async () => { + const { microphone, old, stream, state } = setup(); + old.readyState = "ended"; + + assert.equal(await microphone.switchTo("", "lost", old), "switched"); + + assert.deepEqual(state.opened, [""], "the default is asked for by no id"); + assert.equal(audioIn(stream).length, 1); + assert.notEqual(audioIn(stream)[0], old); + assert.equal(state.switched[0].reason, "lost"); +}); + +// The pick replaced the track that later ended, so the loss is about an input +// the interview no longer uses. +test("a loss queued behind a pick leaves the picked input in use", async () => { + let grant = null; + const { microphone, old, stream, state } = setup({ + // Only the pick waits on its prompt, so a loss that wrongly runs resolves + // and fails the assertions rather than hanging the test. + open: (deviceId) => + grant + ? new FakeMic(deviceId) + : new Promise((resolve) => { + grant = () => resolve(new FakeMic(deviceId)); + }), + }); + + const picked = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + old.readyState = "ended"; + const lost = microphone.switchTo("", "lost", old); + grant(); + + assert.equal(await picked, "switched"); + assert.equal(await lost, "unchanged"); + assert.equal(audioIn(stream)[0].deviceId, "headset"); + assert.deepEqual(state.opened, ["headset"]); +}); + +// LiveKit's replace can be slow, and the interview can end inside it. +test("an interview that ends during the replace keeps nothing open", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + let release = null; + sender.pending = new Promise((resolve) => { + release = resolve; + }); + + const result = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + state.live = false; + release(); + + assert.equal(await result, "idle"); + assert.ok(opened[0].stopped, "the input opened for the switch stayed on"); + assert.deepEqual(audioIn(stream), [old]); + assert.deepEqual(state.switched, []); +}); + +test("a refusal that arrives after the interview ended is not reported", async () => { + let refuse = null; + const { microphone, state } = setup({ + open: () => + new Promise((_, reject) => { + refuse = () => reject(new Error("NotAllowedError")); + }), + }); + + const result = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + state.live = false; + refuse(); + + assert.equal(await result, "idle"); + assert.deepEqual(state.failed, []); +}); + +// Unplugged while the replace was pending: its `ended` fired before anything +// listened, so committing it would leave the interview silent for good. +test("an input that ends during the replace hands back the old one", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + let release = null; + sender.pending = new Promise((resolve) => { + release = resolve; + }); + + const result = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + sender.pending = null; + opened[0].readyState = "ended"; + release(); + + assert.equal(await result, "refused"); + assert.equal(sender.sending, old, "the sender kept the dead input"); + assert.equal(old.stopped, false); + assert.deepEqual(audioIn(stream), [old]); + assert.deepEqual(state.failed, ["chosen"]); + assert.deepEqual(state.switched, []); +}); + +// A device plugged in while the move after a loss is still opening queues a +// retry for the same loss; if the first attempt strands, that retry is the way +// back and must not read as already handled. +test("a retry queued behind a stranded loss still tries", async () => { + let attempts = 0; + let refuse = null; + const { microphone, old, stream, state } = setup({ + open: (deviceId) => { + attempts += 1; + if (attempts > 1) return new FakeMic("headset"); + return new Promise((_, reject) => { + refuse = () => reject(new Error("NotFoundError")); + }); + }, + }); + old.readyState = "ended"; + + const first = microphone.switchTo("", "lost", old); + const retry = microphone.switchTo("", "lost", old); + await new Promise((resolve) => setTimeout(resolve, 0)); + refuse(); + + assert.equal(await first, "stranded"); + assert.equal(await retry, "switched"); + assert.equal(audioIn(stream)[0].deviceId, "headset"); + assert.deepEqual(state.outcomes, ["stranded"]); +}); + +// The track opened as the default reports the alias's id; picking the +// physical row of the same device is not a switch. Two real inputs of one +// interface share a group too, and moving between them is. +test("a pick in the input's own group is unchanged", async () => { + const old = new FakeMic("default"); + old.getSettings = () => ({ deviceId: "default", groupId: "h" }); + const stream = new FakeStream([old]); + const sender = new FakePublished(old); + const opened = []; + const microphone = createMicrophoneSwitch({ + open: async (deviceId) => { + opened.push(deviceId); + return new FakeMic(deviceId); + }, + streamOf: () => stream, + publishedTrack: () => sender, + isLive: () => true, + micEnabled: () => true, + onSwitched: () => {}, + onFailed: () => {}, + }); + + assert.equal( + await microphone.switchTo("headset", "chosen", null, "h"), + "unchanged", + ); + assert.deepEqual(opened, []); + assert.equal( + await microphone.switchTo("laptop", "chosen", null, "l"), + "switched", + ); +}); + +test("two real inputs in one group are different inputs", async () => { + const old = new FakeMic("input-1"); + old.getSettings = () => ({ deviceId: "input-1", groupId: "interface" }); + const stream = new FakeStream([old]); + const sender = new FakePublished(old); + const microphone = createMicrophoneSwitch({ + open: async (deviceId) => new FakeMic(deviceId), + streamOf: () => stream, + publishedTrack: () => sender, + isLive: () => true, + micEnabled: () => true, + onSwitched: () => {}, + onFailed: () => {}, + }); + + assert.equal( + await microphone.switchTo("input-2", "chosen", null, "interface"), + "switched", + ); +}); + +// Dropping an input that still worked is the one stop the trail would not see: +// `stop()` fires no `ended`. One that had already ended recorded its own. +test("a dropped input reports whether it was still working", async () => { + const live = setup(); + live.sender.isMuted = true; + live.sender.refuseUnmute = true; + await live.microphone.switchTo("headset"); + assert.deepEqual(live.state.outcomes, ["stranded"]); + assert.deepEqual(live.state.dropped, [true]); + + const dead = setup({ + open: () => { + throw new Error("NotFoundError"); + }, + }); + dead.old.readyState = "ended"; + await dead.microphone.switchTo("", "lost", dead.old); + assert.deepEqual(dead.state.outcomes, ["stranded"]); + assert.deepEqual(dead.state.dropped, [false]); +}); + +// LiveKit was already listening to the new input when it ended, so it muted the +// publication; handing back the old input must clear that or Jim hears nothing. +test("handing back the old input clears LiveKit's mute flag", async () => { + const opened = []; + const { microphone, old, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + sender.muteOnEnded = true; + let release = null; + sender.pending = new Promise((resolve) => { + release = resolve; + }); + + const result = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + sender.pending = null; + opened[0].readyState = "ended"; + release(); + + assert.equal(await result, "refused"); + assert.equal(sender.sending, old); + assert.equal(sender.isMuted, false, "the restored input stays muted"); + assert.deepEqual(state.outcomes, ["refused"]); +}); + +test("a replace that rejects after sending still hands back", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + sender.rejectAfterSend = true; + + assert.equal(await microphone.switchTo("headset"), "refused"); + assert.equal(sender.sending, old, "LiveKit kept sending the stopped input"); + assert.ok(opened[0].stopped); + assert.deepEqual(audioIn(stream), [old]); + assert.deepEqual(state.outcomes, ["refused"]); +}); + +// The sender is then left on the dead input, so claiming the old one is still +// in use would be false, and keeping it in the stream would make a pick of the +// same device read as unchanged. +test("a failed hand-back says so and lets the same device be picked", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + let release = null; + sender.pending = new Promise((resolve) => { + release = resolve; + }); + + const result = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + sender.pending = null; + sender.refuseAfter = 1; + opened[0].readyState = "ended"; + release(); + + assert.equal(await result, "stranded"); + assert.deepEqual(state.outcomes, ["stranded"]); + assert.ok(old.stopped); + assert.deepEqual(audioIn(stream), []); + + sender.refuseAfter = Infinity; + assert.equal(await microphone.switchTo("laptop"), "switched"); + assert.equal(sender.sending, audioIn(stream)[0]); +}); + +// The unmute after a loss is another await the interview can end inside. +test("an interview that ends during the unmute keeps nothing open", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + old.readyState = "ended"; + sender.isMuted = true; + let release = null; + sender.unmutePending = new Promise((resolve) => { + release = resolve; + }); + + const result = microphone.switchTo("", "lost", old); + await new Promise((resolve) => setTimeout(resolve, 0)); + state.live = false; + release(); + + assert.equal(await result, "idle"); + assert.ok(opened[0].stopped); + assert.deepEqual(audioIn(stream), [old]); + assert.deepEqual(state.failed, []); +}); + +test("a refused unmute leaves no input claimed as in use", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + sender.isMuted = true; + sender.refuseUnmute = true; + + assert.equal(await microphone.switchTo("headset"), "stranded"); + // The hand-back needs the same unmute, so with it refused nothing is sent. + assert.ok(opened[0].stopped); + assert.ok(old.stopped, "the old input is still open but nothing sends it"); + assert.deepEqual(audioIn(stream), []); + assert.deepEqual(state.outcomes, ["stranded"]); + + sender.refuseUnmute = false; + sender.isMuted = false; + assert.equal(await microphone.switchTo("laptop"), "switched"); +}); + +test("an input that ends during the unmute is not committed", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + sender.isMuted = true; + let release = null; + sender.unmutePending = new Promise((resolve) => { + release = resolve; + }); + + const result = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + sender.unmutePending = null; + opened[0].readyState = "ended"; + release(); + + assert.equal(await result, "refused"); + assert.equal(sender.sending, old); + assert.deepEqual(audioIn(stream), [old]); + assert.deepEqual(state.outcomes, ["refused"]); +}); + +// With the old input already dead there is nothing to hand back to, so the +// page must not claim one is still in use. +test("a failed move after a loss strands rather than refuses", async () => { + const { microphone, old, stream, state } = setup({ + open: () => { + throw new Error("NotFoundError"); + }, + }); + old.readyState = "ended"; + + assert.equal(await microphone.switchTo("", "lost", old), "stranded"); + assert.deepEqual(audioIn(stream), []); + assert.deepEqual(state.outcomes, ["stranded"]); +}); + +test("a new input that ends after a loss strands too", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + old.readyState = "ended"; + let release = null; + sender.pending = new Promise((resolve) => { + release = resolve; + }); + + const result = microphone.switchTo("", "lost", old); + await new Promise((resolve) => setTimeout(resolve, 0)); + sender.pending = null; + opened[0].readyState = "ended"; + release(); + + assert.equal(await result, "stranded"); + assert.deepEqual(audioIn(stream), []); + assert.deepEqual(state.outcomes, ["stranded"]); +}); + +// LiveKit mutes the publication when a user-provided track ends, so after a +// loss the interviewer would receive a muted microphone while the page reads +// "Mic on". +test("a move after a loss clears LiveKit's mute flag", async () => { + const { microphone, old, stream, sender } = setup(); + old.readyState = "ended"; + sender.isMuted = true; + + assert.equal(await microphone.switchTo("", "lost", old), "switched"); + + assert.equal(sender.isMuted, false); + assert.equal(audioIn(stream)[0].enabled, true); +}); + +// The flag LiveKit raised after the loss comes down, since left up it is a +// server-side mute; a candidate who had muted themselves stays muted. +test("a muted candidate stays muted after a loss", async () => { + const { microphone, old, stream, sender } = setup({ muted: true }); + old.readyState = "ended"; + sender.isMuted = true; + + await microphone.switchTo("", "lost", old); + + assert.equal(sender.isMuted, false); + assert.deepEqual(sender.liveAfterReplace, [false]); + assert.deepEqual( + sender.liveAfterUnmute, + [false], + "the input went live while the flag came down", + ); + assert.equal(audioIn(stream)[0].enabled, false); +}); + +test("the flag is held for every swap and lifted for anyone", async () => { + const sender = new FakePublished(new FakeMic("laptop")); + + await holdMute(sender); + assert.equal(sender.isMuted, true); + + await liftMute(sender, () => false); + assert.equal(sender.isMuted, false); + assert.deepEqual(sender.liveAfterUnmute, [false]); + + sender.isMuted = true; + await liftMute(sender, () => true); + assert.deepEqual(sender.liveAfterUnmute, [false, true]); + // Nothing to lower: no unmute, so nothing goes live. + await liftMute(sender, () => true); + assert.equal(sender.unmutes, 2); + assert.deepEqual(sender.listeners.unmuted, []); +}); + +// The publication's flag is a server-side mute: the SFU stops forwarding, the +// agent gets no frames, and a candidate who mutes mid-sentence leaves the turn +// open. The button mutes the track only. +test("the Mic button leaves the publication's flag alone", async () => { + const track = new FakeMic("laptop"); + const stream = new FakeStream([track]); + stream.getAudioTracks = () => audioIn(stream); + const sender = new FakePublished(track); + const inFlightCalls = []; + const scope = { + state: { micEnabled: false, localUserStream: stream }, + nodes: { mic: { textContent: "" } }, + publishedMicrophone: () => sender, + microphoneSwitch: () => ({ + setEnabled: (enabled) => inFlightCalls.push(enabled), + }), + setLocalAudioEnabled: (enabled) => { + track.enabled = enabled; + }, + publishIntegrityEvent: () => Promise.resolve(), + console, + }; + const { toggleMicrophone } = loadInterview(["toggleMicrophone"], scope); + + await toggleMicrophone(); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(scope.state.micEnabled, true); + assert.equal(track.enabled, true); + assert.equal(scope.nodes.mic.textContent, "Mic on"); + + await toggleMicrophone(); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(track.enabled, false); + assert.equal(sender.isMuted, false, "the button raised a server-side mute"); + assert.equal(sender.unmutes, 0); + assert.deepEqual(inFlightCalls, [true, false]); +}); + +// Between the open and the commit the new input is in neither the stream nor +// the publication, so a mute pressed then would otherwise leave it live. +test("a mute pressed during a switch reaches the input being handed over", async () => { + const opened = []; + const { microphone, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + let release = null; + sender.pending = new Promise((resolve) => { + release = resolve; + }); + + const result = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + state.muted = true; + microphone.setEnabled(false); + assert.equal(opened[0].enabled, false, "the input in flight stayed live"); + release(); + + assert.equal(await result, "switched"); + // LiveKit enables what it is handed from the publication's flag, so a flag + // left down because the switch began unmuted put the input live here. + assert.deepEqual(sender.liveAfterReplace, [false], "live during the replace"); + assert.deepEqual(sender.liveAfterUnmute, [false]); + assert.equal(audioIn(stream)[0].enabled, false); + // Once committed it is the stream's, and the button reaches it directly. + microphone.setEnabled(true); + assert.equal(opened[0].enabled, false); +}); + +// The hand-back has the same window: the old input is handed to LiveKit again, +// and a mute pressed while that replace is pending must not be undone by it. +test("a mute pressed during the hand-back keeps the old input disabled", async () => { + const { microphone, old, sender, state } = setup(); + sender.rejectAfterSend = true; + const replace = sender.replaceTrack.bind(sender); + let release = null; + sender.replaceTrack = (track, options) => { + if (track === old) + sender.pending = new Promise((resolve) => { + release = resolve; + }); + return replace(track, options); + }; + + const result = microphone.switchTo("headset"); + for (let tick = 0; !release && tick < 100; tick += 1) + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.ok(release, "the hand-back never reached LiveKit"); + state.muted = true; + old.enabled = false; + microphone.setEnabled(false); + release(); + + assert.equal(await result, "refused"); + assert.deepEqual(sender.liveAfterReplace, [false], "live during the replace"); + assert.deepEqual(sender.liveAfterUnmute, [false]); + assert.equal(old.enabled, false); +}); + +// While the swap holds LiveKit's flag, LiveKit has disabled the input still in +// the stream, and the integrity poll would record that as the candidate's. +test("the hold is announced for as long as the swap holds the flag", async (t) => { + const run = async (arrange) => { + const ctx = setup(); + const seen = []; + for (const name of ["mute", "unmute"]) { + const original = ctx.sender[name].bind(ctx.sender); + ctx.sender[name] = (...args) => { + seen.push([name, ctx.state.holding]); + return original(...args); + }; + } + arrange?.(ctx); + const result = await ctx.microphone.switchTo("headset"); + return { result, seen, holding: ctx.state.holding }; + }; + const switched = await run(); + const handedBack = await run(({ sender }) => { + sender.rejectAfterSend = true; + }); + // The new input is sent before the replace rejects, and the hand-back that + // would take it back is refused. + const stranded = await run(({ sender }) => { + sender.rejectAfterSend = true; + sender.refuseAfter = 1; + }); + + for (const [name, outcome, result] of [ + ["switched", switched, "switched"], + ["refused", handedBack, "refused"], + ["stranded", stranded, "stranded"], + ]) { + await t.test(name, () => { + assert.equal(outcome.result, result); + assert.ok(outcome.seen.length > 0, "the flag was never raised"); + assert.ok( + outcome.seen.every(([, holding]) => holding), + `the flag moved while no hold was announced: ${JSON.stringify(outcome.seen)}`, + ); + assert.equal(outcome.holding, false, "the hold outlived the switch"); + }); + } +}); + +test("a throwing callback does not block the next switch", async () => { + const old = new FakeMic("laptop"); + const stream = new FakeStream([old]); + const sender = new FakePublished(old); + let calls = 0; + const microphone = createMicrophoneSwitch({ + open: async (deviceId) => new FakeMic(deviceId), + streamOf: () => stream, + publishedTrack: () => sender, + isLive: () => true, + micEnabled: () => true, + onSwitched: () => { + calls += 1; + if (calls === 1) throw new Error("listener failed"); + }, + onFailed: () => {}, + }); + + await assert.rejects(microphone.switchTo("headset")); + assert.equal(await microphone.switchTo("webcam"), "switched"); +}); + +test("the list offers named inputs only and marks the one in use", () => { + const devices = [ + { kind: "audioinput", deviceId: "", label: "" }, + { kind: "audioinput", deviceId: "laptop", label: "Built-in Microphone" }, + { kind: "audiooutput", deviceId: "speakers", label: "Speakers" }, + { kind: "audioinput", deviceId: "usb", label: "" }, + ]; + + assert.deepEqual(inputOptions(devices, { deviceId: "usb" }), [ + { + deviceId: "laptop", + groupId: "", + label: "Built-in Microphone", + selected: false, + }, + { deviceId: "usb", groupId: "", label: "Microphone 2", selected: true }, + ]); +}); + +// Picking the alias of the input in use reopened it and recorded a switch that +// changed nothing, so the alias rows are not offered. +test("alias rows are left out and the default's group marks its row", () => { + const devices = [ + { kind: "audioinput", deviceId: "default", label: "Default", groupId: "h" }, + { kind: "audioinput", deviceId: "communications", groupId: "h" }, + { kind: "audioinput", deviceId: "headset", label: "Headset", groupId: "h" }, + { kind: "audioinput", deviceId: "laptop", label: "Built-in", groupId: "l" }, + ]; + + const rows = inputOptions(devices, { deviceId: "default", groupId: "h" }); + + // The group travels with each row: it is how the page tells the alias in + // use from a real switch. + assert.deepEqual( + rows.map((row) => [row.deviceId, row.groupId, row.selected]), + [ + ["headset", "h", true], + ["laptop", "l", false], + ], + ); +}); + +// A track opened with no device id can report an id the list does not carry. +// Matching on the label instead keeps the select off its first row, which would +// name a microphone the candidate is not using. +test("the list falls back to the label when the id is not listed", () => { + const devices = [ + { kind: "audioinput", deviceId: "laptop", label: "Built-in Microphone" }, + { kind: "audioinput", deviceId: "usb", label: "USB Headset" }, + ]; + + const selected = (current) => + inputOptions(devices, current) + .filter((option) => option.selected) + .map((option) => option.deviceId); + + const unlisted = { deviceId: "gone", label: "USB Headset" }; + assert.deepEqual(selected(unlisted), ["usb"]); + assert.deepEqual(selected({ ...unlisted, label: "" }), []); +}); + +test("the switch detail keeps a label's separators readable", () => { + assert.equal( + switchDetail("chosen", { label: "USB; Rear=Mic" }), + "reason=chosen;microphone=USB; Rear=Mic", + ); + assert.equal( + switchDetail("lost", { label: "" }), + "reason=lost;microphone=unnamed", + ); +}); + +/// The page's own listing, run against a select that records what it is given +/// and a device list the test releases when it chooses. +function listingPage({ + track = new FakeMic("laptop", "Built-in Microphone"), +} = {}) { + const page = { + live: true, + pending: [], + retried: [], + focused: [], + select: { + hidden: true, + rows: null, + replaceChildren(...rows) { + this.rows = rows; + }, + }, + }; + const stream = new FakeStream(track ? [track] : []); + stream.getAudioTracks = () => audioIn(stream); + const scope = { + inputListing: 0, + state: { localUserStream: stream }, + nodes: { + micInputSelect: page.select, + mic: { focus: () => page.focused.push("mic") }, + }, + microphoneLive: () => page.live, + inputOptions, + navigator: { + mediaDevices: { + enumerateDevices: () => + new Promise((resolve) => page.pending.push(resolve)), + }, + }, + document: { activeElement: null, createElement: () => ({ dataset: {} }) }, + microphoneSwitch: () => ({ + switchTo: (...args) => { + page.retried.push(args); + return Promise.resolve("switched"); + }, + }), + }; + const { refreshAudioInputs, retryMicrophone } = loadInterview( + ["refreshAudioInputs", "retryMicrophone"], + scope, + ); + return { page, scope, refreshAudioInputs, retryMicrophone }; +} + +const twoInputs = [ + { kind: "audioinput", deviceId: "laptop", label: "Built-in Microphone" }, + { kind: "audioinput", deviceId: "usb", label: "USB Headset" }, +]; + +// Teardown hides the select; a listing still in flight must not show it again +// over the report. +test("a listing that finishes after the interview ended paints nothing", async () => { + const { page, refreshAudioInputs } = listingPage(); + + const listing = refreshAudioInputs(); + page.live = false; + page.pending[0](twoInputs); + await listing; + + assert.equal(page.select.rows, null); + assert.equal(page.select.hidden, true); +}); + +// Chrome adds a `default` row, and Windows a `communications` one, for the same +// physical microphone; one device is nothing to choose between. +test("one physical microphone keeps the select hidden", async () => { + const { page, refreshAudioInputs } = listingPage(); + const oneDevice = [ + { kind: "audioinput", deviceId: "default", label: "Default", groupId: "g" }, + { + kind: "audioinput", + deviceId: "communications", + label: "Comms", + groupId: "g", + }, + { kind: "audioinput", deviceId: "laptop", label: "Built-in", groupId: "g" }, + ]; + + const listing = refreshAudioInputs(); + page.pending[0](oneDevice); + await listing; + + assert.deepEqual( + page.select.rows.map((row) => row.value), + ["laptop"], + ); + assert.equal(page.select.hidden, true); +}); + +test("two physical microphones show the select", async () => { + const { page, refreshAudioInputs } = listingPage(); + const twoDevices = [ + { kind: "audioinput", deviceId: "laptop", label: "Built-in", groupId: "a" }, + { kind: "audioinput", deviceId: "usb", label: "USB Headset", groupId: "b" }, + ]; + + const listing = refreshAudioInputs(); + page.pending[0](twoDevices); + await listing; + + assert.equal(page.select.hidden, false); +}); + +test("an older listing that finishes last does not repaint the select", async () => { + const { page, refreshAudioInputs } = listingPage(); + + const older = refreshAudioInputs(); + const newer = refreshAudioInputs(); + page.pending[1](twoInputs); + await newer; + const painted = page.select.rows; + page.pending[0]([twoInputs[0]]); + await older; + + assert.equal(page.select.rows, painted); + assert.equal(page.select.hidden, false); +}); + +// Stranded, the stream has no input. The listing still has to run, or the +// select keeps the failed pick and picking it again fires no `change`. +test("with no input the select offers a placeholder and stays visible", async () => { + const { page, refreshAudioInputs } = listingPage({ track: null }); + const oneDevice = [ + { kind: "audioinput", deviceId: "laptop", label: "Built-in", groupId: "g" }, + ]; + + const listing = refreshAudioInputs(); + page.pending[0](oneDevice); + await listing; + + const [first, ...devices] = page.select.rows; + assert.equal(first.disabled, true); + assert.equal(first.selected, true); + assert.deepEqual( + devices.map((row) => row.value), + ["laptop"], + ); + assert.equal(page.select.hidden, false); +}); + +test("an ended input marks no row as in use", async () => { + const dead = new FakeMic("laptop", "Built-in"); + dead.readyState = "ended"; + const { page, refreshAudioInputs } = listingPage({ track: dead }); + const oneDevice = [ + { kind: "audioinput", deviceId: "laptop", label: "Built-in", groupId: "g" }, + ]; + + const listing = refreshAudioInputs(); + page.pending[0](oneDevice); + await listing; + + assert.equal(page.select.rows[0].disabled, true); + assert.equal(page.select.rows[1].selected, false); + assert.equal(page.select.hidden, false); +}); + +// The loss that failed to move has no `ended` left to follow; a device coming +// back is what gets the candidate heard again. +test("a device change retries the move only while no input is live", () => { + const dead = new FakeMic("laptop"); + dead.readyState = "ended"; + const stranded = listingPage({ track: null }); + const lost = listingPage({ track: dead }); + const fine = listingPage(); + + stranded.retryMicrophone(); + lost.retryMicrophone(); + fine.retryMicrophone(); + + assert.deepEqual(stranded.page.retried, [["", "lost", null]]); + assert.deepEqual(lost.page.retried, [["", "lost", dead]]); + assert.deepEqual(fine.page.retried, []); +}); + +test("two real inputs of one interface both get a row", async () => { + const devices = [ + { kind: "audioinput", deviceId: "in-1", label: "Input 1", groupId: "x" }, + { kind: "audioinput", deviceId: "in-2", label: "Input 2", groupId: "x" }, + ]; + + // Through the page, since counting the select's visibility by group hid + // both rows while the rows themselves were right. + const { page, refreshAudioInputs } = listingPage({ + track: new FakeMic("in-1", "Input 1"), + }); + const listing = refreshAudioInputs(); + page.pending[0](devices); + await listing; + assert.equal(page.select.hidden, false); + + assert.deepEqual( + inputOptions(devices, { deviceId: "in-2", groupId: "x" }).map((row) => [ + row.deviceId, + row.selected, + ]), + [ + ["in-1", false], + ["in-2", true], + ], + ); +}); + +// The default alias of an interface with several inputs could be any of them, +// so its group marks no row and decides nothing. +test("an alias in a group of several inputs names none of them", () => { + const devices = [ + { kind: "audioinput", deviceId: "default", label: "Default", groupId: "x" }, + { kind: "audioinput", deviceId: "in-1", label: "Input 1", groupId: "x" }, + { kind: "audioinput", deviceId: "in-2", label: "Input 2", groupId: "x" }, + ]; + + assert.deepEqual( + inputOptions(devices, { deviceId: "default", groupId: "x" }).filter( + (row) => row.selected, + ), + [], + ); +}); + +test("the picked row's group is passed only when no other row shares it", () => { + const { soleGroupOf } = loadInterview(["soleGroupOf"], {}); + const row = (value, groupId) => ({ value, dataset: { groupId } }); + const select = (picked, rows) => ({ + options: rows, + selectedOptions: [rows.find((each) => each.value === picked)], + }); + const rows = [row("in-1", "x"), row("in-2", "x"), row("headset", "h")]; + + assert.equal(soleGroupOf(select("headset", rows)), "h"); + assert.equal(soleGroupOf(select("in-2", rows)), null); + assert.equal(soleGroupOf(select("x", [row("x", "")])), null); +}); +// The page is what records the dropped input, since the switch only reports it. +test("the page records a working input dropped by a failed switch", () => { + let options = null; + const published = []; + const scope = { + microphoneSwitchInstance: null, + createMicrophoneSwitch: (given) => { + options = given; + return {}; + }, + publishIntegrityEvent: (event) => { + published.push(event); + return Promise.resolve(); + }, + console: { warn: () => {} }, + }; + const { microphoneSwitch } = loadInterview(["microphoneSwitch"], scope, { + stubMissing: true, + }); + microphoneSwitch(); + + options.onFailed("chosen", new Error("x"), "stranded", true); + options.onFailed("lost", new Error("x"), "stranded", false); + options.onFailed("chosen", new Error("x"), "refused", false); + + assert.deepEqual(published, [ + { + type: "MICROPHONE_STOPPED", + source: "microphone", + severity: "high", + detail: "reason=switch_failed", + }, + ]); +}); + +// A live input no row matches, such as a system default whose group no listed +// device shares, is still the microphone in use and is named as such. +test("a live input no row matches is named, not called missing", async () => { + const live = new FakeMic("default", "Default Microphone"); + live.getSettings = () => ({ deviceId: "default", groupId: "virtual" }); + const { page, refreshAudioInputs } = listingPage({ track: live }); + const devices = [ + { + kind: "audioinput", + deviceId: "default", + label: "Default", + groupId: "virtual", + }, + { kind: "audioinput", deviceId: "in-1", label: "Input 1", groupId: "a" }, + { kind: "audioinput", deviceId: "in-2", label: "Input 2", groupId: "b" }, + ]; + + const listing = refreshAudioInputs(); + page.pending[0](devices); + await listing; + + const [first] = page.select.rows; + assert.equal(first.disabled, true); + assert.equal(first.selected, true); + assert.equal(first.textContent, "Default Microphone"); +}); + +// Unplugging down to one input hides the select, and hiding the control that +// has focus drops focus to the body. +test("hiding the focused select moves focus to the Mic button", async () => { + const oneDevice = [ + { kind: "audioinput", deviceId: "laptop", label: "Built-in", groupId: "g" }, + ]; + const run = async (focused) => { + const { page, scope, refreshAudioInputs } = listingPage(); + if (focused) scope.document.activeElement = page.select; + page.select.hidden = false; + const listing = refreshAudioInputs(); + page.pending[0](oneDevice); + await listing; + assert.equal(page.select.hidden, true); + return page.focused; + }; + + assert.deepEqual(await run(true), ["mic"]); + assert.deepEqual(await run(false), []); +}); + +// A status region hidden until it is filled is skipped by screen readers, so +// the words go to one that is always present as well as to the visible note. +test("a note is announced from a region that is never hidden", () => { + const note = { textContent: "", hidden: true }; + const status = { textContent: "", hidden: false }; + const { showInputNote } = loadInterview(["showInputNote"], { + nodes: { micInputNote: note, micInputStatus: status }, + }); + + showInputNote(INPUT_NOTES.refused); + assert.deepEqual( + [note.hidden, note.textContent, status.textContent], + [false, INPUT_NOTES.refused, INPUT_NOTES.refused], + ); + showInputNote(""); + assert.deepEqual( + [note.hidden, note.textContent, status.textContent], + [true, "", ""], + ); + assert.equal(status.hidden, false); + + const markup = read("web/interview.html"); + // Prettier puts each attribute on its own line, so the tag spans several. + const tag = (id) => + markup.match(new RegExp(`<[a-z]+\\s+id="${id}"[^>]*>`))[0]; + assert.match(tag("mic-input-status"), /role="status"/); + assert.doesNotMatch(tag("mic-input-status"), /\shidden[\s>=]/); + assert.doesNotMatch(tag("mic-input-note"), /role=/); +}); + +// LiveKit disables the input still in the stream while a swap holds its flag. +// The trail reads the Mic button for that stretch, so a candidate who never +// muted is not recorded as having switched the microphone off. +test("the integrity trail reads the Mic button while a swap holds the flag", async (t) => { + const integrity = await import("../../web/integrity.js"); + const mic = new FakeMic("laptop"); + const stream = new FakeStream([mic]); + stream.getAudioTracks = () => audioIn(stream); + stream.getVideoTracks = () => []; + const published = []; + const state = { + micEnabled: true, + microphoneSwapHolding: false, + localUserStream: stream, + integrityTrackStates: new Map(), + integrityTrackStateTimer: null, + integrityAnalysisDetail: "x", + }; + integrity.initIntegrity({ + state, + nodes: {}, + updatePresenceBanner: () => {}, + publishIntegrityEvent: (event) => { + published.push(event); + return Promise.resolve(); + }, + endInterview: () => {}, + }); + const microphone = () => + integrity.integrityHeartbeatDetail().match(/microphone=([^;]*)/)[1]; + // The state poll's one-second tick, driven by hand. + let tick = null; + t.mock.method(globalThis, "setInterval", (callback) => { + tick = callback; + return 0; + }); + t.mock.method(globalThis, "clearInterval", () => {}); + integrity.startIntegrityTrackStateMonitor(); + + state.microphoneSwapHolding = true; + mic.enabled = false; + assert.equal(microphone(), "live/on/unmuted"); + tick(); + assert.deepEqual(published, [], "the swap's mute was recorded"); + // A candidate who presses Mic during the swap is reported as muted. + state.micEnabled = false; + assert.equal(microphone(), "live/off/unmuted"); + state.micEnabled = true; + + // Outside a swap the track is what is reported. + state.microphoneSwapHolding = false; + assert.equal(microphone(), "live/off/unmuted"); + tick(); + assert.deepEqual( + published.map((event) => [event.type, event.detail]), + [["MICROPHONE_STATE_CHANGED", "live/off"]], + ); +}); + +test("the page keeps the swap's hold where the integrity trail reads it", () => { + let options = null; + const scope = { + microphoneSwitchInstance: null, + state: { microphoneSwapHolding: false }, + createMicrophoneSwitch: (given) => { + options = given; + return {}; + }, + }; + const { microphoneSwitch } = loadInterview(["microphoneSwitch"], scope, { + stubMissing: true, + }); + microphoneSwitch(); + + options.onHold(true); + assert.equal(scope.state.microphoneSwapHolding, true); + options.onHold(false); + assert.equal(scope.state.microphoneSwapHolding, false); +}); diff --git a/tests/fixtures/integrity-chain.json b/tests/fixtures/integrity-chain.json index c8aeea529..5ddb7eb1c 100644 --- a/tests/fixtures/integrity-chain.json +++ b/tests/fixtures/integrity-chain.json @@ -135,5 +135,17 @@ "444444444444" ], "hash": "a1fefca650b641ce4370bf98be71c940b1f3e79b0bce53b1f3f1fedc194a74fa" + }, + { + "seq": 12, + "prevHash": "a1fefca650b641ce4370bf98be71c940b1f3e79b0bce53b1f3f1fedc194a74fa", + "type": "MICROPHONE_SWITCHED", + "at": "2026-08-18T06:40:30.512Z", + "severity": "info", + "source": "microphone", + "durationMs": 0, + "detail": "reason=chosen;microphone=USB Audio; Rear=Mic (0d8c:0014)", + "sourceEventIds": [], + "hash": "421c6644f15f1acd1c6cd614b55a185f4419cb0ba47d1925d92eeb1e7c2861b3" } ] diff --git a/web/audio-input.js b/web/audio-input.js index 3bb8acced..9fcacf32a 100644 --- a/web/audio-input.js +++ b/web/audio-input.js @@ -1,4 +1,6 @@ -// Microphone choices are limited to preflight, before its stream is published. +// The preflight's microphone picker, which works on the pool before its stream +// is published. Moving the published input once the interview is under way is +// web/mic-switch.js. export function createAudioInput({ mediaDevices, pool, diff --git a/web/devices.js b/web/devices.js index ee010469e..ff958621e 100644 --- a/web/devices.js +++ b/web/devices.js @@ -28,6 +28,39 @@ const CONSTRAINTS = { video: true, }; +/// The microphone constraints with a device named, exactly: a switch the +/// candidate asked for that opened some other input and called it theirs would +/// be worse than one that said it failed. No id asks for the default. +function microphoneConstraints(deviceId) { + if (!deviceId) return CONSTRAINTS.audio; + return { ...CONSTRAINTS.audio, deviceId: { exact: deviceId } }; +} + +// The only capture call in the first-party tree, and the getUserMedia allowlist +// in meet-tab-audio.test.js holds it to that. The preflight pool and a +// mid-interview microphone switch both ask through here, so a switch is a +// second device through the same door rather than a second door. +function capture(mediaDevices, constraints) { + return mediaDevices.getUserMedia(constraints); +} + +/// One live microphone track for an interview already under way, where there +/// is no pool to retry and no preflight to explain a failure: the caller keeps +/// the input it has and says why. Anything else the browser hands back is +/// stopped, so a refusal leaves no device open behind the candidate's back. +export async function openMicrophone(mediaDevices, deviceId) { + const granted = await capture(mediaDevices, { + audio: microphoneConstraints(deviceId), + }); + const tracks = granted.getTracks(); + const track = tracks.find( + (each) => each.kind === "audio" && each.readyState !== "ended", + ); + for (const spare of tracks) if (spare !== track) spare.stop(); + if (!track) throw new Error("no active audio track"); + return track; +} + // `schedule` is injected so a test can fire the retry rather than wait for it. // The bug this guards -- two timers where there should be one -- only shows up // as a count of requests, and a count read off a sleep is a count read off how @@ -106,8 +139,7 @@ export function createDevicePool({ if (device.disabled || device.pending || trackOf(device.kind)) return; device.pending = true; const generation = device.generation; - void mediaDevices - .getUserMedia(device.constraints) + void capture(mediaDevices, device.constraints) .then((granted) => { if ( device.disabled || @@ -181,12 +213,7 @@ export function createDevicePool({ ) return; audio.generation += 1; - audio.constraints = { - audio: { - ...CONSTRAINTS.audio, - ...(deviceId ? { deviceId: { exact: deviceId } } : {}), - }, - }; + audio.constraints = { audio: microphoneConstraints(deviceId) }; audio.error = null; const held = trackOf("audio"); if (held) { diff --git a/web/integrity.js b/web/integrity.js index 12680c701..8eaa64217 100644 --- a/web/integrity.js +++ b/web/integrity.js @@ -101,12 +101,23 @@ export function startIntegrityHeartbeat() { }, INTEGRITY_HEARTBEAT_MS); } +/// While a microphone switch holds LiveKit's mute flag, LiveKit has disabled +/// the input still in the stream. That is the swap's doing, not the +/// candidate's, so for that stretch the Mic button's state is reported instead +/// of the track's, or the trail would read a muted microphone the candidate +/// never muted. +function enabledOf(source, track) { + if (source === "microphone" && state.microphoneSwapHolding) + return state.micEnabled; + return track?.enabled !== false; +} + export function integrityHeartbeatDetail() { const camera = state.localUserStream?.getVideoTracks?.()[0]; const mic = state.localUserStream?.getAudioTracks?.()[0]; - const status = (track) => - `${track?.readyState || "missing"}/${track?.enabled === false ? "off" : "on"}/${track?.muted ? "muted" : "unmuted"}`; - return `camera=${status(camera)};microphone=${status(mic)};analyzer=${state.integrityAnalysisDetail}`; + const status = (source, track) => + `${track?.readyState || "missing"}/${enabledOf(source, track) ? "on" : "off"}/${track?.muted ? "muted" : "unmuted"}`; + return `camera=${status("camera", camera)};microphone=${status("microphone", mic)};analyzer=${state.integrityAnalysisDetail}`; } export function startIntegrityTrackStateMonitor() { @@ -119,7 +130,7 @@ export function startIntegrityTrackStateMonitor() { const poll = () => { for (const [source, track] of tracks()) { if (!track) continue; - const current = `${track.readyState || "missing"}/${track.enabled === false ? "off" : "on"}`; + const current = `${track.readyState || "missing"}/${enabledOf(source, track) ? "on" : "off"}`; const previous = state.integrityTrackStates.get(source); state.integrityTrackStates.set(source, current); if (previous && previous !== current) { diff --git a/web/interview.html b/web/interview.html index 1279e07ad..c97c5cf16 100644 --- a/web/interview.html +++ b/web/interview.html @@ -82,6 +82,16 @@

Loading interview...

+ +
+ + +
diff --git a/web/interview.js b/web/interview.js index b7b1f739b..5f1c62395 100644 --- a/web/interview.js +++ b/web/interview.js @@ -51,8 +51,14 @@ import { isToggleCommentKeystroke, toggleLineComment, } from "./editor.js"; -import { createDevicePool } from "./devices.js"; +import { createDevicePool, openMicrophone } from "./devices.js"; import { createAudioInput } from "./audio-input.js"; +import { + INPUT_NOTES, + createMicrophoneSwitch, + inputOptions, + switchDetail, +} from "./mic-switch.js"; import { createFaceCheck } from "./face-check.js"; import { attachSelfView } from "./self-view.js"; import { createMicMeter, startMediaMeter } from "./mic-meter.js"; @@ -144,6 +150,7 @@ import { PRESENCE_EVENTS, initIntegrity, monitorIntegrityTracks, + watchIntegrityTrack, startIntegrityHeartbeat, startIntegrityWorker, stopIntegrityWorker, @@ -491,6 +498,9 @@ const state = { /// read it from. interviewId: null, micEnabled: true, + /// Up while a microphone switch holds LiveKit's mute flag; web/integrity.js + /// reads the Mic button rather than the track for that stretch. + microphoneSwapHolding: false, localUserStream: null, integrityChain: { seq: 0, hash: "" }, integrityPublish: Promise.resolve(), @@ -646,6 +656,9 @@ const nodes = { preflightOutputNote: document.querySelector("#preflight-output-note"), preflightInput: document.querySelector("#preflight-input"), preflightInputNote: document.querySelector("#preflight-input-note"), + micInputSelect: document.querySelector("#mic-input-select"), + micInputNote: document.querySelector("#mic-input-note"), + micInputStatus: document.querySelector("#mic-input-status"), sidebar: document.querySelector("#interview-sidebar"), sidebarResizer: document.querySelector("#sidebar-resizer"), editorPanel: document.querySelector(".editor-panel"), @@ -1119,6 +1132,14 @@ function bindEvents() { void applyAudioOutput(select.value); }); } + nodes.micInputSelect.addEventListener("change", () => { + void microphoneSwitch().switchTo( + nodes.micInputSelect.value, + "chosen", + null, + soleGroupOf(nodes.micInputSelect), + ); + }); // The checkbox is the only copy of this answer. A mirrored state field had to // be written in two places and read in four, and the one reader that forgot // it published a wrong camera status for the whole interview. @@ -1143,10 +1164,11 @@ function bindEvents() { nodes.meetMode.addEventListener("toggle", () => { if (nodes.meetMode.open) void refreshAudioOutputs(); }); - navigator.mediaDevices?.addEventListener?.( - "devicechange", - () => void refreshAudioOutputs(), - ); + navigator.mediaDevices?.addEventListener?.("devicechange", () => { + void refreshAudioOutputs(); + void refreshAudioInputs(); + retryMicrophone(); + }); nodes.resultsToggle.addEventListener("click", () => setResultsOpen(nodes.resultsBody.hidden), ); @@ -2325,6 +2347,8 @@ async function connectLiveKit(connection, preflight, presenting = false) { nodes.mic.textContent = "Mic on"; updateAgentState(); monitorIntegrityTracks(); + followMicrophoneLoss(state.localUserStream?.getAudioTracks?.()[0]); + void refreshAudioInputs(); await publishIntegrityEvent({ type: "SESSION_START", source: "media", @@ -2450,6 +2474,7 @@ function stopLocalMedia() { stopIntegrityWorker(); stopPreflight({ userStream: state.localUserStream }); state.localUserStream = null; + forgetAudioInputs(); clearInterval(state.integrityHeartbeatTimer); state.integrityHeartbeatTimer = null; clearInterval(state.integrityTrackStateTimer); @@ -2804,6 +2829,9 @@ async function toggleMicrophone() { state.micEnabled = !state.micEnabled; if (state.localUserStream) { setLocalAudioEnabled(state.micEnabled); + // An input a microphone switch is still handing to the sender is in + // neither the stream nor the publication yet. + microphoneSwitch().setEnabled(state.micEnabled); if (!state.micEnabled) { void publishIntegrityEvent({ type: "MICROPHONE_STOPPED", @@ -3287,6 +3315,198 @@ function startTurnRing(stream) { meter.start(); } +/// A switch only ever runs inside a joined room: before it there is no +/// published track to hand a new input to, and after it the device would be +/// opened for an interview that is over. +function microphoneLive() { + return state.phase === "live" && Boolean(state.room); +} + +/// Read through the room on every switch rather than held, because the +/// publication is LiveKit's and the page never owned a handle to it. +function publishedMicrophone() { + const source = window.LivekitClient?.Track?.Source?.Microphone; + return ( + state.room?.localParticipant?.getTrackPublication?.(source)?.track ?? null + ); +} + +/// Built on first use rather than at module scope: the select's listener is +/// bound inside `init`, and a const declared below that call is still in its +/// temporal dead zone while `init` awaits. +let microphoneSwitchInstance = null; + +function microphoneSwitch() { + microphoneSwitchInstance ??= createMicrophoneSwitch({ + open: (deviceId) => openMicrophone(navigator.mediaDevices, deviceId), + streamOf: () => state.localUserStream, + publishedTrack: publishedMicrophone, + isLive: microphoneLive, + micEnabled: () => state.micEnabled, + onHold: (holding) => { + state.microphoneSwapHolding = holding; + }, + onSwitched: (track, reason) => { + // The track the heartbeat, the state poll and the ring read is a new + // object, so each reader is pointed at it here or it keeps reporting the + // one that was stopped. + watchIntegrityTrack(track, "MICROPHONE_STOPPED", "microphone"); + followMicrophoneLoss(track); + startTurnRing(state.localUserStream); + void publishIntegrityEvent({ + type: "MICROPHONE_SWITCHED", + source: "microphone", + severity: "info", + detail: switchDetail(reason, track), + }); + showInputNote(reason === "lost" ? INPUT_NOTES.lost : ""); + void refreshAudioInputs(); + }, + onFailed: (reason, error, outcome, droppedLive) => { + console.warn("codetrial microphone_switch_failed", error); + // The input the interview was using was still working and has been + // dropped, which nothing else in the trail would record: its `stop()` + // fires no `ended`, and the state poll skips a missing track. + if (droppedLive) { + void publishIntegrityEvent({ + type: "MICROPHONE_STOPPED", + source: "microphone", + severity: "high", + detail: "reason=switch_failed", + }); + } + if (reason === "lost") showInputNote(INPUT_NOTES.none); + else if (outcome === "stranded") showInputNote(INPUT_NOTES.stranded); + else showInputNote(INPUT_NOTES.refused); + // A stranded switch dropped the input the ring was reading. + if (outcome === "stranded") startTurnRing(state.localUserStream); + // Puts the select back on the input still in use, rather than on the + // one that was refused. + void refreshAudioInputs(); + }, + }); + return microphoneSwitchInstance; +} + +/// An input that ends under a live interview, unplugged or revoked, leaves the +/// candidate talking into nothing with no prompt to answer. The +/// MICROPHONE_STOPPED its integrity listener publishes stays, because the +/// input really did stop; this only moves the interview to the default. +function followMicrophoneLoss(track) { + track?.addEventListener?.("ended", () => { + if (!microphoneLive()) return; + if (state.localUserStream?.getAudioTracks?.()[0] !== track) return; + void microphoneSwitch().switchTo("", "lost", track); + }); +} + +/// A loss whose move to the default failed leaves nothing listening for the +/// microphone to come back, since the `ended` it followed has already fired. +/// A device appearing is the moment one may be usable again, so the move is +/// tried once more whenever no input is live. +function retryMicrophone() { + if (!microphoneLive()) return; + const held = state.localUserStream?.getAudioTracks?.()[0] ?? null; + if (held && held.readyState !== "ended") return; + void microphoneSwitch().switchTo("", "lost", held); +} + +/// The picked row's group, when no other row shares it. Several real inputs of +/// one audio interface share a group, and then the group cannot say which of +/// them the default alias in use is. +function soleGroupOf(select) { + const group = select.selectedOptions?.[0]?.dataset.groupId; + if (!group) return null; + const rows = [...select.options].filter( + (option) => option.dataset.groupId === group, + ); + return rows.length === 1 ? group : null; +} + +/// Stamps each listing, because `devicechange`, a switch and the join can each +/// start one, and an older listing finishing last would repaint the select +/// with the input that was in use before. +let inputListing = 0; + +async function refreshAudioInputs() { + const request = ++inputListing; + const current = state.localUserStream?.getAudioTracks?.()[0] ?? null; + // No `current` is not a reason to stop: that is the stranded state, where the + // select matters most. + if (!microphoneLive() || !navigator.mediaDevices?.enumerateDevices) return; + let devices = []; + try { + devices = await navigator.mediaDevices.enumerateDevices(); + } catch { + // Listing is a convenience; the input in use keeps working without it. + } + // The interview can end, or a switch replace the input, while the browser + // lists devices; teardown has already hidden the select by then. + if ( + request !== inputListing || + !microphoneLive() || + (state.localUserStream?.getAudioTracks?.()[0] ?? null) !== current + ) + return; + // An ended input is not in use, so it marks no row as the current one. + const inUse = current?.readyState === "live" ? current : null; + const options = inputOptions(devices, { + deviceId: inUse?.getSettings?.().deviceId, + groupId: inUse?.getSettings?.().groupId, + label: inUse?.label, + }); + const rows = options.map((option) => { + const element = document.createElement("option"); + element.value = option.deviceId; + element.dataset.groupId = option.groupId; + element.textContent = option.label; + element.selected = option.selected; + return element; + }); + // With no row for the input in use, the select would show its first device + // as if it were, and picking that row fires no `change`. A disabled + // placeholder in its place makes every device a change. It names the input + // still in use when there is one, such as a system default no row matches, + // so a working microphone does not read as missing. + if (!options.some((option) => option.selected)) { + const placeholder = document.createElement("option"); + placeholder.value = ""; + placeholder.textContent = inUse + ? inUse.label || "System default" + : "Choose a microphone"; + placeholder.disabled = true; + placeholder.selected = true; + rows.unshift(placeholder); + } + nodes.micInputSelect.replaceChildren(...rows); + // One input is nothing to choose between, and a select with a single row + // reads as a control that does nothing. The rows already leave out the + // `default` and `communications` aliases, and keep every real input of a + // group, so they are the count. With no input live it shows regardless, + // since picking is then the way out. + const hide = Boolean(inUse) && options.length < 2; + // Unplugging down to one input can hide the select while it has focus, which + // drops focus to the body; the Mic button beside it takes it first. + if (hide && document.activeElement === nodes.micInputSelect) + nodes.mic.focus({ preventScroll: true }); + nodes.micInputSelect.hidden = hide; +} + +/// Nothing is left to switch once the media stops, so neither the list nor a +/// note about the last switch belongs over the report, and a listing still in +/// flight is told it is stale. +function forgetAudioInputs() { + inputListing += 1; + nodes.micInputSelect.hidden = true; + showInputNote(""); +} + +function showInputNote(text) { + nodes.micInputNote.textContent = text; + nodes.micInputNote.hidden = !text; + nodes.micInputStatus.textContent = text; +} + function applyThinking(thinking) { if (thinking === state.candidateThinking) return; state.candidateThinking = thinking; diff --git a/web/mic-switch.js b/web/mic-switch.js new file mode 100644 index 000000000..2e7fe4d8f --- /dev/null +++ b/web/mic-switch.js @@ -0,0 +1,298 @@ +// Which microphone the interview captures, and moving it to another one +// without leaving the room. +// +// Free of DOM and LiveKit types, like meet-audio.js, so the decisions and the +// order of the swap run under `node --test` instead of being asserted as +// source text. interview.js supplies the stream, the published track, the +// select and the integrity publisher. + +export const INPUT_NOTES = { + refused: + "That microphone could not be opened. Your previous microphone is still in use.", + lost: "Your microphone disconnected. The interview moved to the default microphone.", + none: "Your microphone disconnected and no other microphone could be opened. Plug one in or choose a microphone.", + stranded: + "The microphone could not be switched, and Jim cannot hear you. Choose a microphone again.", +}; + +const ALIAS_IDS = new Set(["default", "communications"]); + +/// Inputs worth offering, the same way `outputOptions` decides outputs: a +/// device with no id is the placeholder a browser reports before a grant, and +/// it would select to "" and switch to whatever the default happens to be. +/// +/// `current` is the track being sent, so the select shows the input in use +/// rather than the one last asked for. Its device id is the first match and +/// its label the fallback: a track opened without a device id can report an +/// id the list does not carry, and an unmatched select shows its first row, +/// which names a microphone the candidate is not using. +export function inputOptions(devices, current) { + // Chrome lists a `default` input beside the physical one, and Windows a + // `communications` one too, all sharing its `groupId`; offered as rows of + // their own, picking the alias of the input in use reopened it and recorded + // a switch that changed nothing. The preflight picker in #411 skips the same + // two ids. Rows are not merged by `groupId` itself, since an audio interface + // or a webcam can put several real inputs in one group. + const inputs = devices.filter( + (device) => + device.kind === "audioinput" && + device.deviceId && + !ALIAS_IDS.has(device.deviceId), + ); + // A track opened as the default reports the alias's id, which no row carries, + // so only then does its group pick the physical row, and only a group with a + // single real input names one. + const byId = inputs.find((device) => device.deviceId === current?.deviceId); + const inGroup = current?.groupId + ? inputs.filter((device) => device.groupId === current.groupId) + : []; + const byGroup = + ALIAS_IDS.has(current?.deviceId) && inGroup.length === 1 + ? inGroup[0] + : null; + const byLabel = current?.label + ? inputs.find((device) => device.label === current.label) + : null; + const selected = byId ?? byGroup ?? byLabel ?? null; + return inputs.map((device, index) => ({ + deviceId: device.deviceId, + groupId: device.groupId || "", + label: device.label || `Microphone ${index + 1}`, + selected: device === selected, + })); +} + +/// The integrity detail for a switch. The label goes last because a device +/// label is the browser's localized text and may itself contain `;` or `=`, +/// which a reader splitting the fields left to right then never reaches. +export function switchDetail(reason, track) { + return `reason=${reason};microphone=${track?.label || "unnamed"}`; +} + +// The publication's own mute flag is a server-side mute: the SFU stops +// forwarding the track, so the agent receives no frames at all, not the +// silence that ends a spoken turn. The Mic button therefore mutes the track +// only, and the flag is raised just around a swap and kept down otherwise. + +/// Raises the flag before LiveKit is handed an input, since LiveKit enables a +/// track it is handed from that flag. It is raised whatever the Mic state: a +/// candidate can press Mic while the replace is pending, and a flag left down +/// would have LiveKit enable the input the button just disabled. +export async function holdMute(published) { + if (!published.isMuted) await published.mute(); +} + +/// Lowers the flag, whether a swap raised it or LiveKit did after a loss, and +/// keeps a muted candidate's input disabled throughout. LiveKit enables the +/// track and announces it in one synchronous step, so the listener puts the +/// button's state back before anything else can run. +export async function liftMute(published, enabled) { + if (!published.isMuted) return; + const keep = () => { + if (published.mediaStreamTrack) + published.mediaStreamTrack.enabled = enabled(); + }; + published.on?.("unmuted", keep); + try { + await published.unmute(); + } finally { + published.off?.("unmuted", keep); + } + keep(); +} + +const audioTrackOf = (stream) => + stream?.getTracks?.().find((track) => track.kind === "audio") ?? null; + +const deviceIdOf = (track) => track?.getSettings?.().deviceId ?? null; +const groupIdOf = (track) => track?.getSettings?.().groupId || null; + +/// `reason` is "chosen" for a pick from the select and "lost" when the input +/// in use ended, in which case `lost` is the track that ended. Every outcome +/// but "switched" leaves the stream, the published track and the device +/// exactly as they were. +/// +/// Serialized like `applyAudioOutput`: a candidate can pick again while the +/// permission prompt for the last pick is still up, and two swaps resolving in +/// either order would leave the stream holding one track while LiveKit sends +/// the other. +export function createMicrophoneSwitch({ + open, + streamOf, + publishedTrack, + isLive, + micEnabled, + onHold = () => {}, + onSwitched, + onFailed, +}) { + let chain = Promise.resolve(); + // The input being handed to the sender. Between the open and the commit it + // is in neither the page's stream nor reach of the Mic button, so a mute + // pressed meanwhile has to be applied to it here. + let inFlight = null; + + const run = async (deviceId, reason, lost, groupId) => { + const stream = streamOf(); + if (!isLive() || !stream) return "idle"; + // Every await below can outlast the interview or the stream it started + // with, and whatever it would then show or keep open belongs to neither. + const current = () => isLive() && streamOf() === stream; + const held = audioTrackOf(stream); + if ( + reason === "chosen" && + held?.readyState === "live" && + (deviceIdOf(held) === deviceId || + (ALIAS_IDS.has(deviceIdOf(held)) && + groupId && + groupIdOf(held) === groupId)) + ) + return "unchanged"; + // A loss queued behind a pick that has since replaced the lost track would + // otherwise move the candidate off the input they just chose. Only another + // track means that: an empty stream is a loss whose earlier recovery + // stranded, and the device that set off this retry may be the way back. + if (reason === "lost" && held && held !== lost) return "unchanged"; + // Checked before the prompt, not after: replacing a track that was never + // published sends nothing, and the candidate would be told the switch + // worked while Jim kept hearing silence. + if (!publishedTrack()) return "idle"; + + // No input is being sent. Dropping the old one from the stream, dead or + // not, keeps the page from claiming it is in use and lets the next pick, + // even of the same device, open a fresh input rather than read as + // unchanged. + const strand = (error) => { + // `stop()` fires no `ended`, so an input still working when it is + // dropped here leaves no trace in the integrity trail unless the caller + // records one; an input that had already ended recorded its own. + const droppedLive = held?.readyState === "live"; + if (held) { + stream.removeTrack(held); + held.stop(); + } + onFailed(reason, error, "stranded", droppedLive); + return "stranded"; + }; + + let track; + try { + track = await open(deviceId); + inFlight = track; + } catch (error) { + if (!current()) return "idle"; + if (held?.readyState !== "live") return strand(error); + onFailed(reason, error, "refused"); + return "refused"; + } + // The prompt can outlast the interview, or the room that published the old + // track, and a device opened for neither would stay on behind the report. + const published = publishedTrack(); + if (!current() || !published) { + track.stop(); + return "idle"; + } + + // Everything that can fail once the sender may hold the new input sits in + // one try, so every failure reaches the hand-back below rather than escaping + // with the stream and the sender on different tracks. + let failure = null; + try { + // `userProvidedTrack` keeps LiveKit from stopping a track this page owns + // when the next switch replaces it. The page stops the old one itself + // below, after the new one is already being sent, so the swap has no gap. + // + // A fresh track starts enabled, and LiveKit enables it again from the + // publication's flag inside the replace, so the flag is held up across + // the swap and the new input is never live before the Mic state says. + track.enabled = micEnabled(); + // Up until the switch is over, the hand-back included: LiveKit's mute + // disables the input still in the stream, which the integrity trail + // would otherwise record as the candidate's. + onHold(true); + await holdMute(published); + await published.replaceTrack(track, { userProvidedTrack: true }); + // Down again afterwards, also when LiveKit raised it after a loss, or the + // agent would hear nothing rather than a muted candidate's silence. + await liftMute(published, micEnabled); + } catch (error) { + failure = error; + } + if (!current()) { + track.stop(); + return "idle"; + } + // An input unplugged during those awaits has already fired `ended`, before + // anything was listening for it, so committing it would leave the + // interview silent with no loss left to follow. + if (!failure && track.readyState === "ended") + failure = new Error("the new input ended during the switch"); + + if (!failure) { + track.enabled = micEnabled(); + if (held) { + stream.removeTrack(held); + // `stop()` fires no `ended`, so the old track's integrity listener + // stays quiet. A switch is not a microphone stopping; the event the + // caller publishes for it says what happened instead. + held.stop(); + } + stream.addTrack(track); + onSwitched(track, reason); + return "switched"; + } + + track.stop(); + // The sender goes back to the input still working, where there is one. A + // rejected replace may already have handed the new input to the sender, + // so the hand-back runs whatever failed; when the sender never left the + // old input, LiveKit returns from replacing a track with itself at once. + let restored = false; + if (held?.readyState === "live") { + try { + held.enabled = micEnabled(); + await holdMute(published); + await published.replaceTrack(held, { userProvidedTrack: true }); + // LiveKit was already listening when the new input ended, so it may + // have muted the publication, which would keep the restored input + // silent to the interviewer while the page reads "Mic on". + await liftMute(published, micEnabled); + held.enabled = micEnabled(); + restored = true; + } catch (error) { + failure = error; + } + if (!current()) return "idle"; + } + if (restored) { + onFailed(reason, failure, "refused"); + return "refused"; + } + return strand(failure); + }; + + return { + /// The Mic button's other half: the input in the stream it reaches itself. + setEnabled(enabled) { + if (inFlight && inFlight.readyState !== "ended") + inFlight.enabled = enabled; + }, + /// `groupId` is the picked row's group, passed only when no other row + /// shares it. It decides "unchanged" only when the input in use was opened + /// under an alias id, which no row carries. + switchTo(deviceId, reason = "chosen", lost = null, groupId = null) { + const next = chain + .then(() => run(deviceId, reason, lost, groupId)) + .finally(() => { + inFlight = null; + // Every way out of a switch, the hand-back and the stranded ones + // included, ends the stretch in which the swap held the flag. + onHold(false); + }); + // A throw from a caller's callback must not wedge every later switch + // behind a rejected promise. + chain = next.catch(() => {}); + return next; + }, + }; +} diff --git a/web/styles.css b/web/styles.css index 46d6a882b..af70cdc14 100644 --- a/web/styles.css +++ b/web/styles.css @@ -1446,6 +1446,25 @@ p { color: white; } +/* Sized as a pill so it sits in the control row beside the Mic button it + belongs to; capped because device labels run long and would push the row's + other controls onto a second line. Bordered like the other form controls, + since a select whose edge is the hairline does not read as one. */ +.mic-input-select { + max-width: 14rem; + min-height: 2rem; + padding: 0.375rem 0.75rem; + border: 1px solid var(--control-border); + border-radius: 999px; + background: var(--inset-surface); + color: var(--sub); + font: inherit; + font-size: 0.75rem; + font-weight: 700; + text-overflow: ellipsis; + cursor: pointer; +} + .tab-row { min-height: 2.5rem; padding: 0 0.75rem;