diff --git a/docs/integrity-evidence.md b/docs/integrity-evidence.md index 6fba4ba4d..5ad7a26ce 100644 --- a/docs/integrity-evidence.md +++ b/docs/integrity-evidence.md @@ -26,6 +26,41 @@ the camera required, because the recording notice describes a video recording. - `declined`: the candidate skipped a camera with no error, or with an error that names none of the causes above. +## Microphone switches + +A candidate can move the interview to another microphone from the interview +page, and the page does so by itself when the input in use ends. Either way, +once the new input is in use, the browser records a signed `MICROPHONE_SWITCHED` +event, because the heartbeat that follows names a live microphone and would +otherwise read as the one the interview started with. Its detail is +`reason=chosen` or `reason=lost`, then `microphone=` and the new device's label, +last because a label is the browser's text and may contain the separators. A +switch is not a microphone stopping, so it records no `MICROPHONE_STOPPED`; an +input that ended before the page switched away from it still records one, since +it did stop. + +A chosen switch that fails leaves the previous input in use and records nothing, +unless the previous input cannot be handed back to the sender either; the page +then drops it, says the interviewer cannot hear the candidate, and asks for a +microphone again. That drop records `MICROPHONE_STOPPED` with +`reason=switch_failed`, because stopping a working input fires no `ended` and +the state poll skips a missing track, so nothing else would. A move after a lost +input that fails leaves the interview with no working microphone; the page says +so, records no switch, and tries the move again whenever a device is plugged in, +while the `MICROPHONE_STOPPED` for the loss stays the last word on the input +that ended. + +While a switch is under way, LiveKit's own mute disables the input still in +use, so that the replacement is not sent before the Mic button says. For that +stretch the heartbeat and the state poll report the Mic button's state instead +of the input's, since a `MICROPHONE_STATE_CHANGED` to `off` there would record +the switch muting a candidate who never pressed Mic. + +The server drops an integrity event whose type it does not list before it +checks the chain, so that event's sequence number is never accepted and every +later event is refused as out of sequence. A new event type is therefore added +to `sanitize_integrity_event` in the same change that starts sending it. + ## Response windows The replay page lists a *response window* for each turn the interviewer took: diff --git a/scripts/gen-wire-fixtures.mjs b/scripts/gen-wire-fixtures.mjs index 082f2f455..88685fcbb 100755 --- a/scripts/gen-wire-fixtures.mjs +++ b/scripts/gen-wire-fixtures.mjs @@ -24,6 +24,7 @@ const { reportRecoveryLimits } = await import( join(ROOT, "web", "report-recovery.js") ); const lib = await import(join(ROOT, "web", "lib.js")); +const { switchDetail } = await import(join(ROOT, "web", "mic-switch.js")); const { ALL_LANGUAGES } = await import( join(ROOT, "web", "compiler-explorer.js") ); @@ -361,6 +362,20 @@ const INTEGRITY_INPUTS = [ "5555555555555", ], }, + // Last, so the rows above keep their hashes. A type the agent's allowlist + // lacks is refused and stops the chain, which is how a switch the browser + // records would otherwise vanish from every report. The detail comes from the + // producer the page uses, with a label carrying the separators it is placed + // last to survive. + { + type: "MICROPHONE_SWITCHED", + source: "microphone", + severity: "info", + at: "2026-08-18T06:40:30.512Z", + detail: switchDetail("chosen", { + label: "USB Audio; Rear=Mic (0d8c:0014)", + }), + }, ]; async function integrityChain() { diff --git a/src/agent/integrity.rs b/src/agent/integrity.rs index dfee02669..3ab66aa1c 100644 --- a/src/agent/integrity.rs +++ b/src/agent/integrity.rs @@ -276,6 +276,12 @@ pub fn sanitize_integrity_event(payload: &serde_json::Value) -> Option "CAMERA_RELEASED_TO_PRESENTER", "CAMERA_NOT_USED" => "CAMERA_NOT_USED", "MICROPHONE_STATE_CHANGED" => "MICROPHONE_STATE_CHANGED", + + // The candidate moved the interview to another input, or the page did + // after the one in use ended. Without it the heartbeat goes on naming a + // live microphone and nothing says it is no longer the one the + // interview started with. + "MICROPHONE_SWITCHED" => "MICROPHONE_SWITCHED", _ => return None, }; let at = text("at")?; diff --git a/tests/browser/devices.test.js b/tests/browser/devices.test.js index 4da35e698..c0d9cd413 100644 --- a/tests/browser/devices.test.js +++ b/tests/browser/devices.test.js @@ -2,7 +2,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { FakeStream, FakeTrack, settle } from "./fake-media.js"; -import { createDevicePool } from "../../web/devices.js"; +import { createDevicePool, openMicrophone } from "../../web/devices.js"; function poolWith(getUserMedia, options = {}) { const changes = { count: 0 }; @@ -571,3 +571,62 @@ test("the pool's own drop of an ended track also removes before notifying", asyn finished = true; pool.cancelRetry(); }); + +// The switch the candidate asked for names its device exactly: opening some +// other input and calling it theirs would be worse than saying it failed. +test("a mid-interview microphone is asked for by exact device", async () => { + const asked = []; + const track = new FakeTrack("audio"); + const opened = await openMicrophone( + { + getUserMedia: async (constraints) => { + asked.push(constraints); + return new FakeStream([track]); + }, + }, + "headset", + ); + + assert.equal(opened, track); + assert.deepEqual(asked, [ + { + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + deviceId: { exact: "headset" }, + }, + }, + ]); +}); + +test("the default microphone is asked for with no device id", async () => { + const asked = []; + await openMicrophone( + { + getUserMedia: async (constraints) => { + asked.push(constraints); + return new FakeStream([new FakeTrack("audio")]); + }, + }, + "", + ); + + assert.equal("deviceId" in asked[0].audio, false); +}); + +test("a grant with no live input is refused and leaves nothing open", async () => { + const spare = new FakeTrack("video"); + const dead = new FakeTrack("audio"); + dead.readyState = "ended"; + + await assert.rejects( + openMicrophone( + { getUserMedia: async () => new FakeStream([dead, spare]) }, + "headset", + ), + /no active audio track/, + ); + assert.ok(spare.stopped, "a track nothing will read was left running"); + assert.ok(dead.stopped); +}); diff --git a/tests/browser/mic-switch.test.js b/tests/browser/mic-switch.test.js new file mode 100644 index 000000000..a6c27c2fe --- /dev/null +++ b/tests/browser/mic-switch.test.js @@ -0,0 +1,1426 @@ +// Run with: node --test tests/browser/mic-switch.test.js +// +// Switching the microphone is an ordering problem: open the new input, hand it +// to the published track, then retire the old one, and back out cleanly at +// every step that can fail. These drive the real switch against fakes for the +// stream and the LiveKit track, so each step is executed rather than read. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { FakeStream, FakeTrack } from "./fake-media.js"; +import { loadInterview, read } from "./source.js"; + +import { + INPUT_NOTES, + createMicrophoneSwitch, + holdMute, + inputOptions, + liftMute, + switchDetail, +} from "../../web/mic-switch.js"; + +/// A microphone that reports the device it opened, which is what the switch +/// compares a pick against and what the select marks as in use. +class FakeMic extends FakeTrack { + constructor(deviceId, label = deviceId) { + super("audio"); + this.deviceId = deviceId; + this.label = label; + } + getSettings() { + return { deviceId: this.deviceId }; + } +} + +/// Stands in for LiveKit's LocalAudioTrack. Like the real one, it re-enables +/// the track it is handed from its own mute flag, which the page never sets. +class FakePublished { + constructor(track) { + this.sending = track; + this.calls = []; + this.refuse = false; + // Refuses every call after this many, so a test can fail only a later one. + this.refuseAfter = Infinity; + // Set by a test to hold the replace open while it changes the session. + this.pending = null; + // Whether each track was enabled at the moment the sender took it. + this.sentEnabled = []; + // LiveKit's own mute flag, which it raises when a user-provided track ends. + this.isMuted = false; + this.liveAfterReplace = []; + this.unmutes = 0; + this.liveAfterUnmute = []; + this.listeners = {}; + } + async replaceTrack(track, options) { + this.calls.push(options); + this.sentEnabled.push(track.enabled); + // LiveKit's setMediaStreamTrack returns at once for the track it already + // holds, so a hand-back to an input never replaced cannot fail. + if (track === this.sending) return; + if (this.pending) await this.pending; + if (this.refuse || this.calls.length > this.refuseAfter) + throw new Error("sender closed"); + // What LiveKit does when a user-provided track it is listening to ends. + if (this.muteOnEnded && track.readyState === "ended") this.isMuted = true; + // LiveKit hands the track to the sender and then resumes upstream, which + // can still reject with the new track already being sent. + // One-shot, so only the replace under test rejects. + if (this.rejectAfterSend) { + this.rejectAfterSend = false; + this.sending = track; + throw new Error("resume upstream failed"); + } + track.enabled = !this.isMuted; + // What the sender was given to send once LiveKit had set it up, which is + // the window a muted candidate must not be live in. + this.liveAfterReplace.push(track.enabled); + this.sending = track; + } + async mute() { + this.isMuted = true; + this.sending.enabled = false; + } + // LiveKit's setTrackMuted: enable the track, then announce it in the same + // synchronous step. + async unmute() { + if (this.unmutePending) await this.unmutePending; + if (this.refuseUnmute) throw new Error("unmute refused"); + this.unmutes += 1; + this.isMuted = false; + this.sending.enabled = true; + for (const listener of [...(this.listeners.unmuted || [])]) listener(this); + // What is left enabled once LiveKit's own step is over. + this.liveAfterUnmute.push(this.sending.enabled); + } + get mediaStreamTrack() { + return this.sending; + } + on(name, listener) { + (this.listeners[name] ||= []).push(listener); + } + off(name, listener) { + this.listeners[name] = (this.listeners[name] || []).filter( + (each) => each !== listener, + ); + } +} + +function setup({ open, live = true, muted = false, published = true } = {}) { + const old = new FakeMic("laptop"); + const stream = new FakeStream([old]); + const sender = published ? new FakePublished(old) : null; + const state = { + live, + muted, + opened: [], + switched: [], + failed: [], + outcomes: [], + dropped: [], + holding: false, + }; + const microphone = createMicrophoneSwitch({ + open: async (deviceId) => { + state.opened.push(deviceId); + return open ? open(deviceId) : new FakeMic(deviceId); + }, + streamOf: () => stream, + publishedTrack: () => sender, + isLive: () => state.live, + micEnabled: () => !state.muted, + onHold: (holding) => { + state.holding = holding; + }, + onSwitched: (track, reason) => state.switched.push({ track, reason }), + onFailed: (reason, error, outcome, droppedLive) => { + state.failed.push(reason); + state.outcomes.push(outcome); + state.dropped.push(droppedLive); + }, + }); + return { microphone, old, stream, sender, state }; +} + +const audioIn = (stream) => + stream.getTracks().filter((t) => t.kind === "audio"); + +test("a chosen input replaces the sent track and retires the old one", async () => { + const { microphone, old, stream, sender, state } = setup(); + + assert.equal(await microphone.switchTo("headset"), "switched"); + + const [now] = audioIn(stream); + assert.equal(now.deviceId, "headset"); + assert.equal(audioIn(stream).length, 1, "the stream holds exactly one input"); + assert.equal(sender.sending, now, "LiveKit sends the input the stream holds"); + assert.deepEqual(sender.calls, [{ userProvidedTrack: true }]); + assert.ok(old.stopped, "the previous device stays open"); + const switched = state.switched.map((s) => [s.reason, s.track.deviceId]); + assert.deepEqual(switched, [["chosen", "headset"]]); +}); + +// The Mic button mutes by disabling the track. LiveKit's replace re-enables +// whatever it is given, so a muted candidate who switches would be live again. +test("a muted microphone stays muted across a switch", async () => { + const { microphone, stream, sender } = setup({ muted: true }); + + await microphone.switchTo("headset"); + + assert.equal(audioIn(stream)[0].enabled, false); + assert.deepEqual( + sender.sentEnabled, + [false], + "the sender took the new input while it was still enabled", + ); + // LiveKit enables the new track from its own flag inside the replace, so the + // flag has to read muted by then, or the input is live until the page + // disables it again. + assert.deepEqual(sender.liveAfterReplace, [false]); + // And down again afterwards, without the input going live as it comes down: + // a raised flag is a server-side mute the agent hears as no frames at all. + assert.equal(sender.isMuted, false); + assert.deepEqual(sender.liveAfterUnmute, [false]); +}); + +test("an input that will not open leaves the current one in use", async () => { + const { microphone, old, stream, sender, state } = setup({ + open: () => { + throw new Error("NotReadableError"); + }, + }); + + assert.equal(await microphone.switchTo("headset"), "refused"); + + assert.deepEqual(audioIn(stream), [old]); + assert.equal(old.stopped, false); + assert.equal(sender.sending, old); + assert.deepEqual(state.failed, ["chosen"]); + assert.deepEqual(state.switched, []); +}); + +test("a refused replace stops the new device and keeps the old", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + sender.refuse = true; + + assert.equal(await microphone.switchTo("headset"), "refused"); + + assert.ok(opened[0].stopped, "the device opened for the switch was left on"); + assert.deepEqual(audioIn(stream), [old]); + assert.equal(old.stopped, false); + assert.deepEqual(state.failed, ["chosen"]); +}); + +// A permission prompt can stay up past the end of the interview. +test("an input granted after the interview ended is stopped, not swapped in", async () => { + let grant = null; + const pending = new FakeMic("headset"); + const { microphone, old, stream, state } = setup({ + open: () => + new Promise((resolve) => { + grant = () => resolve(pending); + }), + }); + + const result = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + state.live = false; + grant(); + + assert.equal(await result, "idle"); + assert.ok(pending.stopped); + assert.deepEqual(audioIn(stream), [old]); + assert.deepEqual(state.switched, []); +}); + +test("nothing is opened while there is no published track to hand it to", async () => { + const { microphone, state } = setup({ published: false }); + + assert.equal(await microphone.switchTo("headset"), "idle"); + assert.deepEqual(state.opened, []); +}); + +test("picking the input already in use opens nothing", async () => { + const { microphone, state } = setup(); + + assert.equal(await microphone.switchTo("laptop"), "unchanged"); + assert.deepEqual(state.opened, []); +}); + +// Two picks while the first prompt is still up. Unserialized, the first could +// resolve last and leave the stream and LiveKit holding different inputs. +test("rapid picks resolve in the order they were made", async () => { + const grants = []; + const tracks = []; + const { microphone, stream, sender } = setup({ + open: (deviceId) => + new Promise((resolve) => { + const track = new FakeMic(deviceId); + tracks.push(track); + grants.push(() => resolve(track)); + }), + }); + + const first = microphone.switchTo("headset"); + const second = microphone.switchTo("webcam"); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(grants.length, 1, "the second pick waits for the first"); + grants[0](); + assert.equal(await first, "switched"); + await new Promise((resolve) => setTimeout(resolve, 0)); + grants[1](); + assert.equal(await second, "switched"); + + assert.equal(audioIn(stream)[0].deviceId, "webcam"); + assert.equal(sender.sending, audioIn(stream)[0]); + assert.ok(tracks[0].stopped, "the input switched away from stays open"); +}); + +test("an input that ended moves the interview to the default", async () => { + const { microphone, old, stream, state } = setup(); + old.readyState = "ended"; + + assert.equal(await microphone.switchTo("", "lost", old), "switched"); + + assert.deepEqual(state.opened, [""], "the default is asked for by no id"); + assert.equal(audioIn(stream).length, 1); + assert.notEqual(audioIn(stream)[0], old); + assert.equal(state.switched[0].reason, "lost"); +}); + +// The pick replaced the track that later ended, so the loss is about an input +// the interview no longer uses. +test("a loss queued behind a pick leaves the picked input in use", async () => { + let grant = null; + const { microphone, old, stream, state } = setup({ + // Only the pick waits on its prompt, so a loss that wrongly runs resolves + // and fails the assertions rather than hanging the test. + open: (deviceId) => + grant + ? new FakeMic(deviceId) + : new Promise((resolve) => { + grant = () => resolve(new FakeMic(deviceId)); + }), + }); + + const picked = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + old.readyState = "ended"; + const lost = microphone.switchTo("", "lost", old); + grant(); + + assert.equal(await picked, "switched"); + assert.equal(await lost, "unchanged"); + assert.equal(audioIn(stream)[0].deviceId, "headset"); + assert.deepEqual(state.opened, ["headset"]); +}); + +// LiveKit's replace can be slow, and the interview can end inside it. +test("an interview that ends during the replace keeps nothing open", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + let release = null; + sender.pending = new Promise((resolve) => { + release = resolve; + }); + + const result = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + state.live = false; + release(); + + assert.equal(await result, "idle"); + assert.ok(opened[0].stopped, "the input opened for the switch stayed on"); + assert.deepEqual(audioIn(stream), [old]); + assert.deepEqual(state.switched, []); +}); + +test("a refusal that arrives after the interview ended is not reported", async () => { + let refuse = null; + const { microphone, state } = setup({ + open: () => + new Promise((_, reject) => { + refuse = () => reject(new Error("NotAllowedError")); + }), + }); + + const result = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + state.live = false; + refuse(); + + assert.equal(await result, "idle"); + assert.deepEqual(state.failed, []); +}); + +// Unplugged while the replace was pending: its `ended` fired before anything +// listened, so committing it would leave the interview silent for good. +test("an input that ends during the replace hands back the old one", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + let release = null; + sender.pending = new Promise((resolve) => { + release = resolve; + }); + + const result = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + sender.pending = null; + opened[0].readyState = "ended"; + release(); + + assert.equal(await result, "refused"); + assert.equal(sender.sending, old, "the sender kept the dead input"); + assert.equal(old.stopped, false); + assert.deepEqual(audioIn(stream), [old]); + assert.deepEqual(state.failed, ["chosen"]); + assert.deepEqual(state.switched, []); +}); + +// A device plugged in while the move after a loss is still opening queues a +// retry for the same loss; if the first attempt strands, that retry is the way +// back and must not read as already handled. +test("a retry queued behind a stranded loss still tries", async () => { + let attempts = 0; + let refuse = null; + const { microphone, old, stream, state } = setup({ + open: (deviceId) => { + attempts += 1; + if (attempts > 1) return new FakeMic("headset"); + return new Promise((_, reject) => { + refuse = () => reject(new Error("NotFoundError")); + }); + }, + }); + old.readyState = "ended"; + + const first = microphone.switchTo("", "lost", old); + const retry = microphone.switchTo("", "lost", old); + await new Promise((resolve) => setTimeout(resolve, 0)); + refuse(); + + assert.equal(await first, "stranded"); + assert.equal(await retry, "switched"); + assert.equal(audioIn(stream)[0].deviceId, "headset"); + assert.deepEqual(state.outcomes, ["stranded"]); +}); + +// The track opened as the default reports the alias's id; picking the +// physical row of the same device is not a switch. Two real inputs of one +// interface share a group too, and moving between them is. +test("a pick in the input's own group is unchanged", async () => { + const old = new FakeMic("default"); + old.getSettings = () => ({ deviceId: "default", groupId: "h" }); + const stream = new FakeStream([old]); + const sender = new FakePublished(old); + const opened = []; + const microphone = createMicrophoneSwitch({ + open: async (deviceId) => { + opened.push(deviceId); + return new FakeMic(deviceId); + }, + streamOf: () => stream, + publishedTrack: () => sender, + isLive: () => true, + micEnabled: () => true, + onSwitched: () => {}, + onFailed: () => {}, + }); + + assert.equal( + await microphone.switchTo("headset", "chosen", null, "h"), + "unchanged", + ); + assert.deepEqual(opened, []); + assert.equal( + await microphone.switchTo("laptop", "chosen", null, "l"), + "switched", + ); +}); + +test("two real inputs in one group are different inputs", async () => { + const old = new FakeMic("input-1"); + old.getSettings = () => ({ deviceId: "input-1", groupId: "interface" }); + const stream = new FakeStream([old]); + const sender = new FakePublished(old); + const microphone = createMicrophoneSwitch({ + open: async (deviceId) => new FakeMic(deviceId), + streamOf: () => stream, + publishedTrack: () => sender, + isLive: () => true, + micEnabled: () => true, + onSwitched: () => {}, + onFailed: () => {}, + }); + + assert.equal( + await microphone.switchTo("input-2", "chosen", null, "interface"), + "switched", + ); +}); + +// Dropping an input that still worked is the one stop the trail would not see: +// `stop()` fires no `ended`. One that had already ended recorded its own. +test("a dropped input reports whether it was still working", async () => { + const live = setup(); + live.sender.isMuted = true; + live.sender.refuseUnmute = true; + await live.microphone.switchTo("headset"); + assert.deepEqual(live.state.outcomes, ["stranded"]); + assert.deepEqual(live.state.dropped, [true]); + + const dead = setup({ + open: () => { + throw new Error("NotFoundError"); + }, + }); + dead.old.readyState = "ended"; + await dead.microphone.switchTo("", "lost", dead.old); + assert.deepEqual(dead.state.outcomes, ["stranded"]); + assert.deepEqual(dead.state.dropped, [false]); +}); + +// LiveKit was already listening to the new input when it ended, so it muted the +// publication; handing back the old input must clear that or Jim hears nothing. +test("handing back the old input clears LiveKit's mute flag", async () => { + const opened = []; + const { microphone, old, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + sender.muteOnEnded = true; + let release = null; + sender.pending = new Promise((resolve) => { + release = resolve; + }); + + const result = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + sender.pending = null; + opened[0].readyState = "ended"; + release(); + + assert.equal(await result, "refused"); + assert.equal(sender.sending, old); + assert.equal(sender.isMuted, false, "the restored input stays muted"); + assert.deepEqual(state.outcomes, ["refused"]); +}); + +test("a replace that rejects after sending still hands back", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + sender.rejectAfterSend = true; + + assert.equal(await microphone.switchTo("headset"), "refused"); + assert.equal(sender.sending, old, "LiveKit kept sending the stopped input"); + assert.ok(opened[0].stopped); + assert.deepEqual(audioIn(stream), [old]); + assert.deepEqual(state.outcomes, ["refused"]); +}); + +// The sender is then left on the dead input, so claiming the old one is still +// in use would be false, and keeping it in the stream would make a pick of the +// same device read as unchanged. +test("a failed hand-back says so and lets the same device be picked", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + let release = null; + sender.pending = new Promise((resolve) => { + release = resolve; + }); + + const result = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + sender.pending = null; + sender.refuseAfter = 1; + opened[0].readyState = "ended"; + release(); + + assert.equal(await result, "stranded"); + assert.deepEqual(state.outcomes, ["stranded"]); + assert.ok(old.stopped); + assert.deepEqual(audioIn(stream), []); + + sender.refuseAfter = Infinity; + assert.equal(await microphone.switchTo("laptop"), "switched"); + assert.equal(sender.sending, audioIn(stream)[0]); +}); + +// The unmute after a loss is another await the interview can end inside. +test("an interview that ends during the unmute keeps nothing open", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + old.readyState = "ended"; + sender.isMuted = true; + let release = null; + sender.unmutePending = new Promise((resolve) => { + release = resolve; + }); + + const result = microphone.switchTo("", "lost", old); + await new Promise((resolve) => setTimeout(resolve, 0)); + state.live = false; + release(); + + assert.equal(await result, "idle"); + assert.ok(opened[0].stopped); + assert.deepEqual(audioIn(stream), [old]); + assert.deepEqual(state.failed, []); +}); + +test("a refused unmute leaves no input claimed as in use", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + sender.isMuted = true; + sender.refuseUnmute = true; + + assert.equal(await microphone.switchTo("headset"), "stranded"); + // The hand-back needs the same unmute, so with it refused nothing is sent. + assert.ok(opened[0].stopped); + assert.ok(old.stopped, "the old input is still open but nothing sends it"); + assert.deepEqual(audioIn(stream), []); + assert.deepEqual(state.outcomes, ["stranded"]); + + sender.refuseUnmute = false; + sender.isMuted = false; + assert.equal(await microphone.switchTo("laptop"), "switched"); +}); + +test("an input that ends during the unmute is not committed", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + sender.isMuted = true; + let release = null; + sender.unmutePending = new Promise((resolve) => { + release = resolve; + }); + + const result = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + sender.unmutePending = null; + opened[0].readyState = "ended"; + release(); + + assert.equal(await result, "refused"); + assert.equal(sender.sending, old); + assert.deepEqual(audioIn(stream), [old]); + assert.deepEqual(state.outcomes, ["refused"]); +}); + +// With the old input already dead there is nothing to hand back to, so the +// page must not claim one is still in use. +test("a failed move after a loss strands rather than refuses", async () => { + const { microphone, old, stream, state } = setup({ + open: () => { + throw new Error("NotFoundError"); + }, + }); + old.readyState = "ended"; + + assert.equal(await microphone.switchTo("", "lost", old), "stranded"); + assert.deepEqual(audioIn(stream), []); + assert.deepEqual(state.outcomes, ["stranded"]); +}); + +test("a new input that ends after a loss strands too", async () => { + const opened = []; + const { microphone, old, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + old.readyState = "ended"; + let release = null; + sender.pending = new Promise((resolve) => { + release = resolve; + }); + + const result = microphone.switchTo("", "lost", old); + await new Promise((resolve) => setTimeout(resolve, 0)); + sender.pending = null; + opened[0].readyState = "ended"; + release(); + + assert.equal(await result, "stranded"); + assert.deepEqual(audioIn(stream), []); + assert.deepEqual(state.outcomes, ["stranded"]); +}); + +// LiveKit mutes the publication when a user-provided track ends, so after a +// loss the interviewer would receive a muted microphone while the page reads +// "Mic on". +test("a move after a loss clears LiveKit's mute flag", async () => { + const { microphone, old, stream, sender } = setup(); + old.readyState = "ended"; + sender.isMuted = true; + + assert.equal(await microphone.switchTo("", "lost", old), "switched"); + + assert.equal(sender.isMuted, false); + assert.equal(audioIn(stream)[0].enabled, true); +}); + +// The flag LiveKit raised after the loss comes down, since left up it is a +// server-side mute; a candidate who had muted themselves stays muted. +test("a muted candidate stays muted after a loss", async () => { + const { microphone, old, stream, sender } = setup({ muted: true }); + old.readyState = "ended"; + sender.isMuted = true; + + await microphone.switchTo("", "lost", old); + + assert.equal(sender.isMuted, false); + assert.deepEqual(sender.liveAfterReplace, [false]); + assert.deepEqual( + sender.liveAfterUnmute, + [false], + "the input went live while the flag came down", + ); + assert.equal(audioIn(stream)[0].enabled, false); +}); + +test("the flag is held for every swap and lifted for anyone", async () => { + const sender = new FakePublished(new FakeMic("laptop")); + + await holdMute(sender); + assert.equal(sender.isMuted, true); + + await liftMute(sender, () => false); + assert.equal(sender.isMuted, false); + assert.deepEqual(sender.liveAfterUnmute, [false]); + + sender.isMuted = true; + await liftMute(sender, () => true); + assert.deepEqual(sender.liveAfterUnmute, [false, true]); + // Nothing to lower: no unmute, so nothing goes live. + await liftMute(sender, () => true); + assert.equal(sender.unmutes, 2); + assert.deepEqual(sender.listeners.unmuted, []); +}); + +// The publication's flag is a server-side mute: the SFU stops forwarding, the +// agent gets no frames, and a candidate who mutes mid-sentence leaves the turn +// open. The button mutes the track only. +test("the Mic button leaves the publication's flag alone", async () => { + const track = new FakeMic("laptop"); + const stream = new FakeStream([track]); + stream.getAudioTracks = () => audioIn(stream); + const sender = new FakePublished(track); + const inFlightCalls = []; + const scope = { + state: { micEnabled: false, localUserStream: stream }, + nodes: { mic: { textContent: "" } }, + publishedMicrophone: () => sender, + microphoneSwitch: () => ({ + setEnabled: (enabled) => inFlightCalls.push(enabled), + }), + setLocalAudioEnabled: (enabled) => { + track.enabled = enabled; + }, + publishIntegrityEvent: () => Promise.resolve(), + console, + }; + const { toggleMicrophone } = loadInterview(["toggleMicrophone"], scope); + + await toggleMicrophone(); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(scope.state.micEnabled, true); + assert.equal(track.enabled, true); + assert.equal(scope.nodes.mic.textContent, "Mic on"); + + await toggleMicrophone(); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(track.enabled, false); + assert.equal(sender.isMuted, false, "the button raised a server-side mute"); + assert.equal(sender.unmutes, 0); + assert.deepEqual(inFlightCalls, [true, false]); +}); + +// Between the open and the commit the new input is in neither the stream nor +// the publication, so a mute pressed then would otherwise leave it live. +test("a mute pressed during a switch reaches the input being handed over", async () => { + const opened = []; + const { microphone, stream, sender, state } = setup({ + open: (deviceId) => { + const track = new FakeMic(deviceId); + opened.push(track); + return track; + }, + }); + let release = null; + sender.pending = new Promise((resolve) => { + release = resolve; + }); + + const result = microphone.switchTo("headset"); + await new Promise((resolve) => setTimeout(resolve, 0)); + state.muted = true; + microphone.setEnabled(false); + assert.equal(opened[0].enabled, false, "the input in flight stayed live"); + release(); + + assert.equal(await result, "switched"); + // LiveKit enables what it is handed from the publication's flag, so a flag + // left down because the switch began unmuted put the input live here. + assert.deepEqual(sender.liveAfterReplace, [false], "live during the replace"); + assert.deepEqual(sender.liveAfterUnmute, [false]); + assert.equal(audioIn(stream)[0].enabled, false); + // Once committed it is the stream's, and the button reaches it directly. + microphone.setEnabled(true); + assert.equal(opened[0].enabled, false); +}); + +// The hand-back has the same window: the old input is handed to LiveKit again, +// and a mute pressed while that replace is pending must not be undone by it. +test("a mute pressed during the hand-back keeps the old input disabled", async () => { + const { microphone, old, sender, state } = setup(); + sender.rejectAfterSend = true; + const replace = sender.replaceTrack.bind(sender); + let release = null; + sender.replaceTrack = (track, options) => { + if (track === old) + sender.pending = new Promise((resolve) => { + release = resolve; + }); + return replace(track, options); + }; + + const result = microphone.switchTo("headset"); + for (let tick = 0; !release && tick < 100; tick += 1) + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.ok(release, "the hand-back never reached LiveKit"); + state.muted = true; + old.enabled = false; + microphone.setEnabled(false); + release(); + + assert.equal(await result, "refused"); + assert.deepEqual(sender.liveAfterReplace, [false], "live during the replace"); + assert.deepEqual(sender.liveAfterUnmute, [false]); + assert.equal(old.enabled, false); +}); + +// While the swap holds LiveKit's flag, LiveKit has disabled the input still in +// the stream, and the integrity poll would record that as the candidate's. +test("the hold is announced for as long as the swap holds the flag", async (t) => { + const run = async (arrange) => { + const ctx = setup(); + const seen = []; + for (const name of ["mute", "unmute"]) { + const original = ctx.sender[name].bind(ctx.sender); + ctx.sender[name] = (...args) => { + seen.push([name, ctx.state.holding]); + return original(...args); + }; + } + arrange?.(ctx); + const result = await ctx.microphone.switchTo("headset"); + return { result, seen, holding: ctx.state.holding }; + }; + const switched = await run(); + const handedBack = await run(({ sender }) => { + sender.rejectAfterSend = true; + }); + // The new input is sent before the replace rejects, and the hand-back that + // would take it back is refused. + const stranded = await run(({ sender }) => { + sender.rejectAfterSend = true; + sender.refuseAfter = 1; + }); + + for (const [name, outcome, result] of [ + ["switched", switched, "switched"], + ["refused", handedBack, "refused"], + ["stranded", stranded, "stranded"], + ]) { + await t.test(name, () => { + assert.equal(outcome.result, result); + assert.ok(outcome.seen.length > 0, "the flag was never raised"); + assert.ok( + outcome.seen.every(([, holding]) => holding), + `the flag moved while no hold was announced: ${JSON.stringify(outcome.seen)}`, + ); + assert.equal(outcome.holding, false, "the hold outlived the switch"); + }); + } +}); + +test("a throwing callback does not block the next switch", async () => { + const old = new FakeMic("laptop"); + const stream = new FakeStream([old]); + const sender = new FakePublished(old); + let calls = 0; + const microphone = createMicrophoneSwitch({ + open: async (deviceId) => new FakeMic(deviceId), + streamOf: () => stream, + publishedTrack: () => sender, + isLive: () => true, + micEnabled: () => true, + onSwitched: () => { + calls += 1; + if (calls === 1) throw new Error("listener failed"); + }, + onFailed: () => {}, + }); + + await assert.rejects(microphone.switchTo("headset")); + assert.equal(await microphone.switchTo("webcam"), "switched"); +}); + +test("the list offers named inputs only and marks the one in use", () => { + const devices = [ + { kind: "audioinput", deviceId: "", label: "" }, + { kind: "audioinput", deviceId: "laptop", label: "Built-in Microphone" }, + { kind: "audiooutput", deviceId: "speakers", label: "Speakers" }, + { kind: "audioinput", deviceId: "usb", label: "" }, + ]; + + assert.deepEqual(inputOptions(devices, { deviceId: "usb" }), [ + { + deviceId: "laptop", + groupId: "", + label: "Built-in Microphone", + selected: false, + }, + { deviceId: "usb", groupId: "", label: "Microphone 2", selected: true }, + ]); +}); + +// Picking the alias of the input in use reopened it and recorded a switch that +// changed nothing, so the alias rows are not offered. +test("alias rows are left out and the default's group marks its row", () => { + const devices = [ + { kind: "audioinput", deviceId: "default", label: "Default", groupId: "h" }, + { kind: "audioinput", deviceId: "communications", groupId: "h" }, + { kind: "audioinput", deviceId: "headset", label: "Headset", groupId: "h" }, + { kind: "audioinput", deviceId: "laptop", label: "Built-in", groupId: "l" }, + ]; + + const rows = inputOptions(devices, { deviceId: "default", groupId: "h" }); + + // The group travels with each row: it is how the page tells the alias in + // use from a real switch. + assert.deepEqual( + rows.map((row) => [row.deviceId, row.groupId, row.selected]), + [ + ["headset", "h", true], + ["laptop", "l", false], + ], + ); +}); + +// A track opened with no device id can report an id the list does not carry. +// Matching on the label instead keeps the select off its first row, which would +// name a microphone the candidate is not using. +test("the list falls back to the label when the id is not listed", () => { + const devices = [ + { kind: "audioinput", deviceId: "laptop", label: "Built-in Microphone" }, + { kind: "audioinput", deviceId: "usb", label: "USB Headset" }, + ]; + + const selected = (current) => + inputOptions(devices, current) + .filter((option) => option.selected) + .map((option) => option.deviceId); + + const unlisted = { deviceId: "gone", label: "USB Headset" }; + assert.deepEqual(selected(unlisted), ["usb"]); + assert.deepEqual(selected({ ...unlisted, label: "" }), []); +}); + +test("the switch detail keeps a label's separators readable", () => { + assert.equal( + switchDetail("chosen", { label: "USB; Rear=Mic" }), + "reason=chosen;microphone=USB; Rear=Mic", + ); + assert.equal( + switchDetail("lost", { label: "" }), + "reason=lost;microphone=unnamed", + ); +}); + +/// The page's own listing, run against a select that records what it is given +/// and a device list the test releases when it chooses. +function listingPage({ + track = new FakeMic("laptop", "Built-in Microphone"), +} = {}) { + const page = { + live: true, + pending: [], + retried: [], + focused: [], + select: { + hidden: true, + rows: null, + replaceChildren(...rows) { + this.rows = rows; + }, + }, + }; + const stream = new FakeStream(track ? [track] : []); + stream.getAudioTracks = () => audioIn(stream); + const scope = { + inputListing: 0, + state: { localUserStream: stream }, + nodes: { + micInputSelect: page.select, + mic: { focus: () => page.focused.push("mic") }, + }, + microphoneLive: () => page.live, + inputOptions, + navigator: { + mediaDevices: { + enumerateDevices: () => + new Promise((resolve) => page.pending.push(resolve)), + }, + }, + document: { activeElement: null, createElement: () => ({ dataset: {} }) }, + microphoneSwitch: () => ({ + switchTo: (...args) => { + page.retried.push(args); + return Promise.resolve("switched"); + }, + }), + }; + const { refreshAudioInputs, retryMicrophone } = loadInterview( + ["refreshAudioInputs", "retryMicrophone"], + scope, + ); + return { page, scope, refreshAudioInputs, retryMicrophone }; +} + +const twoInputs = [ + { kind: "audioinput", deviceId: "laptop", label: "Built-in Microphone" }, + { kind: "audioinput", deviceId: "usb", label: "USB Headset" }, +]; + +// Teardown hides the select; a listing still in flight must not show it again +// over the report. +test("a listing that finishes after the interview ended paints nothing", async () => { + const { page, refreshAudioInputs } = listingPage(); + + const listing = refreshAudioInputs(); + page.live = false; + page.pending[0](twoInputs); + await listing; + + assert.equal(page.select.rows, null); + assert.equal(page.select.hidden, true); +}); + +// Chrome adds a `default` row, and Windows a `communications` one, for the same +// physical microphone; one device is nothing to choose between. +test("one physical microphone keeps the select hidden", async () => { + const { page, refreshAudioInputs } = listingPage(); + const oneDevice = [ + { kind: "audioinput", deviceId: "default", label: "Default", groupId: "g" }, + { + kind: "audioinput", + deviceId: "communications", + label: "Comms", + groupId: "g", + }, + { kind: "audioinput", deviceId: "laptop", label: "Built-in", groupId: "g" }, + ]; + + const listing = refreshAudioInputs(); + page.pending[0](oneDevice); + await listing; + + assert.deepEqual( + page.select.rows.map((row) => row.value), + ["laptop"], + ); + assert.equal(page.select.hidden, true); +}); + +test("two physical microphones show the select", async () => { + const { page, refreshAudioInputs } = listingPage(); + const twoDevices = [ + { kind: "audioinput", deviceId: "laptop", label: "Built-in", groupId: "a" }, + { kind: "audioinput", deviceId: "usb", label: "USB Headset", groupId: "b" }, + ]; + + const listing = refreshAudioInputs(); + page.pending[0](twoDevices); + await listing; + + assert.equal(page.select.hidden, false); +}); + +test("an older listing that finishes last does not repaint the select", async () => { + const { page, refreshAudioInputs } = listingPage(); + + const older = refreshAudioInputs(); + const newer = refreshAudioInputs(); + page.pending[1](twoInputs); + await newer; + const painted = page.select.rows; + page.pending[0]([twoInputs[0]]); + await older; + + assert.equal(page.select.rows, painted); + assert.equal(page.select.hidden, false); +}); + +// Stranded, the stream has no input. The listing still has to run, or the +// select keeps the failed pick and picking it again fires no `change`. +test("with no input the select offers a placeholder and stays visible", async () => { + const { page, refreshAudioInputs } = listingPage({ track: null }); + const oneDevice = [ + { kind: "audioinput", deviceId: "laptop", label: "Built-in", groupId: "g" }, + ]; + + const listing = refreshAudioInputs(); + page.pending[0](oneDevice); + await listing; + + const [first, ...devices] = page.select.rows; + assert.equal(first.disabled, true); + assert.equal(first.selected, true); + assert.deepEqual( + devices.map((row) => row.value), + ["laptop"], + ); + assert.equal(page.select.hidden, false); +}); + +test("an ended input marks no row as in use", async () => { + const dead = new FakeMic("laptop", "Built-in"); + dead.readyState = "ended"; + const { page, refreshAudioInputs } = listingPage({ track: dead }); + const oneDevice = [ + { kind: "audioinput", deviceId: "laptop", label: "Built-in", groupId: "g" }, + ]; + + const listing = refreshAudioInputs(); + page.pending[0](oneDevice); + await listing; + + assert.equal(page.select.rows[0].disabled, true); + assert.equal(page.select.rows[1].selected, false); + assert.equal(page.select.hidden, false); +}); + +// The loss that failed to move has no `ended` left to follow; a device coming +// back is what gets the candidate heard again. +test("a device change retries the move only while no input is live", () => { + const dead = new FakeMic("laptop"); + dead.readyState = "ended"; + const stranded = listingPage({ track: null }); + const lost = listingPage({ track: dead }); + const fine = listingPage(); + + stranded.retryMicrophone(); + lost.retryMicrophone(); + fine.retryMicrophone(); + + assert.deepEqual(stranded.page.retried, [["", "lost", null]]); + assert.deepEqual(lost.page.retried, [["", "lost", dead]]); + assert.deepEqual(fine.page.retried, []); +}); + +test("two real inputs of one interface both get a row", async () => { + const devices = [ + { kind: "audioinput", deviceId: "in-1", label: "Input 1", groupId: "x" }, + { kind: "audioinput", deviceId: "in-2", label: "Input 2", groupId: "x" }, + ]; + + // Through the page, since counting the select's visibility by group hid + // both rows while the rows themselves were right. + const { page, refreshAudioInputs } = listingPage({ + track: new FakeMic("in-1", "Input 1"), + }); + const listing = refreshAudioInputs(); + page.pending[0](devices); + await listing; + assert.equal(page.select.hidden, false); + + assert.deepEqual( + inputOptions(devices, { deviceId: "in-2", groupId: "x" }).map((row) => [ + row.deviceId, + row.selected, + ]), + [ + ["in-1", false], + ["in-2", true], + ], + ); +}); + +// The default alias of an interface with several inputs could be any of them, +// so its group marks no row and decides nothing. +test("an alias in a group of several inputs names none of them", () => { + const devices = [ + { kind: "audioinput", deviceId: "default", label: "Default", groupId: "x" }, + { kind: "audioinput", deviceId: "in-1", label: "Input 1", groupId: "x" }, + { kind: "audioinput", deviceId: "in-2", label: "Input 2", groupId: "x" }, + ]; + + assert.deepEqual( + inputOptions(devices, { deviceId: "default", groupId: "x" }).filter( + (row) => row.selected, + ), + [], + ); +}); + +test("the picked row's group is passed only when no other row shares it", () => { + const { soleGroupOf } = loadInterview(["soleGroupOf"], {}); + const row = (value, groupId) => ({ value, dataset: { groupId } }); + const select = (picked, rows) => ({ + options: rows, + selectedOptions: [rows.find((each) => each.value === picked)], + }); + const rows = [row("in-1", "x"), row("in-2", "x"), row("headset", "h")]; + + assert.equal(soleGroupOf(select("headset", rows)), "h"); + assert.equal(soleGroupOf(select("in-2", rows)), null); + assert.equal(soleGroupOf(select("x", [row("x", "")])), null); +}); +// The page is what records the dropped input, since the switch only reports it. +test("the page records a working input dropped by a failed switch", () => { + let options = null; + const published = []; + const scope = { + microphoneSwitchInstance: null, + createMicrophoneSwitch: (given) => { + options = given; + return {}; + }, + publishIntegrityEvent: (event) => { + published.push(event); + return Promise.resolve(); + }, + console: { warn: () => {} }, + }; + const { microphoneSwitch } = loadInterview(["microphoneSwitch"], scope, { + stubMissing: true, + }); + microphoneSwitch(); + + options.onFailed("chosen", new Error("x"), "stranded", true); + options.onFailed("lost", new Error("x"), "stranded", false); + options.onFailed("chosen", new Error("x"), "refused", false); + + assert.deepEqual(published, [ + { + type: "MICROPHONE_STOPPED", + source: "microphone", + severity: "high", + detail: "reason=switch_failed", + }, + ]); +}); + +// A live input no row matches, such as a system default whose group no listed +// device shares, is still the microphone in use and is named as such. +test("a live input no row matches is named, not called missing", async () => { + const live = new FakeMic("default", "Default Microphone"); + live.getSettings = () => ({ deviceId: "default", groupId: "virtual" }); + const { page, refreshAudioInputs } = listingPage({ track: live }); + const devices = [ + { + kind: "audioinput", + deviceId: "default", + label: "Default", + groupId: "virtual", + }, + { kind: "audioinput", deviceId: "in-1", label: "Input 1", groupId: "a" }, + { kind: "audioinput", deviceId: "in-2", label: "Input 2", groupId: "b" }, + ]; + + const listing = refreshAudioInputs(); + page.pending[0](devices); + await listing; + + const [first] = page.select.rows; + assert.equal(first.disabled, true); + assert.equal(first.selected, true); + assert.equal(first.textContent, "Default Microphone"); +}); + +// Unplugging down to one input hides the select, and hiding the control that +// has focus drops focus to the body. +test("hiding the focused select moves focus to the Mic button", async () => { + const oneDevice = [ + { kind: "audioinput", deviceId: "laptop", label: "Built-in", groupId: "g" }, + ]; + const run = async (focused) => { + const { page, scope, refreshAudioInputs } = listingPage(); + if (focused) scope.document.activeElement = page.select; + page.select.hidden = false; + const listing = refreshAudioInputs(); + page.pending[0](oneDevice); + await listing; + assert.equal(page.select.hidden, true); + return page.focused; + }; + + assert.deepEqual(await run(true), ["mic"]); + assert.deepEqual(await run(false), []); +}); + +// A status region hidden until it is filled is skipped by screen readers, so +// the words go to one that is always present as well as to the visible note. +test("a note is announced from a region that is never hidden", () => { + const note = { textContent: "", hidden: true }; + const status = { textContent: "", hidden: false }; + const { showInputNote } = loadInterview(["showInputNote"], { + nodes: { micInputNote: note, micInputStatus: status }, + }); + + showInputNote(INPUT_NOTES.refused); + assert.deepEqual( + [note.hidden, note.textContent, status.textContent], + [false, INPUT_NOTES.refused, INPUT_NOTES.refused], + ); + showInputNote(""); + assert.deepEqual( + [note.hidden, note.textContent, status.textContent], + [true, "", ""], + ); + assert.equal(status.hidden, false); + + const markup = read("web/interview.html"); + // Prettier puts each attribute on its own line, so the tag spans several. + const tag = (id) => + markup.match(new RegExp(`<[a-z]+\\s+id="${id}"[^>]*>`))[0]; + assert.match(tag("mic-input-status"), /role="status"/); + assert.doesNotMatch(tag("mic-input-status"), /\shidden[\s>=]/); + assert.doesNotMatch(tag("mic-input-note"), /role=/); +}); + +// LiveKit disables the input still in the stream while a swap holds its flag. +// The trail reads the Mic button for that stretch, so a candidate who never +// muted is not recorded as having switched the microphone off. +test("the integrity trail reads the Mic button while a swap holds the flag", async (t) => { + const integrity = await import("../../web/integrity.js"); + const mic = new FakeMic("laptop"); + const stream = new FakeStream([mic]); + stream.getAudioTracks = () => audioIn(stream); + stream.getVideoTracks = () => []; + const published = []; + const state = { + micEnabled: true, + microphoneSwapHolding: false, + localUserStream: stream, + integrityTrackStates: new Map(), + integrityTrackStateTimer: null, + integrityAnalysisDetail: "x", + }; + integrity.initIntegrity({ + state, + nodes: {}, + updatePresenceBanner: () => {}, + publishIntegrityEvent: (event) => { + published.push(event); + return Promise.resolve(); + }, + endInterview: () => {}, + }); + const microphone = () => + integrity.integrityHeartbeatDetail().match(/microphone=([^;]*)/)[1]; + // The state poll's one-second tick, driven by hand. + let tick = null; + t.mock.method(globalThis, "setInterval", (callback) => { + tick = callback; + return 0; + }); + t.mock.method(globalThis, "clearInterval", () => {}); + integrity.startIntegrityTrackStateMonitor(); + + state.microphoneSwapHolding = true; + mic.enabled = false; + assert.equal(microphone(), "live/on/unmuted"); + tick(); + assert.deepEqual(published, [], "the swap's mute was recorded"); + // A candidate who presses Mic during the swap is reported as muted. + state.micEnabled = false; + assert.equal(microphone(), "live/off/unmuted"); + state.micEnabled = true; + + // Outside a swap the track is what is reported. + state.microphoneSwapHolding = false; + assert.equal(microphone(), "live/off/unmuted"); + tick(); + assert.deepEqual( + published.map((event) => [event.type, event.detail]), + [["MICROPHONE_STATE_CHANGED", "live/off"]], + ); +}); + +test("the page keeps the swap's hold where the integrity trail reads it", () => { + let options = null; + const scope = { + microphoneSwitchInstance: null, + state: { microphoneSwapHolding: false }, + createMicrophoneSwitch: (given) => { + options = given; + return {}; + }, + }; + const { microphoneSwitch } = loadInterview(["microphoneSwitch"], scope, { + stubMissing: true, + }); + microphoneSwitch(); + + options.onHold(true); + assert.equal(scope.state.microphoneSwapHolding, true); + options.onHold(false); + assert.equal(scope.state.microphoneSwapHolding, false); +}); diff --git a/tests/fixtures/integrity-chain.json b/tests/fixtures/integrity-chain.json index c8aeea529..5ddb7eb1c 100644 --- a/tests/fixtures/integrity-chain.json +++ b/tests/fixtures/integrity-chain.json @@ -135,5 +135,17 @@ "444444444444" ], "hash": "a1fefca650b641ce4370bf98be71c940b1f3e79b0bce53b1f3f1fedc194a74fa" + }, + { + "seq": 12, + "prevHash": "a1fefca650b641ce4370bf98be71c940b1f3e79b0bce53b1f3f1fedc194a74fa", + "type": "MICROPHONE_SWITCHED", + "at": "2026-08-18T06:40:30.512Z", + "severity": "info", + "source": "microphone", + "durationMs": 0, + "detail": "reason=chosen;microphone=USB Audio; Rear=Mic (0d8c:0014)", + "sourceEventIds": [], + "hash": "421c6644f15f1acd1c6cd614b55a185f4419cb0ba47d1925d92eeb1e7c2861b3" } ] diff --git a/web/audio-input.js b/web/audio-input.js index 3bb8acced..9fcacf32a 100644 --- a/web/audio-input.js +++ b/web/audio-input.js @@ -1,4 +1,6 @@ -// Microphone choices are limited to preflight, before its stream is published. +// The preflight's microphone picker, which works on the pool before its stream +// is published. Moving the published input once the interview is under way is +// web/mic-switch.js. export function createAudioInput({ mediaDevices, pool, diff --git a/web/devices.js b/web/devices.js index ee010469e..ff958621e 100644 --- a/web/devices.js +++ b/web/devices.js @@ -28,6 +28,39 @@ const CONSTRAINTS = { video: true, }; +/// The microphone constraints with a device named, exactly: a switch the +/// candidate asked for that opened some other input and called it theirs would +/// be worse than one that said it failed. No id asks for the default. +function microphoneConstraints(deviceId) { + if (!deviceId) return CONSTRAINTS.audio; + return { ...CONSTRAINTS.audio, deviceId: { exact: deviceId } }; +} + +// The only capture call in the first-party tree, and the getUserMedia allowlist +// in meet-tab-audio.test.js holds it to that. The preflight pool and a +// mid-interview microphone switch both ask through here, so a switch is a +// second device through the same door rather than a second door. +function capture(mediaDevices, constraints) { + return mediaDevices.getUserMedia(constraints); +} + +/// One live microphone track for an interview already under way, where there +/// is no pool to retry and no preflight to explain a failure: the caller keeps +/// the input it has and says why. Anything else the browser hands back is +/// stopped, so a refusal leaves no device open behind the candidate's back. +export async function openMicrophone(mediaDevices, deviceId) { + const granted = await capture(mediaDevices, { + audio: microphoneConstraints(deviceId), + }); + const tracks = granted.getTracks(); + const track = tracks.find( + (each) => each.kind === "audio" && each.readyState !== "ended", + ); + for (const spare of tracks) if (spare !== track) spare.stop(); + if (!track) throw new Error("no active audio track"); + return track; +} + // `schedule` is injected so a test can fire the retry rather than wait for it. // The bug this guards -- two timers where there should be one -- only shows up // as a count of requests, and a count read off a sleep is a count read off how @@ -106,8 +139,7 @@ export function createDevicePool({ if (device.disabled || device.pending || trackOf(device.kind)) return; device.pending = true; const generation = device.generation; - void mediaDevices - .getUserMedia(device.constraints) + void capture(mediaDevices, device.constraints) .then((granted) => { if ( device.disabled || @@ -181,12 +213,7 @@ export function createDevicePool({ ) return; audio.generation += 1; - audio.constraints = { - audio: { - ...CONSTRAINTS.audio, - ...(deviceId ? { deviceId: { exact: deviceId } } : {}), - }, - }; + audio.constraints = { audio: microphoneConstraints(deviceId) }; audio.error = null; const held = trackOf("audio"); if (held) { diff --git a/web/integrity.js b/web/integrity.js index 12680c701..8eaa64217 100644 --- a/web/integrity.js +++ b/web/integrity.js @@ -101,12 +101,23 @@ export function startIntegrityHeartbeat() { }, INTEGRITY_HEARTBEAT_MS); } +/// While a microphone switch holds LiveKit's mute flag, LiveKit has disabled +/// the input still in the stream. That is the swap's doing, not the +/// candidate's, so for that stretch the Mic button's state is reported instead +/// of the track's, or the trail would read a muted microphone the candidate +/// never muted. +function enabledOf(source, track) { + if (source === "microphone" && state.microphoneSwapHolding) + return state.micEnabled; + return track?.enabled !== false; +} + export function integrityHeartbeatDetail() { const camera = state.localUserStream?.getVideoTracks?.()[0]; const mic = state.localUserStream?.getAudioTracks?.()[0]; - const status = (track) => - `${track?.readyState || "missing"}/${track?.enabled === false ? "off" : "on"}/${track?.muted ? "muted" : "unmuted"}`; - return `camera=${status(camera)};microphone=${status(mic)};analyzer=${state.integrityAnalysisDetail}`; + const status = (source, track) => + `${track?.readyState || "missing"}/${enabledOf(source, track) ? "on" : "off"}/${track?.muted ? "muted" : "unmuted"}`; + return `camera=${status("camera", camera)};microphone=${status("microphone", mic)};analyzer=${state.integrityAnalysisDetail}`; } export function startIntegrityTrackStateMonitor() { @@ -119,7 +130,7 @@ export function startIntegrityTrackStateMonitor() { const poll = () => { for (const [source, track] of tracks()) { if (!track) continue; - const current = `${track.readyState || "missing"}/${track.enabled === false ? "off" : "on"}`; + const current = `${track.readyState || "missing"}/${enabledOf(source, track) ? "on" : "off"}`; const previous = state.integrityTrackStates.get(source); state.integrityTrackStates.set(source, current); if (previous && previous !== current) { diff --git a/web/interview.html b/web/interview.html index 1279e07ad..c97c5cf16 100644 --- a/web/interview.html +++ b/web/interview.html @@ -82,6 +82,16 @@

Loading interview...

+ +
+ + +
diff --git a/web/interview.js b/web/interview.js index b7b1f739b..5f1c62395 100644 --- a/web/interview.js +++ b/web/interview.js @@ -51,8 +51,14 @@ import { isToggleCommentKeystroke, toggleLineComment, } from "./editor.js"; -import { createDevicePool } from "./devices.js"; +import { createDevicePool, openMicrophone } from "./devices.js"; import { createAudioInput } from "./audio-input.js"; +import { + INPUT_NOTES, + createMicrophoneSwitch, + inputOptions, + switchDetail, +} from "./mic-switch.js"; import { createFaceCheck } from "./face-check.js"; import { attachSelfView } from "./self-view.js"; import { createMicMeter, startMediaMeter } from "./mic-meter.js"; @@ -144,6 +150,7 @@ import { PRESENCE_EVENTS, initIntegrity, monitorIntegrityTracks, + watchIntegrityTrack, startIntegrityHeartbeat, startIntegrityWorker, stopIntegrityWorker, @@ -491,6 +498,9 @@ const state = { /// read it from. interviewId: null, micEnabled: true, + /// Up while a microphone switch holds LiveKit's mute flag; web/integrity.js + /// reads the Mic button rather than the track for that stretch. + microphoneSwapHolding: false, localUserStream: null, integrityChain: { seq: 0, hash: "" }, integrityPublish: Promise.resolve(), @@ -646,6 +656,9 @@ const nodes = { preflightOutputNote: document.querySelector("#preflight-output-note"), preflightInput: document.querySelector("#preflight-input"), preflightInputNote: document.querySelector("#preflight-input-note"), + micInputSelect: document.querySelector("#mic-input-select"), + micInputNote: document.querySelector("#mic-input-note"), + micInputStatus: document.querySelector("#mic-input-status"), sidebar: document.querySelector("#interview-sidebar"), sidebarResizer: document.querySelector("#sidebar-resizer"), editorPanel: document.querySelector(".editor-panel"), @@ -1119,6 +1132,14 @@ function bindEvents() { void applyAudioOutput(select.value); }); } + nodes.micInputSelect.addEventListener("change", () => { + void microphoneSwitch().switchTo( + nodes.micInputSelect.value, + "chosen", + null, + soleGroupOf(nodes.micInputSelect), + ); + }); // The checkbox is the only copy of this answer. A mirrored state field had to // be written in two places and read in four, and the one reader that forgot // it published a wrong camera status for the whole interview. @@ -1143,10 +1164,11 @@ function bindEvents() { nodes.meetMode.addEventListener("toggle", () => { if (nodes.meetMode.open) void refreshAudioOutputs(); }); - navigator.mediaDevices?.addEventListener?.( - "devicechange", - () => void refreshAudioOutputs(), - ); + navigator.mediaDevices?.addEventListener?.("devicechange", () => { + void refreshAudioOutputs(); + void refreshAudioInputs(); + retryMicrophone(); + }); nodes.resultsToggle.addEventListener("click", () => setResultsOpen(nodes.resultsBody.hidden), ); @@ -2325,6 +2347,8 @@ async function connectLiveKit(connection, preflight, presenting = false) { nodes.mic.textContent = "Mic on"; updateAgentState(); monitorIntegrityTracks(); + followMicrophoneLoss(state.localUserStream?.getAudioTracks?.()[0]); + void refreshAudioInputs(); await publishIntegrityEvent({ type: "SESSION_START", source: "media", @@ -2450,6 +2474,7 @@ function stopLocalMedia() { stopIntegrityWorker(); stopPreflight({ userStream: state.localUserStream }); state.localUserStream = null; + forgetAudioInputs(); clearInterval(state.integrityHeartbeatTimer); state.integrityHeartbeatTimer = null; clearInterval(state.integrityTrackStateTimer); @@ -2804,6 +2829,9 @@ async function toggleMicrophone() { state.micEnabled = !state.micEnabled; if (state.localUserStream) { setLocalAudioEnabled(state.micEnabled); + // An input a microphone switch is still handing to the sender is in + // neither the stream nor the publication yet. + microphoneSwitch().setEnabled(state.micEnabled); if (!state.micEnabled) { void publishIntegrityEvent({ type: "MICROPHONE_STOPPED", @@ -3287,6 +3315,198 @@ function startTurnRing(stream) { meter.start(); } +/// A switch only ever runs inside a joined room: before it there is no +/// published track to hand a new input to, and after it the device would be +/// opened for an interview that is over. +function microphoneLive() { + return state.phase === "live" && Boolean(state.room); +} + +/// Read through the room on every switch rather than held, because the +/// publication is LiveKit's and the page never owned a handle to it. +function publishedMicrophone() { + const source = window.LivekitClient?.Track?.Source?.Microphone; + return ( + state.room?.localParticipant?.getTrackPublication?.(source)?.track ?? null + ); +} + +/// Built on first use rather than at module scope: the select's listener is +/// bound inside `init`, and a const declared below that call is still in its +/// temporal dead zone while `init` awaits. +let microphoneSwitchInstance = null; + +function microphoneSwitch() { + microphoneSwitchInstance ??= createMicrophoneSwitch({ + open: (deviceId) => openMicrophone(navigator.mediaDevices, deviceId), + streamOf: () => state.localUserStream, + publishedTrack: publishedMicrophone, + isLive: microphoneLive, + micEnabled: () => state.micEnabled, + onHold: (holding) => { + state.microphoneSwapHolding = holding; + }, + onSwitched: (track, reason) => { + // The track the heartbeat, the state poll and the ring read is a new + // object, so each reader is pointed at it here or it keeps reporting the + // one that was stopped. + watchIntegrityTrack(track, "MICROPHONE_STOPPED", "microphone"); + followMicrophoneLoss(track); + startTurnRing(state.localUserStream); + void publishIntegrityEvent({ + type: "MICROPHONE_SWITCHED", + source: "microphone", + severity: "info", + detail: switchDetail(reason, track), + }); + showInputNote(reason === "lost" ? INPUT_NOTES.lost : ""); + void refreshAudioInputs(); + }, + onFailed: (reason, error, outcome, droppedLive) => { + console.warn("codetrial microphone_switch_failed", error); + // The input the interview was using was still working and has been + // dropped, which nothing else in the trail would record: its `stop()` + // fires no `ended`, and the state poll skips a missing track. + if (droppedLive) { + void publishIntegrityEvent({ + type: "MICROPHONE_STOPPED", + source: "microphone", + severity: "high", + detail: "reason=switch_failed", + }); + } + if (reason === "lost") showInputNote(INPUT_NOTES.none); + else if (outcome === "stranded") showInputNote(INPUT_NOTES.stranded); + else showInputNote(INPUT_NOTES.refused); + // A stranded switch dropped the input the ring was reading. + if (outcome === "stranded") startTurnRing(state.localUserStream); + // Puts the select back on the input still in use, rather than on the + // one that was refused. + void refreshAudioInputs(); + }, + }); + return microphoneSwitchInstance; +} + +/// An input that ends under a live interview, unplugged or revoked, leaves the +/// candidate talking into nothing with no prompt to answer. The +/// MICROPHONE_STOPPED its integrity listener publishes stays, because the +/// input really did stop; this only moves the interview to the default. +function followMicrophoneLoss(track) { + track?.addEventListener?.("ended", () => { + if (!microphoneLive()) return; + if (state.localUserStream?.getAudioTracks?.()[0] !== track) return; + void microphoneSwitch().switchTo("", "lost", track); + }); +} + +/// A loss whose move to the default failed leaves nothing listening for the +/// microphone to come back, since the `ended` it followed has already fired. +/// A device appearing is the moment one may be usable again, so the move is +/// tried once more whenever no input is live. +function retryMicrophone() { + if (!microphoneLive()) return; + const held = state.localUserStream?.getAudioTracks?.()[0] ?? null; + if (held && held.readyState !== "ended") return; + void microphoneSwitch().switchTo("", "lost", held); +} + +/// The picked row's group, when no other row shares it. Several real inputs of +/// one audio interface share a group, and then the group cannot say which of +/// them the default alias in use is. +function soleGroupOf(select) { + const group = select.selectedOptions?.[0]?.dataset.groupId; + if (!group) return null; + const rows = [...select.options].filter( + (option) => option.dataset.groupId === group, + ); + return rows.length === 1 ? group : null; +} + +/// Stamps each listing, because `devicechange`, a switch and the join can each +/// start one, and an older listing finishing last would repaint the select +/// with the input that was in use before. +let inputListing = 0; + +async function refreshAudioInputs() { + const request = ++inputListing; + const current = state.localUserStream?.getAudioTracks?.()[0] ?? null; + // No `current` is not a reason to stop: that is the stranded state, where the + // select matters most. + if (!microphoneLive() || !navigator.mediaDevices?.enumerateDevices) return; + let devices = []; + try { + devices = await navigator.mediaDevices.enumerateDevices(); + } catch { + // Listing is a convenience; the input in use keeps working without it. + } + // The interview can end, or a switch replace the input, while the browser + // lists devices; teardown has already hidden the select by then. + if ( + request !== inputListing || + !microphoneLive() || + (state.localUserStream?.getAudioTracks?.()[0] ?? null) !== current + ) + return; + // An ended input is not in use, so it marks no row as the current one. + const inUse = current?.readyState === "live" ? current : null; + const options = inputOptions(devices, { + deviceId: inUse?.getSettings?.().deviceId, + groupId: inUse?.getSettings?.().groupId, + label: inUse?.label, + }); + const rows = options.map((option) => { + const element = document.createElement("option"); + element.value = option.deviceId; + element.dataset.groupId = option.groupId; + element.textContent = option.label; + element.selected = option.selected; + return element; + }); + // With no row for the input in use, the select would show its first device + // as if it were, and picking that row fires no `change`. A disabled + // placeholder in its place makes every device a change. It names the input + // still in use when there is one, such as a system default no row matches, + // so a working microphone does not read as missing. + if (!options.some((option) => option.selected)) { + const placeholder = document.createElement("option"); + placeholder.value = ""; + placeholder.textContent = inUse + ? inUse.label || "System default" + : "Choose a microphone"; + placeholder.disabled = true; + placeholder.selected = true; + rows.unshift(placeholder); + } + nodes.micInputSelect.replaceChildren(...rows); + // One input is nothing to choose between, and a select with a single row + // reads as a control that does nothing. The rows already leave out the + // `default` and `communications` aliases, and keep every real input of a + // group, so they are the count. With no input live it shows regardless, + // since picking is then the way out. + const hide = Boolean(inUse) && options.length < 2; + // Unplugging down to one input can hide the select while it has focus, which + // drops focus to the body; the Mic button beside it takes it first. + if (hide && document.activeElement === nodes.micInputSelect) + nodes.mic.focus({ preventScroll: true }); + nodes.micInputSelect.hidden = hide; +} + +/// Nothing is left to switch once the media stops, so neither the list nor a +/// note about the last switch belongs over the report, and a listing still in +/// flight is told it is stale. +function forgetAudioInputs() { + inputListing += 1; + nodes.micInputSelect.hidden = true; + showInputNote(""); +} + +function showInputNote(text) { + nodes.micInputNote.textContent = text; + nodes.micInputNote.hidden = !text; + nodes.micInputStatus.textContent = text; +} + function applyThinking(thinking) { if (thinking === state.candidateThinking) return; state.candidateThinking = thinking; diff --git a/web/mic-switch.js b/web/mic-switch.js new file mode 100644 index 000000000..2e7fe4d8f --- /dev/null +++ b/web/mic-switch.js @@ -0,0 +1,298 @@ +// Which microphone the interview captures, and moving it to another one +// without leaving the room. +// +// Free of DOM and LiveKit types, like meet-audio.js, so the decisions and the +// order of the swap run under `node --test` instead of being asserted as +// source text. interview.js supplies the stream, the published track, the +// select and the integrity publisher. + +export const INPUT_NOTES = { + refused: + "That microphone could not be opened. Your previous microphone is still in use.", + lost: "Your microphone disconnected. The interview moved to the default microphone.", + none: "Your microphone disconnected and no other microphone could be opened. Plug one in or choose a microphone.", + stranded: + "The microphone could not be switched, and Jim cannot hear you. Choose a microphone again.", +}; + +const ALIAS_IDS = new Set(["default", "communications"]); + +/// Inputs worth offering, the same way `outputOptions` decides outputs: a +/// device with no id is the placeholder a browser reports before a grant, and +/// it would select to "" and switch to whatever the default happens to be. +/// +/// `current` is the track being sent, so the select shows the input in use +/// rather than the one last asked for. Its device id is the first match and +/// its label the fallback: a track opened without a device id can report an +/// id the list does not carry, and an unmatched select shows its first row, +/// which names a microphone the candidate is not using. +export function inputOptions(devices, current) { + // Chrome lists a `default` input beside the physical one, and Windows a + // `communications` one too, all sharing its `groupId`; offered as rows of + // their own, picking the alias of the input in use reopened it and recorded + // a switch that changed nothing. The preflight picker in #411 skips the same + // two ids. Rows are not merged by `groupId` itself, since an audio interface + // or a webcam can put several real inputs in one group. + const inputs = devices.filter( + (device) => + device.kind === "audioinput" && + device.deviceId && + !ALIAS_IDS.has(device.deviceId), + ); + // A track opened as the default reports the alias's id, which no row carries, + // so only then does its group pick the physical row, and only a group with a + // single real input names one. + const byId = inputs.find((device) => device.deviceId === current?.deviceId); + const inGroup = current?.groupId + ? inputs.filter((device) => device.groupId === current.groupId) + : []; + const byGroup = + ALIAS_IDS.has(current?.deviceId) && inGroup.length === 1 + ? inGroup[0] + : null; + const byLabel = current?.label + ? inputs.find((device) => device.label === current.label) + : null; + const selected = byId ?? byGroup ?? byLabel ?? null; + return inputs.map((device, index) => ({ + deviceId: device.deviceId, + groupId: device.groupId || "", + label: device.label || `Microphone ${index + 1}`, + selected: device === selected, + })); +} + +/// The integrity detail for a switch. The label goes last because a device +/// label is the browser's localized text and may itself contain `;` or `=`, +/// which a reader splitting the fields left to right then never reaches. +export function switchDetail(reason, track) { + return `reason=${reason};microphone=${track?.label || "unnamed"}`; +} + +// The publication's own mute flag is a server-side mute: the SFU stops +// forwarding the track, so the agent receives no frames at all, not the +// silence that ends a spoken turn. The Mic button therefore mutes the track +// only, and the flag is raised just around a swap and kept down otherwise. + +/// Raises the flag before LiveKit is handed an input, since LiveKit enables a +/// track it is handed from that flag. It is raised whatever the Mic state: a +/// candidate can press Mic while the replace is pending, and a flag left down +/// would have LiveKit enable the input the button just disabled. +export async function holdMute(published) { + if (!published.isMuted) await published.mute(); +} + +/// Lowers the flag, whether a swap raised it or LiveKit did after a loss, and +/// keeps a muted candidate's input disabled throughout. LiveKit enables the +/// track and announces it in one synchronous step, so the listener puts the +/// button's state back before anything else can run. +export async function liftMute(published, enabled) { + if (!published.isMuted) return; + const keep = () => { + if (published.mediaStreamTrack) + published.mediaStreamTrack.enabled = enabled(); + }; + published.on?.("unmuted", keep); + try { + await published.unmute(); + } finally { + published.off?.("unmuted", keep); + } + keep(); +} + +const audioTrackOf = (stream) => + stream?.getTracks?.().find((track) => track.kind === "audio") ?? null; + +const deviceIdOf = (track) => track?.getSettings?.().deviceId ?? null; +const groupIdOf = (track) => track?.getSettings?.().groupId || null; + +/// `reason` is "chosen" for a pick from the select and "lost" when the input +/// in use ended, in which case `lost` is the track that ended. Every outcome +/// but "switched" leaves the stream, the published track and the device +/// exactly as they were. +/// +/// Serialized like `applyAudioOutput`: a candidate can pick again while the +/// permission prompt for the last pick is still up, and two swaps resolving in +/// either order would leave the stream holding one track while LiveKit sends +/// the other. +export function createMicrophoneSwitch({ + open, + streamOf, + publishedTrack, + isLive, + micEnabled, + onHold = () => {}, + onSwitched, + onFailed, +}) { + let chain = Promise.resolve(); + // The input being handed to the sender. Between the open and the commit it + // is in neither the page's stream nor reach of the Mic button, so a mute + // pressed meanwhile has to be applied to it here. + let inFlight = null; + + const run = async (deviceId, reason, lost, groupId) => { + const stream = streamOf(); + if (!isLive() || !stream) return "idle"; + // Every await below can outlast the interview or the stream it started + // with, and whatever it would then show or keep open belongs to neither. + const current = () => isLive() && streamOf() === stream; + const held = audioTrackOf(stream); + if ( + reason === "chosen" && + held?.readyState === "live" && + (deviceIdOf(held) === deviceId || + (ALIAS_IDS.has(deviceIdOf(held)) && + groupId && + groupIdOf(held) === groupId)) + ) + return "unchanged"; + // A loss queued behind a pick that has since replaced the lost track would + // otherwise move the candidate off the input they just chose. Only another + // track means that: an empty stream is a loss whose earlier recovery + // stranded, and the device that set off this retry may be the way back. + if (reason === "lost" && held && held !== lost) return "unchanged"; + // Checked before the prompt, not after: replacing a track that was never + // published sends nothing, and the candidate would be told the switch + // worked while Jim kept hearing silence. + if (!publishedTrack()) return "idle"; + + // No input is being sent. Dropping the old one from the stream, dead or + // not, keeps the page from claiming it is in use and lets the next pick, + // even of the same device, open a fresh input rather than read as + // unchanged. + const strand = (error) => { + // `stop()` fires no `ended`, so an input still working when it is + // dropped here leaves no trace in the integrity trail unless the caller + // records one; an input that had already ended recorded its own. + const droppedLive = held?.readyState === "live"; + if (held) { + stream.removeTrack(held); + held.stop(); + } + onFailed(reason, error, "stranded", droppedLive); + return "stranded"; + }; + + let track; + try { + track = await open(deviceId); + inFlight = track; + } catch (error) { + if (!current()) return "idle"; + if (held?.readyState !== "live") return strand(error); + onFailed(reason, error, "refused"); + return "refused"; + } + // The prompt can outlast the interview, or the room that published the old + // track, and a device opened for neither would stay on behind the report. + const published = publishedTrack(); + if (!current() || !published) { + track.stop(); + return "idle"; + } + + // Everything that can fail once the sender may hold the new input sits in + // one try, so every failure reaches the hand-back below rather than escaping + // with the stream and the sender on different tracks. + let failure = null; + try { + // `userProvidedTrack` keeps LiveKit from stopping a track this page owns + // when the next switch replaces it. The page stops the old one itself + // below, after the new one is already being sent, so the swap has no gap. + // + // A fresh track starts enabled, and LiveKit enables it again from the + // publication's flag inside the replace, so the flag is held up across + // the swap and the new input is never live before the Mic state says. + track.enabled = micEnabled(); + // Up until the switch is over, the hand-back included: LiveKit's mute + // disables the input still in the stream, which the integrity trail + // would otherwise record as the candidate's. + onHold(true); + await holdMute(published); + await published.replaceTrack(track, { userProvidedTrack: true }); + // Down again afterwards, also when LiveKit raised it after a loss, or the + // agent would hear nothing rather than a muted candidate's silence. + await liftMute(published, micEnabled); + } catch (error) { + failure = error; + } + if (!current()) { + track.stop(); + return "idle"; + } + // An input unplugged during those awaits has already fired `ended`, before + // anything was listening for it, so committing it would leave the + // interview silent with no loss left to follow. + if (!failure && track.readyState === "ended") + failure = new Error("the new input ended during the switch"); + + if (!failure) { + track.enabled = micEnabled(); + if (held) { + stream.removeTrack(held); + // `stop()` fires no `ended`, so the old track's integrity listener + // stays quiet. A switch is not a microphone stopping; the event the + // caller publishes for it says what happened instead. + held.stop(); + } + stream.addTrack(track); + onSwitched(track, reason); + return "switched"; + } + + track.stop(); + // The sender goes back to the input still working, where there is one. A + // rejected replace may already have handed the new input to the sender, + // so the hand-back runs whatever failed; when the sender never left the + // old input, LiveKit returns from replacing a track with itself at once. + let restored = false; + if (held?.readyState === "live") { + try { + held.enabled = micEnabled(); + await holdMute(published); + await published.replaceTrack(held, { userProvidedTrack: true }); + // LiveKit was already listening when the new input ended, so it may + // have muted the publication, which would keep the restored input + // silent to the interviewer while the page reads "Mic on". + await liftMute(published, micEnabled); + held.enabled = micEnabled(); + restored = true; + } catch (error) { + failure = error; + } + if (!current()) return "idle"; + } + if (restored) { + onFailed(reason, failure, "refused"); + return "refused"; + } + return strand(failure); + }; + + return { + /// The Mic button's other half: the input in the stream it reaches itself. + setEnabled(enabled) { + if (inFlight && inFlight.readyState !== "ended") + inFlight.enabled = enabled; + }, + /// `groupId` is the picked row's group, passed only when no other row + /// shares it. It decides "unchanged" only when the input in use was opened + /// under an alias id, which no row carries. + switchTo(deviceId, reason = "chosen", lost = null, groupId = null) { + const next = chain + .then(() => run(deviceId, reason, lost, groupId)) + .finally(() => { + inFlight = null; + // Every way out of a switch, the hand-back and the stranded ones + // included, ends the stretch in which the swap held the flag. + onHold(false); + }); + // A throw from a caller's callback must not wedge every later switch + // behind a rejected promise. + chain = next.catch(() => {}); + return next; + }, + }; +} diff --git a/web/styles.css b/web/styles.css index 46d6a882b..af70cdc14 100644 --- a/web/styles.css +++ b/web/styles.css @@ -1446,6 +1446,25 @@ p { color: white; } +/* Sized as a pill so it sits in the control row beside the Mic button it + belongs to; capped because device labels run long and would push the row's + other controls onto a second line. Bordered like the other form controls, + since a select whose edge is the hairline does not read as one. */ +.mic-input-select { + max-width: 14rem; + min-height: 2rem; + padding: 0.375rem 0.75rem; + border: 1px solid var(--control-border); + border-radius: 999px; + background: var(--inset-surface); + color: var(--sub); + font: inherit; + font-size: 0.75rem; + font-weight: 700; + text-overflow: ellipsis; + cursor: pointer; +} + .tab-row { min-height: 2.5rem; padding: 0 0.75rem;