From b4fe940a89d01f65bd7168f6431ea2788e8fd771 Mon Sep 17 00:00:00 2001 From: s4512138 Date: Wed, 21 Oct 2026 17:53:46 +0800 Subject: [PATCH] Choose audio devices during preflight Candidates with multiple audio devices otherwise have to change system settings before joining. Offer explicit choices while retaining system defaults, and require fresh microphone activity and output confirmation when a choice changes so preflight proves the devices used in the interview. Preserve existing output preferences across reloads, falling back to the system default when the selected hardware is unavailable. Closes #206 --- README.md | 8 + tests/browser/audio-input.test.js | 479 ++++++++++++ tests/browser/audio-playback-browser.test.js | 50 +- tests/browser/audio-playback.test.js | 153 ++++ tests/browser/devices.test.js | 160 ++++ tests/browser/jim-volume.test.js | 20 + tests/browser/meet-tab-audio.test.js | 85 +- tests/browser/mic-meter.test.js | 22 + tests/browser/preflight-speech.test.js | 774 ++++++++++++++++++- web/audio-input.js | 70 ++ web/audio-output.js | 217 ++++-- web/audio-playback.js | 11 + web/devices.js | 37 +- web/interview.html | 30 + web/interview.js | 124 ++- web/meet-audio.js | 62 +- web/mic-meter.js | 2 + 17 files changed, 2088 insertions(+), 216 deletions(-) create mode 100644 tests/browser/audio-input.test.js create mode 100644 web/audio-input.js diff --git a/README.md b/README.md index d5e7ec126..b47054515 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,14 @@ it is unavailable or declined; the signed integrity trail and the report record that neutral condition and why. A recorded interview still requires its camera before it can start. +Where the browser supports device selection, preflight offers microphone and +output selectors using System default until another device is chosen. Changing +the microphone releases the previous track and requires a fresh sound check; +the selected track is published to the interview. The output selection applies +to test sounds and Jim's voice, is shared with the Meet sidebar, and is +remembered across reloads. A saved output that is no longer listed falls back +to System default. + Candidates can present the interview in Google Meet by sharing the CodeTrial tab with tab audio enabled. Meet owns the shared tab after that, and face-presence analysis is disabled for the session. See the diff --git a/tests/browser/audio-input.test.js b/tests/browser/audio-input.test.js new file mode 100644 index 000000000..7e711aaf1 --- /dev/null +++ b/tests/browser/audio-input.test.js @@ -0,0 +1,479 @@ +import { after, before, test } from "node:test"; +import assert from "node:assert/strict"; +import { + DEFAULT_RUNTIME_CONFIG, + FAKE_SPEECH_CAPTURE, + launchChromium, + startStaticServer, +} from "./source.js"; + +let browser; +let server; +let base; +before(async () => { + browser = await launchChromium({ + args: [ + "--use-fake-device-for-media-stream", + "--use-fake-ui-for-media-stream", + FAKE_SPEECH_CAPTURE, + ], + }); + if (browser) + ({ server, base } = await startStaticServer({ + runtimeConfig: DEFAULT_RUNTIME_CONFIG, + })); +}); +after(async () => { + await browser?.close(); + await new Promise((resolve) => (server ? server.close(resolve) : resolve())); +}); + +async function preflight(t, { enumeration = true, deviceIds = true } = {}) { + const page = await browser.newPage(); + t.after(() => page.close()); + const errors = []; + page.on("pageerror", (error) => errors.push(error.message)); + t.after(() => assert.deepEqual(errors, [])); + await page.addInitScript( + ({ enumeration, deviceIds }) => { + if (!deviceIds) + navigator.mediaDevices.getSupportedConstraints = () => ({}); + window.inputs = [ + { + kind: "audioinput", + deviceId: "default", + label: "Default microphone", + }, + { kind: "audioinput", deviceId: "usb", label: "USB microphone" }, + { kind: "audioinput", deviceId: "headset", label: "Headset" }, + { kind: "audiooutput", deviceId: "speaker", label: "Speaker" }, + ]; + navigator.mediaDevices.enumerateDevices = enumeration + ? async () => window.inputs + : undefined; + const capture = navigator.mediaDevices.getUserMedia.bind( + navigator.mediaDevices, + ); + window.microphones = []; + window.audioRequests = []; + navigator.mediaDevices.getUserMedia = async (constraints) => { + if (!constraints.audio) { + const stream = await capture(constraints); + window.cameraTrack = stream.getVideoTracks()[0]; + return stream; + } + const deviceId = constraints.audio.deviceId?.exact || ""; + window.audioRequests.push(deviceId); + if (window.refusedInput === deviceId) + throw new DOMException( + "Selected microphone refused", + window.refusedInputError || "NotAllowedError", + ); + // Hardware ids are synthetic; capture real fake-media tracks so the + // production meter and publication code still receive MediaStreamTracks. + const audio = { ...constraints.audio }; + delete audio.deviceId; + const stream = await capture({ audio }); + const track = stream.getAudioTracks()[0]; + track.testDeviceId = deviceId; + const stop = track.stop.bind(track); + track.stop = () => { + const stopped = JSON.parse( + sessionStorage.getItem("microphone-stops") || "[]", + ); + stopped.push(track.id); + sessionStorage.setItem("microphone-stops", JSON.stringify(stopped)); + stop(); + }; + window.microphones.push(track); + if (window.heldInput === deviceId) + await new Promise((resolve) => { + window.releaseInput = resolve; + }); + return stream; + }; + }, + { enumeration, deviceIds }, + ); + await page.goto(`${base}/interview.html?problem=chargeback-pair-match`); + await page.waitForFunction( + (count) => + window.microphones.length && + window.cameraTrack && + document.querySelector("#preflight-input").options.length === count, + enumeration && deviceIds ? 3 : 1, + ); + return page; +} + +test("preflight offers one default and microphone devices without changing initial capture", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + assert.deepEqual( + await page.locator("#preflight-input option").allTextContents(), + ["System default", "USB microphone", "Headset"], + ); + assert.equal(await page.locator("#preflight-input").inputValue(), ""); + assert.deepEqual(await page.evaluate(() => window.audioRequests), [""]); + const original = await page.evaluate(() => window.microphones[0].id); + await page.locator("#preflight-input").selectOption("usb"); + await page.waitForFunction(() => + window.microphones.some((track) => track.testDeviceId === "usb"), + ); + assert.deepEqual( + await page.evaluate(() => ({ + old: window.microphones[0].readyState, + current: window.microphones.at(-1).readyState, + camera: window.cameraTrack.readyState, + })), + { old: "ended", current: "live", camera: "live" }, + ); + assert.notEqual( + await page.evaluate(() => window.microphones.at(-1).id), + original, + ); +}); + +test("switching microphones closes the gate until the replacement proves sound", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + await page.locator("#camera-skip").click(); + await page.locator("#audio-heard").click(); + await page.waitForFunction( + () => !document.querySelector("#audio-check-join").disabled, + ); + await page.evaluate(() => { + window.heldInput = "usb"; + }); + await page.locator("#preflight-input").selectOption("usb"); + await page.waitForFunction(() => Boolean(window.releaseInput)); + assert.equal(await page.locator("#audio-check-join").isDisabled(), true); + assert.equal( + await page + .locator("#audio-step-mic") + .evaluate((node) => node.classList.contains("done")), + false, + ); + assert.equal( + await page + .locator("#audio-meter-fill") + .evaluate((node) => node.style.width), + "0%", + ); + await page.evaluate(() => { + window.heldInput = null; + window.releaseInput(); + }); + await page.waitForFunction( + () => !document.querySelector("#audio-check-join").disabled, + ); + assert.equal(await page.locator("#audio-heard").textContent(), "Confirmed"); +}); + +test("rapid microphone selections discard an earlier pending grant", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + await page.evaluate(() => { + window.heldInput = "usb"; + }); + await page.locator("#preflight-input").selectOption("usb"); + await page.waitForFunction(() => Boolean(window.releaseInput)); + await page.locator("#preflight-input").selectOption("headset"); + assert.deepEqual(await page.evaluate(() => window.audioRequests), [ + "", + "usb", + ]); + await page.evaluate(() => { + window.heldInput = null; + window.releaseInput(); + }); + await page.waitForFunction( + () => window.microphones.at(-1).testDeviceId === "headset", + ); + assert.equal( + await page.evaluate( + () => + window.microphones.find((track) => track.testDeviceId === "usb") + .readyState, + ), + "ended", + ); + assert.equal(await page.locator("#preflight-input").inputValue(), "headset"); + assert.equal( + await page.evaluate(() => window.microphones.at(-1).readyState), + "live", + ); +}); + +test("a browser without device enumeration still captures the default microphone", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, { enumeration: false }); + assert.equal(await page.locator("#preflight-input").isDisabled(), true); + assert.match( + await page.locator("#preflight-input-note").textContent(), + /system default/, + ); + assert.equal( + await page.evaluate(() => window.microphones[0].readyState), + "live", + ); + await page.waitForFunction(() => + document.querySelector("#audio-step-mic").classList.contains("done"), + ); +}); + +test("unsupported device-id constraints keep microphone selection on default", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, { deviceIds: false }); + assert.equal(await page.locator("#preflight-input").isDisabled(), true); + assert.equal(await page.locator("#preflight-input").inputValue(), ""); + assert.deepEqual(await page.evaluate(() => window.audioRequests), [""]); + await page.waitForFunction(() => + document.querySelector("#audio-step-mic").classList.contains("done"), + ); +}); + +test("a refused microphone blocks preflight until a different device succeeds", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + await page.evaluate(() => { + window.refusedInput = "usb"; + }); + await page.locator("#preflight-input").selectOption("usb"); + await page.waitForFunction(() => + document + .querySelector("#audio-check-status") + .textContent.includes("Selected microphone refused"), + ); + assert.equal(await page.locator("#audio-check-join").isDisabled(), true); + assert.equal( + await page.evaluate(() => window.microphones[0].readyState), + "ended", + ); + await page.locator("#preflight-input").selectOption("headset"); + await page.waitForFunction(() => + document.querySelector("#audio-step-mic").classList.contains("done"), + ); + assert.equal( + await page.evaluate(() => window.microphones.at(-1).testDeviceId), + "headset", + ); +}); + +test("failed capture retries preserve device menus until a grant or devicechange", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + await page.evaluate(() => { + const enumerate = navigator.mediaDevices.enumerateDevices; + window.deviceListings = 0; + navigator.mediaDevices.enumerateDevices = async () => { + window.deviceListings += 1; + return enumerate(); + }; + window.inputOption = document.querySelector("#preflight-input").options[1]; + window.outputOption = + document.querySelector("#preflight-output").options[0]; + window.refusedInput = "usb"; + window.refusedInputError = "OverconstrainedError"; + }); + await page.locator("#preflight-input").selectOption("usb"); + await page.waitForFunction( + () => window.audioRequests.filter((id) => id === "usb").length >= 2, + ); + assert.deepEqual( + await page.evaluate(() => ({ + listings: window.deviceListings, + inputRetained: window.inputOption.isConnected, + outputRetained: window.outputOption.isConnected, + })), + { listings: 0, inputRetained: true, outputRetained: true }, + ); + await page.evaluate(() => { + window.inputs.find((device) => device.deviceId === "headset").label = + "Granted headset"; + }); + await page.locator("#preflight-input").selectOption("headset"); + await page.waitForFunction( + () => + document.querySelector('#preflight-input option[value="headset"]') + .textContent === "Granted headset", + ); + await page.evaluate(() => { + window.inputs.find((device) => device.deviceId === "usb").label = + "Reconnected USB"; + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + }); + await page.waitForFunction( + () => + document.querySelector('#preflight-input option[value="usb"]') + .textContent === "Reconnected USB", + ); +}); + +test("unplugging the selected microphone requests default and reconnecting refreshes choices", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + await page.locator("#preflight-input").selectOption("usb"); + await page.waitForFunction( + () => window.microphones.at(-1).testDeviceId === "usb", + ); + await page.evaluate(() => { + window.removedInput = window.inputs.find( + (device) => device.deviceId === "usb", + ); + window.inputs = window.inputs.filter((device) => device.deviceId !== "usb"); + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + }); + await page.waitForFunction( + () => + document.querySelector("#preflight-input").value === "" && + window.microphones.at(-1).testDeviceId === "", + ); + assert.equal( + await page.evaluate( + () => + window.microphones.find((track) => track.testDeviceId === "usb") + .readyState, + ), + "ended", + ); + await page.evaluate(() => { + window.inputs.push(window.removedInput); + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + }); + await page.waitForFunction( + () => document.querySelector("#preflight-input").options.length === 3, + ); + assert.equal(await page.locator("#preflight-input").inputValue(), ""); +}); + +test("microphone enumeration errors recover and ignore an older reply", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + await page.evaluate(() => { + navigator.mediaDevices.enumerateDevices = async () => { + throw new Error("Listing refused"); + }; + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + }); + await page.waitForFunction(() => + document + .querySelector("#preflight-input-note") + .textContent.includes("could not be listed"), + ); + assert.equal( + await page.locator("#preflight-input-note").textContent(), + "Microphones could not be listed. Check your browser permissions and reload the page.", + ); + await page.evaluate(() => { + window.pendingLists = []; + window.holdLists = true; + navigator.mediaDevices.enumerateDevices = () => + window.holdLists + ? new Promise((resolve) => { + window.pendingLists.push(resolve); + }) + : Promise.resolve(window.inputs); + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + window.staleLists = [...window.pendingLists]; + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + for (const resolve of window.pendingLists.slice(window.staleLists.length)) + resolve(window.inputs); + window.holdLists = false; + }); + await page.waitForFunction( + () => document.querySelector("#preflight-input-note").hidden, + ); + await page.evaluate(async () => { + for (const resolve of window.staleLists) resolve([]); + await Promise.resolve(); + }); + assert.equal(await page.locator("#preflight-input option").count(), 3); +}); + +test("the chosen microphone track is the one handed to LiveKit", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + await page.locator("#preflight-input").selectOption("usb"); + await page.locator("#camera-skip").click(); + await page.locator("#audio-heard").click(); + await page.waitForFunction( + () => !document.querySelector("#audio-check-join").disabled, + ); + const selected = await page.evaluate(() => window.microphones.at(-1).id); + await page.route("**/api/token", (route) => + route.fulfill({ + json: { serverUrl: "wss://test.invalid", token: "test", durationMin: 45 }, + }), + ); + await page.evaluate(() => { + const events = window.LivekitClient.RoomEvent; + window.published = []; + window.LivekitClient = { + RoomEvent: events, + Track: { Source: { Microphone: "microphone", Camera: "camera" } }, + Room: class { + localParticipant = { + async publishTrack(track, options) { + window.published.push({ + id: track.id, + kind: track.kind, + deviceId: track.testDeviceId, + source: options.source, + }); + // Stop at publication: the test has no LiveKit service or agent. + throw new Error("Test publication captured"); + }, + }; + on() { + return this; + } + async connect() {} + async disconnect() {} + }, + }; + }); + await page.locator("#audio-check-join").click(); + await page.waitForFunction(() => window.published.length > 0); + assert.deepEqual(await page.evaluate(() => window.published), [ + { id: selected, kind: "audio", deviceId: "usb", source: "microphone" }, + ]); +}); + +test("leaving preflight stops a microphone grant that arrives late", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t); + await page.evaluate(() => { + window.heldInput = "usb"; + }); + await page.locator("#preflight-input").selectOption("usb"); + await page.waitForFunction(() => Boolean(window.releaseInput)); + let releaseLobby; + const lobby = new Promise((resolve) => { + releaseLobby = resolve; + }); + await page.route(`${base}/`, async (route) => { + await lobby; + await route.continue(); + }); + try { + const held = await page.evaluate(() => { + const ids = window.microphones.map((track) => track.id); + document.querySelector("#audio-check-leave").click(); + window.releaseInput(); + return ids; + }); + releaseLobby(); + await page.waitForURL(`${base}/`); + const stopped = await page.evaluate(() => + JSON.parse(sessionStorage.getItem("microphone-stops") || "[]"), + ); + assert.ok(held.length >= 2); + assert.deepEqual( + held.filter((id) => !stopped.includes(id)), + [], + ); + } finally { + releaseLobby(); + } +}); diff --git a/tests/browser/audio-playback-browser.test.js b/tests/browser/audio-playback-browser.test.js index 62c0b33a5..ccb1e3859 100644 --- a/tests/browser/audio-playback-browser.test.js +++ b/tests/browser/audio-playback-browser.test.js @@ -86,9 +86,12 @@ test("interview reconnect wiring recovers blocked Jim audio and clears on discon for (const error of errors) t.diagnostic(`page error: ${error}`); }); await page.addInitScript(() => { + delete HTMLMediaElement.prototype.setSinkId; const nativePlay = HTMLMediaElement.prototype.play; window.blockJim = true; HTMLMediaElement.prototype.play = function () { + if (this.id === "jim-audio" && window.jimMutedAtPlay === undefined) + window.jimMutedAtPlay = this.muted; return this.id === "jim-audio" && window.blockJim ? Promise.reject( new DOMException("Test autoplay denial", "NotAllowedError"), @@ -137,6 +140,8 @@ test("interview reconnect wiring recovers blocked Jim audio and clears on discon await page.waitForFunction( () => !document.querySelector("#audio-check-join").disabled, ); + const outputNote = await page.locator("#preflight-output-note").textContent(); + assert.match(outputNote, /cannot choose an output device/); await page.locator("#audio-check-join").click(); await page.waitForFunction( () => @@ -144,6 +149,14 @@ test("interview reconnect wiring recovers blocked Jim audio and clears on discon document.querySelector("#mic").textContent === "Mic on", ); await page.evaluate(() => { + window.fakeLivekit = window.LivekitClient; + }); + await page.addScriptTag({ + url: `${base}/vendor/livekit-client.js?real-attach`, + }); + await page.evaluate(async () => { + const RemoteAudioTrack = window.LivekitClient.RemoteAudioTrack; + window.LivekitClient = window.fakeLivekit; const room = window.testRoom; const participant = { identity: "interviewer-jim", @@ -152,19 +165,34 @@ test("interview reconnect wiring recovers blocked Jim audio and clears on discon room.remoteParticipants.set(participant.identity, participant); room.emit("Reconnecting"); room.emit("Reconnected"); - window.testTrack = { - kind: "audio", - attach: (audio) => { - // A live track never ends; loop the short fixture so it does not - // report paused again before the wait after resume sees it. - audio.loop = true; - audio.src = "/audio/test-voice.wav"; - }, - detach: () => [document.querySelector("#jim-audio")], + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + window.testTrack = new RemoteAudioTrack( + stream.getAudioTracks()[0], + "test-jim", + ); + const attach = window.testTrack.attach.bind(window.testTrack); + window.testTrack.attach = (audio) => { + const result = attach(audio); + window.jimMutedAtAttach = audio.muted; + return result; }; room.emit("TrackSubscribed", window.testTrack, {}, participant); }); await page.waitForSelector("#audio-playback-notice:not([hidden])"); + assert.deepEqual( + await page.evaluate(() => [window.jimMutedAtAttach, window.jimMutedAtPlay]), + [false, true], + ); + assert.equal( + await page.locator("#preflight-output-note").textContent(), + outputNote, + ); + assert.equal( + await page + .locator("#preflight-output-note") + .evaluate((note) => note.hidden), + false, + ); assert.equal(await page.locator("#jim-stage #resume-audio").count(), 1); await page.locator("#resume-audio").click(); await page.waitForFunction( @@ -172,6 +200,10 @@ test("interview reconnect wiring recovers blocked Jim audio and clears on discon document.querySelector("#audio-playback-notice").hidden && !document.querySelector("#jim-audio").paused, ); + assert.equal( + await page.locator("#jim-audio").evaluate((audio) => audio.muted), + false, + ); assert.equal(await page.evaluate(() => window.testRoom.starts), 1); assert.equal( await page.evaluate(() => document.activeElement.id), diff --git a/tests/browser/audio-playback.test.js b/tests/browser/audio-playback.test.js index 1d2c1c65e..62d49598b 100644 --- a/tests/browser/audio-playback.test.js +++ b/tests/browser/audio-playback.test.js @@ -6,6 +6,7 @@ import { playbackBlocked, } from "../../web/audio-playback.js"; import { loadInterview } from "./source.js"; +import { createJimVolume } from "../../web/audio-output.js"; const blocked = () => new DOMException("Gesture required", "NotAllowedError"); const tick = () => new Promise((resolve) => setImmediate(resolve)); @@ -328,12 +329,158 @@ function audioTrack() { }; } +test("releasing a routing hold detects blocked audible play and offers recovery", async () => { + const route = deferred(); + let allowed = false; + const element = { + volume: 1, + muted: false, + isConnected: false, + play() { + return this.muted || allowed + ? Promise.resolve() + : Promise.reject(blocked()); + }, + }; + let volume; + const f = fixture({ reapply: () => volume.apply() }); + const own = { + jimAudio: null, + jimTrack: null, + audioPlayback: f.playback, + isCurrentAgent: () => true, + applyAudioOutput: async () => {}, + audioOutputSettled: () => route.promise, + setTimeout: () => 1, + clearTimeout() {}, + setBanner() {}, + document: { + createElement: () => element, + body: { appendChild: (audio) => (audio.isConnected = true) }, + }, + }; + volume = own.jimVolume = createJimVolume({ + element: () => own.jimAudio, + storage: null, + }); + interviewFunction("playRemoteAudio", own)({ kind: "audio", attach() {} }, {}); + await tick(); + assert.equal(element.muted, true); + assert.equal(f.notice.hidden, true); + route.resolve(); + await tick(); + assert.equal(element.muted, false); + assert.equal(f.notice.hidden, false); + allowed = true; + f.click(); + await tick(); + assert.equal(f.notice.hidden, true); +}); + +test("a native pause after successful play rechecks permission and drops old listeners", async () => { + const f = fixture(); + let allowed = true; + let attempts = 0; + const audio = Object.assign(new EventTarget(), { + paused: false, + ended: false, + play() { + attempts += 1; + return allowed ? Promise.resolve() : Promise.reject(blocked()); + }, + }); + await f.playback.play(audio); + allowed = false; + audio.paused = true; + audio.dispatchEvent(new Event("pause")); + await tick(); + assert.equal(attempts, 2); + assert.equal(f.notice.hidden, false); + f.playback.forget(audio); + audio.dispatchEvent(new Event("pause")); + await tick(); + assert.equal(attempts, 2); + assert.equal(f.notice.hidden, true); +}); + +test("fresh Jim audio stays muted until the selected output route settles", async () => { + const route = deferred(); + const observed = []; + const element = { muted: false, volume: 1, isConnected: false }; + let expire; + const messages = []; + const own = { + jimAudio: null, + jimTrack: null, + isCurrentAgent: () => true, + audioOutputId: () => "headphones", + setTimeout(callback) { + expire = callback; + return 1; + }, + clearTimeout() {}, + setBanner(source, message) { + messages.push([source, message]); + }, + applyAudioOutput(id) { + assert.equal(id, "headphones"); + return Promise.resolve(); + }, + audioOutputSettled() { + return route.promise; + }, + audioPlayback: { + play(audio) { + observed.push(["play", audio.muted]); + return Promise.resolve(); + }, + }, + document: { + createElement: () => element, + body: { appendChild: (audio) => (audio.isConnected = true) }, + }, + }; + own.jimVolume = createJimVolume({ + element: () => own.jimAudio, + storage: null, + }); + const track = { + kind: "audio", + attach(audio) { + audio.muted = false; + observed.push(["attach", audio.muted]); + }, + }; + const play = interviewFunction("playRemoteAudio", own); + play(track, {}); + assert.deepEqual(observed, [ + ["attach", false], + ["play", true], + ]); + await tick(); + assert.equal(element.muted, true); + expire(); + assert.match(messages.at(-1)[1], /reload/); + route.resolve(); + await tick(); + assert.equal(element.muted, false); + assert.deepEqual(messages.at(-1), ["output", ""]); + play(track, {}); + assert.equal(own.jimAudio, element); + assert.deepEqual(observed.slice(-2), [ + ["attach", false], + ["play", false], + ]); +}); + test("a late unsubscribe for Jim's old track leaves the new track playing and recoverable", async () => { const f = fixture(); const own = { jimAudio: null, jimTrack: null, audioPlayback: f.playback, + applyAudioOutput: async () => {}, + audioOutputSettled: async () => {}, isCurrentAgent: () => true, isAvatarAnalyserTrack: () => false, document: { @@ -354,6 +501,10 @@ test("a late unsubscribe for Jim's old track leaves the new track playing and re body: { appendChild: (element) => (element.isConnected = true) }, }, }; + own.jimVolume = createJimVolume({ + element: () => own.jimAudio, + storage: null, + }); const play = interviewFunction("playRemoteAudio", own); const drop = interviewFunction("dropRemoteAudio", own); const old = audioTrack(); @@ -361,6 +512,7 @@ test("a late unsubscribe for Jim's old track leaves the new track playing and re play(old, {}); await tick(); const element = own.jimAudio; + assert.equal(element.muted, false); // Reconnect order LiveKit is allowed to use: the new subscribe first. play(replacement, {}); await tick(); @@ -368,6 +520,7 @@ test("a late unsubscribe for Jim's old track leaves the new track playing and re drop(old); assert.equal(own.jimAudio, element); assert.equal(element.isConnected, true); + assert.equal(element.muted, false); assert.equal(element.paused, false); assert.equal(f.notice.hidden, false); drop(replacement); diff --git a/tests/browser/devices.test.js b/tests/browser/devices.test.js index 0c8312c48..4da35e698 100644 --- a/tests/browser/devices.test.js +++ b/tests/browser/devices.test.js @@ -44,6 +44,166 @@ test("a granted track of each kind lands in one stream", async () => { assert.equal(pool.errorOf("audio"), null); }); +test("selecting a microphone replaces only audio and preserves processing constraints", async () => { + const asked = []; + const { pool } = poolWith(async (constraints) => { + asked.push(constraints); + return new FakeStream([ + new FakeTrack(constraints.audio ? "audio" : "video"), + ]); + }); + let lost = 0; + pool.configure("audio", { + onLost: () => { + lost += 1; + }, + }); + pool.start(); + await settle(); + const original = pool.trackOf("audio"); + const camera = pool.trackOf("video"); + const stream = pool.stream; + pool.selectMicrophone("usb"); + assert.equal(original.stopped, true); + assert.equal(pool.trackOf("audio"), null); + assert.equal(lost, 1); + await settle(); + assert.deepEqual(asked.at(-1), { + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + deviceId: { exact: "usb" }, + }, + }); + assert.equal(pool.stream, stream); + assert.equal(pool.trackOf("video"), camera); + assert.equal(camera.stopped, false); + assert.notEqual(pool.trackOf("audio"), original); + const count = asked.length; + pool.selectMicrophone("usb"); + assert.equal(asked.length, count); + pool.selectMicrophone(""); + await settle(); + assert.equal(asked.at(-1).audio.deviceId, undefined); + assert.equal(pool.stream.getTracks().length, 2); +}); + +test("rapid microphone choices stop the old grant and request only the latest device", async () => { + const pending = []; + const asked = []; + const { pool } = poolWith((constraints) => { + if (constraints.video) + return Promise.resolve(new FakeStream([new FakeTrack("video")])); + asked.push(constraints.audio.deviceId?.exact || ""); + return new Promise((resolve) => pending.push(resolve)); + }); + pool.start(); + pool.selectMicrophone("usb"); + pool.selectMicrophone("headset"); + assert.deepEqual(asked, [""]); + const abandoned = new FakeTrack("audio"); + pending.shift()(new FakeStream([abandoned])); + await settle(); + assert.equal(abandoned.stopped, true); + assert.equal(pool.trackOf("audio"), null); + assert.deepEqual(asked, ["", "headset"]); + const latest = new FakeTrack("audio"); + pending.shift()(new FakeStream([latest])); + await settle(); + assert.equal(pool.trackOf("audio"), latest); + assert.equal(latest.stopped, false); +}); + +test("an obsolete microphone rejection does not poison the latest choice", async () => { + let rejectOld; + let grantNew; + const { pool } = poolWith((constraints) => { + if (constraints.video) + return Promise.resolve(new FakeStream([new FakeTrack("video")])); + if (!constraints.audio.deviceId) + return new Promise((resolve, reject) => { + rejectOld = reject; + }); + return new Promise((resolve) => { + grantNew = resolve; + }); + }); + pool.start(); + pool.selectMicrophone("usb"); + rejectOld(new Error("Old device refused")); + await settle(); + assert.equal(pool.errorOf("audio"), null); + const selected = new FakeTrack("audio"); + grantNew(new FakeStream([selected])); + await settle(); + assert.equal(pool.trackOf("audio"), selected); +}); + +test("a failed selected microphone retries that device and can return to default", async () => { + const asked = []; + const retries = []; + const { pool } = poolWith( + async (constraints) => { + if (constraints.audio?.deviceId) { + asked.push(constraints.audio.deviceId.exact); + throw new Error("Selected device unavailable"); + } + return new FakeStream([ + new FakeTrack(constraints.audio ? "audio" : "video"), + ]); + }, + { + schedule: (callback) => { + retries.push(callback); + return retries.length; + }, + unschedule: () => {}, + }, + ); + pool.start(); + await settle(); + pool.selectMicrophone("missing"); + await settle(); + assert.equal(pool.trackOf("audio"), null); + assert.match(pool.errorOf("audio"), /Selected device unavailable/); + retries.shift()(); + await settle(); + assert.deepEqual(asked, ["missing", "missing"]); + pool.selectMicrophone(""); + await settle(); + assert.ok(pool.trackOf("audio")); + assert.equal(pool.errorOf("audio"), null); +}); + +test("leaving during a microphone change stops a late grant without requesting again", async () => { + let finished = false; + let grant; + let asked = 0; + const { pool } = poolWith( + (constraints) => { + if (constraints.video) + return Promise.resolve(new FakeStream([new FakeTrack("video")])); + asked += 1; + return new Promise((resolve) => { + grant = resolve; + }); + }, + { isFinished: () => finished }, + ); + pool.start(); + pool.selectMicrophone("usb"); + finished = true; + const late = new FakeTrack("audio"); + grant(new FakeStream([late])); + await settle(); + assert.equal(late.stopped, true); + assert.equal(pool.trackOf("audio"), null); + assert.equal(asked, 1); + pool.selectMicrophone("headset"); + assert.equal(asked, 1); +}); + // The bug this replaces: each device scheduled its own retry and both called // back into the same start, so denying both prompts doubled the number of // in-flight requests every two seconds until the tab died. diff --git a/tests/browser/jim-volume.test.js b/tests/browser/jim-volume.test.js index 5fbfcc849..a31f518cd 100644 --- a/tests/browser/jim-volume.test.js +++ b/tests/browser/jim-volume.test.js @@ -19,6 +19,26 @@ function fixture(stored = null, audio = { volume: 1, muted: false }) { }; } +test("routing hold and candidate mute remain independent through volumechange", () => { + const audio = Object.assign(new EventTarget(), { volume: 1, muted: false }); + const f = fixture("37", audio); + f.volume.setRoutingHeld(true); + assert.equal(audio.muted, true); + assert.equal(f.volume.muted, false); + f.volume.setVolume(45); + audio.muted = false; + audio.dispatchEvent(new Event("volumechange")); + assert.equal(audio.muted, true); + f.volume.toggleMute(); + f.volume.setRoutingHeld(false); + assert.equal(audio.muted, true); + assert.equal(f.volume.muted, true); + f.volume.toggleMute(); + assert.equal(audio.muted, false); + assert.equal(audio.volume, 0.45); + assert.equal(f.storage.getItem(JIM_VOLUME_KEY), "45"); +}); + for (const stored of [ null, "", diff --git a/tests/browser/meet-tab-audio.test.js b/tests/browser/meet-tab-audio.test.js index d5933e733..8a9f648dc 100644 --- a/tests/browser/meet-tab-audio.test.js +++ b/tests/browser/meet-tab-audio.test.js @@ -12,12 +12,11 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { - assertIncludesCompact, firstPartyScripts as webScripts, interviewSource, root, } from "./source.js"; -import { outputAfterRouting, outputOptions } from "../../web/meet-audio.js"; +import { outputOptions } from "../../web/meet-audio.js"; const web = join(root, "web"); const read = (name) => readFileSync(join(web, name), "utf8"); @@ -121,102 +120,49 @@ test("meet-mode getusermedia allowlist", () => { // These execute the decisions instead of reading them. The text assertions // below still guard the wiring, but a wiring assertion cannot tell whether the // logic is right, and both bugs this feature shipped with were logic. -test("sink routing offers only devices the browser has named", () => { +test("output choices include one default and only named device ids", () => { const before = [{ kind: "audiooutput", deviceId: "", label: "" }]; const after = [ + { kind: "audiooutput", deviceId: "default", label: "System speaker" }, { kind: "audiooutput", deviceId: "speakers", label: "Built-in" }, { kind: "audioinput", deviceId: "mic", label: "Mic" }, { kind: "audiooutput", deviceId: "usb", label: "" }, ]; - // Pre-grant placeholder: an empty deviceId renders as a real choice and then - // routes to nothing, so it must not reach the list at all. - assert.deepEqual(outputOptions(before, null), []); + // The explicit default replaces blank placeholders and the browser's default + // alias, so neither can produce a duplicate choice. + assert.deepEqual(outputOptions(before, null), [ + { deviceId: "", label: "System default", selected: true }, + ]); assert.deepEqual(outputOptions(after, "usb"), [ + { deviceId: "", label: "System default", selected: false }, { deviceId: "speakers", label: "Built-in", selected: false }, { deviceId: "usb", label: "Output 2", selected: true }, ]); // An unnamed device still gets a usable label, and inputs never appear. assert.deepEqual( outputOptions(after, null).map((option) => option.deviceId), - ["speakers", "usb"], - ); -}); - -test("sink routing keeps the preference and the real sink in agreement", () => { - // Success: the pick becomes the preference. - assert.equal(outputAfterRouting("old", "new", true), "new"); - // Refusal rolls back. Keeping the refused device would leave the select - // showing a speaker Jim never played through, until the next reload. - assert.equal(outputAfterRouting("old", "new", false), "old"); - // Refused with nothing to fall back to: no preference at all, so the caller - // clears the key rather than persisting the device that just failed. - assert.equal(outputAfterRouting(null, "new", false), null); - // Reapplying a stored preference passes the same id as both, so there is no - // earlier device to fall back to. Returning it would write the refused id - // back and retry it on every reload. - assert.equal( - outputAfterRouting("same", "same", false), - "", - "a refused reapply must clear the preference, not re-store it", - ); - assert.equal( - outputAfterRouting("same", "same", true), - "same", - "and a reapply that works keeps it", + ["", "speakers", "usb"], ); }); // What the executable tests above cannot see: that interview.js actually wires // those decisions to the right nodes and the right track. test("sink routing", () => { - // setSinkId targets Jim's element and nothing else, so the candidate cannot - // accidentally reroute their own microphone monitoring. + // Playback routing must never target the candidate's microphone monitoring. const sinkCalls = [...script.matchAll(/(\w+)\s*\n?\s*\.setSinkId\(/g)].map( (match) => match[1], ); assert.deepEqual( sinkCalls, - ["jimAudio"], - "setSinkId may only target Jim's audio element", + ["target", "jimAudio", "jimAudio"], + "setSinkId targets only the test sounds and Jim's audio", ); assert.match( script, /element\.id = "jim-audio"/, "the remote audio element needs a stable id", ); - assert.match( - script, - /outputOptions\(devices, readStored\(AUDIO_OUTPUT_KEY\)\)/, - "the selector is built from the shared decision", - ); - // Persist before routing. A choice made during the preflight, before Jim's - // track exists, is dropped if it only survives a successful setSinkId. - // `applyAudioOutput` is the queue in front of this; the routing itself is - // what has to get the order right. - const apply = script.slice(script.indexOf("async function routeAudioOutput")); - assert.ok( - apply.indexOf("writeStored(AUDIO_OUTPUT_KEY") < - apply.indexOf(".setSinkId("), - "routing must persist before it routes", - ); - // Serialized and stamped. Two picks in flight can settle in either order, and - // the older one must not write its device over the newer one's. - assertIncludesCompact( - script, - "routingChain = routingChain.then(() => routeAudioOutput(deviceId, ++routingRequest))", - "routing requests are serialized", - ); - assert.match( - apply, - /if \(request !== routingRequest\) return;/, - "and a superseded request does not get the last word on the preference", - ); - assert.match( - apply, - /outputAfterRouting\(previous, deviceId, routed\)/, - "rollback comes from the tested decision", - ); // One element for the session, and it is released on unsubscribe. assert.match( script, @@ -228,11 +174,6 @@ test("sink routing", () => { /TrackUnsubscribed, dropRemoteAudio/, "unsubscribed audio must be detached", ); - // Unsupported browsers get told, not silently downgraded, and the whole row - // hides so the label does not dangle over a control that is gone. - assert.match(script, /"setSinkId" in HTMLMediaElement\.prototype/); - assert.match(script, /meetOutputRow\.hidden = true/); - assert.match(script, /showOutputNote\(OUTPUT_NOTES\.unsupported\)/); }); test("no synthetic microphone", () => { diff --git a/tests/browser/mic-meter.test.js b/tests/browser/mic-meter.test.js index ef1cca861..d0dfc9582 100644 --- a/tests/browser/mic-meter.test.js +++ b/tests/browser/mic-meter.test.js @@ -122,6 +122,28 @@ test("a replacement microphone is proven on its own frames", () => { ); }); +test("callbacks from a replaced microphone cannot prove or fail the current one", () => { + const started = fakeMeter(); + const pool = fakePool(); + const meter = createMicMeter({ + pool, + isFinished: () => false, + onLevel: () => {}, + onFailure: () => assert.fail("a stale meter failed the current microphone"), + startMeter: started.start, + }); + meter.start(); + const previous = started.latest(); + meter.forget(); + meter.start(); + for (let index = 0; index < MIC_CONFIRM_FRAMES; index++) previous.onPeak(0.8); + previous.onError("Old meter stopped"); + assert.equal(meter.peak(), 0); + assert.equal(pool.errorOf("audio"), null); + loudEnough(started); + assert.ok(meter.peak() > 0); +}); + /// Timers a test can run by hand. /// /// The retry is the whole point of these two, and a real `setTimeout` would diff --git a/tests/browser/preflight-speech.test.js b/tests/browser/preflight-speech.test.js index d50db53de..507fa7d05 100644 --- a/tests/browser/preflight-speech.test.js +++ b/tests/browser/preflight-speech.test.js @@ -46,9 +46,10 @@ async function playLoopedSpeech(page) { ); } -async function preflight(t) { +async function preflight(t, initialize) { const page = await browser.newPage(); t.after(() => page.close()); + if (initialize) await page.addInitScript(initialize); await page.addInitScript(() => { const capture = navigator.mediaDevices.getUserMedia.bind( navigator.mediaDevices, @@ -66,6 +67,777 @@ async function preflight(t) { return page; } +for (const reset of [false, true]) { + for (const listing of ["empty", "placeholder", "failure"]) { + test(`saved output survives a pre-grant ${listing} listing and ${reset ? "can return to default" : "routes both test sounds"}`, async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await browser.newPage(); + t.after(() => page.close()); + await page.addInitScript((listing) => { + localStorage.setItem("codetrial:audioOutputId", "headphones"); + const capture = navigator.mediaDevices.getUserMedia.bind( + navigator.mediaDevices, + ); + navigator.mediaDevices.getUserMedia = async (constraints) => { + if (constraints.audio) + await new Promise((resolve) => { + window.grantMicrophone = resolve; + }); + const stream = await capture(constraints); + if (constraints.audio) window.microphoneGranted = true; + return stream; + }; + navigator.mediaDevices.enumerateDevices = async () => { + if (listing === "failure") throw new Error("Listing unavailable"); + if (window.microphoneGranted) + return [ + { + kind: "audiooutput", + deviceId: "headphones", + label: "Headphones", + }, + ]; + return listing === "empty" + ? [] + : [{ kind: "audiooutput", deviceId: "", label: "" }]; + }; + window.probeRoutes = []; + HTMLMediaElement.prototype.setSinkId = async function (id) { + window.probeRoutes.push(["speech", id]); + Object.defineProperty(this, "sinkId", { + configurable: true, + value: id, + }); + }; + AudioContext.prototype.setSinkId = async function (id) { + window.probeRoutes.push(["tone", id]); + Object.defineProperty(this, "sinkId", { + configurable: true, + value: id, + }); + }; + }, listing); + await page.goto(`${base}/interview.html?problem=chargeback-pair-match`); + await page.waitForFunction( + () => + window.grantMicrophone && + !document.querySelector("#preflight-output-note").hidden, + ); + for (const selector of ["#preflight-output", "#meet-output-select"]) { + assert.equal(await page.locator(selector).inputValue(), "headphones"); + assert.equal( + await page.locator(`${selector} option:checked`).textContent(), + "Saved output device", + ); + } + assert.equal( + await page.evaluate(() => + localStorage.getItem("codetrial:audioOutputId"), + ), + "headphones", + ); + await page.locator("#audio-test-tone").click(); + await page.waitForFunction( + () => + document.querySelector("#audio-test-tone").textContent === + "Play it again", + ); + assert.deepEqual( + await page.evaluate(() => + window.probeRoutes.filter(([sound]) => sound === "tone"), + ), + [["tone", "headphones"]], + ); + await page.locator("#audio-test-speech").click(); + await page.waitForFunction( + () => + !document.querySelector("#audio-test-voice").paused || + document.querySelector("#audio-test-voice").ended, + ); + assert.deepEqual( + await page.evaluate(() => + window.probeRoutes.filter(([sound]) => sound === "speech"), + ), + [["speech", "headphones"]], + ); + if (reset) { + await page.locator("#preflight-output").selectOption(""); + for (const selector of ["#preflight-output", "#meet-output-select"]) + assert.equal(await page.locator(selector).inputValue(), ""); + assert.equal( + await page.evaluate(() => + localStorage.getItem("codetrial:audioOutputId"), + ), + null, + ); + await page.locator("#audio-test-speech").click(); + await page.waitForFunction(() => + window.probeRoutes.some( + ([sound, id]) => sound === "speech" && id === "", + ), + ); + } + await page.evaluate(() => window.grantMicrophone()); + await page.waitForFunction(() => window.microphoneGranted); + if (listing !== "failure") { + await page.waitForFunction( + () => + document.querySelector("#preflight-output option[value=headphones]") + ?.textContent === "Headphones", + ); + } + for (const selector of ["#preflight-output", "#meet-output-select"]) + assert.equal( + await page.locator(selector).inputValue(), + reset ? "" : "headphones", + ); + assert.equal( + await page.evaluate(() => + localStorage.getItem("codetrial:audioOutputId"), + ), + reset ? null : "headphones", + ); + }); + } +} + +test("preflight output defaults to the system and routes test sounds to the chosen device", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "default", label: "Default speaker" }, + { kind: "audiooutput", deviceId: "headphones", label: "Headphones" }, + { kind: "audioinput", deviceId: "microphone", label: "Microphone" }, + ]; + window.outputRoutes = []; + HTMLMediaElement.prototype.setSinkId = async function (id) { + window.outputRoutes.push({ target: this.id, id }); + Object.defineProperty(this, "sinkId", { configurable: true, value: id }); + }; + AudioContext.prototype.setSinkId = async function (id) { + window.outputRoutes.push({ target: "tone", id }); + Object.defineProperty(this, "sinkId", { configurable: true, value: id }); + }; + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 2, + ); + assert.equal(await page.locator("#preflight-output").inputValue(), ""); + assert.deepEqual( + await page.locator("#preflight-output option").allTextContents(), + ["System default", "Headphones"], + ); + await page.locator("#audio-heard").click(); + await page.waitForFunction(() => + document.querySelector("#audio-step-output").classList.contains("done"), + ); + await page.locator("#preflight-output").selectOption("headphones"); + assert.equal(await page.locator("#audio-check-join").isDisabled(), true); + assert.equal(await page.locator("#audio-heard").textContent(), "I heard it"); + await page.locator("#audio-test-tone").click(); + await page.waitForFunction(() => + window.outputRoutes.some( + (route) => route.target === "tone" && route.id === "headphones", + ), + ); + await page.locator("#audio-test-speech").click(); + await page.waitForFunction(() => + window.outputRoutes.some( + (route) => + route.target === "audio-test-voice" && route.id === "headphones", + ), + ); + await page.locator("#preflight-output").selectOption(""); + await page.locator("#audio-test-tone").click(); + await page.waitForFunction(() => + window.outputRoutes.some( + (route) => route.target === "tone" && route.id === "", + ), + ); + await page.locator("#audio-test-speech").click(); + await page.waitForFunction(() => + window.outputRoutes.some( + (route) => route.target === "audio-test-voice" && route.id === "", + ), + ); +}); + +test("system-default test sounds leave already-default sinks unchanged", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + window.routeAttempts = 0; + const route = async () => { + window.routeAttempts += 1; + }; + HTMLMediaElement.prototype.setSinkId = route; + AudioContext.prototype.setSinkId = route; + }); + await page.locator("#audio-test-tone").click(); + await page.waitForFunction( + () => + document.querySelector("#audio-test-tone").textContent === + "Play it again", + ); + await page.locator("#audio-test-speech").click(); + await page.waitForFunction( + () => document.querySelector("#audio-test-voice").ended, + ); + assert.equal(await page.evaluate(() => window.routeAttempts), 0); +}); + +async function outputRoutingPage(t) { + const page = await preflight(t, () => { + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "default", label: "Default" }, + { kind: "audiooutput", deviceId: "headphones", label: "Headphones" }, + { kind: "audiooutput", deviceId: "speakers", label: "Speakers" }, + ]; + }); + await page.evaluate(async () => { + window.outputApi = await import("./audio-output.js"); + window.jimRoutes = []; + window.mockJim = { + sinkId: "", + async setSinkId(id) { + window.jimRoutes.push(id); + if (window.holdRoute) + await new Promise((resolve) => { + window.releaseRoute = resolve; + }); + if (id === window.refusedOutput) throw new Error("Device refused"); + this.sinkId = id; + }, + }; + window.outputApi.initAudioOutput({ + nodes: { + meetOutputSelect: document.querySelector("#meet-output-select"), + meetOutputRow: document.querySelector("#meet-output-row"), + meetOutputNote: document.querySelector("#meet-output-note"), + preflightOutput: document.querySelector("#preflight-output"), + preflightOutputNote: document.querySelector("#preflight-output-note"), + }, + jimAudioElement: () => window.mockJim, + }); + await window.outputApi.refreshAudioOutputs(); + }); + return page; +} + +test("both output selectors share choices and reset preflight confirmation", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await outputRoutingPage(t); + assert.deepEqual( + await page.locator("#meet-output-select option").allTextContents(), + await page.locator("#preflight-output option").allTextContents(), + ); + await page.locator("#preflight-output").selectOption("headphones"); + await page.waitForFunction(() => window.mockJim.sinkId === "headphones"); + assert.equal( + await page.locator("#meet-output-select").inputValue(), + "headphones", + ); + await page.locator("#audio-heard").click(); + await page.evaluate(() => { + const select = document.querySelector("#meet-output-select"); + select.value = "speakers"; + select.dispatchEvent(new Event("change")); + }); + await page.waitForFunction(() => window.mockJim.sinkId === "speakers"); + assert.equal( + await page.locator("#preflight-output").inputValue(), + "speakers", + ); + assert.equal(await page.locator("#audio-heard").textContent(), "I heard it"); +}); + +test("a refused Jim route restores both selectors to the actual previous sink", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await outputRoutingPage(t); + await page.locator("#preflight-output").selectOption("speakers"); + await page.waitForFunction(() => window.mockJim.sinkId === "speakers"); + await page.evaluate(() => { + window.refusedOutput = "headphones"; + }); + await page.locator("#preflight-output").selectOption("headphones"); + await page.waitForFunction(() => + document + .querySelector("#preflight-output-note") + .textContent.includes("refused"), + ); + for (const id of ["preflight-output", "meet-output-select"]) { + assert.equal(await page.locator(`#${id}`).inputValue(), "speakers"); + } + assert.equal( + await page.evaluate(() => window.outputApi.audioOutputId()), + "speakers", + ); + assert.equal(await page.evaluate(() => window.mockJim.sinkId), "speakers"); + assert.equal( + await page.locator("#preflight-output-note").textContent(), + "That output device was refused. The previous output or system default is selected.", + ); + const refusal = await page.locator("#preflight-output-note").textContent(); + await page.evaluate(async () => { + await window.outputApi.refreshAudioOutputs(); + window.previousOutputOption = + document.querySelector("#preflight-output").options[0]; + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + }); + await page.waitForFunction(() => !window.previousOutputOption.isConnected); + assert.equal( + await page.locator("#preflight-output-note").textContent(), + refusal, + ); + assert.equal( + await page.evaluate(() => localStorage.getItem("codetrial:audioOutputId")), + "speakers", + ); + await page.evaluate(() => { + window.holdRoute = true; + }); + await page.locator("#preflight-output").selectOption(""); + await page.waitForFunction(() => Boolean(window.releaseRoute)); + assert.match( + await page.locator("#preflight-output-note").textContent(), + /refused/, + ); + await page.evaluate(() => { + window.holdRoute = false; + window.releaseRoute(); + }); + await page.waitForFunction(() => window.mockJim.sinkId === ""); + assert.equal(await page.locator("#preflight-output-note").isHidden(), true); +}); + +test("a pending Jim route finishes before the newest choice and cannot roll it back", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await outputRoutingPage(t); + await page.evaluate(() => { + window.holdRoute = true; + }); + await page.locator("#preflight-output").selectOption("headphones"); + await page.waitForFunction(() => Boolean(window.releaseRoute)); + await page.locator("#preflight-output").selectOption("speakers"); + assert.deepEqual(await page.evaluate(() => window.jimRoutes), ["headphones"]); + await page.evaluate(() => { + window.holdRoute = false; + window.releaseRoute(); + }); + await page.waitForFunction(() => window.mockJim.sinkId === "speakers"); + assert.deepEqual(await page.evaluate(() => window.jimRoutes), [ + "headphones", + "speakers", + ]); + for (const id of ["preflight-output", "meet-output-select"]) { + assert.equal(await page.locator(`#${id}`).inputValue(), "speakers"); + } +}); + +test("a refused Jim route returns to default when its previous sink was unplugged", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await outputRoutingPage(t); + await page.locator("#preflight-output").selectOption("headphones"); + await page.waitForFunction(() => window.mockJim.sinkId === "headphones"); + await page.evaluate(() => { + window.refusedOutput = "speakers"; + window.holdRoute = true; + }); + await page.locator("#preflight-output").selectOption("speakers"); + await page.waitForFunction(() => Boolean(window.releaseRoute)); + await page.evaluate(() => { + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "speakers", label: "Speakers" }, + ]; + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 2, + ); + await page.evaluate(() => { + window.holdRoute = false; + window.releaseRoute(); + }); + await page.waitForFunction(() => + document + .querySelector("#preflight-output-note") + .textContent.includes("refused"), + ); + for (const id of ["preflight-output", "meet-output-select"]) { + assert.equal(await page.locator(`#${id}`).inputValue(), ""); + } + assert.equal(await page.evaluate(() => window.mockJim.sinkId), ""); + assert.equal(await page.evaluate(() => window.outputApi.audioOutputId()), ""); +}); + +test("output survives a replaced Jim element with storage blocked", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await outputRoutingPage(t); + await page.evaluate(() => { + Object.defineProperty(window, "localStorage", { + configurable: true, + get() { + throw new Error("Storage blocked"); + }, + }); + }); + await page.locator("#preflight-output").selectOption("headphones"); + await page.waitForFunction(() => window.mockJim.sinkId === "headphones"); + await page.evaluate(async () => { + window.mockJim = { + sinkId: "", + async setSinkId(id) { + this.sinkId = id; + }, + }; + await window.outputApi.applyAudioOutput(window.outputApi.audioOutputId()); + }); + assert.equal(await page.evaluate(() => window.mockJim.sinkId), "headphones"); + await page.addInitScript(() => { + Object.defineProperty(window, "localStorage", { + get() { + throw new Error("Storage blocked"); + }, + }); + }); + await page.reload(); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 3, + ); + assert.equal(await page.locator("#preflight-output").inputValue(), ""); +}); + +test("output choices survive reload and System default clears the saved preference", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await outputRoutingPage(t); + await page.evaluate(() => localStorage.setItem("reload-check", "available")); + await page.locator("#preflight-output").selectOption("headphones"); + await page.waitForFunction(() => window.mockJim.sinkId === "headphones"); + await page.evaluate(() => { + const select = document.querySelector("#meet-output-select"); + select.value = "speakers"; + select.dispatchEvent(new Event("change")); + }); + await page.waitForFunction(() => window.mockJim.sinkId === "speakers"); + assert.equal( + await page.evaluate(() => localStorage.getItem("codetrial:audioOutputId")), + "speakers", + ); + await page.reload(); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 3, + ); + assert.equal( + await page.locator("#preflight-output").inputValue(), + "speakers", + ); + assert.equal( + await page.locator("#meet-output-select").inputValue(), + "speakers", + ); + assert.equal( + await page.evaluate(() => localStorage.getItem("reload-check")), + "available", + ); + await page.locator("#preflight-output").selectOption(""); + assert.equal( + await page.evaluate(() => localStorage.getItem("codetrial:audioOutputId")), + null, + ); + await page.reload(); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 3, + ); + assert.equal(await page.locator("#preflight-output").inputValue(), ""); + assert.equal(await page.locator("#meet-output-select").inputValue(), ""); +}); + +test("a saved output that is no longer listed falls back to System default", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + localStorage.setItem("codetrial:audioOutputId", "removed"); + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "headphones", label: "Headphones" }, + ]; + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 2, + ); + await page.waitForFunction( + () => localStorage.getItem("codetrial:audioOutputId") === null, + ); + assert.equal(await page.locator("#preflight-output").inputValue(), ""); + assert.equal(await page.locator("#meet-output-select").inputValue(), ""); + assert.equal( + await page.evaluate(() => localStorage.getItem("codetrial:audioOutputId")), + null, + ); +}); + +for (const superseded of ["before routing starts", "during routing"]) { + test(`output settlement follows newer choices ${superseded}`, async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await outputRoutingPage(t); + await page.evaluate((when) => { + window.holdRoute = true; + window.outputSettled = false; + void window.outputApi.applyAudioOutput("headphones"); + window.outputApi.audioOutputSettled().then(() => { + window.outputSettled = true; + }); + if (when === "before routing starts") + void window.outputApi.applyAudioOutput("speakers"); + }, superseded); + await page.waitForFunction(() => Boolean(window.releaseRoute)); + if (superseded === "during routing") { + await page.evaluate(() => { + void window.outputApi.applyAudioOutput("speakers"); + window.releaseRoute(); + }); + await page.waitForFunction(() => window.jimRoutes.length === 2); + } + assert.equal(await page.evaluate(() => window.outputSettled), false); + await page.evaluate(() => { + void window.outputApi.applyAudioOutput(""); + window.releaseRoute(); + }); + await page.waitForFunction(() => window.jimRoutes.at(-1) === ""); + assert.equal(await page.evaluate(() => window.outputSettled), false); + await page.evaluate(() => { + window.holdRoute = false; + window.releaseRoute(); + }); + await page.waitForFunction(() => window.outputSettled); + assert.equal(await page.evaluate(() => window.mockJim.sinkId), ""); + }); +} + +test("output permission hint appears without specific devices and clears when they appear", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "", label: "" }, + ]; + }); + await page.waitForFunction(() => + document + .querySelector("#preflight-output-note") + .textContent.includes("microphone access"), + ); + assert.deepEqual( + await page.locator("#preflight-output option").allTextContents(), + ["System default"], + ); + for (const id of ["preflight-output-note", "meet-output-note"]) { + assert.equal( + await page.locator(`#${id}`).evaluate((note) => note.hidden), + false, + ); + } + await page.evaluate(() => { + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "headphones", label: "Headphones" }, + ]; + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 2, + ); + for (const id of ["preflight-output-note", "meet-output-note"]) { + assert.equal( + await page.locator(`#${id}`).evaluate((note) => note.hidden), + true, + ); + } +}); + +test("unsupported output selection keeps system-default playback available", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + localStorage.setItem("codetrial:audioOutputId", "headphones"); + delete HTMLMediaElement.prototype.setSinkId; + }); + assert.equal(await page.locator("#preflight-output").isDisabled(), true); + assert.match( + await page.locator("#preflight-output-note").textContent(), + /system default/, + ); + await page.locator("#audio-test-speech").click(); + await page.waitForFunction( + () => document.querySelector("#audio-test-voice").ended, + ); + assert.equal(await page.locator("#audio-heard").textContent(), "I heard it"); +}); + +test("output enumeration recovers and ignores an older failed refresh", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + navigator.mediaDevices.enumerateDevices = async () => { + throw new Error("Device list unavailable"); + }; + }); + await page.waitForFunction(() => + document + .querySelector("#preflight-output-note") + .textContent.includes("could not be listed"), + ); + await page.evaluate(() => { + window.lists = []; + window.holdLists = true; + const devices = [ + { kind: "audiooutput", deviceId: "speaker", label: "Speaker" }, + ]; + navigator.mediaDevices.enumerateDevices = () => + window.holdLists + ? new Promise((resolve, reject) => { + window.lists.push({ resolve, reject }); + }) + : Promise.resolve(devices); + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + window.staleLists = [...window.lists]; + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + for (const request of window.lists.slice(window.staleLists.length)) + request.resolve(devices); + window.holdLists = false; + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 2, + ); + await page.evaluate(async () => { + for (const request of window.staleLists) + request.reject(new Error("Stale failure")); + await Promise.resolve(); + }); + assert.equal(await page.locator("#preflight-output-note").isHidden(), true); + assert.deepEqual( + await page.locator("#preflight-output option").allTextContents(), + ["System default", "Speaker"], + ); +}); + +test("removing the selected output restores default and requires confirmation again", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "headphones", label: "Headphones" }, + ]; + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 2, + ); + await page.locator("#preflight-output").selectOption("headphones"); + await page.locator("#audio-heard").click(); + await page.waitForFunction(() => + document.querySelector("#audio-step-output").classList.contains("done"), + ); + await page.evaluate(() => { + navigator.mediaDevices.enumerateDevices = async () => []; + navigator.mediaDevices.dispatchEvent(new Event("devicechange")); + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 1, + ); + assert.equal(await page.locator("#preflight-output").inputValue(), ""); + assert.equal(await page.locator("#audio-heard").textContent(), "I heard it"); + assert.equal(await page.locator("#audio-check-join").isDisabled(), true); + assert.match( + await page.locator("#preflight-output-note").textContent(), + /microphone access/, + ); + assert.equal( + await page + .locator("#preflight-output-note") + .evaluate((note) => note.hidden), + false, + ); +}); + +test("a pending tone route cannot play after selecting away and back", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "headphones", label: "Headphones" }, + ]; + window.routes = []; + window.tones = []; + AudioContext.prototype.setSinkId = async function (id) { + window.routes.push(id); + if (window.routes.length === 1) + await new Promise((resolve) => { + window.releaseRoute = resolve; + }); + this.testSink = id; + Object.defineProperty(this, "sinkId", { configurable: true, value: id }); + }; + const createOscillator = AudioContext.prototype.createOscillator; + AudioContext.prototype.createOscillator = function () { + const oscillator = createOscillator.call(this); + const start = oscillator.start.bind(oscillator); + oscillator.start = () => { + window.tones.push(this.testSink); + start(); + }; + return oscillator; + }; + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 2, + ); + await page.locator("#preflight-output").selectOption("headphones"); + await page.locator("#audio-test-tone").click(); + await page.waitForFunction(() => Boolean(window.releaseRoute)); + await page.locator("#preflight-output").selectOption(""); + await page.locator("#preflight-output").selectOption("headphones"); + await page.locator("#audio-test-tone").click(); + assert.deepEqual(await page.evaluate(() => window.tones), []); + await page.evaluate(() => window.releaseRoute()); + await page.waitForFunction(() => window.tones.length > 0); + assert.deepEqual(await page.evaluate(() => window.tones), ["headphones"]); + assert.deepEqual(await page.evaluate(() => window.routes), ["headphones"]); +}); + +for (const sound of ["tone", "speech"]) { + test(`a refused ${sound} output can be retried without confirming it`, async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await preflight(t, () => { + navigator.mediaDevices.enumerateDevices = async () => [ + { kind: "audiooutput", deviceId: "headphones", label: "Headphones" }, + ]; + window.refuseOutput = true; + window.routedOutput = false; + const route = async () => { + if (window.refuseOutput) + throw new DOMException("Device refused", "NotAllowedError"); + window.routedOutput = true; + }; + HTMLMediaElement.prototype.setSinkId = route; + AudioContext.prototype.setSinkId = route; + }); + await page.waitForFunction( + () => document.querySelector("#preflight-output").options.length === 2, + ); + await page.locator("#preflight-output").selectOption("headphones"); + await page.locator(`#audio-test-${sound}`).click(); + await page.waitForFunction(() => + document + .querySelector("#audio-check-status") + .textContent.includes("Could not"), + ); + assert.equal( + await page.locator("#audio-heard").textContent(), + "I heard it", + ); + await page.evaluate(() => { + window.refuseOutput = false; + }); + await page.locator(`#audio-test-${sound}`).click(); + await page.waitForFunction(() => window.routedOutput); + assert.equal( + await page.locator("#audio-heard").textContent(), + "I heard it", + ); + }); +} + test("preflight speech replays during capture and still requires the candidate's confirmation", async (t) => { if (!browser) return t.skip("playwright chromium unavailable"); const page = await preflight(t); diff --git a/web/audio-input.js b/web/audio-input.js new file mode 100644 index 000000000..3bb8acced --- /dev/null +++ b/web/audio-input.js @@ -0,0 +1,70 @@ +// Microphone choices are limited to preflight, before its stream is published. +export function createAudioInput({ + mediaDevices, + pool, + select, + note, + isFinished, +}) { + let listing = 0; + const showNote = (text) => { + note.textContent = text; + note.hidden = !text; + }; + const supported = + pool.supported && + Boolean(mediaDevices?.enumerateDevices) && + Boolean(mediaDevices?.getSupportedConstraints?.().deviceId); + select.disabled = !supported; + if (!supported) { + showNote("Microphone selection is unavailable. Use the system default."); + } + const refresh = async () => { + if (!supported || isFinished()) return; + const request = ++listing; + try { + const devices = await mediaDevices.enumerateDevices(); + if (isFinished() || request !== listing) return; + const selected = select.value; + const options = [new Option("System default", "")]; + for (const device of devices) { + if ( + device.kind !== "audioinput" || + !device.deviceId || + device.deviceId === "default" + ) + continue; + options.push( + new Option( + device.label || `Microphone ${options.length}`, + device.deviceId, + ), + ); + } + select.replaceChildren(...options); + select.value = options.some((option) => option.value === selected) + ? selected + : ""; + showNote(""); + if (selected && !select.value) pool.selectMicrophone(""); + } catch { + if (!isFinished() && request === listing) + showNote( + "Microphones could not be listed. Check your browser permissions and reload the page.", + ); + } + }; + const change = () => { + if (!isFinished()) pool.selectMicrophone(select.value); + }; + select.addEventListener("change", change); + mediaDevices?.addEventListener?.("devicechange", refresh); + return { + refresh, + close() { + listing += 1; + select.removeEventListener("change", change); + mediaDevices?.removeEventListener?.("devicechange", refresh); + }, + }; +} diff --git a/web/audio-output.js b/web/audio-output.js index d8e792eab..6bbf644cb 100644 --- a/web/audio-output.js +++ b/web/audio-output.js @@ -1,114 +1,162 @@ -/// Which speaker plays the tab's audio, and the two preferences that outlive a -/// reload. -/// -/// No mixer and no synthetic microphone: Meet captures the tab, and CodeTrial -/// only decides which output device the tab's own audio lands on. Split out of -/// `interview.js` because routing, the stored preference and the note the -/// candidate reads about it are one decision made in three places, and the -/// order they happen in is the part that matters. - -import { - OUTPUT_NOTES, - outputAfterRouting, - outputOptions, -} from "./meet-audio.js"; +// One output selection for preflight playback and the interviewer's voice. + +import { OUTPUT_NOTES, outputOptions } from "./meet-audio.js"; let nodes = null; -/// Read through a function rather than held: `playRemoteAudio` creates and -/// drops Jim's element as LiveKit re-subscribes him, and a copy taken at init -/// would route the one from the previous subscription. let jimAudioElement = () => null; +let selectedOutputId = ""; +let outputNote = ""; +let listRequest = 0; +// A pre-grant listing can hide a saved device; only a granted listing can +// prove that it disappeared. +let outputAccessGranted = false; +// Serialize routes so an older setSinkId cannot finish after a newer one. +// Stamp requests to skip superseded choices and ignore their late failures. +let routingRequest = 0; +let routingChain = Promise.resolve(); + +export const AUDIO_OUTPUT_KEY = "codetrial:audioOutputId"; +export const MEET_PRESENTATION_KEY = "codetrial:meetPresentation"; export function initAudioOutput(deps) { ({ nodes, jimAudioElement } = deps); + selectedOutputId = readStored(AUDIO_OUTPUT_KEY) || ""; + renderOutputs(outputOptions([], selectedOutputId)); } -export const AUDIO_OUTPUT_KEY = "codetrial:audioOutputId"; -export const MEET_PRESENTATION_KEY = "codetrial:meetPresentation"; +export function audioOutputId() { + return selectedOutputId; +} -/// Lists output devices so Jim can be routed away from the shared tab's -/// default. No mixer and no synthetic microphone: Meet captures the tab, and -/// CodeTrial only decides which speaker plays the tab's own audio. -export async function refreshAudioOutputs() { +function renderOutputs(options) { + if ( + selectedOutputId && + !outputAccessGranted && + !options.some((option) => option.deviceId === selectedOutputId) + ) { + options = [ + ...options, + { + deviceId: selectedOutputId, + label: "Saved output device", + selected: true, + }, + ]; + } + for (const select of [nodes.meetOutputSelect, nodes.preflightOutput]) { + select.replaceChildren( + ...options.map((option) => { + const element = document.createElement("option"); + element.value = option.deviceId; + element.textContent = option.label; + element.selected = option.selected; + return element; + }), + ); + } +} + +function selectOutput(deviceId) { + const changed = selectedOutputId !== deviceId; + selectedOutputId = deviceId; + writeStored(AUDIO_OUTPUT_KEY, deviceId); + nodes.meetOutputSelect.value = deviceId; + nodes.preflightOutput.value = deviceId; + if (changed) nodes.preflightOutput.dispatchEvent(new Event("outputchange")); +} + +export async function refreshAudioOutputs({ granted = false } = {}) { + if (granted) outputAccessGranted = true; + const request = ++listRequest; const mediaDevices = navigator.mediaDevices; if ( !mediaDevices?.enumerateDevices || !("setSinkId" in HTMLMediaElement.prototype) ) { - // Hide the row, not just the control: a label pointing at a hidden select - // renders as a heading with nothing under it. nodes.meetOutputRow.hidden = true; + nodes.preflightOutput.disabled = true; showOutputNote(OUTPUT_NOTES.unsupported); + if (selectedOutputId) void applyAudioOutput(""); return; } - let devices = []; + nodes.meetOutputRow.hidden = false; + nodes.preflightOutput.disabled = false; try { - devices = await mediaDevices.enumerateDevices(); + const devices = await mediaDevices.enumerateDevices(); + if (request !== listRequest) return; + const options = outputOptions(devices, selectedOutputId); + const missing = + outputAccessGranted && + !options.some((option) => option.deviceId === selectedOutputId); + if (missing) { + options[0].selected = true; + } + renderOutputs(options); + if (outputNote !== OUTPUT_NOTES.refused) + showOutputNote(options.length === 1 ? OUTPUT_NOTES.empty : ""); + if (missing) void applyAudioOutput(""); } catch { - showOutputNote(OUTPUT_NOTES.listFailed); - return; - } - const options = outputOptions(devices, readStored(AUDIO_OUTPUT_KEY)); - nodes.meetOutputSelect.replaceChildren(); - for (const option of options) { - const element = document.createElement("option"); - element.value = option.deviceId; - element.textContent = option.label; - element.selected = option.selected; - nodes.meetOutputSelect.append(element); + if (request === listRequest) showOutputNote(OUTPUT_NOTES.listFailed); } - showOutputNote(options.length ? "" : OUTPUT_NOTES.empty); } -/// The routing request this one has to finish behind, and the number that says -/// which request is the newest. -/// -/// `setSinkId` is asynchronous and the candidate can pick again before it -/// settles. Two in flight can resolve in either order, and the older one would -/// then write its device into the preference and the select, undoing a choice -/// the candidate made after it. Serialized so they resolve in the order they -/// were asked for, and stamped so a request that is no longer the newest -/// routes but does not get the last word on what is stored or shown. -let routingChain = Promise.resolve(); -let routingRequest = 0; - export function applyAudioOutput(deviceId) { - routingChain = routingChain.then(() => - routeAudioOutput(deviceId, ++routingRequest), - ); + deviceId = deviceId || ""; + const request = ++routingRequest; + selectOutput(deviceId); + routingChain = routingChain + .catch(() => {}) + .then(() => routeAudioOutput(deviceId, request)); return routingChain; } +export async function audioOutputSettled() { + let pending; + do { + pending = routingChain; + await pending; + } while (pending !== routingChain); +} + async function routeAudioOutput(deviceId, request) { - if (!deviceId) return; - const previous = readStored(AUDIO_OUTPUT_KEY); - writeStored(AUDIO_OUTPUT_KEY, deviceId); - const jimAudio = jimAudioElement(); - if (!jimAudio || typeof jimAudio.setSinkId !== "function") return; - const routed = await jimAudio - .setSinkId(deviceId) - .then(() => true) - .catch(() => false); - // Cleared, not left alone, when there is no previous device to restore. The - // refused id was written before routing was attempted, so leaving it would - // persist a device Jim never played through: the next reload would pre-select - // it and retry the same refusal. - // A newer pick arrived while this one was routing. It has already routed, so - // Jim plays through whatever the last `setSinkId` chose, and the newest - // request is the one entitled to say so in the preference and the select. if (request !== routingRequest) return; - const stored = outputAfterRouting(previous, deviceId, routed); - writeStored(AUDIO_OUTPUT_KEY, stored); - nodes.meetOutputSelect.value = stored || ""; - // An empty list keeps its explanation: routing succeeding says nothing about - // whether there was anything to choose from. - const note = routed ? "" : OUTPUT_NOTES.refused; - if (note || nodes.meetOutputSelect.length) showOutputNote(note); + // The element changes across subscriptions; read it when routing starts. + const jimAudio = jimAudioElement(); + if (!jimAudio) return; + const previous = jimAudio.sinkId || ""; + try { + if (typeof jimAudio.setSinkId === "function") { + if (jimAudio.sinkId !== deviceId) await jimAudio.setSinkId(deviceId); + } else if (deviceId) throw new Error("Output selection is unavailable"); + if (request === routingRequest && outputNote === OUTPUT_NOTES.refused) + showOutputNote(""); + } catch { + if (request !== routingRequest) return; + // The previous sink may have disappeared while this request was pending. + const fallback = Array.from(nodes.meetOutputSelect.options).some( + (option) => option.value === previous, + ) + ? previous + : ""; + if (fallback !== previous) { + try { + await jimAudio.setSinkId(fallback); + } catch { + // Keep the refusal visible if even the system output is unavailable. + } + } + if (request !== routingRequest) return; + selectOutput(fallback); + showOutputNote(OUTPUT_NOTES.refused); + } } -export function showOutputNote(text) { - nodes.meetOutputNote.textContent = text; - nodes.meetOutputNote.hidden = !text; +function showOutputNote(text) { + outputNote = text; + for (const note of [nodes.meetOutputNote, nodes.preflightOutputNote]) { + note.textContent = text; + note.hidden = !text; + } } /// `localStorage` or `sessionStorage`, or `null` where site data is blocked: @@ -169,13 +217,14 @@ export function createJimVolume({ const stored = readStored(JIM_VOLUME_KEY, storage); let level = /^(?:[1-9][0-9]?|100)$/.test(stored ?? "") ? Number(stored) : 100; let muted = false; + let routingHeld = false; let supported = true; let tested = null; const restore = () => { const audio = element(); if ( audio && - (audio.muted !== muted || + (audio.muted !== (muted || routingHeld) || (supported && !sameVolume(audio.volume, level / 100))) ) apply(); @@ -192,7 +241,7 @@ export function createJimVolume({ // another level synchronously, then restore the requested attenuation. if (fresh) supported = setsVolume(audio, 0.5); if (supported) supported = setsVolume(audio, level / 100); - audio.muted = muted; + audio.muted = muted || routingHeld; if (fresh) audio.addEventListener?.("volumechange", restore); } onChange(); @@ -225,6 +274,10 @@ export function createJimVolume({ return supported; }, apply, + setRoutingHeld(value) { + routingHeld = Boolean(value); + apply(); + }, setVolume, toggleMute() { muted = !muted; diff --git a/web/audio-playback.js b/web/audio-playback.js index 249690cc3..f9e4cb3b3 100644 --- a/web/audio-playback.js +++ b/web/audio-playback.js @@ -31,6 +31,12 @@ export function createAudioPlayback({ let pending = false; let blocked = false; + function paused() { + // Unmuting can pause native playback after its play promise succeeded. + if (element?.paused && !element.ended && !pending && !blocked) + void play(element); + } + function render() { if (!blocked && button.ownerDocument.activeElement === button) focusTarget.focus({ preventScroll: true }); @@ -60,6 +66,7 @@ export function createAudioPlayback({ /// restored, so nothing is announced. function clear() { revision += 1; + element?.removeEventListener?.("pause", paused); element = null; pending = false; blocked = false; @@ -84,6 +91,10 @@ export function createAudioPlayback({ function play(nextElement) { if (!room) return Promise.resolve(); revision += 1; + if (element !== nextElement) { + element?.removeEventListener?.("pause", paused); + nextElement.addEventListener?.("pause", paused); + } element = nextElement; pending = false; reapply(); diff --git a/web/devices.js b/web/devices.js index b931fa35e..ee010469e 100644 --- a/web/devices.js +++ b/web/devices.js @@ -48,6 +48,7 @@ export function createDevicePool({ kind, constraints: { [kind]: CONSTRAINTS[kind] }, pending: false, + generation: 0, disabled: false, error: null, accept: () => true, @@ -87,6 +88,7 @@ export function createDevicePool({ // `pending` matters as much as the shared timer: a retry must not open a // second prompt while the first is still on screen. const requestDevice = (device) => { + if (isFinished()) return; // An ended track never revives, and while it sits in the stream it looks // like a device this pool already holds, so the retry asks for nothing and // the candidate is stranded with no way back. @@ -103,10 +105,15 @@ export function createDevicePool({ if (device.disabled || device.pending || trackOf(device.kind)) return; device.pending = true; + const generation = device.generation; void mediaDevices .getUserMedia(device.constraints) .then((granted) => { - if (device.disabled) { + if ( + device.disabled || + isFinished() || + generation !== device.generation + ) { granted.getTracks().forEach((track) => track.stop()); return; } @@ -122,6 +129,7 @@ export function createDevicePool({ device.onTrack(); }) .catch((error) => { + if (isFinished() || generation !== device.generation) return; // Some browsers leave a DOMException's message empty and name the // failure instead. Keep both: the preflight words are the candidate's // only explanation, and the optional-camera path distinguishes denied @@ -133,6 +141,9 @@ export function createDevicePool({ }) .finally(() => { device.pending = false; + // A selection changed while the browser was answering the old request. + // Reuse the pending guard so repeated changes never open more prompts. + if (generation !== device.generation) requestDevice(device); onChange(); }); }; @@ -162,6 +173,30 @@ export function createDevicePool({ }, start, retry, + selectMicrophone(deviceId) { + const audio = devices.audio; + if ( + isFinished() || + (audio.constraints.audio.deviceId?.exact || "") === deviceId + ) + return; + audio.generation += 1; + audio.constraints = { + audio: { + ...CONSTRAINTS.audio, + ...(deviceId ? { deviceId: { exact: deviceId } } : {}), + }, + }; + audio.error = null; + const held = trackOf("audio"); + if (held) { + stream.removeTrack(held); + held.stop(); + } + audio.onLost(); + start(); + onChange(); + }, disable(kind) { const device = devices[kind]; device.disabled = true; diff --git a/web/interview.html b/web/interview.html index ce46ce5da..58c3d279e 100644 --- a/web/interview.html +++ b/web/interview.html @@ -561,6 +561,21 @@

Devices

+
+ + +
+

Sound — @@ -607,6 +622,21 @@

+
+ + +
+

Microphone — say something and watch the diff --git a/web/interview.js b/web/interview.js index 7080a53b0..b018ba852 100644 --- a/web/interview.js +++ b/web/interview.js @@ -45,6 +45,7 @@ import { toggleLineComment, } from "./editor.js"; import { createDevicePool } from "./devices.js"; +import { createAudioInput } from "./audio-input.js"; import { createFaceCheck } from "./face-check.js"; import { attachSelfView } from "./self-view.js"; import { createMicMeter, startMediaMeter } from "./mic-meter.js"; @@ -119,7 +120,8 @@ import { responseWindowIndex, } from "./replay-feed.js"; import { - AUDIO_OUTPUT_KEY, + audioOutputId, + audioOutputSettled, MEET_PRESENTATION_KEY, applyAudioOutput, createJimVolume, @@ -617,6 +619,10 @@ const nodes = { meetOutputRow: document.querySelector("#meet-output-row"), meetOutputSelect: document.querySelector("#meet-output-select"), meetOutputNote: document.querySelector("#meet-output-note"), + preflightOutput: document.querySelector("#preflight-output"), + preflightOutputNote: document.querySelector("#preflight-output-note"), + preflightInput: document.querySelector("#preflight-input"), + preflightInputNote: document.querySelector("#preflight-input-note"), sidebar: document.querySelector("#interview-sidebar"), sidebarResizer: document.querySelector("#sidebar-resizer"), editorPanel: document.querySelector(".editor-panel"), @@ -1039,9 +1045,11 @@ function bindEvents() { removeCandidateCase(Number(button.dataset.removeCase)); }); void initializeCandidateCases(); - nodes.meetOutputSelect.addEventListener("change", () => { - void applyAudioOutput(nodes.meetOutputSelect.value); - }); + for (const select of [nodes.meetOutputSelect, nodes.preflightOutput]) { + select.addEventListener("change", () => { + void applyAudioOutput(select.value); + }); + } // The checkbox is the only copy of this answer. A mirrored state field had to // be written in two places and read in four, and the one reader that forgot // it published a wrong camera status for the whole interview. @@ -1421,6 +1429,38 @@ function runAudioCheck() { let hint = null; let hintUntil = 0; let finished = false; + const microphoneSelector = createAudioInput({ + mediaDevices, + pool, + select: nodes.preflightInput, + note: nodes.preflightInputNote, + isFinished: () => finished, + }); + let playbackRequest = 0; + let playbackRouting = Promise.resolve(); + const outputChanged = () => { + if (finished) return; + playbackRequest += 1; + outputConfirmed = false; + nodes.audioTestVoice.pause(); + refresh(); + }; + nodes.preflightOutput.addEventListener("outputchange", outputChanged); + // A stale setSinkId can change the sink even when its caller never plays. + // Keep routing ordered, including when the selection returns to the same id. + const routeTestOutput = (target, outputId, request) => { + const routing = playbackRouting.then(async () => { + if (finished || request !== playbackRequest) return false; + if (typeof target.setSinkId === "function") { + if (target.sinkId !== outputId) await target.setSinkId(outputId); + } else if (outputId) { + throw new Error("Output selection is unavailable"); + } + return !finished && request === playbackRequest; + }); + playbackRouting = routing.catch(() => {}); + return routing; + }; // Set once, because refresh() runs on every animation frame: the // candidate is the only reliable output sensor, so confirming is // accepted whenever they click rather than gated on tone timing. @@ -1481,6 +1521,9 @@ function runAudioCheck() { // so a request still pending when this runs never lights the camera. const stopChecks = () => { finished = true; + microphoneSelector.close(); + playbackRequest += 1; + nodes.preflightOutput.removeEventListener("outputchange", outputChanged); nodes.audioTestVoice.pause(); nodes.audioTestVoice.currentTime = 0; meter.stop(); @@ -1525,11 +1568,29 @@ function runAudioCheck() { }; nodes.audioTestTone.addEventListener("click", async () => { + const outputId = audioOutputId(); + const request = ++playbackRequest; nodes.audioTestVoice.pause(); if (!(await unblockOutput())) { showHint("The browser is still blocking audio. Click the tone again."); return; } + try { + if (typeof context.setSinkId !== "function" && outputId) { + showHint( + "This browser cannot route the test tone. Try the test voice instead.", + ); + return; + } + if (!(await routeTestOutput(context, outputId, request))) return; + } catch { + if (finished || request !== playbackRequest) return; + showHint( + "Could not use that output device. Choose another device and try again.", + ); + return; + } + if (finished || request !== playbackRequest) return; const oscillator = context.createOscillator(); const gain = context.createGain(); oscillator.frequency.value = 440; @@ -1546,17 +1607,25 @@ function runAudioCheck() { // speech. A local sample needs no interview session or synthesis service, // and playing it says nothing about whether the candidate actually heard it. nodes.audioTestSpeech.addEventListener("click", async () => { + const outputId = audioOutputId(); + const request = ++playbackRequest; nodes.audioTestSpeech.disabled = true; try { nodes.audioTestVoice.pause(); nodes.audioTestVoice.currentTime = 0; if (nodes.audioTestVoice.error) nodes.audioTestVoice.load(); + if (!(await routeTestOutput(nodes.audioTestVoice, outputId, request))) + return; await nodes.audioTestVoice.play(); - if (finished) return; + if (finished || request !== playbackRequest) return; nodes.audioTestSpeech.textContent = "Play speech again"; showHint('If you hear the test voice, click "I heard it".'); } catch (error) { - if (!finished && error?.name !== "AbortError") { + if ( + !finished && + request === playbackRequest && + error?.name !== "AbortError" + ) { showHint( "Could not play the test voice. Try again and check your audio settings.", ); @@ -1620,7 +1689,13 @@ function runAudioCheck() { // actually carries sound, and it runs for the rest of the preflight. pool.configure("audio", { accept: (track) => Boolean(track), - onTrack: () => meter.start(), + onTrack: () => { + meter.start(); + // A grant reveals labels; failed retries must leave open menus alone + // while the candidate chooses a different device. + void refreshAudioOutputs({ granted: true }); + void microphoneSelector.refresh(); + }, onLost: () => { meter.forget(); nodes.audioMeterFill.style.width = "0%"; @@ -1647,6 +1722,8 @@ function runAudioCheck() { }, }); pool.start(); + void microphoneSelector.refresh(); + void refreshAudioOutputs(); refresh(); }); } @@ -2293,6 +2370,7 @@ function playRemoteAudio(track, participant) { !isCurrentAgent(participant) ) return; + const fresh = !jimAudio; const element = jimAudio ?? document.createElement("audio"); // LiveKit can deliver the old track's unsubscribe after the new track's // subscribe. Its detach pauses every element the track still holds, so the @@ -2303,11 +2381,30 @@ function playRemoteAudio(track, participant) { element.id = "jim-audio"; jimAudio = element; jimTrack = track; - jimVolume.apply(); + // LiveKit unmutes during attach; every volume reapply must retain this hold. + if (fresh) jimVolume.setRoutingHeld(true); + else jimVolume.apply(); if (!element.isConnected) document.body.appendChild(element); void audioPlayback.play(element); // The candidate may have chosen an output before Jim's track arrived. - void applyAudioOutput(readStored(AUDIO_OUTPUT_KEY)); + void applyAudioOutput(audioOutputId()); + if (fresh) { + const pending = setTimeout(() => { + if (jimAudio === element) + setBanner( + "output", + "Audio output selection is taking longer than expected. Check your device, or reload the page to retry.", + ); + }, 5000); + void audioOutputSettled().finally(() => { + clearTimeout(pending); + if (jimAudio !== element) return; + setBanner("output", ""); + jimVolume.setRoutingHeld(false); + // Muted play can succeed without permission to play audibly. + void audioPlayback.play(element); + }); + } attachAvatarAnalyser(track, participant); } @@ -2322,7 +2419,8 @@ function dropRemoteAudio(track) { element.remove(); if (element === jimAudio) { jimAudio = jimTrack = null; - jimVolume.apply(); + jimVolume.setRoutingHeld(false); + setBanner("output", ""); } } // Only if it was the analyser's own track. The guard above already refuses to @@ -2509,8 +2607,9 @@ function setLanguage(language) { /// deferred-text field that only one of the two owners got, and a lock-breaking /// special case for session-ending events. const BANNER_RANK = { - session: 5, - connection: 4, + session: 6, + connection: 5, + output: 4, interviewer: 3, reconnect: 2, face: 1, @@ -2518,6 +2617,7 @@ const BANNER_RANK = { const banners = { session: "", connection: "", + output: "", interviewer: "", reconnect: "", face: "", diff --git a/web/meet-audio.js b/web/meet-audio.js index a0dec281e..cbc7a74a3 100644 --- a/web/meet-audio.js +++ b/web/meet-audio.js @@ -1,9 +1,4 @@ -// Output routing for Google Meet presentation mode. -// -// Meet re-shares the CodeTrial tab, so Jim reaches the interviewer through -// whatever speaker plays this tab. Choosing that speaker is the whole feature, -// and it is all decisions: which devices are worth offering, what to call them, -// and what to keep when the browser refuses one. +// Output choices shared by the preflight and Google Meet controls. // // Free of DOM and Web Audio types on purpose, like audio-check.js, so these // decisions run under `node --test` instead of being asserted as source text; @@ -13,44 +8,33 @@ export const OUTPUT_NOTES = { unsupported: "This browser cannot choose an output device. Use the system default, or switch to Chrome.", listFailed: - "Output devices could not be listed. The system default is in use.", + "Output devices could not be listed. Check the selected device or use the system default.", empty: - "No output devices yet. Finish the media preflight; this list fills in once the browser grants access.", - refused: "That output device was refused. Jim stays on the previous device.", + "Specific output devices may appear after you allow microphone access. You can still use the system default.", + refused: + "That output device was refused. The previous output or system default is selected.", }; /// Devices worth offering. Outputs only, and only those the browser has named: /// before a `getUserMedia` grant it reports a placeholder whose `deviceId` is /// empty, which renders as a real choice and then selects to "", so a candidate /// picking it silently gets nothing. -export function outputOptions(devices, savedId) { - return devices - .filter((device) => device.kind === "audiooutput" && device.deviceId) - .map((device, index) => ({ - deviceId: device.deviceId, - // Labels stay blank until a permission grant; number them so the list is - // still usable rather than a column of empty rows. - label: device.label || `Output ${index + 1}`, - selected: device.deviceId === savedId, - })); -} - -/// Which device the preference should name after an attempt to route. -/// `routed` is whether `setSinkId` resolved. Null means no preference at all. -/// -/// The preference is written before routing is attempted, because a candidate -/// can pick a device during the preflight, long before Jim's track exists. A -/// preference that only survived a successful `setSinkId` would be dropped -/// exactly then, and the select would keep showing a device Jim never used. -/// A refusal therefore has to roll the stored value back, or the select and the -/// real sink disagree until the next reload. -export function outputAfterRouting(previous, requested, routed) { - if (routed) return requested; - // A refusal falls back to the device that was working before, except when - // that is the same device: reapplying a stored preference on reload passes - // the same id as both, so returning `previous` would write the refused id - // straight back and retry it on every reload for the life of the browser - // profile. There is no earlier device to fall back to, so the preference - // clears and the next load uses the system default. - return previous === requested ? "" : previous; +export function outputOptions(devices, selectedId) { + return [ + { deviceId: "", label: "System default", selected: !selectedId }, + ...devices + .filter( + (device) => + device.kind === "audiooutput" && + device.deviceId && + device.deviceId !== "default", + ) + .map((device, index) => ({ + deviceId: device.deviceId, + // Labels stay blank until a permission grant; number them so the list is + // still usable rather than a column of empty rows. + label: device.label || `Output ${index + 1}`, + selected: device.deviceId === selectedId, + })), + ]; } diff --git a/web/mic-meter.js b/web/mic-meter.js index 33129928e..41db0c159 100644 --- a/web/mic-meter.js +++ b/web/mic-meter.js @@ -66,6 +66,7 @@ export function createMicMeter({ return startMeter( pool.stream, (level) => { + if (isFinished() || mine !== generation) return; pool.setError("audio", null); recent.push(level); if (recent.length > MIC_CONFIRM_FRAMES) recent.shift(); @@ -75,6 +76,7 @@ export function createMicMeter({ onLevel(level); }, (error) => { + if (isFinished() || mine !== generation) return; pool.setError("audio", error); forgetPeak(); onFailure(error);