diff --git a/.github/workflows/closura-arc-ci.yml b/.github/workflows/closura-arc-ci.yml index 6ab8a3312e27c..2b69665f86da3 100644 --- a/.github/workflows/closura-arc-ci.yml +++ b/.github/workflows/closura-arc-ci.yml @@ -176,6 +176,7 @@ jobs: apps/server/src/mcp/OrchestratorMcpService.test.ts \ apps/server/src/cli/thread.test.ts \ apps/web/src/composerDraftStore.test.ts \ + packages/client-runtime/src/voice-input/controller.test.ts \ apps/web/src/components/chat/composerSend.logic.test.ts \ apps/web/src/components/chat/chatViewSend.test.ts \ apps/web/src/components/chat/composerImagePersistence.test.ts \ @@ -208,6 +209,40 @@ jobs: src/features/home/pendingTaskRoute.test.tsx \ src/features/threads/NewTaskDraftRouteScreen.test.tsx + - name: Provide Chromium libraries for the voice latency browser test + shell: bash + run: | + set -euo pipefail + if test -e /usr/lib/x86_64-linux-gnu/libnspr4.so; then + printf 'Chromium libraries present\n' + exit 0 + fi + dir="$RUNNER_TEMP/chromium-libs" + mkdir -p "$dir/lists/partial" "$dir/cache/archives/partial" "$dir/root" + apt_opts=(-o Debug::NoLocking=1 -o Dir::State::Lists="$dir/lists" -o Dir::Cache="$dir/cache") + apt-get "${apt_opts[@]}" -qq update + apt-get "${apt_opts[@]}" -qq install --download-only --no-install-recommends -y \ + libasound2t64 libatk-bridge2.0-0t64 libatk1.0-0t64 libatspi2.0-0t64 libcairo2 \ + libcups2t64 libdbus-1-3 libdrm2 libgbm1 libglib2.0-0t64 libnspr4 libnss3 \ + libpango-1.0-0 libx11-6 libxcb1 libxcomposite1 libxdamage1 libxext6 libxfixes3 \ + libxkbcommon0 libxrandr2 + for deb in "$dir"/cache/archives/*.deb; do dpkg-deb -x "$deb" "$dir/root"; done + lib_dir="$dir/root/usr/lib/x86_64-linux-gnu" + test -f "$lib_dir/libnspr4.so" + printf 'LD_LIBRARY_PATH=%s\n' "$lib_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" >> "$GITHUB_ENV" + + - name: Test voice press-to-recording and release-to-text stay under 400 ms + shell: bash + working-directory: apps/web + env: + PLAYWRIGHT_BROWSERS_PATH: ${{ runner.temp }}/playwright-browsers + run: | + set -euo pipefail + vp exec playwright install --only-shell chromium + vp test run --project browser -t "records within 400 ms" \ + src/speech/voiceInputLatency.browser.test.ts \ + src/speech/useEnvironmentSpeechInput.browser.test.tsx + - name: Build native T3 server shell: bash run: vp run --filter t3 build:bundle diff --git a/apps/web/src/speech/browserPcmCapture.ts b/apps/web/src/speech/browserPcmCapture.ts index 63480d7c06e6a..efb31b563900f 100644 --- a/apps/web/src/speech/browserPcmCapture.ts +++ b/apps/web/src/speech/browserPcmCapture.ts @@ -8,9 +8,36 @@ import { requestMicrophone } from "./requestMicrophone"; import workletUrl from "./pcmWorklet.ts?worker&url"; type PreparedBrowserTranscription = PreparedVoiceTranscription & { - readonly streaming?: VoiceStreamingSession & { readonly feed: (pcm: Float32Array) => void }; + readonly streaming?: VoiceStreamingSession & { + readonly feed: (pcm: Float32Array) => void; + readonly cancel: () => void; + }; }; +let warmAudio: { context: AudioContext; ready: Promise } | undefined; +let capturing = false; + +export function warmBrowserAudioCapture(): Promise { + if ( + warmAudio && + warmAudio.context instanceof AudioContext && + warmAudio.context.state !== "closed" + ) + return warmAudio.ready; + const context = new AudioContext(); + const ready = context.audioWorklet.addModule(workletUrl).then(async () => { + if (context.state === "running" && !capturing) await context.suspend(); + return context; + }); + const entry = { context, ready }; + warmAudio = entry; + ready.catch(() => { + if (warmAudio === entry) warmAudio = undefined; + void context.close().catch(() => undefined); + }); + return ready; +} + export function createBrowserPcmCapture( input: { readonly getMicrophoneId: () => string | Promise; @@ -24,7 +51,9 @@ export function createBrowserPcmCapture( let recordingUri: string | null = null; let chunks: Float32Array[] = []; let audioContext: AudioContext | undefined; + let source: MediaStreamAudioSourceNode | undefined; let worklet: AudioWorkletNode | undefined; + let captureFinished = false; let durationTimer: ReturnType | undefined; let signal: AbortSignal | undefined; let feed: ((pcm: Float32Array) => void) | undefined; @@ -35,13 +64,18 @@ export function createBrowserPcmCapture( const cleanupCapture = () => { if (durationTimer) clearTimeout(durationTimer); durationTimer = undefined; + source?.disconnect(); + source = undefined; worklet?.disconnect(); - worklet?.port.close(); + if (worklet) worklet.port.onmessage = null; + worklet?.port.postMessage("release", []); worklet = undefined; stream?.getTracks().forEach((track) => track.stop()); stream = undefined; - void audioContext?.close(); + if (audioContext && audioContext.state !== "closed") + void audioContext.suspend().catch(() => undefined); audioContext = undefined; + capturing = false; stopped?.resolve(); stopped = undefined; chunks = []; @@ -53,13 +87,18 @@ export function createBrowserPcmCapture( get uri() { return recordingUri; }, - prepareToRecordAsync: async () => { - const captureSignal = signal; + prepareToRecordAsync: async (requestedSignal) => { + const captureSignal = requestedSignal ?? signal; if (!captureSignal) throw new Error("Voice transcription is not prepared."); captureSignal.throwIfAborted(); + signal = captureSignal; if (recordingUri) URL.revokeObjectURL(recordingUri); recordingUri = null; chunks = []; + captureFinished = false; + capturing = true; + const warming = warmBrowserAudioCapture(); + warming.catch(() => undefined); const microphoneId = await input.getMicrophoneId(); captureSignal.throwIfAborted(); const captured = await new Promise((resolve, reject) => { @@ -84,9 +123,8 @@ export function createBrowserPcmCapture( } stream = captured; try { - const context = new AudioContext(); + const context = await warming; audioContext = context; - await context.audioWorklet.addModule(workletUrl); captureSignal.throwIfAborted(); const node = new AudioWorkletNode(context, "t3-pcm-capture", { channelCount: 1 }); worklet = node; @@ -117,7 +155,8 @@ export function createBrowserPcmCapture( track.addEventListener("ended", () => input.onError("The microphone disconnected."), { once: true, }); - context.createMediaStreamSource(captured).connect(node); + source = context.createMediaStreamSource(captured); + source.connect(node); // The processor emits silence, keeping the graph active without microphone playback. node.connect(context.destination); await context.resume(); @@ -153,6 +192,7 @@ export function createBrowserPcmCapture( const blob = new Blob(chunks, { type: "application/octet-stream" }); recordingUri = URL.createObjectURL(blob); } + captureFinished = true; } finally { clearTimeout(timeout); cleanupCapture(); @@ -168,11 +208,22 @@ export function createBrowserPcmCapture( if (recordingUri === uri) recordingUri = null; }, transcriber: { - prepare: async (options: VoiceTranscriptionOptions) => { + prepare: async ( + options: VoiceTranscriptionOptions, + ): Promise => { const prepared = await prepare(options); options.signal.throwIfAborted(); signal = options.signal; + if (prepared.streaming && captureFinished) { + prepared.streaming.cancel(); + const { streaming: _streaming, ...buffered } = prepared; + return buffered; + } feed = prepared.streaming?.feed; + if (feed) { + for (const chunk of chunks) feed(chunk); + chunks = []; + } return prepared; }, }, diff --git a/apps/web/src/speech/browserVoiceInput.browser.test.ts b/apps/web/src/speech/browserVoiceInput.browser.test.ts index 93e11ad6d5801..1e91193820216 100644 --- a/apps/web/src/speech/browserVoiceInput.browser.test.ts +++ b/apps/web/src/speech/browserVoiceInput.browser.test.ts @@ -33,9 +33,11 @@ function capture() { class { audioWorklet = { addModule: async () => {} }; createMediaStreamSource() { - return { connect() {} }; + return { connect() {}, disconnect() {} }; } + state = "running"; resume = async () => {}; + suspend = async () => {}; close = async () => {}; }, ), diff --git a/apps/web/src/speech/pcmWorklet.ts b/apps/web/src/speech/pcmWorklet.ts index ff6962ce530d0..dcad64c93ff70 100644 --- a/apps/web/src/speech/pcmWorklet.ts +++ b/apps/web/src/speech/pcmWorklet.ts @@ -12,6 +12,7 @@ class PcmCaptureProcessor extends AudioWorkletProcessor { private length = 0; private recording = false; private flushed = false; + private released = false; private samples = 0; private levelSamples = 0; private levelEnergy = 0; @@ -23,6 +24,7 @@ class PcmCaptureProcessor extends AudioWorkletProcessor { this.meterOnly = options?.processorOptions?.meterOnly === true; this.port.onmessage = ({ data }) => { if (data === "start") this.recording = true; + if (data === "release") this.released = true; if (data === "stop") { this.recording = false; if (!this.meterOnly && !this.flushed) this.append(this.resampler.flush()); @@ -55,6 +57,7 @@ class PcmCaptureProcessor extends AudioWorkletProcessor { } process(inputs: Float32Array[][]): boolean { + if (this.released || this.flushed) return false; if (!this.recording) return true; const channels = inputs[0]; if (!channels?.[0]) return true; diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index b5418bcb409da..2d464ec3e681b 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -40,6 +40,7 @@ const mocks = vi.hoisted(() => ({ primaryEnvironmentId: "primary-env", statusRequests: [] as string[], statusUnavailable: false, + statusHold: null as Promise | null, modelState: "ready" as "ready" | "missing-model", statusModelId: "test-model", models: [] as EnvironmentSpeechModel[], @@ -116,8 +117,9 @@ vi.mock("../components/ui/toast", () => ({ toastManager: { add: vi.fn() } })); vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({ ...(await importOriginal()), getEnvironmentSpeechStatus: (prepared: PreparedConnection) => - Effect.sync(() => { + Effect.promise(async () => { mocks.statusRequests.push(prepared.environmentId); + if (mocks.statusHold) await mocks.statusHold; if (mocks.statusUnavailable) throw new Error("Temporarily offline"); return { supported: true, @@ -218,6 +220,7 @@ function setup(options: { readonly destination?: string | null } = {}) { mocks.prepared.clear(); mocks.statusRequests = []; mocks.statusUnavailable = false; + mocks.statusHold = null; mocks.modelState = "ready"; mocks.statusModelId = "test-model"; mocks.models = []; @@ -801,6 +804,25 @@ it("shares one status probe per destination across every mounted editor", async expect(mocks.statusRequests).toEqual(["env-a"]); }); +it("records within 400 ms of the press while the status refresh is still unanswered", async () => { + setup({ destination: "env-a" }); + await using _screen = await render(); + expect(mocks.voice.available).toBe(true); + let answer!: () => void; + mocks.statusHold = new Promise((resolve) => { + answer = resolve; + }); + const started = mocks.voice.start(); + await vi.waitFor(() => expect(mocks.voice.state.phase).toBe("recording"), { + timeout: 400, + interval: 1, + }); + answer(); + await act(() => started); + await act(() => mocks.voice.stop()); + expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything()); +}); + it("lets the user retry after a status failure without reselecting the destination", async () => { setup({ destination: "env-a" }); mocks.statusUnavailable = true; diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index 9078a16b7d2c9..dbff4547e7601 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -38,6 +38,7 @@ import { import { useEnvironment } from "../state/environments"; import { useEnvironmentScope, usePreparedConnection } from "../state/session"; import { chooseRecordingMicrophone } from "./browserMicrophones"; +import { warmBrowserAudioCapture } from "./browserPcmCapture"; import { createBrowserVoiceInputPlatform } from "./browserVoiceInput"; import { preferredSpeechLanguages } from "./preferredSpeechLanguages"; import { runEnvironmentSpeech } from "./runEnvironmentSpeech"; @@ -313,6 +314,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { }; controller = new VoiceInputController({ retryFailedRecording: true, + captureDuringPreparation: true, recorder: platform.recorder, getTranscriber: () => platform.transcriber, requestPermission: async () => ({ granted: true, canAskAgain: true }), @@ -340,6 +342,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { }, }); controllerRef.current = controller; + if (typeof AudioContext !== "undefined") void warmBrowserAudioCapture().catch(() => undefined); return () => { disposed = true; pendingSubmissionRef.current = null; @@ -472,6 +475,27 @@ export function useEnvironmentSpeechInput(input: HookInput) { latestRef.current.input.ownerKey === owner && !latestRef.current.input.disabled; setSetup(INITIAL_SETUP); + const knownStatus = status?.prepared === prepared ? status.value : null; + if (knownStatus?.supported && knownStatus.state !== "missing-model") { + setLevel(0); + void probeSpeechStatus(prepared, true).then( + (latest) => { + if (!stillCurrent()) return; + setStatus({ prepared, value: latest }); + if (latest.supported && latest.state !== "missing-model") return; + controller.cancel(); + if (latest.supported) return loadSetupCatalog(prepared, latest.modelId, request); + toastManager.add({ + type: "error", + title: "Voice input is not available", + description: latest.reason, + }); + }, + () => undefined, + ); + await controller.start(); + return; + } setQueuedStart(prepared); let latestStatus: EnvironmentSpeechStatus; try { diff --git a/apps/web/src/speech/voiceInputLatency.browser.test.ts b/apps/web/src/speech/voiceInputLatency.browser.test.ts new file mode 100644 index 0000000000000..8354cb3af799d --- /dev/null +++ b/apps/web/src/speech/voiceInputLatency.browser.test.ts @@ -0,0 +1,26 @@ +import { it, vi } from "vite-plus/test"; + +import { createVoiceLatencyHarness } from "../../test/voiceLatencyHarness"; + +vi.mock("../connection/runtime", async () => { + const [{ Atom }, { layerRemoteHttpClient }] = await Promise.all([ + import("effect/reactivity"), + import("@t3tools/client-runtime/rpc"), + ]); + return { + connectionAtomRuntime: Atom.runtime( + layerRemoteHttpClient((input, init) => globalThis.fetch(input, init)), + ), + }; +}); + +it.each([{ streaming: false }, { streaming: true }])( + "records within 400 ms of the press while the server is still answering, and inserts text within 400 ms of release: %j", + async ({ streaming }) => { + using voice = await createVoiceLatencyHarness({ streaming }); + await voice.press(); + await voice.captureAudio(); + await voice.connectServer(); + await voice.release(); + }, +); diff --git a/apps/web/test/voiceLatencyHarness.ts b/apps/web/test/voiceLatencyHarness.ts new file mode 100644 index 0000000000000..9e780d66bcbfd --- /dev/null +++ b/apps/web/test/voiceLatencyHarness.ts @@ -0,0 +1,161 @@ +import { BearerConnectionTarget } from "@t3tools/client-runtime/connection"; +import { VoiceInputController } from "@t3tools/client-runtime/voice-input"; +import { + DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + EnvironmentId, + type EnvironmentSpeechStatus, +} from "@t3tools/contracts"; +import { expect, vi } from "vite-plus/test"; + +import { warmBrowserAudioCapture } from "../src/speech/browserPcmCapture"; +import { createBrowserVoiceInputPlatform } from "../src/speech/browserVoiceInput"; +import { stubGlobal } from "./render"; + +const PRESS_TO_RECORDING_BUDGET_MS = 400; +const RELEASE_TO_TEXT_BUDGET_MS = 400; + +const prepared = (() => { + const target = new BearerConnectionTarget({ + environmentId: EnvironmentId.make("latency-transcription"), + label: "Latency transcription", + connectionId: "latency", + }); + return { + environmentId: target.environmentId, + label: target.label, + target, + httpBaseUrl: "https://transcription.example.test", + socketUrl: "wss://transcription.example.test/ws", + httpAuthorization: { _tag: "Bearer" as const, token: "audio-grant" }, + }; +})(); + +function speechStatus(streaming: boolean) { + return { + supported: true, + state: "ready", + modelId: "latency-model", + model: "Latency model", + size: 1, + supportsStreaming: streaming, + supportsBufferedStreaming: streaming, + supportsTranslation: false, + languages: ["en"], + supportsLanguageDetection: false, + acceleration: "cpu", + modelUnloadTimeout: "never", + gpuDevices: [], + benchmarkTier: "cpu", + } satisfies EnvironmentSpeechStatus; +} + +export async function createVoiceLatencyHarness(options: { + readonly streaming: boolean; + readonly transcript?: string; +}) { + const resources = new DisposableStack(); + const transcript = options.transcript ?? "dictated"; + let openServer!: () => void; + const serverOpened = new Promise((resolve) => { + openServer = resolve; + }); + const sockets: EventTarget[] = []; + resources.use( + stubGlobal( + "WebSocket", + class extends EventTarget { + constructor() { + super(); + sockets.push(this); + void serverOpened.then(() => this.dispatchEvent(new Event("open"))); + } + send(data: string | Uint8Array) { + const reply = (message: unknown) => + queueMicrotask(() => + this.dispatchEvent(new MessageEvent("message", { data: JSON.stringify(message) })), + ); + if (typeof data !== "string") return reply({ type: "update", revision: 0, text: null }); + const { type } = JSON.parse(data) as { type: string }; + if (type === "start") reply({ type: "ready" }); + if (type === "finish") reply({ type: "finished", text: transcript }); + } + close() {} + }, + ), + ); + const fetch = globalThis.fetch; + resources.use( + vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + const request = new Request(input, init); + if (request.url === `${prepared.httpBaseUrl}/api/voice/status`) { + await serverOpened; + return Response.json(speechStatus(options.streaming)); + } + if (request.url.includes("/api/auth/websocket-ticket/")) + return Response.json({ ticket: "stream-ticket", expiresAt: "2030-01-01T00:00:00.000Z" }); + if (request.url === `${prepared.httpBaseUrl}/api/voice/transcribe`) + return Response.json({ text: transcript }); + return fetch(input, init); + }), + ); + let meterUpdates = 0; + const platform = createBrowserVoiceInputPlatform({ + prepared, + hasAudioConsent: () => true, + getStreamingEnabled: () => options.streaming, + getTranscriptionOptions: () => DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + getMicrophoneId: () => "", + onLevel: () => { + meterUpdates += 1; + }, + onDurationLimit() {}, + onText() {}, + onError() {}, + }); + const commitDraft = vi.fn<(text: string) => void>(); + const controller = new VoiceInputController({ + recorder: platform.recorder, + retryFailedRecording: true, + captureDuringPreparation: true, + getTranscriber: () => platform.transcriber, + requestPermission: async () => ({ granted: true, canAskAgain: true }), + configureRecording: async () => {}, + releaseRecording: async () => platform.cancelRecording(), + deleteRecording: platform.deleteRecording, + readDraft: () => ({ ownerKey: "draft", text: "", selection: { start: 0, end: 0 } }), + commitDraft, + onStateChange() {}, + }); + resources.defer(() => controller.dispose()); + await warmBrowserAudioCapture(); + + return { + controller, + commitDraft, + transcript, + openServer, + press: async () => { + void controller.start(); + await vi.waitFor(() => expect(controller.currentState.phase).toBe("recording"), { + timeout: PRESS_TO_RECORDING_BUDGET_MS, + interval: 1, + }); + }, + captureAudio: async () => { + const seen = meterUpdates; + await vi.waitFor(() => expect(meterUpdates).toBeGreaterThan(seen + 1)); + }, + connectServer: async () => { + openServer(); + if (options.streaming) await vi.waitFor(() => expect(sockets).toHaveLength(1)); + }, + release: async () => { + void controller.stop(); + await vi.waitFor(() => expect(commitDraft.mock.lastCall?.[0]).toBe(transcript), { + timeout: RELEASE_TO_TEXT_BUDGET_MS, + interval: 1, + }); + }, + [Symbol.dispose]: () => resources.dispose(), + }; +} diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index 1a55244ddc9a3..a691b89b4fb20 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -140,7 +140,16 @@ const browserTestProject = { browser: { enabled: true, headless: true, - provider: playwright(), + provider: playwright({ + launchOptions: { + args: [ + "--use-fake-ui-for-media-stream", + "--use-fake-device-for-media-stream", + "--autoplay-policy=no-user-gesture-required", + "--disable-audio-output", + ], + }, + }), instances: [{ browser: "chromium" }], screenshotFailures: false, commands: { diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts index 15f5fea3aac76..f7775bc9f712a 100644 --- a/packages/client-runtime/src/voice-input/controller.test.ts +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -84,6 +84,93 @@ function createHarness( }; } +describe("voice input press latency", () => { + beforeEach(() => resetVoiceInputGlobalsForTests()); + + const PRESS_TO_RECORDING_BUDGET_MS = 400; + + it("reaches recording within 400 ms while transcription preparation is still pending", async () => { + const preparation = deferred(); + const harness = createHarness({ + captureDuringPreparation: true, + getTranscriber: () => ({ prepare: () => preparation.promise }), + }); + const started = harness.controller.start(); + await vi.waitFor(() => expect(harness.controller.currentState.phase).toBe("recording"), { + timeout: PRESS_TO_RECORDING_BUDGET_MS, + interval: 1, + }); + await started; + expect(harness.recorder.record).toHaveBeenCalledOnce(); + + const stopped = harness.controller.stop(); + expect(harness.controller.currentState.phase).toBe("transcribing"); + preparation.resolve(preparedTranscription(async () => "late text")); + await stopped; + expect(harness.commits.map(({ text }) => text)).toEqual(["hello late text"]); + expect(harness.controller.currentState.phase).toBe("idle"); + }); + + it("stops the recording when preparation fails after capture started, and retries the same audio", async () => { + const preparation = deferred(); + const held: { uri: { uri: string | null } | null } = { uri: null }; + const releaseRecording = vi.fn(async () => { + if (held.uri) held.uri.uri = null; + }); + const prepare = vi + .fn() + .mockReturnValueOnce(preparation.promise) + .mockResolvedValueOnce(preparedTranscription(async () => "kept text")); + const harness = createHarness({ + captureDuringPreparation: true, + retryFailedRecording: true, + releaseRecording, + getTranscriber: () => ({ prepare }), + }); + held.uri = harness.recorder; + await harness.controller.start(); + expect(harness.controller.currentState.phase).toBe("recording"); + preparation.reject( + new VoiceTranscriptionError({ code: "unavailable", message: "Server is offline." }), + ); + await vi.waitFor(() => + expect(harness.controller.currentState).toEqual({ + phase: "error", + error: "Server is offline.", + errorAction: "retry", + }), + ); + await vi.waitFor(() => expect(releaseRecording).toHaveBeenCalled()); + expect(harness.recorder.stop).toHaveBeenCalled(); + expect(harness.commits).toEqual([]); + expect(harness.deleted).toEqual([]); + + await harness.controller.start(); + expect(harness.recorder.record).toHaveBeenCalledOnce(); + expect(harness.commits.map(({ text }) => text)).toEqual(["hello kept text"]); + expect(harness.deleted).toEqual(["file:///voice.m4a"]); + }); + + it("never records when preparation fails before capture is ready", async () => { + const capture = deferred(); + const harness = createHarness({ + captureDuringPreparation: true, + getTranscriber: () => ({ + prepare: async () => { + throw new VoiceTranscriptionError({ code: "unavailable", message: "No model." }); + }, + }), + }); + harness.recorder.prepareToRecordAsync.mockReturnValueOnce(capture.promise); + const started = harness.controller.start(); + await Promise.resolve(); + capture.resolve(undefined); + await started; + expect(harness.recorder.record).not.toHaveBeenCalled(); + expect(harness.controller.currentState.error).toBe("No model."); + }); +}); + describe("environment voice input", () => { beforeEach(() => resetVoiceInputGlobalsForTests()); diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts index d093b30a6497a..2094c97565123 100644 --- a/packages/client-runtime/src/voice-input/controller.ts +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -78,6 +78,7 @@ export type VoiceInputControllerDependencies = { readonly deleteRecording: (uri: string) => void; /** Keep a recording whose transcription failed so the next start retries it without recording again. */ readonly retryFailedRecording?: boolean; + readonly captureDuringPreparation?: boolean; /** * Optional cleanup of the transcript before insertion. Returning null skips it for this * recording; a failure falls back to the raw transcript. @@ -259,6 +260,7 @@ export class VoiceInputController { private operationToken = 0; private sessionToken: VoiceInputSession | null = null; private transcription: PreparedVoiceTranscription | null = null; + private preparation: Promise | null = null; private liveTranscription: Promise<{ text: string | null } | { error: unknown }> | null = null; private transcriptionAbortController: AbortController | null = null; private postProcessingAbortController: AbortController | null = null; @@ -271,7 +273,7 @@ export class VoiceInputController { private retryRecording: { uri: string; ownerKey: string; - transcription: PreparedVoiceTranscription; + transcription: PreparedVoiceTranscription | null; } | null = null; constructor(dependencies: VoiceInputControllerDependencies) { @@ -316,10 +318,27 @@ export class VoiceInputController { this.deleteOwnedRecordings(); } if (this.retryRecording) { - this.recordingUri = this.retryRecording.uri; + const retry = this.retryRecording; + this.recordingUri = retry.uri; this.capturedDraft = initiatingDraft; - this.transcription = this.retryRecording.transcription; this.retryRecording = null; + if (retry.transcription) { + this.transcription = retry.transcription; + } else { + const retryTranscriber = this.dependencies.getTranscriber(); + try { + if (!retryTranscriber) throw new Error("Voice transcription is not available."); + this.transcription = await runTranscriptionOperation(() => + retryTranscriber.prepare({ signal: abortController.signal }), + ); + } catch (error) { + if (!this.isCurrent(operationToken)) return; + this.retryRecording = retry; + this.setError(preparationErrorMessage(error), "retry"); + return; + } + if (!this.isCurrent(operationToken)) return; + } await this.finishRecording(true, this.recordingUri); return; } @@ -339,6 +358,16 @@ export class VoiceInputController { return; } + if (this.dependencies.captureDuringPreparation) { + await this.captureDuringPreparation( + transcriber, + abortController.signal, + initiatingDraft, + operationToken, + ); + return; + } + try { this.transcription = await runTranscriptionOperation(() => transcriber.prepare({ signal: abortController.signal }), @@ -399,6 +428,51 @@ export class VoiceInputController { } } + private async captureDuringPreparation( + transcriber: VoiceTranscriber, + signal: AbortSignal, + initiatingDraft: VoiceDraftSnapshot, + operationToken: number, + ): Promise { + let failure: { readonly error: unknown } | null = null; + const preparation = runTranscriptionOperation(() => transcriber.prepare({ signal })); + this.preparation = preparation; + preparation.catch((error: unknown) => { + failure = { error }; + }); + const failIfUnprepared = () => { + if (!failure) return false; + this.setError(preparationErrorMessage(failure.error), "retry"); + return true; + }; + + await this.dependencies.configureRecording(); + this.recordingConfigured = true; + if (!this.isCurrent(operationToken) || failIfUnprepared()) return; + await this.dependencies.recorder.prepareToRecordAsync(signal); + if (!this.isCurrent(operationToken) || failIfUnprepared()) return; + this.recordingUri = this.dependencies.recorder.uri; + this.rememberRecordingUri(this.recordingUri); + + const capturedDraft = this.dependencies.readDraft(); + if (!capturedDraft || capturedDraft.ownerKey !== initiatingDraft.ownerKey) { + this.setError("This draft is no longer available.", "retry"); + return; + } + this.capturedDraft = capturedDraft; + this.dependencies.recorder.record({ forDuration: VOICE_RECORDING_LIMIT_SECONDS }); + this.setState({ phase: "recording", error: null, errorAction: null }); + void preparation.then( + (transcription) => { + if (this.preparation === preparation) this.transcription = transcription; + }, + () => { + if (this.isCurrent(operationToken) && this.state.phase === "recording") + void this.finishRecording(false, null); + }, + ); + } + stop(): Promise { if (this.state.phase !== "recording") return Promise.resolve(); return this.finishRecording(false, null); @@ -507,11 +581,28 @@ export class VoiceInputController { try { if (!alreadyStopped) await this.dependencies.recorder.stop(); - await this.releaseAudioSession(); this.recordingUri = completedUri ?? this.dependencies.recorder.uri ?? this.recordingUri; this.rememberRecordingUri(this.recordingUri); + await this.releaseAudioSession(); if (!this.isCurrent(operationToken)) return; - const transcription = this.transcription; + let transcription = this.transcription; + if (!transcription && this.preparation) { + try { + transcription = await this.preparation; + } catch (error) { + if (this.isCurrent(operationToken)) { + if (this.dependencies.retryFailedRecording && this.recordingUri && this.capturedDraft) + this.retryRecording = { + uri: this.recordingUri, + ownerKey: this.capturedDraft.ownerKey, + transcription: null, + }; + this.setError(preparationErrorMessage(error), "retry"); + } + return; + } + if (!this.isCurrent(operationToken)) return; + } const recordingUri = this.recordingUri; if ( !transcription || @@ -651,6 +742,7 @@ export class VoiceInputController { this.sessionToken = null; this.capturedDraft = null; this.transcription = null; + this.preparation = null; this.transcriptionAbortController = null; this.postProcessingAbortController = null; this.skipPostProcessingRequested = false;