Skip to content
35 changes: 35 additions & 0 deletions .github/workflows/closura-arc-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ jobs:
apps/server/src/mcp/OrchestratorMcpService.test.ts \
apps/server/src/cli/thread.test.ts \
apps/web/src/composerDraftStore.test.ts \
packages/client-runtime/src/voice-input/controller.test.ts \
apps/web/src/components/chat/composerSend.logic.test.ts \
apps/web/src/components/chat/chatViewSend.test.ts \
apps/web/src/components/chat/composerImagePersistence.test.ts \
Expand Down Expand Up @@ -208,6 +209,40 @@ jobs:
src/features/home/pendingTaskRoute.test.tsx \
src/features/threads/NewTaskDraftRouteScreen.test.tsx

- name: Provide Chromium libraries for the voice latency browser test
shell: bash
run: |
set -euo pipefail
if test -e /usr/lib/x86_64-linux-gnu/libnspr4.so; then
printf 'Chromium libraries present\n'
exit 0
fi
dir="$RUNNER_TEMP/chromium-libs"
mkdir -p "$dir/lists/partial" "$dir/cache/archives/partial" "$dir/root"
apt_opts=(-o Debug::NoLocking=1 -o Dir::State::Lists="$dir/lists" -o Dir::Cache="$dir/cache")
apt-get "${apt_opts[@]}" -qq update
apt-get "${apt_opts[@]}" -qq install --download-only --no-install-recommends -y \
libasound2t64 libatk-bridge2.0-0t64 libatk1.0-0t64 libatspi2.0-0t64 libcairo2 \
libcups2t64 libdbus-1-3 libdrm2 libgbm1 libglib2.0-0t64 libnspr4 libnss3 \
libpango-1.0-0 libx11-6 libxcb1 libxcomposite1 libxdamage1 libxext6 libxfixes3 \
libxkbcommon0 libxrandr2
for deb in "$dir"/cache/archives/*.deb; do dpkg-deb -x "$deb" "$dir/root"; done
lib_dir="$dir/root/usr/lib/x86_64-linux-gnu"
test -f "$lib_dir/libnspr4.so"
printf 'LD_LIBRARY_PATH=%s\n' "$lib_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" >> "$GITHUB_ENV"

- name: Test voice press-to-recording and release-to-text stay under 400 ms
shell: bash
working-directory: apps/web
env:
PLAYWRIGHT_BROWSERS_PATH: ${{ runner.temp }}/playwright-browsers
run: |
set -euo pipefail
vp exec playwright install --only-shell chromium
vp test run --project browser -t "records within 400 ms" \
src/speech/voiceInputLatency.browser.test.ts \
src/speech/useEnvironmentSpeechInput.browser.test.tsx

- name: Build native T3 server
shell: bash
run: vp run --filter t3 build:bundle
Expand Down
69 changes: 60 additions & 9 deletions apps/web/src/speech/browserPcmCapture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,36 @@ import { requestMicrophone } from "./requestMicrophone";
import workletUrl from "./pcmWorklet.ts?worker&url";

type PreparedBrowserTranscription = PreparedVoiceTranscription & {
readonly streaming?: VoiceStreamingSession & { readonly feed: (pcm: Float32Array) => void };
readonly streaming?: VoiceStreamingSession & {
readonly feed: (pcm: Float32Array) => void;
readonly cancel: () => void;
};
};

let warmAudio: { context: AudioContext; ready: Promise<AudioContext> } | undefined;
let capturing = false;

export function warmBrowserAudioCapture(): Promise<AudioContext> {
if (
warmAudio &&
warmAudio.context instanceof AudioContext &&
warmAudio.context.state !== "closed"
)
return warmAudio.ready;
const context = new AudioContext();
const ready = context.audioWorklet.addModule(workletUrl).then(async () => {
if (context.state === "running" && !capturing) await context.suspend();
return context;
});
const entry = { context, ready };
warmAudio = entry;
ready.catch(() => {
if (warmAudio === entry) warmAudio = undefined;
void context.close().catch(() => undefined);
});
return ready;
}

export function createBrowserPcmCapture(
input: {
readonly getMicrophoneId: () => string | Promise<string>;
Expand All @@ -24,7 +51,9 @@ export function createBrowserPcmCapture(
let recordingUri: string | null = null;
let chunks: Float32Array<ArrayBuffer>[] = [];
let audioContext: AudioContext | undefined;
let source: MediaStreamAudioSourceNode | undefined;
let worklet: AudioWorkletNode | undefined;
let captureFinished = false;
let durationTimer: ReturnType<typeof setTimeout> | undefined;
let signal: AbortSignal | undefined;
let feed: ((pcm: Float32Array) => void) | undefined;
Expand All @@ -35,13 +64,18 @@ export function createBrowserPcmCapture(
const cleanupCapture = () => {
if (durationTimer) clearTimeout(durationTimer);
durationTimer = undefined;
source?.disconnect();
source = undefined;
worklet?.disconnect();
worklet?.port.close();
if (worklet) worklet.port.onmessage = null;
worklet?.port.postMessage("release", []);
worklet = undefined;
stream?.getTracks().forEach((track) => track.stop());
stream = undefined;
void audioContext?.close();
if (audioContext && audioContext.state !== "closed")
void audioContext.suspend().catch(() => undefined);
audioContext = undefined;
capturing = false;
stopped?.resolve();
stopped = undefined;
chunks = [];
Expand All @@ -53,13 +87,18 @@ export function createBrowserPcmCapture(
get uri() {
return recordingUri;
},
prepareToRecordAsync: async () => {
const captureSignal = signal;
prepareToRecordAsync: async (requestedSignal) => {
const captureSignal = requestedSignal ?? signal;
if (!captureSignal) throw new Error("Voice transcription is not prepared.");
captureSignal.throwIfAborted();
signal = captureSignal;
if (recordingUri) URL.revokeObjectURL(recordingUri);
recordingUri = null;
chunks = [];
captureFinished = false;
capturing = true;
const warming = warmBrowserAudioCapture();
warming.catch(() => undefined);
const microphoneId = await input.getMicrophoneId();
captureSignal.throwIfAborted();
const captured = await new Promise<MediaStream>((resolve, reject) => {
Expand All @@ -84,9 +123,8 @@ export function createBrowserPcmCapture(
}
stream = captured;
try {
const context = new AudioContext();
const context = await warming;
audioContext = context;
await context.audioWorklet.addModule(workletUrl);
captureSignal.throwIfAborted();
const node = new AudioWorkletNode(context, "t3-pcm-capture", { channelCount: 1 });
worklet = node;
Expand Down Expand Up @@ -117,7 +155,8 @@ export function createBrowserPcmCapture(
track.addEventListener("ended", () => input.onError("The microphone disconnected."), {
once: true,
});
context.createMediaStreamSource(captured).connect(node);
source = context.createMediaStreamSource(captured);
source.connect(node);
// The processor emits silence, keeping the graph active without microphone playback.
node.connect(context.destination);
await context.resume();
Expand Down Expand Up @@ -153,6 +192,7 @@ export function createBrowserPcmCapture(
const blob = new Blob(chunks, { type: "application/octet-stream" });
recordingUri = URL.createObjectURL(blob);
}
captureFinished = true;
} finally {
clearTimeout(timeout);
cleanupCapture();
Expand All @@ -168,11 +208,22 @@ export function createBrowserPcmCapture(
if (recordingUri === uri) recordingUri = null;
},
transcriber: {
prepare: async (options: VoiceTranscriptionOptions) => {
prepare: async (
options: VoiceTranscriptionOptions,
): Promise<PreparedBrowserTranscription> => {
const prepared = await prepare(options);
options.signal.throwIfAborted();
signal = options.signal;
if (prepared.streaming && captureFinished) {
prepared.streaming.cancel();
const { streaming: _streaming, ...buffered } = prepared;
return buffered;
}
feed = prepared.streaming?.feed;
if (feed) {
for (const chunk of chunks) feed(chunk);
chunks = [];
}
return prepared;
},
},
Expand Down
4 changes: 3 additions & 1 deletion apps/web/src/speech/browserVoiceInput.browser.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,11 @@ function capture() {
class {
audioWorklet = { addModule: async () => {} };
createMediaStreamSource() {
return { connect() {} };
return { connect() {}, disconnect() {} };
}
state = "running";
resume = async () => {};
suspend = async () => {};
close = async () => {};
},
),
Expand Down
3 changes: 3 additions & 0 deletions apps/web/src/speech/pcmWorklet.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ class PcmCaptureProcessor extends AudioWorkletProcessor {
private length = 0;
private recording = false;
private flushed = false;
private released = false;
private samples = 0;
private levelSamples = 0;
private levelEnergy = 0;
Expand All @@ -23,6 +24,7 @@ class PcmCaptureProcessor extends AudioWorkletProcessor {
this.meterOnly = options?.processorOptions?.meterOnly === true;
this.port.onmessage = ({ data }) => {
if (data === "start") this.recording = true;
if (data === "release") this.released = true;
if (data === "stop") {
this.recording = false;
if (!this.meterOnly && !this.flushed) this.append(this.resampler.flush());
Expand Down Expand Up @@ -55,6 +57,7 @@ class PcmCaptureProcessor extends AudioWorkletProcessor {
}

process(inputs: Float32Array[][]): boolean {
if (this.released || this.flushed) return false;
if (!this.recording) return true;
const channels = inputs[0];
if (!channels?.[0]) return true;
Expand Down
24 changes: 23 additions & 1 deletion apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ const mocks = vi.hoisted(() => ({
primaryEnvironmentId: "primary-env",
statusRequests: [] as string[],
statusUnavailable: false,
statusHold: null as Promise<void> | null,
modelState: "ready" as "ready" | "missing-model",
statusModelId: "test-model",
models: [] as EnvironmentSpeechModel[],
Expand Down Expand Up @@ -116,8 +117,9 @@ vi.mock("../components/ui/toast", () => ({ toastManager: { add: vi.fn() } }));
vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
...(await importOriginal<typeof import("@t3tools/client-runtime/voice-input")>()),
getEnvironmentSpeechStatus: (prepared: PreparedConnection) =>
Effect.sync(() => {
Effect.promise(async () => {
mocks.statusRequests.push(prepared.environmentId);
if (mocks.statusHold) await mocks.statusHold;
if (mocks.statusUnavailable) throw new Error("Temporarily offline");
return {
supported: true,
Expand Down Expand Up @@ -218,6 +220,7 @@ function setup(options: { readonly destination?: string | null } = {}) {
mocks.prepared.clear();
mocks.statusRequests = [];
mocks.statusUnavailable = false;
mocks.statusHold = null;
mocks.modelState = "ready";
mocks.statusModelId = "test-model";
mocks.models = [];
Expand Down Expand Up @@ -801,6 +804,25 @@ it("shares one status probe per destination across every mounted editor", async
expect(mocks.statusRequests).toEqual(["env-a"]);
});

it("records within 400 ms of the press while the status refresh is still unanswered", async () => {
setup({ destination: "env-a" });
await using _screen = await render(<Probe />);
expect(mocks.voice.available).toBe(true);
let answer!: () => void;
mocks.statusHold = new Promise((resolve) => {
answer = resolve;
});
const started = mocks.voice.start();
await vi.waitFor(() => expect(mocks.voice.state.phase).toBe("recording"), {
timeout: 400,
interval: 1,
});
answer();
await act(() => started);
await act(() => mocks.voice.stop());
expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything());
});

it("lets the user retry after a status failure without reselecting the destination", async () => {
setup({ destination: "env-a" });
mocks.statusUnavailable = true;
Expand Down
24 changes: 24 additions & 0 deletions apps/web/src/speech/useEnvironmentSpeechInput.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import {
import { useEnvironment } from "../state/environments";
import { useEnvironmentScope, usePreparedConnection } from "../state/session";
import { chooseRecordingMicrophone } from "./browserMicrophones";
import { warmBrowserAudioCapture } from "./browserPcmCapture";
import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
import { preferredSpeechLanguages } from "./preferredSpeechLanguages";
import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
Expand Down Expand Up @@ -313,6 +314,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
};
controller = new VoiceInputController({
retryFailedRecording: true,
captureDuringPreparation: true,
recorder: platform.recorder,
getTranscriber: () => platform.transcriber,
requestPermission: async () => ({ granted: true, canAskAgain: true }),
Expand Down Expand Up @@ -340,6 +342,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
},
});
controllerRef.current = controller;
if (typeof AudioContext !== "undefined") void warmBrowserAudioCapture().catch(() => undefined);
return () => {
disposed = true;
pendingSubmissionRef.current = null;
Expand Down Expand Up @@ -472,6 +475,27 @@ export function useEnvironmentSpeechInput(input: HookInput) {
latestRef.current.input.ownerKey === owner &&
!latestRef.current.input.disabled;
setSetup(INITIAL_SETUP);
const knownStatus = status?.prepared === prepared ? status.value : null;
if (knownStatus?.supported && knownStatus.state !== "missing-model") {
setLevel(0);
void probeSpeechStatus(prepared, true).then(
(latest) => {
if (!stillCurrent()) return;
setStatus({ prepared, value: latest });
if (latest.supported && latest.state !== "missing-model") return;
controller.cancel();
if (latest.supported) return loadSetupCatalog(prepared, latest.modelId, request);
toastManager.add({
type: "error",
title: "Voice input is not available",
description: latest.reason,
});
},
() => undefined,
);
await controller.start();
return;
}
setQueuedStart(prepared);
let latestStatus: EnvironmentSpeechStatus;
try {
Expand Down
26 changes: 26 additions & 0 deletions apps/web/src/speech/voiceInputLatency.browser.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import { it, vi } from "vite-plus/test";

import { createVoiceLatencyHarness } from "../../test/voiceLatencyHarness";

vi.mock("../connection/runtime", async () => {
const [{ Atom }, { layerRemoteHttpClient }] = await Promise.all([
import("effect/reactivity"),
import("@t3tools/client-runtime/rpc"),
]);
return {
connectionAtomRuntime: Atom.runtime(
layerRemoteHttpClient((input, init) => globalThis.fetch(input, init)),
),
};
});

it.each([{ streaming: false }, { streaming: true }])(
"records within 400 ms of the press while the server is still answering, and inserts text within 400 ms of release: %j",
async ({ streaming }) => {
using voice = await createVoiceLatencyHarness({ streaming });
await voice.press();
await voice.captureAudio();
await voice.connectServer();
await voice.release();
},
);
Loading
Loading