From 7bd2e4446a0a561d85713bcdbf8798381676fd60 Mon Sep 17 00:00:00 2001 From: nullStack65 <112618179+nullStack65@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:07:22 +0000 Subject: [PATCH 1/2] fix(web): voice setup picks a backend first and never dead-ends Transcribe with now lists backends (AssemblyAI recommended, OpenAI, Grok, OpenRouter, Local model) instead of environments, and runs the choice on this device's environment by default. Overriding the environment is an optional Advanced row that appears only when more than one environment can run the backend, so there is no "environment without a backend" state. The setup dialog no longer says "Connect ... to continue" with only Cancel: an offline environment offers Open Environments, and a dialog with nothing to do says Close. The server also accepts an AssemblyAI key from ASSEMBLYAI_API_KEY in its launch environment (for example injected from the vault), reports it as host-managed so the UI offers nothing to type or remove, and drops it from its own environment so agent subprocesses don't inherit it. A key saved in settings still wins. First-time dictation offers AssemblyAI straight away. Model: claude-opus-5-5 Task: fix Agent: claude-code Co-Authored-By: Claude Opus 5.5 --- apps/server/src/speech/CloudSpeech.test.ts | 52 +++ apps/server/src/speech/CloudSpeech.ts | 32 +- .../CloudVoiceConfiguration.browser.test.tsx | 105 +++++- .../settings/CloudVoiceConfiguration.tsx | 99 +++--- .../VoiceSettingsPanel.browser.test.tsx | 5 +- .../settings/VoiceSettingsPanel.tsx | 298 ++++++++++-------- .../src/components/settings/settingsSearch.ts | 2 +- apps/web/src/speech/voiceBackends.test.ts | 41 +++ apps/web/src/speech/voiceBackends.ts | 58 ++++ docs/user/faq.md | 2 +- docs/user/voice-input.md | 22 +- packages/contracts/src/speech.ts | 2 + 12 files changed, 532 insertions(+), 186 deletions(-) create mode 100644 apps/web/src/speech/voiceBackends.test.ts create mode 100644 apps/web/src/speech/voiceBackends.ts diff --git a/apps/server/src/speech/CloudSpeech.test.ts b/apps/server/src/speech/CloudSpeech.test.ts index 1a1054f8d251..a5f27502dec7 100644 --- a/apps/server/src/speech/CloudSpeech.test.ts +++ b/apps/server/src/speech/CloudSpeech.test.ts @@ -1,3 +1,4 @@ +// @effect-diagnostics processEnvInEffect:off - checks that the host key leaves the live environment. import { it } from "@effect/vitest"; import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, @@ -5,6 +6,7 @@ import { type SpeechTranscriptionOptions, } from "@t3tools/contracts"; import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import * as ConfigProvider from "effect/ConfigProvider"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; @@ -302,3 +304,53 @@ it.effect( expect(yield* Effect.flip(transcribe(1600))).toMatchObject({ reason: "missing-key" }); }), ); + +it.effect( + "a host-injected AssemblyAI key works without settings and never outranks a saved key", + () => + Effect.gen(function* () { + const upstream = vi.fn(async () => Response.json({ text: "hello" })); + process.env.ASSEMBLYAI_API_KEY = "host-fixture"; + const service = yield* CloudSpeech.CloudSpeech.pipe( + Effect.provide( + CloudSpeech.layer.pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide(ServerSettings.ServerSettingsService.layerTest()), + Layer.provide(Layer.mock(SecretResolver.SecretResolver)({})), + Layer.provide( + ConfigProvider.layer( + ConfigProvider.fromEnv({ env: { ASSEMBLYAI_API_KEY: " host-fixture\n" } }), + ), + ), + ), + ), + ); + // Agent subprocesses must not inherit the key. + expect(process.env.ASSEMBLYAI_API_KEY).toBeUndefined(); + const authorization = () => + new Headers(upstream.mock.calls.at(-1)![1]?.headers).get("authorization"); + const transcribe = () => + service + .transcribe("assemblyai", new Uint8Array(new Float32Array(1600).buffer), { + ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + }) + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); + expect(yield* service.configured).toEqual({ + openai: false, + xai: false, + openrouter: false, + assemblyai: true, + hostManaged: ["assemblyai"], + }); + expect(yield* transcribe()).toBe("hello"); + expect(authorization()).toBe("host-fixture"); + yield* service.setKey("assemblyai", Redacted.make("saved-fixture")); + expect((yield* service.configured).hostManaged).toBeUndefined(); + yield* transcribe(); + expect(authorization()).toBe("saved-fixture"); + yield* service.setKey("assemblyai", null); + expect((yield* service.configured).hostManaged).toEqual(["assemblyai"]); + yield* transcribe(); + expect(authorization()).toBe("host-fixture"); + }), +); diff --git a/apps/server/src/speech/CloudSpeech.ts b/apps/server/src/speech/CloudSpeech.ts index b846bff387df..5c6e56890e8f 100644 --- a/apps/server/src/speech/CloudSpeech.ts +++ b/apps/server/src/speech/CloudSpeech.ts @@ -14,10 +14,12 @@ import { } from "@t3tools/contracts"; import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; import { cloudSpeechLanguages, effectiveSpeechLanguage } from "@t3tools/shared/speech"; +import * as Config from "effect/Config"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Redacted from "effect/Redacted"; import * as Schema from "effect/Schema"; import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"; @@ -35,6 +37,11 @@ const endpoints = { }; const OPENROUTER_CATALOG_URL = "https://openrouter.ai/api/v1/models?output_modalities=transcription&zdr=true"; +/** + * Keys a host injects at launch (for example `infisical run -- t3`), so nobody types or sees them. + * A key saved in settings still wins. + */ +const HOST_KEY_VARIABLES = { assemblyai: "ASSEMBLYAI_API_KEY" } as const; const isApiKey = Schema.is(CloudSpeechApiKey); const isModelId = Schema.is(CloudSpeechModelId); const sameCredential = Schema.toEquivalence(SettingsSecretValue); @@ -140,15 +147,34 @@ export const layer = Layer.effect( const settings = yield* ServerSettings.ServerSettingsService; const resolver = yield* SecretResolver.SecretResolver; const http = yield* HttpClient.HttpClient; - const credentials = settings.getSettings.pipe( + const hostAssemblyAi = Option.filter( + Option.map(yield* Config.Redacted(HOST_KEY_VARIABLES.assemblyai).pipe(Config.option), (key) => + Redacted.value(key).trim(), + ), + isApiKey, + ); + // Agent subprocesses inherit this environment; only this service should hold the key. + // @effect-diagnostics-next-line processEnvInEffect:off + delete process.env[HOST_KEY_VARIABLES.assemblyai]; + const savedCredentials = settings.getSettings.pipe( Effect.map((value) => value.cloudSpeechCredentials), ); - const configured = credentials.pipe( + const credentials = savedCredentials.pipe( + Effect.map((saved) => + !saved.assemblyai && Option.isSome(hostAssemblyAi) + ? { ...saved, assemblyai: hostAssemblyAi.value } + : saved, + ), + ); + const configured = savedCredentials.pipe( Effect.map(({ openai, xai, openrouter, assemblyai }) => ({ openai: Boolean(openai), xai: Boolean(xai), openrouter: Boolean(openrouter), - assemblyai: Boolean(assemblyai), + assemblyai: Boolean(assemblyai) || Option.isSome(hostAssemblyAi), + ...(!assemblyai && Option.isSome(hostAssemblyAi) + ? { hostManaged: ["assemblyai" as const] } + : {}), })), ); const execute = (request: HttpClientRequest.HttpClientRequest) => diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx index b5eddbcc10df..955d5aee4ae7 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx @@ -46,6 +46,7 @@ const grantKey = "t3code:voice-audio-grant:v1"; async function withVoicePanel( route: (url: URL, request: RequestInfo | URL, init?: RequestInit) => Promise | null, body: (screen: Awaited>) => Promise, + { connected = true, path = "/" }: { connected?: boolean; path?: string } = {}, ) { const target = new PrimaryConnectionTarget({ environmentId: id, @@ -53,12 +54,13 @@ async function withVoicePanel( httpBaseUrl: window.location.origin, wsBaseUrl: "ws://localhost", }); - const prepared = Option.some({ + const connection = Option.some({ ...target, target, socketUrl: target.wsBaseUrl, httpAuthorization: null, }); + const prepared = connected ? connection : Option.none(); using cleanup = new DisposableStack(); cleanup.defer(() => chooseAudioDestination(null)); const fetch = globalThis.fetch; @@ -76,6 +78,7 @@ async function withVoicePanel( {}}> , + { router: path }, ); await body(screen); }, @@ -99,13 +102,13 @@ it("cancelling a provider's setup keeps the current destination and writes nothi }, async (screen) => { const destination = screen.getByRole("combobox", { name: "Transcribe with" }); - await expect.element(destination).toHaveTextContent("Laptop"); + await expect.element(destination).toHaveTextContent("Local model"); await destination.click(); - await page.getByRole("option", { name: "OpenAI via Laptop" }).click(); + await page.getByRole("option", { name: "OpenAI" }).click(); const dialog = page.getByRole("dialog"); await dialog.getByRole("button", { name: "Cancel" }).click(); await expect.element(dialog).not.toBeInTheDocument(); - await expect.element(destination).toHaveTextContent("Laptop"); + await expect.element(destination).toHaveTextContent("Local model"); expect(writes).toEqual([]); }, ); @@ -132,7 +135,7 @@ it("enables the chosen provider only after a successful setup", async () => { async (screen) => { const destination = screen.getByRole("combobox", { name: "Transcribe with" }); await destination.click(); - await page.getByRole("option", { name: "OpenAI via Laptop" }).click(); + await page.getByRole("option", { name: "OpenAI" }).click(); const dialog = page.getByRole("dialog"); await dialog.getByRole("combobox", { name: "OpenAI API key source" }).click(); await page.getByRole("option", { name: "Work vault" }).click(); @@ -144,7 +147,7 @@ it("enables the chosen provider only after a successful setup", async () => { await dialog.getByLabelText("OpenAI API key reference").click(); await userEvent.keyboard("{Enter}"); await expect.element(dialog).not.toBeInTheDocument(); - await expect.element(destination).toHaveTextContent("OpenAI via Laptop"); + await expect.element(destination).toHaveTextContent("OpenAI"); expect(writes).toEqual([{ key: { provider: "work", reference: "op://Work/Voice/key" } }]); }, ); @@ -186,12 +189,12 @@ it("uses a provider's saved key without writing it, and turns dictation off", as async (screen) => { const destination = screen.getByRole("combobox", { name: "Transcribe with" }); await destination.click(); - await page.getByRole("option", { name: "Grok Voice Transcribe via Laptop" }).click(); + await page.getByRole("option", { name: "Grok Voice Transcribe" }).click(); await page .getByRole("dialog") .getByRole("button", { name: "Use Grok Voice Transcribe" }) .click(); - await expect.element(destination).toHaveTextContent("Grok Voice Transcribe via Laptop"); + await expect.element(destination).toHaveTextContent("Grok Voice Transcribe"); await destination.click(); await page.getByRole("option", { name: "Off", exact: true }).click(); await expect.element(destination).toHaveTextContent("Off"); @@ -230,7 +233,7 @@ it("OpenRouter needs an exact model, and changing it writes no credential", asyn async (screen) => { const destination = screen.getByRole("combobox", { name: "Transcribe with" }); await destination.click(); - await page.getByRole("option", { name: "OpenRouter via Laptop" }).click(); + await page.getByRole("option", { name: "OpenRouter" }).click(); const dialog = page.getByRole("dialog"); const save = dialog.getByRole("button", { name: "Save and enable" }); await expect.element(save).toBeDisabled(); @@ -246,7 +249,7 @@ it("OpenRouter needs an exact model, and changing it writes no credential", asyn await key.fill(""); await save.click(); await expect.element(dialog).not.toBeInTheDocument(); - await expect.element(destination).toHaveTextContent("OpenRouter via Laptop"); + await expect.element(destination).toHaveTextContent("OpenRouter"); expect(grantModel()).toBe("other/listen-2"); expect(writes).toEqual([]); await screen.getByRole("button", { name: "Manage OpenRouter" }).click(); @@ -256,9 +259,89 @@ it("OpenRouter needs an exact model, and changing it writes no credential", asyn await page.getByRole("option", { name: /Acme: Transcribe 1/ }).click(); await dialog.getByRole("button", { name: "Save" }).click(); await expect.element(dialog).not.toBeInTheDocument(); - await expect.element(destination).toHaveTextContent("OpenRouter via Laptop"); + await expect.element(destination).toHaveTextContent("OpenRouter"); expect(grantModel()).toBe("acme/transcribe-1"); expect(writes).toEqual([]); }, ); }); + +/** Every step either offers a real next action or names nothing the user can't find. */ +function expectNoDeadEnd(dialog: ReturnType) { + const element = dialog.element(); + const buttons = [...element.querySelectorAll("button")].map((button) => + button.textContent.trim(), + ); + const forward = buttons.filter((label) => label && !["Cancel", "Close"].includes(label)); + expect(forward.length).toBeGreaterThan(0); + for (const [, name] of element.textContent.matchAll( + /\b(?:press|click|tap|hit|open)\s+([A-Z][\w ]*?)(?=[ .,]|$)/g, + )) + expect(buttons.some((label) => label.includes(name!))).toBe(true); + if (/\bcontinue\b/i.test(element.textContent)) expect(buttons).toContain("Continue"); +} + +it("a backend on a disconnected environment points to a real control, never a missing Continue", async () => { + chooseAudioDestination(null); + await withVoicePanel( + () => null, + async (screen) => { + await screen.getByRole("combobox", { name: "Transcribe with" }).click(); + await page.getByRole("option", { name: /AssemblyAI/ }).click(); + const dialog = page.getByRole("dialog"); + await expect + .element(dialog.getByText("Laptop isn’t connected.", { exact: false })) + .toBeVisible(); + expectNoDeadEnd(dialog); + await dialog.getByRole("button", { name: "Open Environments" }).click(); + await expect.element(dialog).not.toBeInTheDocument(); + expect(screen.router?.state.location.pathname).toBe("/settings/connections"); + }, + { connected: false }, + ); +}); + +it("AssemblyAI with a host-provided key enables in one step, with nothing to type or remove", async () => { + // Never chosen, unlike an explicit Off. + window.localStorage.removeItem(grantKey); + const writes: unknown[] = []; + await withVoicePanel( + (url) => { + if (url.pathname === "/api/voice/cloud/keys") + return Promise.resolve( + Response.json({ + openai: false, + xai: false, + assemblyai: true, + hostManaged: ["assemblyai"], + }), + ); + if (url.pathname.endsWith("/key")) writes.push(url.pathname); + return null; + }, + async (screen) => { + // The composer's mic sends first-time users here; the recommended backend is offered at once. + const dialog = page.getByRole("dialog"); + await expect.element(dialog.getByText("Use AssemblyAI for dictation?")).toBeVisible(); + await expect + .element( + dialog.getByText("was started with, so there’s nothing to enter", { exact: false }), + ) + .toBeVisible(); + expect(dialog.element().querySelector("input")).toBeNull(); + expectNoDeadEnd(dialog); + await dialog.getByRole("button", { name: "Use AssemblyAI" }).click(); + await expect.element(dialog).not.toBeInTheDocument(); + const destination = screen.getByRole("combobox", { name: "Transcribe with" }); + await expect.element(destination).toHaveTextContent("AssemblyAI"); + await screen.getByRole("button", { name: "Manage key" }).click(); + await expect.element(dialog).toBeVisible(); + expect(dialog.element().querySelector("input")).toBeNull(); + await expect + .element(dialog.getByRole("button", { name: "Remove key…" })) + .not.toBeInTheDocument(); + expect(writes).toEqual([]); + }, + { path: "/settings/voice#voice-transcription-destination" }, + ); +}); diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx index b3579f5def75..4db3d9e0d270 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx @@ -29,6 +29,7 @@ import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import type { HttpClient } from "effect/http"; import { Atom, AsyncResult } from "effect/reactivity"; +import { useNavigate } from "@tanstack/react-router"; import { useId, useState } from "react"; import { isSecretsFeatureEnabled } from "../../env"; import { @@ -164,6 +165,7 @@ export function CloudVoiceConfiguration({ const company = CLOUD_SPEECH_PROVIDER_NAMES[provider]; const routed = provider === "openrouter"; const formId = useId(); + const navigate = useNavigate(); const prepared = Option.getOrNull(usePreparedConnection(id)); const canUse = useEnvironmentScope(id, AuthOrchestrationOperateScope); const canManage = useEnvironmentScope(id, AuthSettingsWriteScope); @@ -183,6 +185,9 @@ export function CloudVoiceConfiguration({ const [error, setError] = useState(""); const [model, setModel] = useState(currentModel ?? ""); const saved = AsyncResult.isSuccess(presence) ? presence.value?.[provider] : undefined; + // The host's launch environment supplies this key, so there is nothing to type or remove. + const hostManaged = + AsyncResult.isSuccess(presence) && presence.value?.hostManaged?.includes(provider) === true; const existing = environment.serverConfig?.settings?.cloudSpeechCredentials?.[provider]; const sources = isSecretsFeatureEnabled && environment.serverConfig?.secretReferences @@ -210,7 +215,8 @@ export function CloudVoiceConfiguration({ ); const connected = canUse && prepared !== null; // Fixed providers edit the key only when nothing is saved or the device already uses them. - const editingKey = canManage && saved !== undefined && (routed || active || !saved); + const editingKey = + canManage && saved !== undefined && !hostManaged && (routed || active || !saved); const listedModels = AsyncResult.isSuccess(models) ? (models.value?.models ?? null) : null; const modelChosen = isModelId(model) && (listedModels?.some((m) => m.id === model) ?? false); const keyReady = draft.trim() ? credential !== undefined : Boolean(saved); @@ -309,6 +315,50 @@ export function CloudVoiceConfiguration({ ); + const primaryAction = !connected ? ( + + ) : remove ? ( + + ) : routed ? ( + saved !== undefined && (saved || canManage) ? ( + + ) : null + ) : editingKey ? ( + + ) : saved && !active ? ( + + ) : null; + return ( {!connected ? ( -

Connect to {environment.label} with dictation access to continue.

+

+ {prepared === null + ? `${environment.label} isn’t connected. Open Environments to reconnect it.` + : `This device’s access to ${environment.label} doesn’t include dictation. Open Environments to pair it again with full access.`} +

) : AsyncResult.isFailure(presence) && saved === undefined ? ( - {connected && - (remove ? ( - - ) : routed ? ( - saved !== undefined && (saved || canManage) ? ( - - ) : null - ) : editingKey ? ( - - ) : saved && !active ? ( - - ) : null)} + {primaryAction}
diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx index 4c8172333ff9..c0fa50c25c8f 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx @@ -579,8 +579,9 @@ it("keeps a missing destination selected without claiming it cannot transcribe", ); await expect .element(screen.getByRole("combobox", { name: "Transcribe with" })) - .toHaveTextContent("Chosen environment (unavailable)"); - await expect.element(screen.getByText(/can’t transcribe audio/)).not.toBeInTheDocument(); + .toHaveTextContent("Local model (unavailable)"); + await expect.element(screen.getByText(/can’t run a local model/)).not.toBeInTheDocument(); + await expect.element(screen.getByText(/Choose a backend to set it up again/)).toBeVisible(); }); it("shows the model's effective language and restores the saved preference when supported again", async () => { diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index a2c8a682263e..cef32631364a 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -15,7 +15,8 @@ import { AuthFilesystemReadScope, AuthSettingsWriteScope, CLOUD_SPEECH_LABELS, - CloudSpeechProvider, + CLOUD_SPEECH_PROVIDER_NAMES, + type CloudSpeechProvider, DEFAULT_SPEECH_POST_PROCESSING_PROMPT, type EnvironmentId, type EnvironmentSpeechModel, @@ -49,6 +50,8 @@ import { UnlinkIcon, XIcon, } from "lucide-react"; +import { useAtomValue } from "@effect/atom-react"; +import { useLocation } from "@tanstack/react-router"; import { useEffect, useRef, useState, type ReactNode } from "react"; import { useStableCallback } from "~/hooks/useStableCallback"; @@ -69,6 +72,14 @@ import { chooseTranscriptCleanup, useCleanupGrant, } from "../../speech/voiceConsent"; +import { + defaultVoiceEnvironment, + RECOMMENDED_VOICE_BACKEND, + VOICE_BACKENDS, + voiceBackendEnvironments, + type VoiceBackend, +} from "../../speech/voiceBackends"; +import { primaryEnvironmentIdAtom } from "../../state/primaryEnvironment"; import { updateClientSettings, useClientSettings } from "../../hooks/useSettings"; import { runEnvironmentSpeech } from "../../speech/runEnvironmentSpeech"; import { @@ -107,15 +118,7 @@ import { MenuSeparator, MenuTrigger, } from "../ui/menu"; -import { - Select, - SelectGroup, - SelectGroupLabel, - SelectItem, - SelectPopup, - SelectTrigger, - SelectValue, -} from "../ui/select"; +import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select"; import { Switch } from "../ui/switch"; import { Textarea } from "../ui/textarea"; import { toastManager } from "../ui/toast"; @@ -139,6 +142,16 @@ import { useScopedModelDisabledReason } from "./useScopedModelAvailability"; import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings"; const OFF = "off"; +const UNAVAILABLE = "unavailable"; +const VOICE_BACKEND_LABELS: Record = { + ...CLOUD_SPEECH_LABELS, + local: "Local model", +}; +function VoiceBackendIcon({ backend }: { backend: VoiceBackend | null }) { + if (backend === null) return ; + if (backend === "local") return ; + return ; +} function CloudSpeechLogo({ provider }: { provider: CloudSpeechProvider }) { if (provider === "assemblyai") return ; if (provider === "openrouter") @@ -722,24 +735,20 @@ function useVoiceModels(prepared: PreparedConnection | null) { type VoiceModels = ReturnType; -function AdvancedSpeechSettings({ +/** Local-model rows for the Advanced fold; render only when the environment supports speech. */ +function LocalSpeechAdvancedRows({ prepared, catalog, + supported, }: { prepared: PreparedConnection; catalog: VoiceModels; + supported: Extract; }) { const settings = useClientSettings(); - const { run, controlsDisabled, status } = catalog; - if (!status?.supported) return null; - const supported = status; + const { run, controlsDisabled } = catalog; return ( - + <> } /> - + ); } @@ -1218,11 +1227,46 @@ export function VoiceSettingsPanel() { const settingsScope = useOptionalSettingsScope(); const destination = useVoiceTranscriptionDestination(); const canManageKey = useEnvironmentScope(destination.environmentId, AuthSettingsWriteScope); - const [pendingCloud, setPendingCloud] = useState<{ + const { environments } = useEnvironments(); + const primaryEnvironmentId = useAtomValue(primaryEnvironmentIdAtom); + const hash = useLocation({ select: (location) => location.hash.replace(/^#/, "") }); + // The composer's mic links here on first use; remember it before the page consumes the hash. + const [offerRecommended, setOfferRecommended] = useState( + () => hash === "voice-transcription-destination", + ); + const candidates = environments.map((environment) => ({ + environment, + environmentId: environment.environmentId, + connected: environment.connection.phase === "available", + capabilities: environment.serverConfig?.environment.capabilities, + })); + const defaultEnvironment = (backend: VoiceBackend) => + defaultVoiceEnvironment(backend, candidates, { + current: destination.environmentId, + primary: primaryEnvironmentId, + }); + const [chosenCloud, setChosenCloud] = useState<{ provider: CloudSpeechProvider; environmentId: EnvironmentId; } | null>(null); - const { environments } = useEnvironments(); + // Until the user acts, a first-time visit offers the recommended backend straight away. + const recommended = RECOMMENDED_VOICE_BACKEND; + const recommendedEnvironment = + offerRecommended && destination.offerFirstRunSetup && recommended !== "local" + ? defaultEnvironment(recommended) + : null; + const pendingCloud = + chosenCloud ?? + (recommendedEnvironment && recommended !== "local" + ? { provider: recommended, environmentId: recommendedEnvironment } + : null); + const chooseBackend = (backend: VoiceBackend, environmentId = defaultEnvironment(backend)) => { + setOfferRecommended(false); + if (!environmentId) return; + if (backend !== "local") return setChosenCloud({ provider: backend, environmentId }); + setChosenCloud(null); + chooseAudioDestination({ kind: "environment", environmentId }); + }; const [microphones, setMicrophones] = useState([]); const prepared = destination.cloudProvider ? null : destination.prepared; const catalog = useVoiceModels(prepared); @@ -1256,28 +1300,9 @@ export function VoiceSettingsPanel() { const order = microphoneOrder(settings); const selectedMicrophone = preferredMicrophoneId(order, microphones); - const environmentChoices = environments.map((environment) => ({ - environment, - value: JSON.stringify(["environment", environment.environmentId]), - })); - const cloudGroups = environments - .filter( - (environment) => environment.serverConfig?.environment.capabilities.cloudVoiceTranscription, - ) - .map((environment) => ({ - environment, - choices: CloudSpeechProvider.literals.map((provider) => ({ - environment, - provider, - value: JSON.stringify(["cloud", environment.environmentId, provider]), - })), - })); - const cloudChoices = cloudGroups.flatMap((group) => group.choices); - const destinationValue = destination.cloudProvider - ? JSON.stringify(["cloud", destination.environmentId, destination.cloudProvider]) - : destination.environmentId - ? JSON.stringify(["environment", destination.environmentId]) - : OFF; + const backend: VoiceBackend | null = + destination.cloudProvider ?? (destination.environmentId ? "local" : null); + const backendEnvironments = backend ? voiceBackendEnvironments(backend, candidates) : []; const cloudEnvironment = environments.find( (env) => env.environmentId === pendingCloud?.environmentId, ); @@ -1292,6 +1317,17 @@ export function VoiceSettingsPanel() { chosenEnvironment?.serverConfig != null && chosenEnvironment.serverConfig.environment.capabilities.voiceTranscription !== true; const model = prepared && modelStatus?.supported ? modelStatus : null; + const backendValue = !backend ? OFF : chosenEnvironment ? backend : UNAVAILABLE; + const destinationDescription = + backend && !chosenEnvironment + ? "The environment this used is no longer available. Choose a backend to set it up again." + : chosenEnvironment && chosenUnsupported + ? `${chosenEnvironment.label} can’t run a local model. Choose another backend.` + : chosenEnvironment && destination.cloudProvider + ? `Audio goes to ${CLOUD_SPEECH_PROVIDER_NAMES[destination.cloudProvider]} through ${chosenEnvironment.label}.` + : chosenEnvironment + ? `A speech model runs on ${chosenEnvironment.label}.` + : "Where your speech is turned into text."; return ( {destination.cloudProvider && @@ -1499,7 +1491,7 @@ export function VoiceSettingsPanel() { aria-label={manageLabel} onClick={() => { if (destination.cloudProvider) - setPendingCloud({ + setChosenCloud({ provider: destination.cloudProvider, environmentId: chosenEnvironment.environmentId, }); @@ -1524,7 +1516,10 @@ export function VoiceSettingsPanel() { destination.environmentId === pendingCloud.environmentId && destination.cloudProvider === pendingCloud.provider } - close={() => setPendingCloud(null)} + close={() => { + setOfferRecommended(false); + setChosenCloud(null); + }} /> )} {(model || destination.cloudProvider) && ( @@ -1605,7 +1600,64 @@ export function VoiceSettingsPanel() { projectName={project?.title ?? null} /> ) : null} - {prepared && } + {backend && (backendEnvironments.length > 1 || (prepared && model)) && ( + + {backendEnvironments.length > 1 && ( + { + if (value && value !== UNAVAILABLE) + chooseBackend(backend, value as EnvironmentId); + }} + > + + + {chosenEnvironment?.label ?? "Unavailable environment"} + + + + {backendEnvironments.map(({ environment }) => ( + + + + {environment.label} + + + ))} + + + } + /> + )} + {prepared && model && ( + + )} + + )} diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index 937e37dd8ec3..de2491ae3755 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -544,7 +544,7 @@ export const SETTINGS_SEARCH_ITEMS = [ title: "Transcribe with", to: "/settings/voice", searchTerms: [ - "send microphone audio dictation destination speech voice stt transcription environment machine consent privacy cloud OpenAI Grok OpenRouter", + "send microphone audio dictation destination speech voice stt transcription backend environment machine consent privacy cloud AssemblyAI OpenAI Grok OpenRouter", ], }, { diff --git a/apps/web/src/speech/voiceBackends.test.ts b/apps/web/src/speech/voiceBackends.test.ts new file mode 100644 index 000000000000..fe2446ec98b5 --- /dev/null +++ b/apps/web/src/speech/voiceBackends.test.ts @@ -0,0 +1,41 @@ +import { EnvironmentId } from "@t3tools/contracts"; +import { expect, it } from "vite-plus/test"; +import { defaultVoiceEnvironment, voiceBackendEnvironments } from "./voiceBackends"; + +const laptop = EnvironmentId.make("laptop"); +const server = EnvironmentId.make("server"); +const both = { voiceTranscription: true, cloudVoiceTranscription: true }; +const environments = [ + { environmentId: server, connected: true, capabilities: both }, + { environmentId: laptop, connected: true, capabilities: { cloudVoiceTranscription: true } }, +]; + +it("a backend only runs where an environment supports it", () => { + expect(voiceBackendEnvironments("local", environments).map((e) => e.environmentId)).toEqual([ + server, + ]); + expect(voiceBackendEnvironments("assemblyai", environments)).toHaveLength(2); +}); + +it("picking a backend keeps the current environment, else prefers this device's own", () => { + expect( + defaultVoiceEnvironment("assemblyai", environments, { current: server, primary: laptop }), + ).toBe(server); + expect( + defaultVoiceEnvironment("assemblyai", environments, { current: null, primary: laptop }), + ).toBe(laptop); + // The current environment can't run a local model, so the backend moves to one that can. + expect(defaultVoiceEnvironment("local", environments, { current: laptop, primary: laptop })).toBe( + server, + ); +}); + +it("an offline primary environment yields to a connected one", () => { + const offline = environments.map((environment) => + environment.environmentId === laptop ? { ...environment, connected: false } : environment, + ); + expect(defaultVoiceEnvironment("openai", offline, { current: null, primary: laptop })).toBe( + server, + ); + expect(defaultVoiceEnvironment("openai", [], { current: null, primary: laptop })).toBeNull(); +}); diff --git a/apps/web/src/speech/voiceBackends.ts b/apps/web/src/speech/voiceBackends.ts new file mode 100644 index 000000000000..ea848a3553b5 --- /dev/null +++ b/apps/web/src/speech/voiceBackends.ts @@ -0,0 +1,58 @@ +import type { CloudSpeechProvider, EnvironmentId } from "@t3tools/contracts"; + +/** What turns speech into text. The environment that runs it is a separate, defaulted choice. */ +export type VoiceBackend = CloudSpeechProvider | "local"; + +/** Menu order. This fork recommends AssemblyAI; its key comes from the host's vault. */ +export const VOICE_BACKENDS: ReadonlyArray = [ + "assemblyai", + "openai", + "xai", + "openrouter", + "local", +]; +export const RECOMMENDED_VOICE_BACKEND: VoiceBackend = "assemblyai"; + +export interface VoiceEnvironmentCandidate { + readonly environmentId: EnvironmentId; + readonly connected: boolean; + readonly capabilities: + | { readonly voiceTranscription?: boolean; readonly cloudVoiceTranscription?: boolean } + | undefined; +} + +export function voiceBackendEnvironments( + backend: VoiceBackend, + environments: ReadonlyArray, +): E[] { + return environments.filter( + (environment) => + environment.capabilities?.[ + backend === "local" ? "voiceTranscription" : "cloudVoiceTranscription" + ] === true, + ); +} + +/** + * The environment a backend runs on unless the user overrides it: the current one, then this + * device's own environment, then the first connected one. + */ +export function defaultVoiceEnvironment( + backend: VoiceBackend, + environments: ReadonlyArray, + preferred: { readonly current: EnvironmentId | null; readonly primary: EnvironmentId | null }, +): EnvironmentId | null { + const candidates = voiceBackendEnvironments(backend, environments); + const pick = (id: EnvironmentId | null) => + id === null ? undefined : candidates.find((environment) => environment.environmentId === id); + const primary = pick(preferred.primary); + return ( + ( + pick(preferred.current) ?? + (primary?.connected ? primary : undefined) ?? + candidates.find((environment) => environment.connected) ?? + primary ?? + candidates[0] + )?.environmentId ?? null + ); +} diff --git a/docs/user/faq.md b/docs/user/faq.md index 5c5b9a8d8368..04e4521b48ea 100644 --- a/docs/user/faq.md +++ b/docs/user/faq.md @@ -77,7 +77,7 @@ Open **Settings → Worktrees → Cleanup** and turn on automatic worktree clean ## Is there voice input? -Yes. On iPhone and Android, dictation uses on-device speech by default; you can opt in to a paired environment instead. Web and desktop require you to choose an environment first. The transcript goes into your draft for review. See [Voice input](./voice-input.md). +Yes. On iPhone and Android, dictation uses on-device speech by default; you can opt in to a paired environment instead. Web and desktop require you to choose a transcription backend first. The transcript goes into your draft for review. See [Voice input](./voice-input.md). ## Does it work on Windows, Mac, and Linux? diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md index 71989004169e..8270b6a80215 100644 --- a/docs/user/voice-input.md +++ b/docs/user/voice-input.md @@ -12,20 +12,26 @@ to a named environment on that phone, open **Settings → Voice → Transcribe o ## Choose where audio goes -On web and desktop, you choose where transcription runs. -Nothing is sent until you choose one: open **Settings → Voice** and set -**Transcribe with** to the environment that should transcribe for you. -On a new device, the microphone or dictation shortcut opens that setting from the composer. +On web and desktop, you choose what transcribes your speech. +Nothing is sent until you choose: open **Settings → Voice** and set +**Transcribe with** to a backend: **AssemblyAI**, **OpenAI**, **Grok Voice Transcribe**, +**OpenRouter**, or **Local model** for a speech model you download. +On a new device, the microphone or dictation shortcut opens that setting from the composer +and offers **AssemblyAI** straight away. +The backend runs on your own environment unless you pick another under +**Advanced → Transcription environment**, which appears when more than one environment can run it. Microphone audio recorded on this device is sent to that environment over your authenticated connection, transcribed there, and discarded. The choice is saved on this device only, so every computer and browser you use asks for it once. Set it back to **Off** at any time; a recording in progress is discarded. If the chosen environment goes offline, voice input waits for it. T3 Code never -substitutes another environment. +substitutes another environment. If you pick a backend while its environment is offline, +choose **Open Environments** to reconnect it. -For cloud dictation, choose **OpenAI** or **Grok Voice Transcribe** via the environment -you want to use. If it needs a key, enter one and choose **Save and enable**. +Cloud backends need an API key on the environment. If the environment was started +with one, such as `ASSEMBLYAI_API_KEY` loaded from your vault, just choose **Use AssemblyAI**; +there is nothing to type. Otherwise enter a key and choose **Save and enable**. You can also choose a [secret provider](secrets.md) instead of **Value** and enter a reference. Unlock the vault on that environment before dictating. Anyone with dictation access to the environment can use the key, but only this device opts in. @@ -33,7 +39,7 @@ If a key is already saved, just confirm the provider. After you stop, the environment forwards your recording and any vocabulary hints to the selected provider, which bills its API account. No local model download or chat subscription is needed. Switch back to **This device** on mobile or **Off** on -web and desktop to stop sharing. Removing the key stops that provider for everyone +web and desktop to stop sharing. Removing a saved key stops that provider for everyone on the environment. **OpenRouter** works the same way but has no default model: pick one from its live diff --git a/packages/contracts/src/speech.ts b/packages/contracts/src/speech.ts index ea9e331eae64..8c83eaba221c 100644 --- a/packages/contracts/src/speech.ts +++ b/packages/contracts/src/speech.ts @@ -28,6 +28,8 @@ export const CloudSpeechKeys = Schema.Struct({ xai: Schema.Boolean, openrouter: Schema.optionalKey(Schema.Boolean), assemblyai: Schema.optionalKey(Schema.Boolean), + /** Providers whose key comes from the host's launch environment, not settings. */ + hostManaged: Schema.optionalKey(Schema.Array(CloudSpeechProvider)), }); export type CloudSpeechKeys = typeof CloudSpeechKeys.Type; From 1de0437f378ab2f71ea971fc85178137e8a88f09 Mon Sep 17 00:00:00 2001 From: nullStack65 Date: Wed, 18 Nov 2026 22:41:41 +0000 Subject: [PATCH 2/2] fix(web): treat only a connected environment as live for voice defaults The voice panel compared connection.phase to "available", which means a known environment that is not connected, so the first-connected rule never picked a live environment. Move the mapping into voiceEnvironmentCandidates, compare against "connected", fix the browser fixture, and add a two-environment test. Model: claude-opus-5-5 Task: fix Agent: claude-code Harness: t3-claude Co-Authored-By: Claude Opus 5.5 --- .../CloudVoiceConfiguration.browser.test.tsx | 2 +- .../settings/VoiceSettingsPanel.tsx | 8 ++----- apps/web/src/speech/voiceBackends.test.ts | 18 +++++++++++++++- apps/web/src/speech/voiceBackends.ts | 21 +++++++++++++++++++ 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx index 955d5aee4ae7..aeb09849ebc2 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx @@ -22,7 +22,7 @@ vi.mock("../../state/environments", async (importOriginal) => ({ { environmentId: "voice-host", label: "Laptop", - connection: { phase: "available" }, + connection: { phase: "connected" }, serverConfig: { secretReferences: true, settings: { diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index cef32631364a..0530dfe35938 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -74,6 +74,7 @@ import { } from "../../speech/voiceConsent"; import { defaultVoiceEnvironment, + voiceEnvironmentCandidates, RECOMMENDED_VOICE_BACKEND, VOICE_BACKENDS, voiceBackendEnvironments, @@ -1234,12 +1235,7 @@ export function VoiceSettingsPanel() { const [offerRecommended, setOfferRecommended] = useState( () => hash === "voice-transcription-destination", ); - const candidates = environments.map((environment) => ({ - environment, - environmentId: environment.environmentId, - connected: environment.connection.phase === "available", - capabilities: environment.serverConfig?.environment.capabilities, - })); + const candidates = voiceEnvironmentCandidates(environments); const defaultEnvironment = (backend: VoiceBackend) => defaultVoiceEnvironment(backend, candidates, { current: destination.environmentId, diff --git a/apps/web/src/speech/voiceBackends.test.ts b/apps/web/src/speech/voiceBackends.test.ts index fe2446ec98b5..bac9e3ebbcd3 100644 --- a/apps/web/src/speech/voiceBackends.test.ts +++ b/apps/web/src/speech/voiceBackends.test.ts @@ -1,6 +1,10 @@ import { EnvironmentId } from "@t3tools/contracts"; import { expect, it } from "vite-plus/test"; -import { defaultVoiceEnvironment, voiceBackendEnvironments } from "./voiceBackends"; +import { + defaultVoiceEnvironment, + voiceBackendEnvironments, + voiceEnvironmentCandidates, +} from "./voiceBackends"; const laptop = EnvironmentId.make("laptop"); const server = EnvironmentId.make("server"); @@ -39,3 +43,15 @@ it("an offline primary environment yields to a connected one", () => { ); expect(defaultVoiceEnvironment("openai", [], { current: null, primary: laptop })).toBeNull(); }); + +it("only a connected phase counts as live, so a known but unconnected primary yields", () => { + const cloud = { environment: { capabilities: { cloudVoiceTranscription: true } } }; + const candidates = voiceEnvironmentCandidates([ + { environmentId: laptop, connection: { phase: "available" }, serverConfig: cloud }, + { environmentId: server, connection: { phase: "connected" }, serverConfig: cloud }, + ]); + expect(candidates.map((candidate) => candidate.connected)).toEqual([false, true]); + expect(defaultVoiceEnvironment("assemblyai", candidates, { current: null, primary: laptop })).toBe( + server, + ); +}); diff --git a/apps/web/src/speech/voiceBackends.ts b/apps/web/src/speech/voiceBackends.ts index ea848a3553b5..06149225aa06 100644 --- a/apps/web/src/speech/voiceBackends.ts +++ b/apps/web/src/speech/voiceBackends.ts @@ -21,6 +21,27 @@ export interface VoiceEnvironmentCandidate { | undefined; } +/** An environment as the voice settings see it; only a `connected` phase means it is live. */ +export interface VoiceEnvironmentSource { + readonly environmentId: EnvironmentId; + readonly connection: { readonly phase: string }; + readonly serverConfig?: + | { readonly environment: { readonly capabilities?: VoiceEnvironmentCandidate["capabilities"] } } + | null + | undefined; +} + +export function voiceEnvironmentCandidates( + environments: ReadonlyArray, +) { + return environments.map((environment) => ({ + environment, + environmentId: environment.environmentId, + connected: environment.connection.phase === "connected", + capabilities: environment.serverConfig?.environment.capabilities, + })); +} + export function voiceBackendEnvironments( backend: VoiceBackend, environments: ReadonlyArray,