From 9376334df5e925a72a73babccb16053d6e0a306e Mon Sep 17 00:00:00 2001 From: Rodrigo Date: Fri, 24 Jul 2026 05:24:12 +0300 Subject: [PATCH 01/12] Add live transcription --- .env.example | 4 +- AGENTS.md | 6 + ARCHITECTURE.md | 18 ++- README.md | 3 +- agent/channels/transcription.ts | 25 +++ app/public/pcm-processor.js | 16 ++ bun.lock | 2 + components/composer/audio-controls.tsx | 153 ++++++++++++++++++ components/{session => composer}/composer.tsx | 90 +++++++---- components/composer/waveform.tsx | 90 +++++++++++ components/session/session-start.tsx | 2 +- components/session/session-view.tsx | 2 +- lib/audio.ts | 69 ++++++++ lib/transcription.ts | 60 +++++++ package.json | 2 + 15 files changed, 500 insertions(+), 42 deletions(-) create mode 100644 agent/channels/transcription.ts create mode 100644 app/public/pcm-processor.js create mode 100644 components/composer/audio-controls.tsx rename components/{session => composer}/composer.tsx (51%) create mode 100644 components/composer/waveform.tsx create mode 100644 lib/audio.ts create mode 100644 lib/transcription.ts diff --git a/.env.example b/.env.example index f8839e4..fb136f9 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,5 @@ # Created by `convex dev`; used by the browser and the eve persistence hook. VITE_CONVEX_URL=https://your-deployment.convex.cloud -# Used by eve's default Vercel AI Gateway provider. -AI_GATEWAY_API_KEY=your-ai-gateway-key +# Used only to mint short-lived live transcription tokens. +TRANSCRIPTION_AI_GATEWAY_API_KEY=your-ai-gateway-key diff --git a/AGENTS.md b/AGENTS.md index 598ae23..0731b9a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,6 +22,12 @@ of the product. - Keep functions and components small, linear, and responsible for one thing. If a unit must understand unrelated or partially defined data, fix the boundary or data model. +- Compose sibling capabilities in their nearest common parent. A component owns only + the behavior implied by its name; do not move unrelated actions into it to hide + coordination. An optional feature must be removable by deleting its import and + composition node without breaking sibling capabilities. +- Express UI variants with focused components and early returns. Do not accumulate + JSX in mutable variables or turn one component into a dispatcher for unrelated UI. - Keep one source of truth and derive the rest. Model state with one explicit status, not overlapping booleans or synchronization effects. - Keep logic above JSX. Avoid ternaries and boolean chains in markup. Use diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6a3273d..91076b8 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -37,6 +37,9 @@ Eve service ────────────────────── Convex (sessions, turns) ──▶ subscribed browser cache Browser ── workspace and preview HTTP ──▶ Eve channels ──▶ same sandbox + +Browser ── short-lived transcription token ──▶ Eve channel ──▶ AI Gateway + └────────────────── live PCM over WebSocket ─────────────▶ ``` The HTTP channels are narrow product adapters. Eve still creates the session and @@ -48,7 +51,7 @@ sandbox, runs the agent, serializes turns, and owns the durable event stream. session-bound sandbox. - **Convex** owns the durable product index and compact completed-turn checkpoints. - **The browser runtime** owns only in-flight events, optimistic input, view state, - and caches derived from Eve or Convex. + microphone capture, live transcripts, and caches derived from Eve or Convex. - **Eve channels** expose narrow browser-to-sandbox operations that Eve does not provide directly. They do not become a second application backend. - **Git metadata** describes the repository currently found in a session workspace. @@ -153,13 +156,14 @@ tracked in [docs/eve-improvements.md](./docs/eve-improvements.md). generated directories, caps the tree at 10,000 paths, and refuses binary or text files over 200 KiB. -The current workspace routes are: +The current browser-facing product channel routes are: ```text GET /eve/v1/workspace/:sessionId GET /eve/v1/workspace/:sessionId/file?path=... GET /eve/v1/workspace/:sessionId/command GET /eve/v1/workspace/:sessionId/download +POST /eve/v1/transcription ``` Here `sessionId` is Eve's durable session ID, not the app's public session ID. @@ -174,6 +178,10 @@ Here `sessionId` is Eve's durable session ID, not the app's public session ID. Its header shows the selected GitHub repository when the workspace has one. The read-only workspace contains breadcrumbs, a keyboard-accessible tree, and a highlighted source viewer. File tool activity can open the corresponding file. +- **Composer** captures microphone PCM through a browser-only adapter with no AI + dependency. A separate adapter streams it to AI Gateway with a short-lived token; + its server route uses a dedicated Gateway key so Eve remains on OIDC. Audio is + never recorded or persisted. - **Activity** projects Eve events into reasoning, tool calls, live Bash output, file diffs, and elapsed time. - **Session management** includes responsive sidebar navigation, rename, and delete. @@ -223,6 +231,7 @@ convex/ schema, session operations, and checkpoint persistence lib/ lowest-level non-component modules and runtime/vendor adapters components/ ui/ generic visual primitives + composer/ message input, voice controls, waveform, and transcription lifecycle code/ Pierre-backed source and diff facades session/ conversation, activity, navigation, and preview control workspace/ file navigation, tree, queries, and panel @@ -242,6 +251,10 @@ Layer rules: receive one stable shape instead of repeating defensive parsing. - Feature components may import `ui/`, `lib/`, generated Convex APIs, and sibling or lower feature facades. `app/` wires them together; nothing imports from `app/`. +- Parents compose sibling capabilities and own only the coordination between them. + Feature components own the behavior named by their boundary and never absorb + unrelated sibling actions. Removing an optional feature at its composition site + must leave unrelated workflows intact. - Vendor renderers stay behind `components/code/` or the workspace feature boundary. Consumers do not depend on Pierre directly. - There are no barrel files. Modules export only what a real consumer uses. @@ -253,6 +266,7 @@ Layer rules: The runtime remains intentionally short: - `eve`, `@vercel/sandbox`, and `@vercel/oidc` for the agent and sandbox +- `ai` and `@ai-sdk/gateway` for live transcription and short-lived browser tokens - `convex`, `@convex-dev/react-query`, and TanStack Query for durable reactive data - React 19, React Router, and Zustand for the browser runtime - Tailwind 4, `@shadcn/react`, Lucide, and Streamdown for the interface diff --git a/README.md b/README.md index ba3f5fb..a35922c 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,8 @@ bunx vercel env pull .env.local bun run dev ``` -Set `AI_GATEWAY_API_KEY` in `.env.local` only when Vercel OIDC is unavailable. +Set `TRANSCRIPTION_AI_GATEWAY_API_KEY` in `.env.local` to enable voice input. +Eve continues to use Vercel OIDC, independently from this key. Open [http://localhost:5173](http://localhost:5173). diff --git a/agent/channels/transcription.ts b/agent/channels/transcription.ts new file mode 100644 index 0000000..9194b0b --- /dev/null +++ b/agent/channels/transcription.ts @@ -0,0 +1,25 @@ +import { createGateway } from "@ai-sdk/gateway"; +import { defineChannel, POST } from "eve/channels"; + +const model = "openai/gpt-realtime-whisper"; +const headers = { "Cache-Control": "no-store" }; +const apiKey = process.env.TRANSCRIPTION_AI_GATEWAY_API_KEY; +const transcription = apiKey ? createGateway({ apiKey }).experimental_transcription : undefined; + +export default defineChannel({ + routes: [ + POST("/eve/v1/transcription", async () => { + if (!transcription) { + return Response.json({ error: "Voice input is unavailable." }, { headers, status: 503 }); + } + + try { + const { token } = await transcription.getToken({ model }); + return Response.json({ model, token }, { headers }); + } catch (error) { + console.error("Could not create transcription token", error); + return Response.json({ error: "Voice input is unavailable." }, { headers, status: 503 }); + } + }), + ], +}); diff --git a/app/public/pcm-processor.js b/app/public/pcm-processor.js new file mode 100644 index 0000000..a9ff674 --- /dev/null +++ b/app/public/pcm-processor.js @@ -0,0 +1,16 @@ +class PcmProcessor extends AudioWorkletProcessor { + process(inputs) { + const input = inputs[0]?.[0]; + if (!input) return true; + + const pcm = new Int16Array(input.length); + for (let index = 0; index < input.length; index += 1) { + const sample = Math.max(-1, Math.min(1, input[index])); + pcm[index] = sample < 0 ? sample * 0x8000 : sample * 0x7fff; + } + this.port.postMessage(pcm.buffer, [pcm.buffer]); + return true; + } +} + +registerProcessor("pcm-processor", PcmProcessor); diff --git a/bun.lock b/bun.lock index 1cf1bef..9c4f8da 100644 --- a/bun.lock +++ b/bun.lock @@ -5,6 +5,7 @@ "": { "name": "eve-code", "dependencies": { + "@ai-sdk/gateway": "^4.0.0", "@convex-dev/react-query": "^0.1.0", "@fontsource-variable/geist": "^5.2.9", "@pierre/diffs": "^1.2.12", @@ -14,6 +15,7 @@ "@tanstack/react-query": "^5.101.2", "@vercel/oidc": "^3.8.0", "@vercel/sandbox": "^2.3.0", + "ai": "^7.0.0", "convex": "^1.42.2", "diff": "^9.0.0", "eve": "^0.24.6", diff --git a/components/composer/audio-controls.tsx b/components/composer/audio-controls.tsx new file mode 100644 index 0000000..4092e05 --- /dev/null +++ b/components/composer/audio-controls.tsx @@ -0,0 +1,153 @@ +import { Mic, Square } from "lucide-react"; +import { useEffect, useRef, useState } from "react"; + +import { Waveform } from "@/components/composer/waveform"; +import { Button } from "@/components/ui/button"; +import { startTranscription, type Transcription } from "@/lib/transcription"; + +type AudioControlsProps = { + readonly disabled: boolean; + readonly onChange: (value: string) => void; + readonly value: string; +}; + +type Recording = Transcription | "busy" | "error" | undefined; + +function message(prefix: string, transcript: string): string { + return [prefix, transcript].filter(Boolean).join(" ").trim(); +} + +function RecordingWaveform({ recording }: { readonly recording: Recording }) { + if (typeof recording !== "object") return null; + return ; +} + +function RecordButton({ + disabled, + onStart, + onStop, + recording, +}: { + readonly disabled: boolean; + readonly onStart: () => void; + readonly onStop: () => void; + readonly recording: Recording; +}) { + if (typeof recording === "object") { + return ( + + ); + } + + const isBusy = recording === "busy"; + const isUnavailable = recording === "error"; + const label = isUnavailable ? "Retry voice input" : "Start voice input"; + const title = isUnavailable ? "Voice input is unavailable." : undefined; + const buttonDisabled = disabled || isBusy; + return ( + + ); +} + +export function AudioControls({ disabled, onChange, value }: AudioControlsProps) { + const [recording, setRecording] = useState(); + const controller = useRef(undefined); + const prefix = useRef(""); + const transcript = useRef(""); + + useEffect( + () => () => { + controller.current?.abort(); + controller.current = undefined; + }, + [], + ); + + useEffect(() => { + if (!disabled || !controller.current) return; + const activeController = controller.current; + controller.current = undefined; + activeController.abort(); + setRecording(undefined); + }, [disabled]); + + function fail(activeController: AbortController): void { + if (controller.current !== activeController) return; + activeController.abort(); + controller.current = undefined; + setRecording("error"); + } + + async function start(): Promise { + if (controller.current) return; + const activeController = new AbortController(); + controller.current = activeController; + prefix.current = value.trim(); + transcript.current = ""; + setRecording("busy"); + + try { + const live = await startTranscription((delta) => { + transcript.current += delta; + onChange(message(prefix.current, transcript.current)); + }, activeController.signal); + activeController.signal.throwIfAborted(); + setRecording(live); + void live.text.catch(() => fail(activeController)); + } catch { + fail(activeController); + } + } + + async function stop(): Promise { + if (typeof recording !== "object") return; + const live = recording; + const activeController = controller.current; + if (!activeController) return; + setRecording("busy"); + + try { + live.stop(); + const finalTranscript = (await live.text).trim() || transcript.current; + const nextValue = message(prefix.current, finalTranscript); + if (controller.current !== activeController) return; + activeController.abort(); + controller.current = undefined; + setRecording(undefined); + onChange(nextValue); + } catch { + fail(activeController); + } + } + + return ( +
+ + void start()} + onStop={() => void stop()} + recording={recording} + /> +
+ ); +} diff --git a/components/session/composer.tsx b/components/composer/composer.tsx similarity index 51% rename from components/session/composer.tsx rename to components/composer/composer.tsx index f1183eb..9961fca 100644 --- a/components/session/composer.tsx +++ b/components/composer/composer.tsx @@ -1,6 +1,7 @@ import { ArrowUp, Square } from "lucide-react"; import { type FormEvent, type KeyboardEvent, useEffect, useRef } from "react"; +import { AudioControls } from "@/components/composer/audio-controls"; import { Button } from "@/components/ui/button"; import { useComposerStore } from "@/lib/composer-store"; @@ -11,14 +12,57 @@ type ComposerProps = { readonly onStop?: () => void; }; +function SubmitButton({ + disabled, + isGenerating, + onStop, +}: { + readonly disabled: boolean; + readonly isGenerating: boolean; + readonly onStop?: () => void; +}) { + if (isGenerating) { + return ( + + ); + } + + return ( + + ); +} + +function handleKeyDown(event: KeyboardEvent): void { + if (event.key !== "Enter" || event.shiftKey || event.nativeEvent.isComposing) return; + + event.preventDefault(); + event.currentTarget.form?.requestSubmit(); +} + export function Composer({ disabled = false, isGenerating = false, onSend, onStop, }: ComposerProps) { - const draft = useComposerStore((state) => state.draft); - const setDraft = useComposerStore((state) => state.setDraft); + const value = useComposerStore((state) => state.draft); + const onChange = useComposerStore((state) => state.setDraft); const textareaRef = useRef(null); useEffect(() => { @@ -27,19 +71,14 @@ export function Composer({ function handleSubmit(event: FormEvent): void { event.preventDefault(); - const message = draft.trim(); - if (!message) return; - - setDraft(""); + const message = value.trim(); + if (!message || disabled || isGenerating) return; + onChange(""); onSend(message); } - function handleKeyDown(event: KeyboardEvent): void { - if (event.key !== "Enter" || event.shiftKey || event.nativeEvent.isComposing) return; - - event.preventDefault(); - event.currentTarget.form?.requestSubmit(); - } + const audioDisabled = disabled || isGenerating; + const submitDisabled = disabled || !value.trim(); return (
@@ -55,35 +94,16 @@ export function Composer({ className="max-h-48 min-h-16 w-full resize-none overflow-y-auto bg-transparent px-2 py-1 outline-none [field-sizing:content] placeholder:text-muted-foreground disabled:cursor-not-allowed disabled:opacity-50" disabled={disabled} id="message-input" - onChange={(event) => setDraft(event.target.value)} + onChange={(event) => onChange(event.target.value)} onKeyDown={handleKeyDown} placeholder="Message eve-code" ref={textareaRef} rows={1} - value={draft} + value={value} />
- {isGenerating && onStop && ( - - )} - {!isGenerating && ( - - )} + +
diff --git a/components/composer/waveform.tsx b/components/composer/waveform.tsx new file mode 100644 index 0000000..2dadf72 --- /dev/null +++ b/components/composer/waveform.tsx @@ -0,0 +1,90 @@ +import { useEffect, useRef } from "react"; + +type WaveformProps = { + readonly stream: MediaStream; +}; + +export function Waveform({ stream }: WaveformProps) { + const canvasRef = useRef(null); + const timeRef = useRef(null); + + useEffect(() => { + const canvas = canvasRef.current; + const time = timeRef.current; + const drawing = canvas?.getContext("2d"); + if (!canvas || !time || !drawing) return; + + const element = canvas; + const clock = time; + const context = drawing; + const audio = new AudioContext(); + const analyser = audio.createAnalyser(); + const source = audio.createMediaStreamSource(stream); + const samples = new Float32Array(analyser.fftSize); + const levels: number[] = []; + const color = getComputedStyle(element).color; + let elapsed = -1; + let frame = 0; + + analyser.fftSize = 256; + source.connect(analyser); + void audio.resume().catch(() => undefined); + + function draw(): void { + const ratio = window.devicePixelRatio || 1; + const width = Math.max(1, Math.round(element.clientWidth * ratio)); + const height = Math.max(1, Math.round(element.clientHeight * ratio)); + if (element.width !== width || element.height !== height) { + element.width = width; + element.height = height; + } + + analyser.getFloatTimeDomainData(samples); + let level = 0; + for (const sample of samples) level = Math.max(level, Math.abs(sample)); + + const columns = Math.ceil(width / ratio); + levels.push(level); + levels.splice(0, Math.max(0, levels.length - columns)); + context.clearRect(0, 0, width, height); + context.fillStyle = color; + for (let index = 0; index < levels.length; index += 1) { + const barHeight = Math.max(1, (levels[index] ?? 0) * height * 0.9); + context.fillRect( + (columns - levels.length + index) * ratio, + (height - barHeight) / 2, + ratio, + barHeight, + ); + } + + const nextElapsed = Math.floor(audio.currentTime); + if (nextElapsed !== elapsed) { + elapsed = nextElapsed; + clock.textContent = `${Math.floor(elapsed / 60)}:${String(elapsed % 60).padStart(2, "0")}`; + } + frame = requestAnimationFrame(draw); + } + + frame = requestAnimationFrame(draw); + return () => { + cancelAnimationFrame(frame); + source.disconnect(); + void audio.close(); + }; + }, [stream]); + + return ( + <> + + + + ); +} diff --git a/components/session/session-start.tsx b/components/session/session-start.tsx index a4b9bd3..1c4579c 100644 --- a/components/session/session-start.tsx +++ b/components/session/session-start.tsx @@ -1,7 +1,7 @@ import { ArrowLeft, ArrowRight, FilePlus2, GitFork, type LucideIcon } from "lucide-react"; import { type FormEvent, type ReactNode, useRef, useState } from "react"; -import { Composer } from "@/components/session/composer"; +import { Composer } from "@/components/composer/composer"; import { Button } from "@/components/ui/button"; import { type GitRepository, parseGitHubRepository } from "@/lib/github"; diff --git a/components/session/session-view.tsx b/components/session/session-view.tsx index c672bfe..afd9e11 100644 --- a/components/session/session-view.tsx +++ b/components/session/session-view.tsx @@ -1,7 +1,7 @@ import { Activity, lazy, type ReactNode, Suspense, useCallback, useState } from "react"; +import { Composer } from "@/components/composer/composer"; import { CommandLogsProvider } from "@/components/session/command-logs"; -import { Composer } from "@/components/session/composer"; import { Conversation } from "@/components/session/conversation"; import { PageHeader } from "@/components/session/page-header"; import { type StoredSession, useSession } from "@/components/session/use-session"; diff --git a/lib/audio.ts b/lib/audio.ts new file mode 100644 index 0000000..3702854 --- /dev/null +++ b/lib/audio.ts @@ -0,0 +1,69 @@ +const sampleRate = 24_000; + +export type MicrophonePCMStream = { + readonly audioStream: ReadableStream; + readonly mediaStream: MediaStream; + readonly sampleRate: number; + readonly stop: () => Promise; +}; + +export async function createMicrophonePCMStream(): Promise { + const mediaStream = await navigator.mediaDevices.getUserMedia({ + audio: { + autoGainControl: true, + channelCount: 1, + echoCancellation: true, + noiseSuppression: true, + }, + }); + const context = new AudioContext({ sampleRate }); + + try { + await context.audioWorklet.addModule("/pcm-processor.js"); + } catch (error) { + for (const track of mediaStream.getTracks()) track.stop(); + void context.close(); + throw error; + } + + const source = context.createMediaStreamSource(mediaStream); + const processor = new AudioWorkletNode(context, "pcm-processor"); + const silent = context.createGain(); + silent.gain.value = 0; + source.connect(processor).connect(silent).connect(context.destination); + + let controller!: ReadableStreamDefaultController; + let closed = false; + + async function cleanup(): Promise { + if (closed) return; + closed = true; + processor.port.onmessage = null; + source.disconnect(); + processor.disconnect(); + silent.disconnect(); + for (const track of mediaStream.getTracks()) track.stop(); + await context.close(); + } + + const audioStream = new ReadableStream({ + cancel: cleanup, + start(nextController) { + controller = nextController; + processor.port.onmessage = ({ data }: MessageEvent) => { + if (!closed) controller.enqueue(new Uint8Array(data)); + }; + }, + }); + + await context.resume(); + return { + audioStream, + mediaStream, + sampleRate: context.sampleRate, + async stop() { + if (!closed) controller.close(); + await cleanup(); + }, + }; +} diff --git a/lib/transcription.ts b/lib/transcription.ts new file mode 100644 index 0000000..2e2f987 --- /dev/null +++ b/lib/transcription.ts @@ -0,0 +1,60 @@ +import { createGateway, experimental_streamTranscribe as streamTranscribe } from "ai"; + +import { createMicrophonePCMStream } from "@/lib/audio"; + +export type Transcription = { + readonly stop: () => void; + readonly stream: MediaStream; + readonly text: Promise; +}; + +type Token = { + readonly model: string; + readonly token: string; +}; + +async function requestToken(signal: AbortSignal): Promise { + const response = await fetch("/eve/v1/transcription", { method: "POST", signal }); + const body = (await response.json().catch(() => ({}))) as Partial; + if (!response.ok || typeof body.model !== "string" || typeof body.token !== "string") { + throw new Error("Voice input is unavailable."); + } + return { model: body.model, token: body.token }; +} + +export async function startTranscription( + onDelta: (delta: string) => void, + signal: AbortSignal, +): Promise { + const { model, token } = await requestToken(signal); + const microphone = await createMicrophonePCMStream(); + if (signal.aborted) { + await microphone.stop(); + signal.throwIfAborted(); + } + + const stop = () => void microphone.stop(); + signal.addEventListener("abort", stop, { once: true }); + + const result = streamTranscribe({ + abortSignal: signal, + audio: microphone.audioStream, + inputAudioFormat: { rate: microphone.sampleRate, type: "audio/pcm" }, + model: createGateway({ apiKey: token }).transcription(model), + }); + + const text = (async () => { + try { + for await (const part of result.fullStream) { + if (part.type === "transcript-delta") onDelta(part.delta); + if (part.type === "error") throw part.error; + } + return result.text; + } finally { + signal.removeEventListener("abort", stop); + await microphone.stop(); + } + })(); + + return { stop, stream: microphone.mediaStream, text }; +} diff --git a/package.json b/package.json index 80a96ee..3ccc553 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "test:watch": "vitest" }, "dependencies": { + "@ai-sdk/gateway": "^4.0.0", "@convex-dev/react-query": "^0.1.0", "@fontsource-variable/geist": "^5.2.9", "@pierre/diffs": "^1.2.12", @@ -26,6 +27,7 @@ "@tanstack/react-query": "^5.101.2", "@vercel/oidc": "^3.8.0", "@vercel/sandbox": "^2.3.0", + "ai": "^7.0.0", "convex": "^1.42.2", "diff": "^9.0.0", "eve": "^0.24.6", From 8a4cfe1cfe6dbf333a8c24d74e123a2dab3c9556 Mon Sep 17 00:00:00 2001 From: Rodrigo Date: Fri, 24 Jul 2026 05:34:37 +0300 Subject: [PATCH 02/12] Speed up voice input --- components/composer/audio-controls.tsx | 2 +- lib/transcription.ts | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/components/composer/audio-controls.tsx b/components/composer/audio-controls.tsx index 4092e05..a608459 100644 --- a/components/composer/audio-controls.tsx +++ b/components/composer/audio-controls.tsx @@ -63,7 +63,7 @@ function RecordButton({ title={title} variant="ghost" > -