From 922efe641c17dd2927eaa625b88de30f77fc4773 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:48:51 +0100 Subject: [PATCH 1/4] feat(asr): mark the mic's copy of the call as echo, live and in the final pass Without headphones the mic hears the remote side, so every remote sentence also became a line credited to the user. The fold already hides a line marked echo, but nothing wrote the mark. One narrow rule now marks it on both paths: at least 3 words, the call line first, the mic line within 1.5 s of its end, and at least 70 % of its words in the call line. The line stays in the log. --- docs/DESIGN.md | 2 +- docs/TESTING.md | 2 +- docs/TRAPS.md | 1 + src/core/log/echo.ts | 46 ++++++++++++++++++++++++++ src/main/asr/finalize-worker.ts | 6 ++++ src/main/asr/live-worker.ts | 31 ++++++++++++++++++ tests/asr-final.test.ts | 30 +++++++++++++++++ tests/asr-live.test.ts | 49 +++++++++++++++++++++++++++- tests/echo.test.ts | 57 +++++++++++++++++++++++++++++++++ 9 files changed, 221 insertions(+), 3 deletions(-) create mode 100644 src/core/log/echo.ts create mode 100644 tests/echo.test.ts diff --git a/docs/DESIGN.md b/docs/DESIGN.md index d818ba10..87fb81f3 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -255,7 +255,7 @@ A call's streaming model loads in the live Worker when the call starts, and its 4. **Pad** every span to at least 0.5 s before decoding, with the zeros added **after** the speech, in one function shared by every path (live, final, single file). Short words vanished under a 0.3 s engine floor in hark; that floor has not been measured on sherpa-onnx with Parakeet yet. The padding evidence from the streaming zipformer is mixed and does not transfer: a fresh stream of that model dropped the first words *until* it had 2 to 8 s of leading audio, so on that model leading audio restores words rather than losing them. The 0.2 s "Yes." fixture in [TRAPS.md](TRAPS.md) runs against Parakeet, with leading and trailing padding as separate conditions, before this rule counts as measured. 5. **Decode** with the offline recognizer, greedy by default. With `asr.parakeet.decoding` `beam`, decoding uses `modified_beam_search` and the call's current decode list is passed to `createStream(list)` when the model is a transducer. A word added mid-call is in the list from the next segment. Emit a `seg` to main, which assigns `seq`, computes wall time and appends. 6. **Provisional line.** While a segment is open, re-decode it every second (bounded by the 12 s window). It is published with a 3 s expiry and is never written to the log. On x64 the re-decode waits for new speech (`provisionalInPause` is off there): one that falls in the pause that closes the segment would hold the decoder while the committed line waits for it, and on four cores that cost 0.4 s a line. -7. **Echo.** Without headphones the mic hears the call. A mic segment whose text overlaps a call segment within ±1.5 s by at least 70 % of tokens is marked `echo: true` (kept in the log, hidden from views and packs). Real echo cancellation on the recognizer's copy of the mic (`webrtc-audio-processing`, BSD-3) is an M2 option. +7. **Echo.** Without headphones the mic hears the call. A mic segment whose text overlaps a call segment within ±1.5 s by at least 70 % of tokens is marked `echo: true` (kept in the log, hidden from views and packs). So that a read-back of a number or a name stays the user's, the mic segment needs at least 3 tokens and the call segment must start first; the rule is one function (`src/core/log/echo.ts`) used live, whichever channel commits a line, and by the final pass. Real echo cancellation on the recognizer's copy of the mic (`webrtc-audio-processing`, BSD-3) is an M2 option. Targets, measured in M0: committed line within 1.5 s of the end of an utterance on Apple silicon and within 2.5 s on 4-core x64, for lines after the first 10 s of a cold call (ROADMAP G6); first provisional words within 1.5 s of speech; both channels together under 25 % of real time on Apple Silicon and under 50 % on a 4-core x64 laptop. diff --git a/docs/TESTING.md b/docs/TESTING.md index 259e7846..beec4a3b 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -67,7 +67,7 @@ The suites that exist, one line each, so a reader knows where a new test goes. - **Log and fold** (`events`, `log-io`, `fold`, `clock`): append-only writes, torn last lines, the one sort order, revisions win, wall-clock rendering. The fold is the only reader of raw events, so its tests cover every view. - **Call machine** (`call-machine`, `capture-*`): start answers 201 only once audio is written, bounded teardown, restarts in the same folder, crash reconciliation, health verdicts. -- **Speech** (`asr-live`, `asr-final`, `asr-speakers`, `asr-models`, `first-run`): padding of short spans, the whole-timeline rule of the final pass, echo marking, cluster naming, checksummed resumable downloads. +- **Speech** (`asr-live`, `asr-final`, `echo`, `asr-speakers`, `asr-models`, `first-run`): padding of short spans, the whole-timeline rule of the final pass, echo marking (the rule's bounds in `echo`, its writers on the live path and in the final pass), cluster naming, checksummed resumable downloads. - **Query** (`query`, `query-traps`, `bench-pack`, `eval/replay`): packs stay inside the budget, recency and BM25 find the answer, citations are wall-clock, pack p95 under 50 ms on a synthetic 3-hour call. - **Notes and providers** (`notes`, `memo`, `reenhance`, `llm-harness`, `llm-http`, `session-reuse`): citations are checked, the harness is spawned with no tools, usage-limit exits fall back to excerpts, sessions are reused. - **Vocabulary** (`vocab-*`, `bpe-vocab`): decode lists, read-time correction with raw text kept, approval before anything is learned. diff --git a/docs/TRAPS.md b/docs/TRAPS.md index 80e4de0a..b65386e0 100644 --- a/docs/TRAPS.md +++ b/docs/TRAPS.md @@ -50,6 +50,7 @@ Traps that belong to the user's own downstream pipelines (archive formats, word- - **A live line that starts inside the line before it** [LV-1] (M1). Given `asr.segmentPause` under 0.7 s on the Parakeet live path, a line closed `postRoll` (0.2 s) after its last speech, and the next one opened `preRoll` (0.5 s) before its first, so the two overlapped by 0.7 s minus the pause: 0.4 s at a 0.3 s pause, and the next line could start with the end of the last word of the line before. The next line must never start before the last closed line ended, on the same channel and part. Test: `tests/asr-live.test.ts` `[LV-1]`, a 0.3 s pause (the second line starts where the first ended) with the default pause as the control (its pre-roll is whole). - **A live decode that throws shifts the rest of the part** [LV-2] (M1). Given a one-off native error in a Parakeet live decode, the error left the channel's unfinished VAD window where it was before the audio message, so every later line on that channel was cut from audio one message too early, and the line was lost with only an app-log error. A decode that throws must lose only its own line, with a warning that names its span (the final pass decodes that audio again), and the channel's timeline must stay with its audio whatever a decode throws, a provisional re-decode included. Test: `tests/asr-live.test.ts` `[LV-2]`, a recognizer that throws once, on a closing decode and on a provisional one: the other lines have the times of a run with no error. - **A failing final pass that holds the start's catch-up** [FN-1] (M1). Given a final pass that fails by running out of its budget (Qwen on a machine with no GPU and a long call), the start's catch-up runs it again at every start, for its whole budget, and every call that ended later waited behind it. A call whose last pass failed must run after every call that has not had a pass, and must still run, so audio that becomes readable later still gets its final layer. Test: `tests/asr-final-catchup.e2e.test.ts` `[FN-1]`, a failed call that the start meets first and a call with no pass: the call with no pass starts first and both end `done`. +- **Mic hears the call** [ECHO-1] (M1). Given a call on laptop speakers, the mic hears the remote side, so each remote sentence became a `you` line as well as a call line: the user was credited with the other side's words in every view, pack, share and agent answer. The schema and the fold already hid a line marked `echo`, but nothing wrote the mark. A mic line must be marked `echo: true`, in a revision of its own on the live path (checked when either channel writes a line, since a call line can close after the mic line that heard it) and on revision 1 in the final pass, by one rule: at least 3 words, the call line started first, the mic line starts no later than 1.5 s after the call line ends, and at least 70 % of its words are in the call line. The rule is narrow so a read-back of a number or a name stays the user's. The line stays in the log; only views leave it out. Tests: `tests/echo.test.ts` `[ECHO-1]`, each bound with the case just inside and just outside it; `tests/asr-live.test.ts` and `tests/asr-final.test.ts` `[ECHO-1]`, the same speech on both channels (the mic line marked and out of the view, with either line closing first) and different speech as the positive control (no mark). ## Event log and clocks diff --git a/src/core/log/echo.ts b/src/core/log/echo.ts new file mode 100644 index 00000000..f63635d3 --- /dev/null +++ b/src/core/log/echo.ts @@ -0,0 +1,46 @@ +/** + * Echo marking (DESIGN 3.1 step 7): without headphones the mic hears the call, so a remote + * sentence comes back as a `you` line too. A mic line that repeats a call line is marked + * `echo: true`: kept in the log, hidden from views and packs. One rule for the live path and the + * final pass. + * + * The rule must not hide the user's own speech, so it is narrow: a person often reads back a + * number or a name, which is short and said after the other side finished. A mic line is echo only + * when it has at least `ECHO_MIN_TOKENS` words, the call line started first (or at the same time), + * the mic line starts no later than `ECHO_WINDOW_MS` after the call line ends, and at least + * `ECHO_MIN_SHARE` of the mic line's words are in the call line. + */ + +import { tokenize } from "../vocab/correct.ts"; + +/** How far after the call line ends a mic line may start and still be its echo. */ +export const ECHO_WINDOW_MS = 1500; +/** The share of the mic line's words that must be in the call line, in tenths: 7 is 70 %. */ +export const ECHO_MIN_SHARE_TENTHS = 7; +/** A shorter mic line is never echo: two words read back are the user's. */ +export const ECHO_MIN_TOKENS = 3; + +/** What the rule reads of a line: its wall times and its text (null when retracted). */ +export interface EchoLine { + w0: number; + w1: number; + text: string | null; +} + +/** Whether the mic line `mic` is the mic hearing the call line `call`. */ +export function isEcho(mic: EchoLine, call: EchoLine): boolean { + if (call.w0 > mic.w0) return false; + if (mic.w0 > call.w1 + ECHO_WINDOW_MS) return false; + const said = tokenize(mic.text ?? "").map((t) => t.folded); + if (said.length < ECHO_MIN_TOKENS) return false; + const pool = new Map(); + for (const t of tokenize(call.text ?? "")) pool.set(t.folded, (pool.get(t.folded) ?? 0) + 1); + let found = 0; + for (const w of said) { + const n = pool.get(w) ?? 0; + if (n === 0) continue; + found++; + pool.set(w, n - 1); + } + return found * 10 >= said.length * ECHO_MIN_SHARE_TENTHS; +} diff --git a/src/main/asr/finalize-worker.ts b/src/main/asr/finalize-worker.ts index d2e0e694..cc68de06 100644 --- a/src/main/asr/finalize-worker.ts +++ b/src/main/asr/finalize-worker.ts @@ -49,6 +49,7 @@ import { closeSync, openSync, readSync } from "node:fs"; import { join } from "node:path"; import { workerData } from "node:worker_threads"; +import { type EchoLine, isEcho } from "../../core/log/echo.ts"; import type { Channel, EventDraft, LogEvent } from "../../core/log/events.ts"; import { type CallView, fold } from "../../core/log/fold.ts"; import { readLog } from "../../core/log/reader.ts"; @@ -717,6 +718,11 @@ export async function runFinalPass( }; const closePart = (p: number, lines: Line[]) => { lines.sort((a, b) => (a.w0 as number) - (b.w0 as number) || (a.ch === "mic" ? -1 : 1)); + // Echo (DESIGN 3.1 step 7), by the live path's rule: a mic line that repeats a call line. + const calls = lines.filter((l) => l.ch === "call") as EchoLine[]; + for (const l of lines) { + if (l.ch === "mic" && calls.some((c) => isEcho(l as EchoLine, c))) l.echo = true; + } for (const l of lines) { layer.push({ ...l, id: `f${String(nextFinal).padStart(6, "0")}` } as EventDraft); nextFinal++; diff --git a/src/main/asr/live-worker.ts b/src/main/asr/live-worker.ts index 306d8c22..ade8e35c 100644 --- a/src/main/asr/live-worker.ts +++ b/src/main/asr/live-worker.ts @@ -61,6 +61,7 @@ */ import { workerData } from "node:worker_threads"; +import { isEcho } from "../../core/log/echo.ts"; import type { Channel, EventDraft, LogEvent } from "../../core/log/events.ts"; import type { CallView } from "../../core/log/fold.ts"; import { CHANNELS, type Clock, realClock, withDeadline } from "../capture/engine.ts"; @@ -1882,8 +1883,13 @@ interface HostCall { live?: LiveChoice | null; /** The call's second pass, asked once with the engine; null for none. */ upgrade?: HostUpgrade | null; + /** The ids of the lines written in the last `ECHO_RECENT_MS`, for the echo check. */ + recent: string[]; } +/** How far back a written line is kept to be compared with the other channel's next lines. */ +const ECHO_RECENT_MS = 10_000; + /** Worker answers that end a use of the models (`LiveAsr.releaseIdle`). */ const USE_ENDS: ReadonlySet = new Set([ "decoded", @@ -2430,6 +2436,7 @@ export class LiveAsr { parts: new Map(), lagLevel: 0, superseded: false, + recent: [], }; this.calls.set(callId, c); this.current = c; @@ -2718,6 +2725,30 @@ export class LiveAsr { c.access.view.provisional.commit(m.ch, w1); if (c.upgrade && m.key !== undefined) c.upgrade.keys.set(m.key, id); if (m.held && m.key !== undefined) c.held.set(m.key, id); + this.markEcho(c, id); + } + + /** + * Echo (DESIGN 3.1 step 7): a mic line that repeats a call line (`isEcho`) gets a revision + * carrying `echo: true` alone. Checked whichever channel just wrote, since a call line can close + * after the mic line that heard it. Never writes `echo: false`, and leaves a line a person + * touched as it is. + */ + private markEcho(c: HostCall, id: string): void { + const view = c.access.view; + const s = view.segment(id); + if (!s) return; + c.recent = c.recent.filter((r) => (view.segment(r)?.w1 ?? 0) >= s.w0 - ECHO_RECENT_MS); + for (const other of c.recent) { + const o = view.segment(other); + if (!o || o.ch === s.ch) continue; + const mic = s.ch === "mic" ? s : o; + const call = s.ch === "mic" ? o : s; + if (mic.echo || mic.revisions.some((r) => r.by !== undefined)) continue; + if (!isEcho(mic, call)) continue; + c.access.record({ type: "seg", id: mic.id, rev: mic.rev + 1, echo: true }); + } + c.recent.push(id); } /** diff --git a/tests/asr-final.test.ts b/tests/asr-final.test.ts index 889d05ec..c0621f64 100644 --- a/tests/asr-final.test.ts +++ b/tests/asr-final.test.ts @@ -172,6 +172,36 @@ describe("[T1.9, T4.19] Short spans lose their words: the final path pads too", }); }); +describe("[ECHO-1] Mic hears the call: the final pass marks it too", () => { + const remote = ["we", "should", "move", "the", "build", "to", "new", "box"]; + const part = (micWords: string[]) => ({ + 1: { + mic: concat(silence(1.1), speak(micWords), silence(2)), + call: concat(silence(1), speak(remote, { voice: 3 }), silence(2.1)), + }, + }); + + test("the mic's copy of a call line is written with echo and leaves the final view", async () => { + const r = await run(callLog([1]), part(remote)); + const byCh = [...r.segs].sort((a, b) => (a.ch as string).localeCompare(b.ch as string)); + expect(byCh.map((s) => [s.ch, s.text, s.echo])).toEqual([ + ["call", remote.join(" "), undefined], + ["mic", remote.join(" "), true], + ]); + const view = fold( + callLog([1], (b) => { + for (const s of r.segs) b.add(s as EventDraft); + }), + ); + expect(view.lines("final").map((l) => l.ch)).toEqual(["call"]); + }); + + test("positive control: the user's own words at the same time are not echo", async () => { + const r = await run(callLog([1]), part(["hello", "world", "thanks", "meeting", "today"])); + expect(r.segs.map((s) => s.echo)).toEqual([undefined, undefined]); + }); +}); + describe("speakers across the call", () => { test("diarization runs over all parts at once: one voice has one label in every part", async () => { const part = (v: number, w: string[]) => ({ diff --git a/tests/asr-live.test.ts b/tests/asr-live.test.ts index 2d50d58c..64a396ac 100644 --- a/tests/asr-live.test.ts +++ b/tests/asr-live.test.ts @@ -7,7 +7,7 @@ import { afterEach, describe, expect, test } from "bun:test"; import { join } from "node:path"; import type { Channel, LogEvent, Seg } from "../src/core/log/events.ts"; -import { PROVISIONAL_TTL_MS } from "../src/core/log/fold.ts"; +import { fold, PROVISIONAL_TTL_MS } from "../src/core/log/fold.ts"; import type { ModelSpec } from "../src/main/asr/engine.ts"; import type { LiveChoice } from "../src/main/asr/live-engines.ts"; import { @@ -763,6 +763,53 @@ describe("the host writes the live layer", () => { }); }); +describe("[ECHO-1] Mic hears the call", () => { + const remote = ["we", "should", "move", "the", "build", "to", "new", "box"]; + /** Plays `mic` and `call` from 1 s and 1.1 s, and returns the call's segs and folded view. */ + async function play(micWords: string[], callWords: string[]) { + const r = rig(); + await startCall(r); + const call = concat(silence(1), speak(callWords, { voice: 3 }), silence(3)); + const mic = concat(silence(1.1), speak(micWords), silence(3)); + r.engine.last.play(mic, call); + await r.mgr.stop(); + return { s: segs(r.events), view: fold(r.events) }; + } + + for (const [name, micWords] of [ + ["the mic line closes first", remote.slice(0, 4)], + ["the call line closes first", remote], + ] as const) { + test(`${name}: the mic line gets an echo revision, stays in the log and leaves the view`, async () => { + const { s, view } = await play([...micWords], remote); + const micLine = s.find((x) => x.ch === "mic" && x.rev === 1) as Seg; + const callLine = s.find((x) => x.ch === "call" && x.rev === 1) as Seg; + expect(micLine.text).toBe(micWords.join(" ")); + // The order the case is about: which line was written first. + expect(micLine.seq < callLine.seq).toBe(micWords.length < remote.length); + expect(s.filter((x) => x.echo !== undefined)).toMatchObject([ + { id: micLine.id, rev: 2, echo: true }, + ]); + // Only `echo` in the revision: the text, speaker and times stay revision 1's. + const rev2 = s.find((x) => x.echo !== undefined) as Seg; + expect(Object.keys(rev2).sort()).toEqual(["echo", "id", "rev", "seq", "t", "type"]); + expect(view.lines("live").map((l) => l.ch)).toEqual(["call"]); + const all = view.lines("live", { includeEcho: true }); + expect(all.map((l) => [l.ch, l.echo, l.text])).toContainEqual([ + "mic", + true, + micWords.join(" "), + ]); + }); + } + + test("positive control: the user's own words at the same time get no echo revision", async () => { + const { s, view } = await play(["hello", "world", "thanks", "meeting", "today"], remote); + expect(s.filter((x) => x.echo !== undefined)).toEqual([]); + expect(view.lines("live").map((l) => l.ch)).toEqual(["call", "mic"]); + }); +}); + describe("calls that overlap or outlive their end", () => { test("[T0.9] a call started while the last one is still stopping: the last one's open line lands before its call.ended", async () => { const r = rig(); diff --git a/tests/echo.test.ts b/tests/echo.test.ts new file mode 100644 index 00000000..3b000f0a --- /dev/null +++ b/tests/echo.test.ts @@ -0,0 +1,57 @@ +/** + * The echo rule (src/core/log/echo.ts, DESIGN 3.1 step 7) on its own: each bound with the case + * just inside it and the case just outside, so a rule that drifts either way fails. + */ + +import { describe, expect, test } from "bun:test"; +import { type EchoLine, isEcho } from "../src/core/log/echo.ts"; + +const T = 1_000_000; +const call: EchoLine = { w0: T, w1: T + 2000, text: "We ship on Friday, after the review." }; +const mic = (o: Partial = {}): EchoLine => ({ + w0: T + 400, + w1: T + 2400, + text: "we ship on friday after the review", + ...o, +}); + +describe("[ECHO-1] Mic hears the call", () => { + test("the same words on the mic 0.4 s after the call line are echo", () => { + expect(isEcho(mic(), call)).toBe(true); + }); + + test("time: a mic line starting 1.5 s after the call line ends is echo; 1.6 s is not", () => { + expect(isEcho(mic({ w0: T + 3500, w1: T + 5500 }), call)).toBe(true); + expect(isEcho(mic({ w0: T + 3600, w1: T + 5600 }), call)).toBe(false); + }); + + test("order: the call line must start first; a mic line that starts before it is the user's", () => { + expect(isEcho(mic({ w0: T, w1: T + 2000 }), call)).toBe(true); + expect(isEcho(mic({ w0: T - 1, w1: T + 2000 }), call)).toBe(false); + }); + + test("share: 7 of 10 mic words in the call line is echo; 6 of 10 is not", () => { + const ten: EchoLine = { w0: T, w1: T + 3000, text: "one two three four five six seven" }; + const seven = "one two three four five six seven eight nine ten"; + const six = "one two three four five six eight nine ten zero"; + expect(isEcho(mic({ text: seven }), ten)).toBe(true); + expect(isEcho(mic({ text: six }), ten)).toBe(false); + }); + + test("length: three words read back can be echo; two never are", () => { + const read: EchoLine = { w0: T, w1: T + 2000, text: "the code is four four two" }; + expect(isEcho(mic({ text: "four four two" }), read)).toBe(true); + expect(isEcho(mic({ text: "four two" }), read)).toBe(false); + }); + + test("a word counts as many times as the call line says it, not more", () => { + const once: EchoLine = { w0: T, w1: T + 2000, text: "no ok then" }; + expect(isEcho(mic({ text: "no no no ok" }), once)).toBe(false); + }); + + test("different words at the same time are not echo, and a retracted line never is", () => { + expect(isEcho(mic({ text: "can you hear me now" }), call)).toBe(false); + expect(isEcho(mic({ text: null }), call)).toBe(false); + expect(isEcho(mic(), { ...call, text: null })).toBe(false); + }); +}); From dedaa9d3412830e4ce8b8fde10d21597d1a92e2c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:50:06 +0100 Subject: [PATCH 2/4] test(asr): make the call line close first in the echo case that needs it --- tests/asr-live.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/asr-live.test.ts b/tests/asr-live.test.ts index 64a396ac..d7d5e5d6 100644 --- a/tests/asr-live.test.ts +++ b/tests/asr-live.test.ts @@ -778,7 +778,8 @@ describe("[ECHO-1] Mic hears the call", () => { for (const [name, micWords] of [ ["the mic line closes first", remote.slice(0, 4)], - ["the call line closes first", remote], + // The user says two words of their own after the echo: 8 of 10 words are the call's. + ["the call line closes first", [...remote, "ok", "great"]], ] as const) { test(`${name}: the mic line gets an echo revision, stays in the log and leaves the view`, async () => { const { s, view } = await play([...micWords], remote); From 3ff94e6927b7fd7dd7ffdf74ad341654b6d765ca Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:54:06 +0100 Subject: [PATCH 3/4] test(asr): write the call line before the mic's open echo line in the echo case that needs it --- tests/asr-live.test.ts | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/asr-live.test.ts b/tests/asr-live.test.ts index d7d5e5d6..d3190cf4 100644 --- a/tests/asr-live.test.ts +++ b/tests/asr-live.test.ts @@ -766,28 +766,34 @@ describe("the host writes the live layer", () => { describe("[ECHO-1] Mic hears the call", () => { const remote = ["we", "should", "move", "the", "build", "to", "new", "box"]; /** Plays `mic` and `call` from 1 s and 1.1 s, and returns the call's segs and folded view. */ - async function play(micWords: string[], callWords: string[]) { + async function play(micWords: string[], callWords: string[], micGap?: number) { const r = rig(); await startCall(r); const call = concat(silence(1), speak(callWords, { voice: 3 }), silence(3)); - const mic = concat(silence(1.1), speak(micWords), silence(3)); - r.engine.last.play(mic, call); + const mic = concat(silence(1.1), speak(micWords, { gapSeconds: micGap }), silence(3)); + // The host sends each channel's queued audio in turn, so to let the call line be written + // before a mic line that is still open, the audio goes in two halves: the call line's first. + const cut = call.length; + r.engine.last.play(mic.subarray(0, cut), call); + await settle(r, () => segs(r.events).some((x) => x.ch === "call")); + const rest = mic.subarray(cut); + if (rest.length > 0) r.engine.last.play(rest, silence(rest.length / RATE)); await r.mgr.stop(); return { s: segs(r.events), view: fold(r.events) }; } - for (const [name, micWords] of [ - ["the mic line closes first", remote.slice(0, 4)], - // The user says two words of their own after the echo: 8 of 10 words are the call's. - ["the call line closes first", [...remote, "ok", "great"]], + for (const [name, micWords, micGap, micFirst] of [ + ["the mic line is written first", remote.slice(0, 4), undefined, true], + // The mic's copy drawn out (0.5 s between words) closes seconds after the call line. + ["the call line is written first", remote, 0.5, false], ] as const) { test(`${name}: the mic line gets an echo revision, stays in the log and leaves the view`, async () => { - const { s, view } = await play([...micWords], remote); + const { s, view } = await play([...micWords], remote, micGap); const micLine = s.find((x) => x.ch === "mic" && x.rev === 1) as Seg; const callLine = s.find((x) => x.ch === "call" && x.rev === 1) as Seg; expect(micLine.text).toBe(micWords.join(" ")); // The order the case is about: which line was written first. - expect(micLine.seq < callLine.seq).toBe(micWords.length < remote.length); + expect(micLine.seq < callLine.seq).toBe(micFirst); expect(s.filter((x) => x.echo !== undefined)).toMatchObject([ { id: micLine.id, rev: 2, echo: true }, ]); From 42a2e097c33de9050922bb2e7c60b63ae64e8aa8 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:00:56 +0100 Subject: [PATCH 4/4] fix(asr): read the call lines near a mic line together when marking echo The final pass cuts the call channel at every diarized turn and the mic channel only at pauses, so a mic line holding the echo of two remote turns matched neither call line alone and came back in the view after the final pass. Both paths now pool the call lines the mic line may have heard. --- docs/DESIGN.md | 2 +- docs/TRAPS.md | 2 +- src/core/log/echo.ts | 21 +++++++++++++++++++++ src/main/asr/finalize-worker.ts | 7 ++++--- src/main/asr/live-worker.ts | 23 +++++++++++------------ tests/asr-final.test.ts | 19 +++++++++++++++++++ tests/asr-live.test.ts | 23 +++++++++++++++++++++++ tests/echo.test.ts | 27 ++++++++++++++++++++++++++- 8 files changed, 106 insertions(+), 18 deletions(-) diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 87fb81f3..3666bc14 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -255,7 +255,7 @@ A call's streaming model loads in the live Worker when the call starts, and its 4. **Pad** every span to at least 0.5 s before decoding, with the zeros added **after** the speech, in one function shared by every path (live, final, single file). Short words vanished under a 0.3 s engine floor in hark; that floor has not been measured on sherpa-onnx with Parakeet yet. The padding evidence from the streaming zipformer is mixed and does not transfer: a fresh stream of that model dropped the first words *until* it had 2 to 8 s of leading audio, so on that model leading audio restores words rather than losing them. The 0.2 s "Yes." fixture in [TRAPS.md](TRAPS.md) runs against Parakeet, with leading and trailing padding as separate conditions, before this rule counts as measured. 5. **Decode** with the offline recognizer, greedy by default. With `asr.parakeet.decoding` `beam`, decoding uses `modified_beam_search` and the call's current decode list is passed to `createStream(list)` when the model is a transducer. A word added mid-call is in the list from the next segment. Emit a `seg` to main, which assigns `seq`, computes wall time and appends. 6. **Provisional line.** While a segment is open, re-decode it every second (bounded by the 12 s window). It is published with a 3 s expiry and is never written to the log. On x64 the re-decode waits for new speech (`provisionalInPause` is off there): one that falls in the pause that closes the segment would hold the decoder while the committed line waits for it, and on four cores that cost 0.4 s a line. -7. **Echo.** Without headphones the mic hears the call. A mic segment whose text overlaps a call segment within ±1.5 s by at least 70 % of tokens is marked `echo: true` (kept in the log, hidden from views and packs). So that a read-back of a number or a name stays the user's, the mic segment needs at least 3 tokens and the call segment must start first; the rule is one function (`src/core/log/echo.ts`) used live, whichever channel commits a line, and by the final pass. Real echo cancellation on the recognizer's copy of the mic (`webrtc-audio-processing`, BSD-3) is an M2 option. +7. **Echo.** Without headphones the mic hears the call. A mic segment whose text overlaps the call segments within ±1.5 s, taken together, by at least 70 % of tokens is marked `echo: true` (kept in the log, hidden from views and packs). So that a read-back of a number or a name stays the user's, the mic segment needs at least 3 tokens and the call segment must start first; the rule is one function (`src/core/log/echo.ts`) used live, whichever channel commits a line, and by the final pass. Real echo cancellation on the recognizer's copy of the mic (`webrtc-audio-processing`, BSD-3) is an M2 option. Targets, measured in M0: committed line within 1.5 s of the end of an utterance on Apple silicon and within 2.5 s on 4-core x64, for lines after the first 10 s of a cold call (ROADMAP G6); first provisional words within 1.5 s of speech; both channels together under 25 % of real time on Apple Silicon and under 50 % on a 4-core x64 laptop. diff --git a/docs/TRAPS.md b/docs/TRAPS.md index b65386e0..3ce2b75d 100644 --- a/docs/TRAPS.md +++ b/docs/TRAPS.md @@ -50,7 +50,7 @@ Traps that belong to the user's own downstream pipelines (archive formats, word- - **A live line that starts inside the line before it** [LV-1] (M1). Given `asr.segmentPause` under 0.7 s on the Parakeet live path, a line closed `postRoll` (0.2 s) after its last speech, and the next one opened `preRoll` (0.5 s) before its first, so the two overlapped by 0.7 s minus the pause: 0.4 s at a 0.3 s pause, and the next line could start with the end of the last word of the line before. The next line must never start before the last closed line ended, on the same channel and part. Test: `tests/asr-live.test.ts` `[LV-1]`, a 0.3 s pause (the second line starts where the first ended) with the default pause as the control (its pre-roll is whole). - **A live decode that throws shifts the rest of the part** [LV-2] (M1). Given a one-off native error in a Parakeet live decode, the error left the channel's unfinished VAD window where it was before the audio message, so every later line on that channel was cut from audio one message too early, and the line was lost with only an app-log error. A decode that throws must lose only its own line, with a warning that names its span (the final pass decodes that audio again), and the channel's timeline must stay with its audio whatever a decode throws, a provisional re-decode included. Test: `tests/asr-live.test.ts` `[LV-2]`, a recognizer that throws once, on a closing decode and on a provisional one: the other lines have the times of a run with no error. - **A failing final pass that holds the start's catch-up** [FN-1] (M1). Given a final pass that fails by running out of its budget (Qwen on a machine with no GPU and a long call), the start's catch-up runs it again at every start, for its whole budget, and every call that ended later waited behind it. A call whose last pass failed must run after every call that has not had a pass, and must still run, so audio that becomes readable later still gets its final layer. Test: `tests/asr-final-catchup.e2e.test.ts` `[FN-1]`, a failed call that the start meets first and a call with no pass: the call with no pass starts first and both end `done`. -- **Mic hears the call** [ECHO-1] (M1). Given a call on laptop speakers, the mic hears the remote side, so each remote sentence became a `you` line as well as a call line: the user was credited with the other side's words in every view, pack, share and agent answer. The schema and the fold already hid a line marked `echo`, but nothing wrote the mark. A mic line must be marked `echo: true`, in a revision of its own on the live path (checked when either channel writes a line, since a call line can close after the mic line that heard it) and on revision 1 in the final pass, by one rule: at least 3 words, the call line started first, the mic line starts no later than 1.5 s after the call line ends, and at least 70 % of its words are in the call line. The rule is narrow so a read-back of a number or a name stays the user's. The line stays in the log; only views leave it out. Tests: `tests/echo.test.ts` `[ECHO-1]`, each bound with the case just inside and just outside it; `tests/asr-live.test.ts` and `tests/asr-final.test.ts` `[ECHO-1]`, the same speech on both channels (the mic line marked and out of the view, with either line closing first) and different speech as the positive control (no mark). +- **Mic hears the call** [ECHO-1] (M1). Given a call on laptop speakers, the mic hears the remote side, so each remote sentence became a `you` line as well as a call line: the user was credited with the other side's words in every view, pack, share and agent answer. The schema and the fold already hid a line marked `echo`, but nothing wrote the mark. A mic line must be marked `echo: true`, in a revision of its own on the live path (checked when either channel writes a line, since a call line can close after the mic line that heard it) and on revision 1 in the final pass, by one rule: at least 3 words, the call line started first, the mic line starts no later than 1.5 s after the call line ends, and at least 70 % of its words are in the call line. The call lines near a mic line count together, since the channels are not cut in the same places: the final pass cuts the call channel at every turn between two remote speakers, and a pause can close a call line where the mic's copy runs on; one call line at a time left such a mic line visible after the final pass while the live path had hidden it. The rule is narrow so a read-back of a number or a name stays the user's. The line stays in the log; only views leave it out. Tests: `tests/echo.test.ts` `[ECHO-1]`, each bound with the case just inside and just outside it; `tests/asr-live.test.ts` and `tests/asr-final.test.ts` `[ECHO-1]`, the same speech on both channels (the mic line marked and out of the view, with either line closing first, and with the call cut in two where the mic is not) and different speech as the positive control (no mark). ## Event log and clocks diff --git a/src/core/log/echo.ts b/src/core/log/echo.ts index f63635d3..481c6f96 100644 --- a/src/core/log/echo.ts +++ b/src/core/log/echo.ts @@ -44,3 +44,24 @@ export function isEcho(mic: EchoLine, call: EchoLine): boolean { } return found * 10 >= said.length * ECHO_MIN_SHARE_TENTHS; } + +/** + * Whether the mic line `mic` is the mic hearing the call lines near it, taken together. The two + * channels are not cut in the same places (the final pass cuts the call channel at every turn + * between two remote speakers, and a pause can close a call line where the mic's copy runs on), so + * one mic line can hold the echo of several call lines. The call lines that may have been heard are + * the ones that start before the mic line ends and end at most `ECHO_WINDOW_MS` before it starts; + * they are read as one line, from the first one's start to the last end. + */ +export function isEchoOfCalls(mic: EchoLine, calls: readonly EchoLine[]): boolean { + const near = calls + .filter((c) => c.w1 + ECHO_WINDOW_MS >= mic.w0 && c.w0 <= mic.w1) + .sort((a, b) => a.w0 - b.w0); + const first = near[0]; + if (!first) return false; + return isEcho(mic, { + w0: first.w0, + w1: Math.max(...near.map((c) => c.w1)), + text: near.map((c) => c.text ?? "").join(" "), + }); +} diff --git a/src/main/asr/finalize-worker.ts b/src/main/asr/finalize-worker.ts index cc68de06..4f800220 100644 --- a/src/main/asr/finalize-worker.ts +++ b/src/main/asr/finalize-worker.ts @@ -49,7 +49,7 @@ import { closeSync, openSync, readSync } from "node:fs"; import { join } from "node:path"; import { workerData } from "node:worker_threads"; -import { type EchoLine, isEcho } from "../../core/log/echo.ts"; +import { type EchoLine, isEchoOfCalls } from "../../core/log/echo.ts"; import type { Channel, EventDraft, LogEvent } from "../../core/log/events.ts"; import { type CallView, fold } from "../../core/log/fold.ts"; import { readLog } from "../../core/log/reader.ts"; @@ -718,10 +718,11 @@ export async function runFinalPass( }; const closePart = (p: number, lines: Line[]) => { lines.sort((a, b) => (a.w0 as number) - (b.w0 as number) || (a.ch === "mic" ? -1 : 1)); - // Echo (DESIGN 3.1 step 7), by the live path's rule: a mic line that repeats a call line. + // Echo (DESIGN 3.1 step 7), by the live path's rule: a mic line that repeats the call lines + // near it. Taken together, since only the call channel is cut at diarized turns. const calls = lines.filter((l) => l.ch === "call") as EchoLine[]; for (const l of lines) { - if (l.ch === "mic" && calls.some((c) => isEcho(l as EchoLine, c))) l.echo = true; + if (l.ch === "mic" && isEchoOfCalls(l as EchoLine, calls)) l.echo = true; } for (const l of lines) { layer.push({ ...l, id: `f${String(nextFinal).padStart(6, "0")}` } as EventDraft); diff --git a/src/main/asr/live-worker.ts b/src/main/asr/live-worker.ts index ade8e35c..f43fbb3b 100644 --- a/src/main/asr/live-worker.ts +++ b/src/main/asr/live-worker.ts @@ -61,7 +61,7 @@ */ import { workerData } from "node:worker_threads"; -import { isEcho } from "../../core/log/echo.ts"; +import { isEchoOfCalls } from "../../core/log/echo.ts"; import type { Channel, EventDraft, LogEvent } from "../../core/log/events.ts"; import type { CallView } from "../../core/log/fold.ts"; import { CHANNELS, type Clock, realClock, withDeadline } from "../capture/engine.ts"; @@ -2729,26 +2729,25 @@ export class LiveAsr { } /** - * Echo (DESIGN 3.1 step 7): a mic line that repeats a call line (`isEcho`) gets a revision - * carrying `echo: true` alone. Checked whichever channel just wrote, since a call line can close - * after the mic line that heard it. Never writes `echo: false`, and leaves a line a person - * touched as it is. + * Echo (DESIGN 3.1 step 7): a mic line that repeats the call lines near it (`isEchoOfCalls`) + * gets a revision carrying `echo: true` alone. Checked whichever channel just wrote, since a call + * line can close after the mic line that heard it. Never writes `echo: false`, and leaves a line + * a person touched as it is. */ private markEcho(c: HostCall, id: string): void { const view = c.access.view; const s = view.segment(id); if (!s) return; c.recent = c.recent.filter((r) => (view.segment(r)?.w1 ?? 0) >= s.w0 - ECHO_RECENT_MS); - for (const other of c.recent) { - const o = view.segment(other); - if (!o || o.ch === s.ch) continue; - const mic = s.ch === "mic" ? s : o; - const call = s.ch === "mic" ? o : s; + c.recent.push(id); + const lines = c.recent.flatMap((r) => view.segment(r) ?? []); + const calls = lines.filter((l) => l.ch === "call"); + const mics = s.ch === "mic" ? [s] : lines.filter((l) => l.ch === "mic"); + for (const mic of mics) { if (mic.echo || mic.revisions.some((r) => r.by !== undefined)) continue; - if (!isEcho(mic, call)) continue; + if (!isEchoOfCalls(mic, calls)) continue; c.access.record({ type: "seg", id: mic.id, rev: mic.rev + 1, echo: true }); } - c.recent.push(id); } /** diff --git a/tests/asr-final.test.ts b/tests/asr-final.test.ts index c0621f64..28c17eef 100644 --- a/tests/asr-final.test.ts +++ b/tests/asr-final.test.ts @@ -200,6 +200,25 @@ describe("[ECHO-1] Mic hears the call: the final pass marks it too", () => { const r = await run(callLog([1]), part(["hello", "world", "thanks", "meeting", "today"])); expect(r.segs.map((s) => s.echo)).toEqual([undefined, undefined]); }); + + test("two remote speakers: the call is cut at the turn, the mic is not, and the mic line is still echo", async () => { + const a = ["we", "should", "move", "the", "build"]; + const b = ["to", "new", "box", "thanks", "meeting"]; + const r = await run(callLog([1]), { + 1: { + mic: concat(silence(1.1), speak([...a, ...b]), silence(2)), + call: concat(silence(1), speak(a, { voice: 1 }), speak(b, { voice: 4 }), silence(2.1)), + }, + }); + // The case the test is about: two call lines, one mic line. + expect(r.segs.filter((s) => s.ch === "call").map((s) => s.text)).toEqual([ + a.join(" "), + b.join(" "), + ]); + expect(r.segs.filter((s) => s.ch === "mic").map((s) => [s.text, s.echo])).toEqual([ + [[...a, ...b].join(" "), true], + ]); + }); }); describe("speakers across the call", () => { diff --git a/tests/asr-live.test.ts b/tests/asr-live.test.ts index d3190cf4..46278d41 100644 --- a/tests/asr-live.test.ts +++ b/tests/asr-live.test.ts @@ -810,6 +810,29 @@ describe("[ECHO-1] Mic hears the call", () => { }); } + test("a pause cuts the call line where the mic's copy runs on: the mic line is still echo", async () => { + const r = rig(); + await startCall(r); + const a = remote.slice(0, 4); + const b = remote.slice(4); + const call = concat(silence(1), speak(a, { voice: 3 }), silence(1.2), speak(b, { voice: 3 })); + const mic = concat(silence(1.1), speak(remote), silence(1.1)); + r.engine.last.play(mic, call); + r.engine.last.play(silence(3), silence(3)); + await r.mgr.stop(); + const s = segs(r.events); + // The case the test is about: two call lines, one mic line. + expect(s.filter((x) => x.ch === "call" && x.rev === 1).map((x) => x.text)).toEqual([ + a.join(" "), + b.join(" "), + ]); + const micLine = s.find((x) => x.ch === "mic" && x.rev === 1) as Seg; + expect(micLine.text).toBe(remote.join(" ")); + expect(s.filter((x) => x.echo !== undefined)).toMatchObject([ + { id: micLine.id, rev: 2, echo: true }, + ]); + }); + test("positive control: the user's own words at the same time get no echo revision", async () => { const { s, view } = await play(["hello", "world", "thanks", "meeting", "today"], remote); expect(s.filter((x) => x.echo !== undefined)).toEqual([]); diff --git a/tests/echo.test.ts b/tests/echo.test.ts index 3b000f0a..751407a4 100644 --- a/tests/echo.test.ts +++ b/tests/echo.test.ts @@ -4,7 +4,7 @@ */ import { describe, expect, test } from "bun:test"; -import { type EchoLine, isEcho } from "../src/core/log/echo.ts"; +import { type EchoLine, isEcho, isEchoOfCalls } from "../src/core/log/echo.ts"; const T = 1_000_000; const call: EchoLine = { w0: T, w1: T + 2000, text: "We ship on Friday, after the review." }; @@ -55,3 +55,28 @@ describe("[ECHO-1] Mic hears the call", () => { expect(isEcho(mic(), { ...call, text: null })).toBe(false); }); }); + +describe("[ECHO-1] Mic hears the call: one mic line over several call lines", () => { + // Two remote turns, cut apart on the call channel and heard as one line on the mic. + const a: EchoLine = { w0: T, w1: T + 1800, text: "we should move the build" }; + const b: EchoLine = { w0: T + 1900, w1: T + 3700, text: "to new box thanks meeting" }; + const both = mic({ w0: T + 100, w1: T + 3800, text: `${a.text} ${b.text}` }); + + test("the call lines are read together: neither alone holds 70 % of the mic line", () => { + expect(isEcho(both, a)).toBe(false); + expect(isEcho(both, b)).toBe(false); + expect(isEchoOfCalls(both, [b, a])).toBe(true); + }); + + test("a call line is pooled only while the mic line may have heard it", () => { + // Positive control: the same two lines, one ending more than 1.5 s before the mic starts. + const old = { ...a, w0: a.w0 - 5000, w1: a.w1 - 5000 }; + expect(isEchoOfCalls(both, [old, b])).toBe(false); + // A call line that starts after the mic line ends is not pooled either. + const late = { ...a, w0: both.w1 + 1, w1: both.w1 + 1800 }; + expect(isEchoOfCalls(both, [late, b])).toBe(false); + // And the earliest pooled call line must start first, as for one line. + expect(isEchoOfCalls({ ...both, w0: a.w0 - 1 }, [a, b])).toBe(false); + expect(isEchoOfCalls(both, [])).toBe(false); + }); +});