Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 79 additions & 0 deletions .github/scripts/js/tts-text.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
// how replies get cut into TTS chunks, and which Kokoro language
// each chunk is routed to. the cutting decides when the first audio
// plays and how big a chunk gets, the routing whether a chunk is
// read by the right G2P at all. nothing else in CI runs this code,
// node --check only parses it.
//
// run: node --test .github/scripts/js
import { register } from 'node:module';
import { test } from 'node:test';
import assert from 'node:assert/strict';

register('./dom-stub-hooks.mjs', import.meta.url);

globalThis.window = globalThis;
const TTS = await import('../../../webapp/js/voice/tts.js');
const { detectKokoroLang } = await import('../../../webapp/js/voice/kokoro-lang.js');

function cuts(text, first) {
const r = TTS.nextChunk(text, first);
return r ? r.chunk : null;
}

test('English chunking is unchanged', () => {
assert.equal(cuts('Hello there. How are', false), 'Hello there.');
assert.equal(cuts('Oh, hi', true), null);
assert.equal(cuts('Well, I think, maybe', true), 'Well, I think,');
// a run of dots is her trailing off, never a cut
assert.equal(cuts('Wait... what', false), null);
assert.equal(cuts('No full stop yet', false), null);
});

test('fullwidth sentence ends cut a chunk', () => {
assert.equal(cuts('你好!今天天氣很好。', false), '你好!');
assert.equal(cuts('今天天氣很好。我們', false), '今天天氣很好。');
assert.equal(cuts('真的嗎?好', false), '真的嗎?');
assert.equal(cuts('嗯~我想想', false), '嗯~');
});

test('the first chunk soft-splits Chinese once a few hanzi are in', () => {
assert.equal(cuts('今天天氣很好,我們去散步吧', true), '今天天氣很好,');
// two hanzi isn't enough, same as "Oh," in English
assert.equal(cuts('你好,', true), null);
assert.equal(cuts('我覺得;也許', true), '我覺得;');
});

test('Kokoro routes by script', () => {
assert.equal(detectKokoroLang('你好,今天天氣很好。'), 'mandarin');
assert.equal(detectKokoroLang('我是Jun,很高興認識你。'), 'mandarin');
assert.equal(detectKokoroLang('Hello there.'), 'english');
assert.equal(detectKokoroLang('OK!'), 'english');
assert.equal(detectKokoroLang(''), 'english');
});

test('Japanese stays off Mandarin, kanji or not', () => {
assert.equal(detectKokoroLang('今日はいい天気ですね。'), 'english');
assert.equal(detectKokoroLang('カタカナ漢字'), 'english');
assert.equal(detectKokoroLang('ありがとう'), 'english');
});

test('predictLang names what Kokoro should warm', () => {
TTS.setEnabled(true);
TTS.setEngine('kokoro');
TTS.setLang('auto');
assert.equal(TTS.predictLang('你好嗎'), 'mandarin');
assert.equal(TTS.predictLang('how are you'), null);
TTS.setLang('mandarin');
assert.equal(TTS.predictLang('how are you'), 'mandarin');
TTS.setLang('english');
assert.equal(TTS.predictLang('你好嗎'), null);
});

test('pocket-tts prediction is untouched', () => {
TTS.setEngine('pockettts');
TTS.setLang('auto');
assert.equal(TTS.predictLang('what are you doing with the cat'), 'english');
assert.equal(TTS.predictLang('je ne sais pas pour les chats'), 'french_24l');
TTS.setLang('english');
assert.equal(TTS.predictLang('je ne sais pas'), null);
});
4 changes: 3 additions & 1 deletion docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -290,10 +290,12 @@ Tokens are appended to a buffer. When a sentence-ending character (`.`, `!`, `?`

Each queued sentence triggers an immediate `fetch POST /api/tts.php`. The PHP endpoint validates the request (text ≤ 2000 chars, voice pattern, speed range, engine) and forwards it to the voice sidecar. Responses arrive out-of-order since synthesis time varies by sentence length.

The sidecar (`tts/server.py`, code under `tts/sidecar/`) fronts two engines chosen per request by the `engine` field: **kokoro** (Kokoro-82M, default, ~27 EN voices, needs espeak-ng) and **pockettts** (kyutai pocket-tts, ~100M, CPU-friendly, English + 5 languages). For pocket-tts the language is a separate axis from the voice: it's baked into the model weights, so the request's `lang` field (english, french_24l, german_24l, italian, portuguese, spanish_24l) selects a different checkpoint that actually pronounces that language and resolves the chosen voice to that language's embedding. Only one pocket-tts language stays resident; switching `lang` reloads (a full checkpoint load, a few seconds off HF_HOME-cached weights), which is the cost the auto-language path below is built to hide. `TTS_DEVICE` (`cpu`|`cuda`|`auto`) picks the torch device; Kokoro pre-warms one utterance at startup while pocket-tts loads lazily on its first request. `GET /voices` exposes both engines' voice lists and defaults, plus pocket-tts's selectable languages, so the UI can offer an engine + voice + language picker.
The sidecar (`tts/server.py`, code under `tts/sidecar/`) fronts two engines chosen per request by the `engine` field: **kokoro** (Kokoro-82M, default, 15 EN voices plus 4 Mandarin ones, needs espeak-ng) and **pockettts** (kyutai pocket-tts, ~100M, CPU-friendly, English + 5 languages). For pocket-tts the language is a separate axis from the voice: it's baked into the model weights, so the request's `lang` field (english, french_24l, german_24l, italian, portuguese, spanish_24l) selects a different checkpoint that actually pronounces that language and resolves the chosen voice to that language's embedding. Only one pocket-tts language stays resident; switching `lang` reloads (a full checkpoint load, a few seconds off HF_HOME-cached weights), which is the cost the auto-language path below is built to hide. `TTS_DEVICE` (`cpu`|`cuda`|`auto`) picks the torch device; Kokoro pre-warms one utterance at startup while pocket-tts loads lazily on its first request. `GET /voices` exposes both engines' voice lists and defaults, plus pocket-tts's selectable languages, so the UI can offer an engine + voice + language picker.

The language picker also has an **Auto-detect** option (the default), which routes each reply to the right pocket-tts language while keeping reloads rare and off the critical path. On send, `js/voice/tts.js` runs a dependency-free stopword detector over **Anon's message** to predict the reply's language, falling back to the *previous* conversation language (not English) when the message is too short or ambiguous to call. That prediction drives two things in parallel: `app.js` fires `POST /api/tts.php?action=warm` → the sidecar's `/warm` (which preloads that language's checkpoint and voice state off-thread, so the reload overlaps LLM generation instead of stalling the first audio chunk), and the client locks the reply to the predicted language. As the reply streams, the detector *verifies* the prediction against the actual text - only a confident disagreement in the opening ~40 characters switches the language (warming the corrected model), and once locked it never flips again, so a stray foreign word mid-sentence can't trigger a mid-reply reload. The result: a monolingual conversation reloads zero times after the first turn, and a genuine language switch reloads once, hidden behind generation. Detection is purely client-side; the sidecar only ever sees a concrete language id, and `chat.php` gets no language hint at all - an earlier "reply in {language}" line in the live context was removed because the sticky fallback made it self-latching. Reply language is the model's to infer from the conversation.

Kokoro has one model for every language, so its languages are only G2Ps. `KOKORO_LANGUAGES` in `tts/sidecar/config.py` is the registry: an id (what the client sends as `lang`), the Kokoro `lang_code`, the voices made for it with a default, the modules its G2P imports, and optionally the script only it writes. English is the pipeline the pre-warm loads; any other language gets its `KPipeline` built on first use around that same `KModel` (`get_kokoro_pipeline()` in `tts/sidecar/tts.py`), so a second language costs a G2P, not a second copy of the weights. `GET /voices` only lists a language whose modules `importlib.util.find_spec` can find, and only adds `languages` to the kokoro entry when there are two or more, so without `misaki[zh]` the language row stays hidden exactly as before. A voice that isn't the language's own is swapped for its default; a chunk with none of a language's script (`OK!` under Mandarin) goes through the English G2P in the same voice, because Mandarin's would read the letters as noise; a G2P that fails to build is remembered and that language falls back to English until restart. Kokoro's **Auto-detect** is per chunk rather than per reply, since switching costs nothing: `js/voice/kokoro-lang.js` routes a chunk by its characters (hanzi without kana is Mandarin, anything else English) and `tts.js` sends it with the voice picked for that language. The settings keep one voice per language in `tts.voices`; under Auto the voice menu is greyed and each language speaks in its own. `predictLang()` names a non-English language from Anon's message so `/warm` can build its G2P and fetch the voice before the first chunk. The chunker in `tts.js` also cuts on fullwidth `。!?~` and soft-splits on `,;:`, counting each hanzi as a word, so a Chinese reply starts playing after its first sentence instead of arriving as one chunk. Adding a Kokoro language is an entry in `KOKORO_LANGUAGES`, its misaki extra pinned in `tts/requirements.txt`, and a rule in `kokoro-lang.js`.

Results are decoded into `AudioBuffer`s and inserted into a `Map` keyed by submission index. A playback cursor advances only when the buffer at the current index is ready, ensuring sentences always play in the order they were generated even if a later sentence finishes synthesis faster.

### Audio graph
Expand Down
3 changes: 3 additions & 0 deletions tts/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@
# interpreter in that window on purpose, the docker image is
# 3.11.
kokoro==0.9.4
# kokoro's own misaki pin. [zh] is the Mandarin G2P (jieba, pypinyin,
# cn2an). without it Mandarin just isn't offered, English is untouched.
misaki[zh]==0.9.4
# spacy (kokoro -> misaki) pulls this in unpinned. 3.0.14 kills
# en_core_web_sm with OverflowError in _flush_specials, so no voice
preshed==3.0.13
Expand Down
31 changes: 31 additions & 0 deletions tts/sidecar/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,37 @@
"bf_alice", "bf_emma", "bf_isabella", "bf_lily",
]

# the languages Kokoro gets offered in. one entry each, and that's
# the whole registry: id is what the client sends as `lang` (the
# same field pocket-tts uses, tts.php already checks its shape),
# lang_code picks Kokoro's G2P, voices are the ones made for it.
# needs lists the modules that G2P imports. a language missing any
# of them is left out of /voices, so the menu never offers it.
# script, when set, is the characters only that language writes:
# a chunk without any goes through the English G2P instead, in the
# same voice, since this G2P would read latin letters as noise.
#
# adding a language: an entry here, its misaki extra pinned in
# requirements.txt, and a rule in js/voice/kokoro-lang.js so
# auto-detect can find it. female voices only, like the EN list
# above. the 4 Mandarin ones are all graded D upstream, pick by ear.
KOKORO_LANGUAGES = [
{
"id": "english", "label": "English", "lang_code": "a",
"voices": KOKORO_VOICES, "default": KOKORO_DEFAULT,
"needs": (), "script": None,
},
{
"id": "mandarin", "label": "Mandarin Chinese", "lang_code": "z",
"voices": ["zf_xiaobei", "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi"],
"default": "zf_xiaobei",
"needs": ("jieba", "pypinyin", "cn2an"),
"script": r"[\u3400-\u9fff\uf900-\ufaff]",
},
]
KOKORO_DEFAULT_LANG = "english"
KOKORO_LANG_BY_ID = {lang["id"]: lang for lang in KOKORO_LANGUAGES}

# pocket-tts voice names pick timbre, not language. each has an
# embedding for every POCKET_LANGUAGES entry. giovanni, lola,
# juergen, rafael and estelle keep the accent of their non-English
Expand Down
11 changes: 7 additions & 4 deletions tts/sidecar/state.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@
# them. the loaders in tts.py and karaoke.py fill them in. whisper
# isn't here, it never gets unloaded.
pipeline = None
# Kokoro's other languages, lang_code -> KPipeline. each one is only
# a G2P on top of pipeline's model, so they go when it goes.
kokoro_g2p = {}
pocket_model = None
pocket_lang = None
pocket_states = {}
Expand All @@ -36,7 +39,7 @@

def loaded_engines():
live = []
if pipeline is not None: live.append("kokoro")
if pipeline is not None or kokoro_g2p: live.append("kokoro")
if pocket_model is not None: live.append("pockettts")
if separator is not None: live.append("demucs")
return live
Expand All @@ -55,11 +58,11 @@ def free_torch():
def unload(names):
# caller holds lock. drops the model refs, then gc and
# empty_cache hand the VRAM back.
global pipeline, pocket_model, pocket_lang, pocket_states, separator
global pipeline, kokoro_g2p, pocket_model, pocket_lang, pocket_states, separator
freed = []
for name in names:
if name == "kokoro" and pipeline is not None:
pipeline = None; freed.append(name)
if name == "kokoro" and (pipeline is not None or kokoro_g2p):
pipeline = None; kokoro_g2p = {}; freed.append(name)
elif name == "pockettts" and pocket_model is not None:
pocket_model = None; pocket_lang = None; pocket_states = {}; freed.append(name)
elif name == "demucs" and separator is not None:
Expand Down
Loading
Loading