diff --git a/.github/scripts/js/tts-text.test.mjs b/.github/scripts/js/tts-text.test.mjs new file mode 100644 index 00000000..f61ebea6 --- /dev/null +++ b/.github/scripts/js/tts-text.test.mjs @@ -0,0 +1,79 @@ +// how replies get cut into TTS chunks, and which Kokoro language +// each chunk is routed to. the cutting decides when the first audio +// plays and how big a chunk gets, the routing whether a chunk is +// read by the right G2P at all. nothing else in CI runs this code, +// node --check only parses it. +// +// run: node --test .github/scripts/js +import { register } from 'node:module'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +register('./dom-stub-hooks.mjs', import.meta.url); + +globalThis.window = globalThis; +const TTS = await import('../../../webapp/js/voice/tts.js'); +const { detectKokoroLang } = await import('../../../webapp/js/voice/kokoro-lang.js'); + +function cuts(text, first) { + const r = TTS.nextChunk(text, first); + return r ? r.chunk : null; +} + +test('English chunking is unchanged', () => { + assert.equal(cuts('Hello there. How are', false), 'Hello there.'); + assert.equal(cuts('Oh, hi', true), null); + assert.equal(cuts('Well, I think, maybe', true), 'Well, I think,'); + // a run of dots is her trailing off, never a cut + assert.equal(cuts('Wait... what', false), null); + assert.equal(cuts('No full stop yet', false), null); +}); + +test('fullwidth sentence ends cut a chunk', () => { + assert.equal(cuts('你好!今天天氣很好。', false), '你好!'); + assert.equal(cuts('今天天氣很好。我們', false), '今天天氣很好。'); + assert.equal(cuts('真的嗎?好', false), '真的嗎?'); + assert.equal(cuts('嗯~我想想', false), '嗯~'); +}); + +test('the first chunk soft-splits Chinese once a few hanzi are in', () => { + assert.equal(cuts('今天天氣很好,我們去散步吧', true), '今天天氣很好,'); + // two hanzi isn't enough, same as "Oh," in English + assert.equal(cuts('你好,', true), null); + assert.equal(cuts('我覺得;也許', true), '我覺得;'); +}); + +test('Kokoro routes by script', () => { + assert.equal(detectKokoroLang('你好,今天天氣很好。'), 'mandarin'); + assert.equal(detectKokoroLang('我是Jun,很高興認識你。'), 'mandarin'); + assert.equal(detectKokoroLang('Hello there.'), 'english'); + assert.equal(detectKokoroLang('OK!'), 'english'); + assert.equal(detectKokoroLang(''), 'english'); +}); + +test('Japanese stays off Mandarin, kanji or not', () => { + assert.equal(detectKokoroLang('今日はいい天気ですね。'), 'english'); + assert.equal(detectKokoroLang('カタカナ漢字'), 'english'); + assert.equal(detectKokoroLang('ありがとう'), 'english'); +}); + +test('predictLang names what Kokoro should warm', () => { + TTS.setEnabled(true); + TTS.setEngine('kokoro'); + TTS.setLang('auto'); + assert.equal(TTS.predictLang('你好嗎'), 'mandarin'); + assert.equal(TTS.predictLang('how are you'), null); + TTS.setLang('mandarin'); + assert.equal(TTS.predictLang('how are you'), 'mandarin'); + TTS.setLang('english'); + assert.equal(TTS.predictLang('你好嗎'), null); +}); + +test('pocket-tts prediction is untouched', () => { + TTS.setEngine('pockettts'); + TTS.setLang('auto'); + assert.equal(TTS.predictLang('what are you doing with the cat'), 'english'); + assert.equal(TTS.predictLang('je ne sais pas pour les chats'), 'french_24l'); + TTS.setLang('english'); + assert.equal(TTS.predictLang('je ne sais pas'), null); +}); diff --git a/docs/architecture.md b/docs/architecture.md index e60d05ee..ad2c77b1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -290,10 +290,12 @@ Tokens are appended to a buffer. When a sentence-ending character (`.`, `!`, `?` Each queued sentence triggers an immediate `fetch POST /api/tts.php`. The PHP endpoint validates the request (text ≤ 2000 chars, voice pattern, speed range, engine) and forwards it to the voice sidecar. Responses arrive out-of-order since synthesis time varies by sentence length. -The sidecar (`tts/server.py`, code under `tts/sidecar/`) fronts two engines chosen per request by the `engine` field: **kokoro** (Kokoro-82M, default, ~27 EN voices, needs espeak-ng) and **pockettts** (kyutai pocket-tts, ~100M, CPU-friendly, English + 5 languages). For pocket-tts the language is a separate axis from the voice: it's baked into the model weights, so the request's `lang` field (english, french_24l, german_24l, italian, portuguese, spanish_24l) selects a different checkpoint that actually pronounces that language and resolves the chosen voice to that language's embedding. Only one pocket-tts language stays resident; switching `lang` reloads (a full checkpoint load, a few seconds off HF_HOME-cached weights), which is the cost the auto-language path below is built to hide. `TTS_DEVICE` (`cpu`|`cuda`|`auto`) picks the torch device; Kokoro pre-warms one utterance at startup while pocket-tts loads lazily on its first request. `GET /voices` exposes both engines' voice lists and defaults, plus pocket-tts's selectable languages, so the UI can offer an engine + voice + language picker. +The sidecar (`tts/server.py`, code under `tts/sidecar/`) fronts two engines chosen per request by the `engine` field: **kokoro** (Kokoro-82M, default, 15 EN voices plus 4 Mandarin ones, needs espeak-ng) and **pockettts** (kyutai pocket-tts, ~100M, CPU-friendly, English + 5 languages). For pocket-tts the language is a separate axis from the voice: it's baked into the model weights, so the request's `lang` field (english, french_24l, german_24l, italian, portuguese, spanish_24l) selects a different checkpoint that actually pronounces that language and resolves the chosen voice to that language's embedding. Only one pocket-tts language stays resident; switching `lang` reloads (a full checkpoint load, a few seconds off HF_HOME-cached weights), which is the cost the auto-language path below is built to hide. `TTS_DEVICE` (`cpu`|`cuda`|`auto`) picks the torch device; Kokoro pre-warms one utterance at startup while pocket-tts loads lazily on its first request. `GET /voices` exposes both engines' voice lists and defaults, plus pocket-tts's selectable languages, so the UI can offer an engine + voice + language picker. The language picker also has an **Auto-detect** option (the default), which routes each reply to the right pocket-tts language while keeping reloads rare and off the critical path. On send, `js/voice/tts.js` runs a dependency-free stopword detector over **Anon's message** to predict the reply's language, falling back to the *previous* conversation language (not English) when the message is too short or ambiguous to call. That prediction drives two things in parallel: `app.js` fires `POST /api/tts.php?action=warm` → the sidecar's `/warm` (which preloads that language's checkpoint and voice state off-thread, so the reload overlaps LLM generation instead of stalling the first audio chunk), and the client locks the reply to the predicted language. As the reply streams, the detector *verifies* the prediction against the actual text - only a confident disagreement in the opening ~40 characters switches the language (warming the corrected model), and once locked it never flips again, so a stray foreign word mid-sentence can't trigger a mid-reply reload. The result: a monolingual conversation reloads zero times after the first turn, and a genuine language switch reloads once, hidden behind generation. Detection is purely client-side; the sidecar only ever sees a concrete language id, and `chat.php` gets no language hint at all - an earlier "reply in {language}" line in the live context was removed because the sticky fallback made it self-latching. Reply language is the model's to infer from the conversation. +Kokoro has one model for every language, so its languages are only G2Ps. `KOKORO_LANGUAGES` in `tts/sidecar/config.py` is the registry: an id (what the client sends as `lang`), the Kokoro `lang_code`, the voices made for it with a default, the modules its G2P imports, and optionally the script only it writes. English is the pipeline the pre-warm loads; any other language gets its `KPipeline` built on first use around that same `KModel` (`get_kokoro_pipeline()` in `tts/sidecar/tts.py`), so a second language costs a G2P, not a second copy of the weights. `GET /voices` only lists a language whose modules `importlib.util.find_spec` can find, and only adds `languages` to the kokoro entry when there are two or more, so without `misaki[zh]` the language row stays hidden exactly as before. A voice that isn't the language's own is swapped for its default; a chunk with none of a language's script (`OK!` under Mandarin) goes through the English G2P in the same voice, because Mandarin's would read the letters as noise; a G2P that fails to build is remembered and that language falls back to English until restart. Kokoro's **Auto-detect** is per chunk rather than per reply, since switching costs nothing: `js/voice/kokoro-lang.js` routes a chunk by its characters (hanzi without kana is Mandarin, anything else English) and `tts.js` sends it with the voice picked for that language. The settings keep one voice per language in `tts.voices`; under Auto the voice menu is greyed and each language speaks in its own. `predictLang()` names a non-English language from Anon's message so `/warm` can build its G2P and fetch the voice before the first chunk. The chunker in `tts.js` also cuts on fullwidth `。!?~` and soft-splits on `,;:`, counting each hanzi as a word, so a Chinese reply starts playing after its first sentence instead of arriving as one chunk. Adding a Kokoro language is an entry in `KOKORO_LANGUAGES`, its misaki extra pinned in `tts/requirements.txt`, and a rule in `kokoro-lang.js`. + Results are decoded into `AudioBuffer`s and inserted into a `Map` keyed by submission index. A playback cursor advances only when the buffer at the current index is ready, ensuring sentences always play in the order they were generated even if a later sentence finishes synthesis faster. ### Audio graph diff --git a/tts/requirements.txt b/tts/requirements.txt index 692094f3..ad4c1da2 100644 --- a/tts/requirements.txt +++ b/tts/requirements.txt @@ -7,6 +7,9 @@ # interpreter in that window on purpose, the docker image is # 3.11. kokoro==0.9.4 +# kokoro's own misaki pin. [zh] is the Mandarin G2P (jieba, pypinyin, +# cn2an). without it Mandarin just isn't offered, English is untouched. +misaki[zh]==0.9.4 # spacy (kokoro -> misaki) pulls this in unpinned. 3.0.14 kills # en_core_web_sm with OverflowError in _flush_specials, so no voice preshed==3.0.13 diff --git a/tts/sidecar/config.py b/tts/sidecar/config.py index 88262021..49f95576 100644 --- a/tts/sidecar/config.py +++ b/tts/sidecar/config.py @@ -11,6 +11,37 @@ "bf_alice", "bf_emma", "bf_isabella", "bf_lily", ] +# the languages Kokoro gets offered in. one entry each, and that's +# the whole registry: id is what the client sends as `lang` (the +# same field pocket-tts uses, tts.php already checks its shape), +# lang_code picks Kokoro's G2P, voices are the ones made for it. +# needs lists the modules that G2P imports. a language missing any +# of them is left out of /voices, so the menu never offers it. +# script, when set, is the characters only that language writes: +# a chunk without any goes through the English G2P instead, in the +# same voice, since this G2P would read latin letters as noise. +# +# adding a language: an entry here, its misaki extra pinned in +# requirements.txt, and a rule in js/voice/kokoro-lang.js so +# auto-detect can find it. female voices only, like the EN list +# above. the 4 Mandarin ones are all graded D upstream, pick by ear. +KOKORO_LANGUAGES = [ + { + "id": "english", "label": "English", "lang_code": "a", + "voices": KOKORO_VOICES, "default": KOKORO_DEFAULT, + "needs": (), "script": None, + }, + { + "id": "mandarin", "label": "Mandarin Chinese", "lang_code": "z", + "voices": ["zf_xiaobei", "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi"], + "default": "zf_xiaobei", + "needs": ("jieba", "pypinyin", "cn2an"), + "script": r"[\u3400-\u9fff\uf900-\ufaff]", + }, +] +KOKORO_DEFAULT_LANG = "english" +KOKORO_LANG_BY_ID = {lang["id"]: lang for lang in KOKORO_LANGUAGES} + # pocket-tts voice names pick timbre, not language. each has an # embedding for every POCKET_LANGUAGES entry. giovanni, lola, # juergen, rafael and estelle keep the accent of their non-English diff --git a/tts/sidecar/state.py b/tts/sidecar/state.py index 42abd478..c023133b 100644 --- a/tts/sidecar/state.py +++ b/tts/sidecar/state.py @@ -13,6 +13,9 @@ # them. the loaders in tts.py and karaoke.py fill them in. whisper # isn't here, it never gets unloaded. pipeline = None +# Kokoro's other languages, lang_code -> KPipeline. each one is only +# a G2P on top of pipeline's model, so they go when it goes. +kokoro_g2p = {} pocket_model = None pocket_lang = None pocket_states = {} @@ -36,7 +39,7 @@ def loaded_engines(): live = [] - if pipeline is not None: live.append("kokoro") + if pipeline is not None or kokoro_g2p: live.append("kokoro") if pocket_model is not None: live.append("pockettts") if separator is not None: live.append("demucs") return live @@ -55,11 +58,11 @@ def free_torch(): def unload(names): # caller holds lock. drops the model refs, then gc and # empty_cache hand the VRAM back. - global pipeline, pocket_model, pocket_lang, pocket_states, separator + global pipeline, kokoro_g2p, pocket_model, pocket_lang, pocket_states, separator freed = [] for name in names: - if name == "kokoro" and pipeline is not None: - pipeline = None; freed.append(name) + if name == "kokoro" and (pipeline is not None or kokoro_g2p): + pipeline = None; kokoro_g2p = {}; freed.append(name) elif name == "pockettts" and pocket_model is not None: pocket_model = None; pocket_lang = None; pocket_states = {}; freed.append(name) elif name == "demucs" and separator is not None: diff --git a/tts/sidecar/tts.py b/tts/sidecar/tts.py index 3f732a69..f6bf7ef8 100644 --- a/tts/sidecar/tts.py +++ b/tts/sidecar/tts.py @@ -1,4 +1,6 @@ +import importlib.util import logging +import re import threading from typing import Annotated @@ -9,7 +11,8 @@ from . import state from .audio import to_wav -from .config import (DEFAULT_ENGINE, KOKORO_DEFAULT, KOKORO_SAMPLE_RATE, KOKORO_VOICES, POCKET_DEFAULT, +from .config import (DEFAULT_ENGINE, KOKORO_DEFAULT, KOKORO_DEFAULT_LANG, KOKORO_LANG_BY_ID, KOKORO_LANGUAGES, + KOKORO_SAMPLE_RATE, KOKORO_VOICES, POCKET_DEFAULT, POCKET_DEFAULT_LANG, POCKET_LANG_IDS, POCKET_LANGUAGES, POCKET_VOICES, TTS_ENGINES, TTS_MAX_QUEUE, TTS_QUEUE_WAIT_S) from .devices import get_device @@ -24,6 +27,15 @@ _pocket_load_lock = threading.Lock() _tts_waiting = 0 +# serialises building Kokoro's non-English G2Ps, same idea as the +# pocket lock. a lang_code lands in _kokoro_g2p_failed when its G2P +# won't build (the misaki extra isn't installed, usually). that +# doesn't fix itself without a restart, so stop retrying and stop +# logging it on every chunk. +_kokoro_load_lock = threading.Lock() +_kokoro_g2p_failed = set() +_kokoro_scripts = {lang["id"]: re.compile(lang["script"]) for lang in KOKORO_LANGUAGES if lang["script"]} + def get_pipeline(): # loaded late so a failure at import (missing espeak-ng, @@ -37,6 +49,41 @@ def get_pipeline(): return state.pipeline +def kokoro_languages(): + # the KOKORO_LANGUAGES whose G2P modules are installed. find_spec + # only looks for them, it imports nothing, so /voices stays cheap. + return [lang for lang in KOKORO_LANGUAGES + if all(importlib.util.find_spec(m) is not None for m in lang["needs"])] + + +def get_kokoro_pipeline(lang): + # the pipeline for one KOKORO_LANGUAGES entry, or None when its + # G2P can't be built. never part of the pre-warm, so a missing + # extra can't slow down or break startup. + english = get_pipeline() + code = lang["lang_code"] + if code == english.lang_code: + return english + g2p = state.kokoro_g2p.get(code) + if g2p is not None or code in _kokoro_g2p_failed: + return g2p + with _kokoro_load_lock: + g2p = state.kokoro_g2p.get(code) + if g2p is None and code not in _kokoro_g2p_failed: + try: + from kokoro import KPipeline + log.info("loading Kokoro %s G2P (lang_code=%r), sharing the loaded model...", lang["label"], code) + # model= hands it the weights already loaded, only the + # G2P is new. no second copy in memory. + g2p = KPipeline(lang_code=code, repo_id="hexgrad/Kokoro-82M", model=english.model) + state.kokoro_g2p[code] = g2p + log.info("Kokoro %s ready.", lang["label"]) + except Exception: + _kokoro_g2p_failed.add(code) + log.exception("Kokoro %s unavailable, it gets the English G2P until restart", lang["label"]) + return g2p + + def get_pocket_model(language=POCKET_DEFAULT_LANG): # weights only land in HF_HOME once somebody picks pocket-tts. # one checkpoint at a time, so a language change reloads it and @@ -84,8 +131,9 @@ class TTSReq(BaseModel): # has no rate speed: float = Field(default=1.0, ge=0.5, le=2.0) engine: str = DEFAULT_ENGINE - # pocket-tts only. Kokoro ignores it, it speaks american english - # and that's that. + # a POCKET_LANGUAGES id for pocket-tts, a KOKORO_LANGUAGES id for + # Kokoro. each engine puts an id it doesn't know back to its + # default, both of which are english. lang: str = POCKET_DEFAULT_LANG @@ -99,7 +147,7 @@ class WarmReq(BaseModel): def voices(): return { "engines": { - "kokoro": {"voices": KOKORO_VOICES, "default": KOKORO_DEFAULT}, + "kokoro": _kokoro_voices_info(), "pockettts": { "voices": POCKET_VOICES, "default": POCKET_DEFAULT, @@ -111,15 +159,49 @@ def voices(): } -def synth_kokoro(text, voice, speed): - voice = voice if voice in KOKORO_VOICES else KOKORO_DEFAULT +def _kokoro_voices_info(): + info = {"voices": KOKORO_VOICES, "default": KOKORO_DEFAULT} + # one language is no choice, so no `languages` and the client + # keeps the row hidden, same as before any of this existed. + langs = kokoro_languages() + if len(langs) > 1: + info["languages"] = [{"id": lang["id"], "label": lang["label"], "voices": lang["voices"], + "default": lang["default"]} for lang in langs] + info["default_language"] = KOKORO_DEFAULT_LANG + return info + + +def _kokoro_chunks(pipeline, text, voice, speed): chunks = [] - for _gs, _ps, audio in get_pipeline()(text, voice=voice, speed=speed): + for _gs, _ps, audio in pipeline(text, voice=voice, speed=speed): if audio is None: continue if hasattr(audio, "detach"): audio = audio.detach().cpu().numpy() chunks.append(np.asarray(audio, dtype=np.float32)) + return chunks + + +def synth_kokoro(text, voice, speed, lang_id=KOKORO_DEFAULT_LANG): + lang = KOKORO_LANG_BY_ID.get(lang_id) or KOKORO_LANG_BY_ID[KOKORO_DEFAULT_LANG] + voice = voice if voice in lang["voices"] else lang["default"] + pipeline = None + script = _kokoro_scripts.get(lang["id"]) + # "OK!" in a Mandarin chat has nothing this G2P can read, it + # turns latin letters into noise. the English G2P in the same + # voice gives accented but real English. + if script is None or script.search(text): + pipeline = get_kokoro_pipeline(lang) + chunks = None + if pipeline is not None: + try: + chunks = _kokoro_chunks(pipeline, text, voice, speed) + except Exception: + if pipeline is state.pipeline: + raise + log.exception("Kokoro %s synth failed, retrying this chunk with the English G2P", lang["label"]) + if chunks is None: + chunks = _kokoro_chunks(get_pipeline(), text, voice, speed) if not chunks: return None return np.concatenate(chunks), KOKORO_SAMPLE_RATE @@ -171,7 +253,7 @@ def tts(req: TTSReq): if engine == "pockettts": result = synth_pocket(req.text, req.voice, req.lang) else: - result = synth_kokoro(req.text, req.voice, req.speed) + result = synth_kokoro(req.text, req.voice, req.speed, req.lang) finally: state.end_use() finally: @@ -193,7 +275,11 @@ def warm(req: WarmReq): # the client warms pocket-tts language weights and voice state # while Jun is still writing, so the multi second reload is # done before /tts asks. a language change is still a full - # reload. Kokoro has no language checkpoints, nothing to warm. + # reload. Kokoro has one model for every language, so its warm + # only builds a non-English G2P and fetches the voice, a second or + # so the first chunk doesn't have to wait for. + if req.engine == "kokoro": + return _warm_kokoro(req) if req.engine != "pockettts": return {"ok": True, "warmed": None} language = req.lang if req.lang in POCKET_LANG_IDS else POCKET_DEFAULT_LANG @@ -210,3 +296,28 @@ def warm(req: WarmReq): finally: state.tts_slots.release() return {"ok": True, "warmed": language} + + +def _warm_kokoro(req): + lang = KOKORO_LANG_BY_ID.get(req.lang) + if lang is None or lang["id"] == KOKORO_DEFAULT_LANG: + return {"ok": True, "warmed": None} + voice = req.voice if req.voice in lang["voices"] else lang["default"] + if not _acquire_tts_slot(): + return _tts_busy() + warmed = None + try: + state.begin_use("kokoro") + try: + pipeline = get_kokoro_pipeline(lang) + if pipeline is not None: + pipeline.load_voice(voice) + warmed = lang["id"] + except Exception: + # best effort, the first real chunk just loads it instead + log.exception("Kokoro warm for %s failed", lang["label"]) + finally: + state.end_use() + finally: + state.tts_slots.release() + return {"ok": True, "warmed": warmed} diff --git a/webapp/date.html b/webapp/date.html index fbec7ee3..e905de8d 100644 --- a/webapp/date.html +++ b/webapp/date.html @@ -48,6 +48,6 @@