diff --git a/.github/scripts/js/tts-text.test.mjs b/.github/scripts/js/tts-text.test.mjs new file mode 100644 index 00000000..f61ebea6 --- /dev/null +++ b/.github/scripts/js/tts-text.test.mjs @@ -0,0 +1,79 @@ +// how replies get cut into TTS chunks, and which Kokoro language +// each chunk is routed to. the cutting decides when the first audio +// plays and how big a chunk gets, the routing whether a chunk is +// read by the right G2P at all. nothing else in CI runs this code, +// node --check only parses it. +// +// run: node --test .github/scripts/js +import { register } from 'node:module'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +register('./dom-stub-hooks.mjs', import.meta.url); + +globalThis.window = globalThis; +const TTS = await import('../../../webapp/js/voice/tts.js'); +const { detectKokoroLang } = await import('../../../webapp/js/voice/kokoro-lang.js'); + +function cuts(text, first) { + const r = TTS.nextChunk(text, first); + return r ? r.chunk : null; +} + +test('English chunking is unchanged', () => { + assert.equal(cuts('Hello there. How are', false), 'Hello there.'); + assert.equal(cuts('Oh, hi', true), null); + assert.equal(cuts('Well, I think, maybe', true), 'Well, I think,'); + // a run of dots is her trailing off, never a cut + assert.equal(cuts('Wait... what', false), null); + assert.equal(cuts('No full stop yet', false), null); +}); + +test('fullwidth sentence ends cut a chunk', () => { + assert.equal(cuts('你好!今天天氣很好。', false), '你好!'); + assert.equal(cuts('今天天氣很好。我們', false), '今天天氣很好。'); + assert.equal(cuts('真的嗎?好', false), '真的嗎?'); + assert.equal(cuts('嗯~我想想', false), '嗯~'); +}); + +test('the first chunk soft-splits Chinese once a few hanzi are in', () => { + assert.equal(cuts('今天天氣很好,我們去散步吧', true), '今天天氣很好,'); + // two hanzi isn't enough, same as "Oh," in English + assert.equal(cuts('你好,', true), null); + assert.equal(cuts('我覺得;也許', true), '我覺得;'); +}); + +test('Kokoro routes by script', () => { + assert.equal(detectKokoroLang('你好,今天天氣很好。'), 'mandarin'); + assert.equal(detectKokoroLang('我是Jun,很高興認識你。'), 'mandarin'); + assert.equal(detectKokoroLang('Hello there.'), 'english'); + assert.equal(detectKokoroLang('OK!'), 'english'); + assert.equal(detectKokoroLang(''), 'english'); +}); + +test('Japanese stays off Mandarin, kanji or not', () => { + assert.equal(detectKokoroLang('今日はいい天気ですね。'), 'english'); + assert.equal(detectKokoroLang('カタカナ漢字'), 'english'); + assert.equal(detectKokoroLang('ありがとう'), 'english'); +}); + +test('predictLang names what Kokoro should warm', () => { + TTS.setEnabled(true); + TTS.setEngine('kokoro'); + TTS.setLang('auto'); + assert.equal(TTS.predictLang('你好嗎'), 'mandarin'); + assert.equal(TTS.predictLang('how are you'), null); + TTS.setLang('mandarin'); + assert.equal(TTS.predictLang('how are you'), 'mandarin'); + TTS.setLang('english'); + assert.equal(TTS.predictLang('你好嗎'), null); +}); + +test('pocket-tts prediction is untouched', () => { + TTS.setEngine('pockettts'); + TTS.setLang('auto'); + assert.equal(TTS.predictLang('what are you doing with the cat'), 'english'); + assert.equal(TTS.predictLang('je ne sais pas pour les chats'), 'french_24l'); + TTS.setLang('english'); + assert.equal(TTS.predictLang('je ne sais pas'), null); +}); diff --git a/docs/architecture.md b/docs/architecture.md index e60d05ee..ad2c77b1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -290,10 +290,12 @@ Tokens are appended to a buffer. When a sentence-ending character (`.`, `!`, `?` Each queued sentence triggers an immediate `fetch POST /api/tts.php`. The PHP endpoint validates the request (text ≤ 2000 chars, voice pattern, speed range, engine) and forwards it to the voice sidecar. Responses arrive out-of-order since synthesis time varies by sentence length. -The sidecar (`tts/server.py`, code under `tts/sidecar/`) fronts two engines chosen per request by the `engine` field: **kokoro** (Kokoro-82M, default, ~27 EN voices, needs espeak-ng) and **pockettts** (kyutai pocket-tts, ~100M, CPU-friendly, English + 5 languages). For pocket-tts the language is a separate axis from the voice: it's baked into the model weights, so the request's `lang` field (english, french_24l, german_24l, italian, portuguese, spanish_24l) selects a different checkpoint that actually pronounces that language and resolves the chosen voice to that language's embedding. Only one pocket-tts language stays resident; switching `lang` reloads (a full checkpoint load, a few seconds off HF_HOME-cached weights), which is the cost the auto-language path below is built to hide. `TTS_DEVICE` (`cpu`|`cuda`|`auto`) picks the torch device; Kokoro pre-warms one utterance at startup while pocket-tts loads lazily on its first request. `GET /voices` exposes both engines' voice lists and defaults, plus pocket-tts's selectable languages, so the UI can offer an engine + voice + language picker. +The sidecar (`tts/server.py`, code under `tts/sidecar/`) fronts two engines chosen per request by the `engine` field: **kokoro** (Kokoro-82M, default, 15 EN voices plus 4 Mandarin ones, needs espeak-ng) and **pockettts** (kyutai pocket-tts, ~100M, CPU-friendly, English + 5 languages). For pocket-tts the language is a separate axis from the voice: it's baked into the model weights, so the request's `lang` field (english, french_24l, german_24l, italian, portuguese, spanish_24l) selects a different checkpoint that actually pronounces that language and resolves the chosen voice to that language's embedding. Only one pocket-tts language stays resident; switching `lang` reloads (a full checkpoint load, a few seconds off HF_HOME-cached weights), which is the cost the auto-language path below is built to hide. `TTS_DEVICE` (`cpu`|`cuda`|`auto`) picks the torch device; Kokoro pre-warms one utterance at startup while pocket-tts loads lazily on its first request. `GET /voices` exposes both engines' voice lists and defaults, plus pocket-tts's selectable languages, so the UI can offer an engine + voice + language picker. The language picker also has an **Auto-detect** option (the default), which routes each reply to the right pocket-tts language while keeping reloads rare and off the critical path. On send, `js/voice/tts.js` runs a dependency-free stopword detector over **Anon's message** to predict the reply's language, falling back to the *previous* conversation language (not English) when the message is too short or ambiguous to call. That prediction drives two things in parallel: `app.js` fires `POST /api/tts.php?action=warm` → the sidecar's `/warm` (which preloads that language's checkpoint and voice state off-thread, so the reload overlaps LLM generation instead of stalling the first audio chunk), and the client locks the reply to the predicted language. As the reply streams, the detector *verifies* the prediction against the actual text - only a confident disagreement in the opening ~40 characters switches the language (warming the corrected model), and once locked it never flips again, so a stray foreign word mid-sentence can't trigger a mid-reply reload. The result: a monolingual conversation reloads zero times after the first turn, and a genuine language switch reloads once, hidden behind generation. Detection is purely client-side; the sidecar only ever sees a concrete language id, and `chat.php` gets no language hint at all - an earlier "reply in {language}" line in the live context was removed because the sticky fallback made it self-latching. Reply language is the model's to infer from the conversation. +Kokoro has one model for every language, so its languages are only G2Ps. `KOKORO_LANGUAGES` in `tts/sidecar/config.py` is the registry: an id (what the client sends as `lang`), the Kokoro `lang_code`, the voices made for it with a default, the modules its G2P imports, and optionally the script only it writes. English is the pipeline the pre-warm loads; any other language gets its `KPipeline` built on first use around that same `KModel` (`get_kokoro_pipeline()` in `tts/sidecar/tts.py`), so a second language costs a G2P, not a second copy of the weights. `GET /voices` only lists a language whose modules `importlib.util.find_spec` can find, and only adds `languages` to the kokoro entry when there are two or more, so without `misaki[zh]` the language row stays hidden exactly as before. A voice that isn't the language's own is swapped for its default; a chunk with none of a language's script (`OK!` under Mandarin) goes through the English G2P in the same voice, because Mandarin's would read the letters as noise; a G2P that fails to build is remembered and that language falls back to English until restart. Kokoro's **Auto-detect** is per chunk rather than per reply, since switching costs nothing: `js/voice/kokoro-lang.js` routes a chunk by its characters (hanzi without kana is Mandarin, anything else English) and `tts.js` sends it with the voice picked for that language. The settings keep one voice per language in `tts.voices`; under Auto the voice menu is greyed and each language speaks in its own. `predictLang()` names a non-English language from Anon's message so `/warm` can build its G2P and fetch the voice before the first chunk. The chunker in `tts.js` also cuts on fullwidth `。!?~` and soft-splits on `,;:`, counting each hanzi as a word, so a Chinese reply starts playing after its first sentence instead of arriving as one chunk. Adding a Kokoro language is an entry in `KOKORO_LANGUAGES`, its misaki extra pinned in `tts/requirements.txt`, and a rule in `kokoro-lang.js`. + Results are decoded into `AudioBuffer`s and inserted into a `Map` keyed by submission index. A playback cursor advances only when the buffer at the current index is ready, ensuring sentences always play in the order they were generated even if a later sentence finishes synthesis faster. ### Audio graph diff --git a/tts/requirements.txt b/tts/requirements.txt index 692094f3..ad4c1da2 100644 --- a/tts/requirements.txt +++ b/tts/requirements.txt @@ -7,6 +7,9 @@ # interpreter in that window on purpose, the docker image is # 3.11. kokoro==0.9.4 +# kokoro's own misaki pin. [zh] is the Mandarin G2P (jieba, pypinyin, +# cn2an). without it Mandarin just isn't offered, English is untouched. +misaki[zh]==0.9.4 # spacy (kokoro -> misaki) pulls this in unpinned. 3.0.14 kills # en_core_web_sm with OverflowError in _flush_specials, so no voice preshed==3.0.13 diff --git a/tts/sidecar/config.py b/tts/sidecar/config.py index 88262021..49f95576 100644 --- a/tts/sidecar/config.py +++ b/tts/sidecar/config.py @@ -11,6 +11,37 @@ "bf_alice", "bf_emma", "bf_isabella", "bf_lily", ] +# the languages Kokoro gets offered in. one entry each, and that's +# the whole registry: id is what the client sends as `lang` (the +# same field pocket-tts uses, tts.php already checks its shape), +# lang_code picks Kokoro's G2P, voices are the ones made for it. +# needs lists the modules that G2P imports. a language missing any +# of them is left out of /voices, so the menu never offers it. +# script, when set, is the characters only that language writes: +# a chunk without any goes through the English G2P instead, in the +# same voice, since this G2P would read latin letters as noise. +# +# adding a language: an entry here, its misaki extra pinned in +# requirements.txt, and a rule in js/voice/kokoro-lang.js so +# auto-detect can find it. female voices only, like the EN list +# above. the 4 Mandarin ones are all graded D upstream, pick by ear. +KOKORO_LANGUAGES = [ + { + "id": "english", "label": "English", "lang_code": "a", + "voices": KOKORO_VOICES, "default": KOKORO_DEFAULT, + "needs": (), "script": None, + }, + { + "id": "mandarin", "label": "Mandarin Chinese", "lang_code": "z", + "voices": ["zf_xiaobei", "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi"], + "default": "zf_xiaobei", + "needs": ("jieba", "pypinyin", "cn2an"), + "script": r"[\u3400-\u9fff\uf900-\ufaff]", + }, +] +KOKORO_DEFAULT_LANG = "english" +KOKORO_LANG_BY_ID = {lang["id"]: lang for lang in KOKORO_LANGUAGES} + # pocket-tts voice names pick timbre, not language. each has an # embedding for every POCKET_LANGUAGES entry. giovanni, lola, # juergen, rafael and estelle keep the accent of their non-English diff --git a/tts/sidecar/state.py b/tts/sidecar/state.py index 42abd478..c023133b 100644 --- a/tts/sidecar/state.py +++ b/tts/sidecar/state.py @@ -13,6 +13,9 @@ # them. the loaders in tts.py and karaoke.py fill them in. whisper # isn't here, it never gets unloaded. pipeline = None +# Kokoro's other languages, lang_code -> KPipeline. each one is only +# a G2P on top of pipeline's model, so they go when it goes. +kokoro_g2p = {} pocket_model = None pocket_lang = None pocket_states = {} @@ -36,7 +39,7 @@ def loaded_engines(): live = [] - if pipeline is not None: live.append("kokoro") + if pipeline is not None or kokoro_g2p: live.append("kokoro") if pocket_model is not None: live.append("pockettts") if separator is not None: live.append("demucs") return live @@ -55,11 +58,11 @@ def free_torch(): def unload(names): # caller holds lock. drops the model refs, then gc and # empty_cache hand the VRAM back. - global pipeline, pocket_model, pocket_lang, pocket_states, separator + global pipeline, kokoro_g2p, pocket_model, pocket_lang, pocket_states, separator freed = [] for name in names: - if name == "kokoro" and pipeline is not None: - pipeline = None; freed.append(name) + if name == "kokoro" and (pipeline is not None or kokoro_g2p): + pipeline = None; kokoro_g2p = {}; freed.append(name) elif name == "pockettts" and pocket_model is not None: pocket_model = None; pocket_lang = None; pocket_states = {}; freed.append(name) elif name == "demucs" and separator is not None: diff --git a/tts/sidecar/tts.py b/tts/sidecar/tts.py index 3f732a69..f6bf7ef8 100644 --- a/tts/sidecar/tts.py +++ b/tts/sidecar/tts.py @@ -1,4 +1,6 @@ +import importlib.util import logging +import re import threading from typing import Annotated @@ -9,7 +11,8 @@ from . import state from .audio import to_wav -from .config import (DEFAULT_ENGINE, KOKORO_DEFAULT, KOKORO_SAMPLE_RATE, KOKORO_VOICES, POCKET_DEFAULT, +from .config import (DEFAULT_ENGINE, KOKORO_DEFAULT, KOKORO_DEFAULT_LANG, KOKORO_LANG_BY_ID, KOKORO_LANGUAGES, + KOKORO_SAMPLE_RATE, KOKORO_VOICES, POCKET_DEFAULT, POCKET_DEFAULT_LANG, POCKET_LANG_IDS, POCKET_LANGUAGES, POCKET_VOICES, TTS_ENGINES, TTS_MAX_QUEUE, TTS_QUEUE_WAIT_S) from .devices import get_device @@ -24,6 +27,15 @@ _pocket_load_lock = threading.Lock() _tts_waiting = 0 +# serialises building Kokoro's non-English G2Ps, same idea as the +# pocket lock. a lang_code lands in _kokoro_g2p_failed when its G2P +# won't build (the misaki extra isn't installed, usually). that +# doesn't fix itself without a restart, so stop retrying and stop +# logging it on every chunk. +_kokoro_load_lock = threading.Lock() +_kokoro_g2p_failed = set() +_kokoro_scripts = {lang["id"]: re.compile(lang["script"]) for lang in KOKORO_LANGUAGES if lang["script"]} + def get_pipeline(): # loaded late so a failure at import (missing espeak-ng, @@ -37,6 +49,41 @@ def get_pipeline(): return state.pipeline +def kokoro_languages(): + # the KOKORO_LANGUAGES whose G2P modules are installed. find_spec + # only looks for them, it imports nothing, so /voices stays cheap. + return [lang for lang in KOKORO_LANGUAGES + if all(importlib.util.find_spec(m) is not None for m in lang["needs"])] + + +def get_kokoro_pipeline(lang): + # the pipeline for one KOKORO_LANGUAGES entry, or None when its + # G2P can't be built. never part of the pre-warm, so a missing + # extra can't slow down or break startup. + english = get_pipeline() + code = lang["lang_code"] + if code == english.lang_code: + return english + g2p = state.kokoro_g2p.get(code) + if g2p is not None or code in _kokoro_g2p_failed: + return g2p + with _kokoro_load_lock: + g2p = state.kokoro_g2p.get(code) + if g2p is None and code not in _kokoro_g2p_failed: + try: + from kokoro import KPipeline + log.info("loading Kokoro %s G2P (lang_code=%r), sharing the loaded model...", lang["label"], code) + # model= hands it the weights already loaded, only the + # G2P is new. no second copy in memory. + g2p = KPipeline(lang_code=code, repo_id="hexgrad/Kokoro-82M", model=english.model) + state.kokoro_g2p[code] = g2p + log.info("Kokoro %s ready.", lang["label"]) + except Exception: + _kokoro_g2p_failed.add(code) + log.exception("Kokoro %s unavailable, it gets the English G2P until restart", lang["label"]) + return g2p + + def get_pocket_model(language=POCKET_DEFAULT_LANG): # weights only land in HF_HOME once somebody picks pocket-tts. # one checkpoint at a time, so a language change reloads it and @@ -84,8 +131,9 @@ class TTSReq(BaseModel): # has no rate speed: float = Field(default=1.0, ge=0.5, le=2.0) engine: str = DEFAULT_ENGINE - # pocket-tts only. Kokoro ignores it, it speaks american english - # and that's that. + # a POCKET_LANGUAGES id for pocket-tts, a KOKORO_LANGUAGES id for + # Kokoro. each engine puts an id it doesn't know back to its + # default, both of which are english. lang: str = POCKET_DEFAULT_LANG @@ -99,7 +147,7 @@ class WarmReq(BaseModel): def voices(): return { "engines": { - "kokoro": {"voices": KOKORO_VOICES, "default": KOKORO_DEFAULT}, + "kokoro": _kokoro_voices_info(), "pockettts": { "voices": POCKET_VOICES, "default": POCKET_DEFAULT, @@ -111,15 +159,49 @@ def voices(): } -def synth_kokoro(text, voice, speed): - voice = voice if voice in KOKORO_VOICES else KOKORO_DEFAULT +def _kokoro_voices_info(): + info = {"voices": KOKORO_VOICES, "default": KOKORO_DEFAULT} + # one language is no choice, so no `languages` and the client + # keeps the row hidden, same as before any of this existed. + langs = kokoro_languages() + if len(langs) > 1: + info["languages"] = [{"id": lang["id"], "label": lang["label"], "voices": lang["voices"], + "default": lang["default"]} for lang in langs] + info["default_language"] = KOKORO_DEFAULT_LANG + return info + + +def _kokoro_chunks(pipeline, text, voice, speed): chunks = [] - for _gs, _ps, audio in get_pipeline()(text, voice=voice, speed=speed): + for _gs, _ps, audio in pipeline(text, voice=voice, speed=speed): if audio is None: continue if hasattr(audio, "detach"): audio = audio.detach().cpu().numpy() chunks.append(np.asarray(audio, dtype=np.float32)) + return chunks + + +def synth_kokoro(text, voice, speed, lang_id=KOKORO_DEFAULT_LANG): + lang = KOKORO_LANG_BY_ID.get(lang_id) or KOKORO_LANG_BY_ID[KOKORO_DEFAULT_LANG] + voice = voice if voice in lang["voices"] else lang["default"] + pipeline = None + script = _kokoro_scripts.get(lang["id"]) + # "OK!" in a Mandarin chat has nothing this G2P can read, it + # turns latin letters into noise. the English G2P in the same + # voice gives accented but real English. + if script is None or script.search(text): + pipeline = get_kokoro_pipeline(lang) + chunks = None + if pipeline is not None: + try: + chunks = _kokoro_chunks(pipeline, text, voice, speed) + except Exception: + if pipeline is state.pipeline: + raise + log.exception("Kokoro %s synth failed, retrying this chunk with the English G2P", lang["label"]) + if chunks is None: + chunks = _kokoro_chunks(get_pipeline(), text, voice, speed) if not chunks: return None return np.concatenate(chunks), KOKORO_SAMPLE_RATE @@ -171,7 +253,7 @@ def tts(req: TTSReq): if engine == "pockettts": result = synth_pocket(req.text, req.voice, req.lang) else: - result = synth_kokoro(req.text, req.voice, req.speed) + result = synth_kokoro(req.text, req.voice, req.speed, req.lang) finally: state.end_use() finally: @@ -193,7 +275,11 @@ def warm(req: WarmReq): # the client warms pocket-tts language weights and voice state # while Jun is still writing, so the multi second reload is # done before /tts asks. a language change is still a full - # reload. Kokoro has no language checkpoints, nothing to warm. + # reload. Kokoro has one model for every language, so its warm + # only builds a non-English G2P and fetches the voice, a second or + # so the first chunk doesn't have to wait for. + if req.engine == "kokoro": + return _warm_kokoro(req) if req.engine != "pockettts": return {"ok": True, "warmed": None} language = req.lang if req.lang in POCKET_LANG_IDS else POCKET_DEFAULT_LANG @@ -210,3 +296,28 @@ def warm(req: WarmReq): finally: state.tts_slots.release() return {"ok": True, "warmed": language} + + +def _warm_kokoro(req): + lang = KOKORO_LANG_BY_ID.get(req.lang) + if lang is None or lang["id"] == KOKORO_DEFAULT_LANG: + return {"ok": True, "warmed": None} + voice = req.voice if req.voice in lang["voices"] else lang["default"] + if not _acquire_tts_slot(): + return _tts_busy() + warmed = None + try: + state.begin_use("kokoro") + try: + pipeline = get_kokoro_pipeline(lang) + if pipeline is not None: + pipeline.load_voice(voice) + warmed = lang["id"] + except Exception: + # best effort, the first real chunk just loads it instead + log.exception("Kokoro warm for %s failed", lang["label"]) + finally: + state.end_use() + finally: + state.tts_slots.release() + return {"ok": True, "warmed": warmed} diff --git a/webapp/date.html b/webapp/date.html index fbec7ee3..e905de8d 100644 --- a/webapp/date.html +++ b/webapp/date.html @@ -48,6 +48,6 @@

General

+
Voice
-
Which voice to use.
+
Which voice to use. On Kokoro, each language keeps its own.
-
Speed
diff --git a/webapp/js/app.js b/webapp/js/app.js index 46fc6efd..39d05263 100644 --- a/webapp/js/app.js +++ b/webapp/js/app.js @@ -7,7 +7,7 @@ // imports. renumber the whole graph. and no example version // numbers in comments, a bulk renumber rewrites those too. -import { showAuthScreen } from './app/auth-screen.js?v=79'; +import { showAuthScreen } from './app/auth-screen.js?v=80'; import * as Auth from './core/auth.js?v=1'; import { loadScripts } from './core/loader.js?v=1'; // side effect only. it sets the phone-ui class and the viewport @@ -22,7 +22,7 @@ if (!me) { // main.js's module graph side by side. nothing in that graph // touches PIXI or marked at import, boot() is the first to const [{ boot }] = await Promise.all([ - import('./app/main.js?v=69'), + import('./app/main.js?v=70'), loadScripts([ ['vendor/pixi.min.js', 'vendor/live2dcubismcore.min.js', 'vendor/marked.umd.js', 'vendor/purify.min.js?v=5'], diff --git a/webapp/js/app/auth-screen.js b/webapp/js/app/auth-screen.js index 1cfd8d5d..da686ac2 100644 --- a/webapp/js/app/auth-screen.js +++ b/webapp/js/app/auth-screen.js @@ -1,4 +1,4 @@ -import * as Prefs from '../core/prefs.js?v=6'; +import * as Prefs from '../core/prefs.js?v=7'; import * as Auth from '../core/auth.js?v=1'; const authScreen = document.getElementById('authScreen'); diff --git a/webapp/js/app/cabinet.js b/webapp/js/app/cabinet.js index 315ef59d..200aa00e 100644 --- a/webapp/js/app/cabinet.js +++ b/webapp/js/app/cabinet.js @@ -2,9 +2,9 @@ import * as Feed from './consolidation-feed.js?v=3'; import * as Live2D from '../live2d/live2d.js?v=90'; -import * as VoiceMode from '../voice/voicemode.js?v=90'; +import * as VoiceMode from '../voice/voicemode.js?v=91'; import * as Achievements from '../achievements/achievements.js?v=8'; -import { showFaceBubble } from './face-bubble.js?v=91'; +import { showFaceBubble } from './face-bubble.js?v=92'; import { escapeHtml } from '../core/util.js?v=1'; const DRAWERS = [ diff --git a/webapp/js/app/cards.js b/webapp/js/app/cards.js index f88c277a..d6884375 100644 --- a/webapp/js/app/cards.js +++ b/webapp/js/app/cards.js @@ -10,7 +10,7 @@ import * as Names from '../core/names.js?v=1'; import * as Live2D from '../live2d/live2d.js?v=90'; import * as Scene from '../trip/scene.js?v=1'; -import * as VoiceMode from '../voice/voicemode.js?v=90'; +import * as VoiceMode from '../voice/voicemode.js?v=91'; import * as Achievements from '../achievements/achievements.js?v=8'; const SUITS = ['♠', '♥', '♦', '♣']; diff --git a/webapp/js/app/chat.js b/webapp/js/app/chat.js index dcc25d61..236d2277 100644 --- a/webapp/js/app/chat.js +++ b/webapp/js/app/chat.js @@ -1,25 +1,25 @@ -import { armIdleAfterReply, cancelActiveIdleNudge, cancelAutoReset, cancelIdleNudge, composerPlaceholder, consolidating, fleeActive, reportActivity, resetIdleNudge, scheduleAutoReset, setCancelActiveIdleNudge, setConsolidating, showConsolidatingBubble, startFleeLock, syncConsolidationStatus } from './consolidation.js?v=122'; +import { armIdleAfterReply, cancelActiveIdleNudge, cancelAutoReset, cancelIdleNudge, composerPlaceholder, consolidating, fleeActive, reportActivity, resetIdleNudge, scheduleAutoReset, setCancelActiveIdleNudge, setConsolidating, showConsolidatingBubble, startFleeLock, syncConsolidationStatus } from './consolidation.js?v=123'; import { chatInput, messagesEl, modelSelect, reasoningSelect, sendBtn, sendButtonIdleMarkup, sendButtonStopMarkup, thinkChk } from './dom.js?v=77'; -import { announceMobileReply, hideFaceBubble, scheduleFaceBubbleHide, setLatestAssistantReply, showFaceBubble } from './face-bubble.js?v=91'; +import { announceMobileReply, hideFaceBubble, scheduleFaceBubbleHide, setLatestAssistantReply, showFaceBubble } from './face-bubble.js?v=92'; import { appendDebugReply, appendDebugThinking, beginDebugTurn, logToolStatus, setDebugPrompt } from './logging.js?v=78'; import { loadMood } from './mood.js?v=91'; -import { refreshSidebar } from './sidebar.js?v=122'; -import { makeNameFilter, makeStreamBuffer } from './stream-filters.js?v=101'; +import { refreshSidebar } from './sidebar.js?v=123'; +import { makeNameFilter, makeStreamBuffer } from './stream-filters.js?v=102'; import { localTimeString, phoneMode } from '../core/util.js?v=1'; import * as Names from '../core/names.js?v=1'; import * as ui from '../core/ui.js?v=1'; import { apiJson } from '../core/api.js?v=1'; import * as ChatAPI from '../core/chat-api.js?v=7'; -import * as Mods from '../mods/mods.js?v=99'; -import * as Outfit from '../outfit/outfit.js?v=98'; -import * as Cards from './cards.js?v=34'; +import * as Mods from '../mods/mods.js?v=100'; +import * as Outfit from '../outfit/outfit.js?v=99'; +import * as Cards from './cards.js?v=35'; import * as Achievements from '../achievements/achievements.js?v=8'; import * as History from './history.js?v=2'; -import * as TTS from '../voice/tts.js?v=90'; -import * as Voice from '../voice/voice.js?v=91'; -import * as VoiceMode from '../voice/voicemode.js?v=90'; +import * as TTS from '../voice/tts.js?v=91'; +import * as Voice from '../voice/voice.js?v=92'; +import * as VoiceMode from '../voice/voicemode.js?v=91'; import { DevHud, abortFn, currentConversationId, messages, setAbortFn, setRenderVoiceDraft, setStopActiveStream, stopActiveStream } from './session.js?v=1'; -import { appendMsg, followLatest, renderMarkdown, updateEmptyState } from './messages.js?v=20'; +import { appendMsg, followLatest, renderMarkdown, updateEmptyState } from './messages.js?v=21'; import { setComposerText } from './composer.js?v=3'; const THUMBS_UP_ICON = ''; diff --git a/webapp/js/app/consolidation.js b/webapp/js/app/consolidation.js index 1623bf2b..0d660718 100644 --- a/webapp/js/app/consolidation.js +++ b/webapp/js/app/consolidation.js @@ -1,14 +1,14 @@ import { abortFn, currentConversationId } from './session.js?v=1'; -import { runChat } from './chat.js?v=50'; +import { runChat } from './chat.js?v=51'; import { chatInput, consolidationActions, consolidationBanner, consolidationSub, consolidationTitle, devNoIdleChk, fleeEtaEl, fleeOverlay, fleeReasonEl, interruptPassBtn, sendBtn, sitWithHerBtn, voiceChk } from './dom.js?v=77'; import * as Feed from './consolidation-feed.js?v=3'; -import * as Cabinet from './cabinet.js?v=12'; -import { showFaceBubble } from './face-bubble.js?v=91'; +import * as Cabinet from './cabinet.js?v=13'; +import { showFaceBubble } from './face-bubble.js?v=92'; import { logAction } from './logging.js?v=78'; import { escapeHtml, formatElapsed } from '../core/util.js?v=1'; import * as Names from '../core/names.js?v=1'; -import * as TTS from '../voice/tts.js?v=90'; -import * as Voice from '../voice/voice.js?v=91'; +import * as TTS from '../voice/tts.js?v=91'; +import * as Voice from '../voice/voice.js?v=92'; import * as Live2D from '../live2d/live2d.js?v=90'; const BUSY_LINES = [ diff --git a/webapp/js/app/devhud.js b/webapp/js/app/devhud.js index a7f6787d..02297f84 100644 --- a/webapp/js/app/devhud.js +++ b/webapp/js/app/devhud.js @@ -1,7 +1,7 @@ import { api } from '../core/api.js?v=1'; import { clamp } from '../core/util.js?v=1'; import * as Feed from './consolidation-feed.js?v=3'; -import { runPassNow } from './consolidation.js?v=122'; +import { runPassNow } from './consolidation.js?v=123'; const STORAGE_KEY = 'omega.devhud.v1'; const GEOM_KEY = 'omega.devhud.geom.v1'; diff --git a/webapp/js/app/face-bubble.js b/webapp/js/app/face-bubble.js index 1fca729d..a89c0912 100644 --- a/webapp/js/app/face-bubble.js +++ b/webapp/js/app/face-bubble.js @@ -1,7 +1,7 @@ import { mobileConversationTitle, mobileReplyStatus, sidebarBackground, stageEl } from './dom.js?v=77'; import { phoneMode, visualRect } from '../core/util.js?v=1'; import * as Names from '../core/names.js?v=1'; -import * as TTS from '../voice/tts.js?v=90'; +import * as TTS from '../voice/tts.js?v=91'; import * as Live2D from '../live2d/live2d.js?v=90'; let currentConversationTitle = 'New conversation'; diff --git a/webapp/js/app/main.js b/webapp/js/app/main.js index 1e62f8c6..7a074404 100644 --- a/webapp/js/app/main.js +++ b/webapp/js/app/main.js @@ -1,32 +1,32 @@ -import { IDLE_AFTER_REPLY_MS, TYPING_POLL_MS, consolidating, reportActivity, resetIdleNudge, runPassNow, scheduleIdleNudge, showConsolidatingBubble, syncConsolidationStatus } from './consolidation.js?v=122'; +import { IDLE_AFTER_REPLY_MS, TYPING_POLL_MS, consolidating, reportActivity, resetIdleNudge, runPassNow, scheduleIdleNudge, showConsolidatingBubble, syncConsolidationStatus } from './consolidation.js?v=123'; import { chatInput, devNoIdleChk, missingParamsEl, modelSelect, narrowSidebarQuery, reasoningSelect, sendBtn, siteVolumeInput, stageEl, thinkChk } from './dom.js?v=77'; -import { faceBubble, hideFaceBubble, latestAssistantReply, restartFaceBubbleHide, scheduleFaceBubblePosition, setLatestAssistantReply } from './face-bubble.js?v=91'; +import { faceBubble, hideFaceBubble, latestAssistantReply, restartFaceBubbleHide, scheduleFaceBubblePosition, setLatestAssistantReply } from './face-bubble.js?v=92'; import { logAction, logMissing, setStageStatus } from './logging.js?v=78'; import { loadMood } from './mood.js?v=91'; -import { applyRoleGates, setSiteVolume, syncThinkToggle, updateSiteVolumeLabel, wireNameSettings } from './settings.js?v=127'; -import { loadConversation, refreshSidebar, setSidebarOpen } from './sidebar.js?v=122'; +import { applyRoleGates, setSiteVolume, syncThinkToggle, updateSiteVolumeLabel, wireNameSettings } from './settings.js?v=128'; +import { loadConversation, refreshSidebar, setSidebarOpen } from './sidebar.js?v=123'; import { escapeHtml, mealNow } from '../core/util.js?v=1'; -import { wireTts } from './wire-tts.js?v=126'; -import { wireVoice } from './wire-voice.js?v=127'; -import { WELCOME_TIERS, fetchWelcome, playWelcome, previewWelcome } from './welcome.js?v=98'; +import { wireTts } from './wire-tts.js?v=127'; +import { wireVoice } from './wire-voice.js?v=128'; +import { WELCOME_TIERS, fetchWelcome, playWelcome, previewWelcome } from './welcome.js?v=99'; import * as Names from '../core/names.js?v=1'; import * as AwayNotify from './away-notify.js?v=5'; -import * as Prefs from '../core/prefs.js?v=6'; +import * as Prefs from '../core/prefs.js?v=7'; import * as ui from '../core/ui.js?v=1'; import * as ChatAPI from '../core/chat-api.js?v=7'; import * as MobileViewport from '../core/viewport.js?v=1'; -import * as Actions from '../live2d/actions.js?v=98'; +import * as Actions from '../live2d/actions.js?v=99'; import * as Live2D from '../live2d/live2d.js?v=90'; -import * as ModelTouch from '../live2d/touch.js?v=101'; -import * as Outfit from '../outfit/outfit.js?v=98'; -import { playIntro } from '../wardrobe/reactions.js?v=88'; +import * as ModelTouch from '../live2d/touch.js?v=102'; +import * as Outfit from '../outfit/outfit.js?v=99'; +import { playIntro } from '../wardrobe/reactions.js?v=89'; import * as BootFX from './boot-fx.js?v=2'; -import * as Cards from './cards.js?v=34'; +import * as Cards from './cards.js?v=35'; import * as History from './history.js?v=2'; import { startSkybox } from '../trip/skybox.js?v=2'; import * as TripLoader from '../trip/trip-loader.js?v=1'; import { DevHud, abortFn, currentConversationId, setDevHud } from './session.js?v=1'; -import { rerenderBubbleStream, sendMessage, sendTouchEvent } from './chat.js?v=50'; +import { rerenderBubbleStream, sendMessage, sendTouchEvent } from './chat.js?v=51'; import { setComposerText } from './composer.js?v=3'; import * as Achievements from '../achievements/achievements.js?v=8'; @@ -142,7 +142,7 @@ export async function boot(me) { .then(r => r.ok ? r.json() : null).catch(() => null); // devhud.js owns Ctrl+Shift+D, so only admins get that module - if (currentUser?.role === 'admin') setDevHud(await import('./devhud.js?v=103')); + if (currentUser?.role === 'admin') setDevHud(await import('./devhud.js?v=104')); startSkybox(); // marked is a global and app.js's loadScripts only brings it in // side by side with this module. at module scope it's not there diff --git a/webapp/js/app/messages.js b/webapp/js/app/messages.js index b22c4f46..3813999e 100644 --- a/webapp/js/app/messages.js +++ b/webapp/js/app/messages.js @@ -1,5 +1,5 @@ import { messagesEl, messagesEmpty } from './dom.js?v=77'; -import { hideFaceBubble } from './face-bubble.js?v=91'; +import { hideFaceBubble } from './face-bubble.js?v=92'; import { escapeHtml } from '../core/util.js?v=1'; import { commandLength } from './composer.js?v=3'; diff --git a/webapp/js/app/settings.js b/webapp/js/app/settings.js index 8294858a..2369436d 100644 --- a/webapp/js/app/settings.js +++ b/webapp/js/app/settings.js @@ -1,15 +1,15 @@ import { abortFn, currentConversationId } from './session.js?v=1'; -import { sendMessage } from './chat.js?v=50'; -import { cancelIdleNudge } from './consolidation.js?v=122'; +import { sendMessage } from './chat.js?v=51'; +import { cancelIdleNudge } from './consolidation.js?v=123'; import { closeSettingsBtn, devNoIdleChk, drawerBackdrop, modelSelect, openSettingsBtn, reasoningSelect, sendBtn, siteVolumeInput, thinkChk, ttsChk, ttsSpeedInput, voiceChk, voiceSilenceInput } from './dom.js?v=77'; import { logAction } from './logging.js?v=78'; import { loadMood, setMoodEditingEnabled } from './mood.js?v=91'; -import { loadConversation, setSidebarOpen } from './sidebar.js?v=122'; +import { loadConversation, setSidebarOpen } from './sidebar.js?v=123'; import { loadTimeline } from './timeline.js?v=3'; import * as Names from '../core/names.js?v=1'; -import * as Prefs from '../core/prefs.js?v=6'; +import * as Prefs from '../core/prefs.js?v=7'; import * as ui from '../core/ui.js?v=1'; -import * as TTS from '../voice/tts.js?v=90'; +import * as TTS from '../voice/tts.js?v=91'; export function syncThinkToggle() { thinkChk.disabled = reasoningSelect.value === 'auto'; diff --git a/webapp/js/app/sidebar.js b/webapp/js/app/sidebar.js index 4d9e4b80..19b2c0cd 100644 --- a/webapp/js/app/sidebar.js +++ b/webapp/js/app/sidebar.js @@ -1,18 +1,18 @@ import { currentConversationId, messages, setCurrentConversationId, stopActiveStream } from './session.js?v=1'; -import { appendMsg, renderMarkdown, scrollToLatest, updateEmptyState } from './messages.js?v=20'; +import { appendMsg, renderMarkdown, scrollToLatest, updateEmptyState } from './messages.js?v=21'; import { dropDraft, useDraftFor } from './composer.js?v=3'; -import { addReplyActions, discardActiveResponse } from './chat.js?v=50'; -import { setConversationTitle } from './face-bubble.js?v=91'; -import { announceMobileReply, faceBubble, hideFaceBubble, latestAssistantReply, scheduleFaceBubbleHide, setLatestAssistantReply, showFaceBubble } from './face-bubble.js?v=91'; -import { IDLE_AFTER_JOIN_MS, cancelAutoReset, reportActivity, resetIdleNudge, scheduleIdleNudge } from './consolidation.js?v=122'; +import { addReplyActions, discardActiveResponse } from './chat.js?v=51'; +import { setConversationTitle } from './face-bubble.js?v=92'; +import { announceMobileReply, faceBubble, hideFaceBubble, latestAssistantReply, scheduleFaceBubbleHide, setLatestAssistantReply, showFaceBubble } from './face-bubble.js?v=92'; +import { IDLE_AFTER_JOIN_MS, cancelAutoReset, reportActivity, resetIdleNudge, scheduleIdleNudge } from './consolidation.js?v=123'; import { chatInput, conversationSidebar, messagesEl, mobileConversationTitle, mobileMenuBtn, narrowSidebarQuery, reloadPromptBtn, resetLive2DBtn, sidebarBackdrop, sidebarBackground } from './dom.js?v=77'; import { logAction } from './logging.js?v=78'; -import { makeStreamBuffer } from './stream-filters.js?v=101'; +import { makeStreamBuffer } from './stream-filters.js?v=102'; import { escapeHtml, phoneMode } from '../core/util.js?v=1'; import * as Names from '../core/names.js?v=1'; import * as ui from '../core/ui.js?v=1'; import * as History from './history.js?v=2'; -import * as TTS from '../voice/tts.js?v=90'; +import * as TTS from '../voice/tts.js?v=91'; import * as Live2D from '../live2d/live2d.js?v=90'; const conversationTitles = new Map(); diff --git a/webapp/js/app/stream-filters.js b/webapp/js/app/stream-filters.js index e35e772e..d692ccc1 100644 --- a/webapp/js/app/stream-filters.js +++ b/webapp/js/app/stream-filters.js @@ -1,7 +1,7 @@ import { logAction } from './logging.js?v=78'; import { noteEmotionTint } from './mood.js?v=91'; import * as Names from '../core/names.js?v=1'; -import * as Actions from '../live2d/actions.js?v=98'; +import * as Actions from '../live2d/actions.js?v=99'; const MARK_RE = /\[\s*(?:A(?:CTIONS?)?|TOOL)\s*:/i; const PARTIAL_RE = /\[\s*(?:A(?:C(?:T(?:I(?:O(?:N(?:S)?)?)?)?)?)?|T(?:O(?:O(?:L)?)?)?)?\s*$/i; diff --git a/webapp/js/app/welcome.js b/webapp/js/app/welcome.js index b093b2f7..7060bac1 100644 --- a/webapp/js/app/welcome.js +++ b/webapp/js/app/welcome.js @@ -1,11 +1,11 @@ import { chatInput, sendBtn } from './dom.js?v=77'; -import { replayFaceBubbleIntro, scheduleFaceBubbleHide, showFaceBubble } from './face-bubble.js?v=91'; +import { replayFaceBubbleIntro, scheduleFaceBubbleHide, showFaceBubble } from './face-bubble.js?v=92'; import { loadMood } from './mood.js?v=91'; import { escapeHtml } from '../core/util.js?v=1'; import * as Names from '../core/names.js?v=1'; import { inject } from '../trip/scene.js?v=1'; -import * as TTS from '../voice/tts.js?v=90'; -import * as VoiceMode from '../voice/voicemode.js?v=90'; +import * as TTS from '../voice/tts.js?v=91'; +import * as VoiceMode from '../voice/voicemode.js?v=91'; import * as Live2D from '../live2d/live2d.js?v=90'; // the phone's slide in welcome.css is .6s after a .15s delay diff --git a/webapp/js/app/wire-tts.js b/webapp/js/app/wire-tts.js index b7ebf724..c26aa958 100644 --- a/webapp/js/app/wire-tts.js +++ b/webapp/js/app/wire-tts.js @@ -1,12 +1,13 @@ -import { IDLE_AFTER_REPLY_MS, scheduleIdleNudge } from './consolidation.js?v=122'; +import { IDLE_AFTER_REPLY_MS, scheduleIdleNudge } from './consolidation.js?v=123'; import { ttsChk, ttsEngineSelect, ttsLangRow, ttsLangSelect, ttsSpeedInput, ttsVoiceSelect } from './dom.js?v=77'; -import { finishPendingFaceBubbleHide } from './face-bubble.js?v=91'; +import { finishPendingFaceBubbleHide } from './face-bubble.js?v=92'; import { logAction } from './logging.js?v=78'; -import { syncVoiceDeps, updateTtsSpeedLabel } from './settings.js?v=127'; +import { syncVoiceDeps, updateTtsSpeedLabel } from './settings.js?v=128'; import { escapeHtml } from '../core/util.js?v=1'; import * as Names from '../core/names.js?v=1'; -import * as Prefs from '../core/prefs.js?v=6'; -import * as TTS from '../voice/tts.js?v=90'; +import * as Prefs from '../core/prefs.js?v=7'; +import * as TTS from '../voice/tts.js?v=91'; +import { PREVIEW_LINES } from '../voice/kokoro-lang.js?v=1'; export async function wireTts() { TTS.setLogger(logAction); @@ -19,16 +20,56 @@ export async function wireTts() { const savedLang = localStorage.getItem('tts.lang') || ''; const savedSpeed = parseFloat(localStorage.getItem('tts.speed') || '1.0'); const savedEngine = localStorage.getItem('tts.engine') || 'kokoro'; + // Kokoro's voice per language, {english: 'af_heart', ...}. one + // string pref like the rest, so prefs.js syncs it as is. + let langVoices = {}; + try { langVoices = JSON.parse(localStorage.getItem('tts.voices') || '{}') || {}; } catch { langVoices = {}; } TTS.setSpeed(savedSpeed); if (ttsSpeedInput) ttsSpeedInput.value = String(savedSpeed); updateTtsSpeedLabel(); let engines = {}; - function populateVoices(engineKey, preferred) { + // a language that brings its own voice list (Kokoro does, + // pocket-tts voices fit every language) or null + function langWithVoices(engineKey, langId) { + const langs = (engines[engineKey] && engines[engineKey].languages) || []; + return langs.find(l => l.id === langId && Array.isArray(l.voices)) || null; + } + function perLangVoices(engineKey) { + const langs = (engines[engineKey] && engines[engineKey].languages) || []; + return langs.some(l => Array.isArray(l.voices)); + } + // every language gets a voice that's really one of its own, the + // saved one if it still is, otherwise the language's default + function syncLangVoices(engineKey) { + if (!perLangVoices(engineKey)) { TTS.setLangVoices({}); return; } + for (const l of engines[engineKey].languages) { + if (!l.voices.includes(langVoices[l.id])) langVoices[l.id] = l.default || l.voices[0] || ''; + } + TTS.setLangVoices(langVoices); + } + + function populateVoices(engineKey, preferred, langId) { const info = engines[engineKey] || { voices: [], default: '' }; - const voices = info.voices || []; + // auto-detect with per-language voices: each language speaks in + // the voice picked for it, so there's no single one to show. + // pick a language to change its voice. + if (langId === 'auto' && perLangVoices(engineKey)) { + if (ttsVoiceSelect) { + ttsVoiceSelect.innerHTML = ''; + ttsVoiceSelect.disabled = true; + } + const def = langVoices[info.default_language] || info.default || ''; + TTS.setVoice(def); + return def; + } + if (ttsVoiceSelect) ttsVoiceSelect.disabled = false; + const lang = langWithVoices(engineKey, langId); + const voices = (lang ? lang.voices : info.voices) || []; + const fallback = lang ? lang.default : info.default; + if (lang) preferred = langVoices[lang.id] || preferred; const def = (preferred && voices.includes(preferred)) ? preferred - : (voices.includes(info.default) ? info.default : (voices[0] || '')); + : (voices.includes(fallback) ? fallback : (voices[0] || '')); if (ttsVoiceSelect) { ttsVoiceSelect.innerHTML = voices.map(name => ``).join(''); @@ -38,19 +79,25 @@ export async function wireTts() { return def; } - // language is a pocket-tts thing ONLY. the row stays hidden for engines - // that don't hand us a `languages` list. + // the row stays hidden for engines that don't hand us a + // `languages` list (Kokoro without a second language installed) function populateLanguages(engineKey, preferred) { const info = engines[engineKey] || {}; const baseLangs = info.languages || []; if (ttsLangRow) ttsLangRow.hidden = baseLangs.length === 0; - if (!baseLangs.length) { TTS.setLang(''); return ''; } + // no choice means the engine's own default, not whatever the + // last engine left behind + if (!baseLangs.length) { TTS.setLang('english'); return ''; } // 'auto' is a fake language that only exists in the client. - // TTS works out each reply's language and sends a real id. it's - // the default so this just works with zero setup. + // TTS works out the language from the text and sends a real id. + // pocket-tts defaults to it so it just works with zero setup. + // Kokoro starts on its default language, English, so nothing + // changes for anyone who never opens this menu: under auto its + // voice menu is per language and would start out greyed. const langs = [{ id: 'auto', label: 'Auto-detect' }, ...baseLangs]; const ids = langs.map(l => l.id); - const def = (preferred && ids.includes(preferred)) ? preferred : 'auto'; + const fresh = perLangVoices(engineKey) && info.default_language ? info.default_language : 'auto'; + const def = (preferred && ids.includes(preferred)) ? preferred : fresh; if (ttsLangSelect) { ttsLangSelect.innerHTML = langs.map(l => ``).join(''); @@ -72,8 +119,9 @@ export async function wireTts() { : (engines[v.default_engine] ? v.default_engine : Object.keys(engines)[0] || 'kokoro'); TTS.setEngine(engineKey); if (ttsEngineSelect) ttsEngineSelect.value = engineKey; - const def = populateVoices(engineKey, savedVoice); - populateLanguages(engineKey, savedLang); + syncLangVoices(engineKey); + const langDef = populateLanguages(engineKey, savedLang); + const def = populateVoices(engineKey, savedVoice, langDef); const count = (engines[engineKey] && engines[engineKey].voices || []).length; if (count) logAction('ok', `TTS ready: ${engineKey}, ${count} voices (default ${def})`); } catch (e) { @@ -85,10 +133,11 @@ export async function wireTts() { const engineKey = ttsEngineSelect.value; TTS.setEngine(engineKey); localStorage.setItem('tts.engine', engineKey); - const def = populateVoices(engineKey, ''); - localStorage.setItem('tts.voice', def); + syncLangVoices(engineKey); const langDef = populateLanguages(engineKey, localStorage.getItem('tts.lang') || ''); if (langDef) localStorage.setItem('tts.lang', langDef); + const def = populateVoices(engineKey, '', langDef); + localStorage.setItem('tts.voice', def); Prefs.pushToServer(); }); } @@ -107,8 +156,16 @@ export async function wireTts() { } if (ttsVoiceSelect) { ttsVoiceSelect.addEventListener('change', () => { + if (!ttsVoiceSelect.value) return; TTS.setVoice(ttsVoiceSelect.value); localStorage.setItem('tts.voice', ttsVoiceSelect.value); + const engineKey = ttsEngineSelect ? ttsEngineSelect.value : ''; + const lang = langWithVoices(engineKey, ttsLangSelect ? ttsLangSelect.value : ''); + if (lang) { + langVoices[lang.id] = ttsVoiceSelect.value; + TTS.setLangVoices(langVoices); + localStorage.setItem('tts.voices', JSON.stringify(langVoices)); + } Prefs.pushToServer(); }); } @@ -121,7 +178,12 @@ export async function wireTts() { ttsChk?.addEventListener('change', () => { if (!ttsChk.checked) settle(); }); previewBtn.addEventListener('click', () => { if (previewing && TTS.isSpeaking()) { TTS.stop(); settle(); return; } - const ok = TTS.speak(`Hi ${Names.getPlayer()}, do you like this voice?`, { + // in the voice's own language, or the sidecar gives an English + // line to the English G2P and you'd hear accented English + const engineKey = ttsEngineSelect ? ttsEngineSelect.value : ''; + const lang = langWithVoices(engineKey, ttsLangSelect ? ttsLangSelect.value : ''); + const line = (lang && PREVIEW_LINES[lang.id]) || `Hi ${Names.getPlayer()}, do you like this voice?`; + const ok = TTS.speak(line, { onDone: settle, onError: settle, }); @@ -134,6 +196,12 @@ export async function wireTts() { ttsLangSelect.addEventListener('change', () => { TTS.setLang(ttsLangSelect.value); localStorage.setItem('tts.lang', ttsLangSelect.value); + // Kokoro's voice menu follows the language + const engineKey = ttsEngineSelect ? ttsEngineSelect.value : ''; + if (perLangVoices(engineKey)) { + const def = populateVoices(engineKey, '', ttsLangSelect.value); + localStorage.setItem('tts.voice', def); + } Prefs.pushToServer(); }); } diff --git a/webapp/js/app/wire-voice.js b/webapp/js/app/wire-voice.js index f4cdf7c5..0038e7ca 100644 --- a/webapp/js/app/wire-voice.js +++ b/webapp/js/app/wire-voice.js @@ -1,13 +1,13 @@ import { renderVoiceDraft, stopActiveStream } from './session.js?v=1'; -import { sendAudioFromVoice, sendFromVoice } from './chat.js?v=50'; +import { sendAudioFromVoice, sendFromVoice } from './chat.js?v=51'; import { voiceBargeChk, voiceChk, voiceHearAllChk, voiceSilenceInput, voiceState } from './dom.js?v=77'; -import { hideFaceBubble } from './face-bubble.js?v=91'; +import { hideFaceBubble } from './face-bubble.js?v=92'; import { logAction } from './logging.js?v=78'; -import { syncVoiceDeps, updateVoiceSilenceLabel } from './settings.js?v=127'; -import * as Prefs from '../core/prefs.js?v=6'; +import { syncVoiceDeps, updateVoiceSilenceLabel } from './settings.js?v=128'; +import * as Prefs from '../core/prefs.js?v=7'; import * as ui from '../core/ui.js?v=1'; -import * as Voice from '../voice/voice.js?v=91'; -import * as VoiceMode from '../voice/voicemode.js?v=90'; +import * as Voice from '../voice/voice.js?v=92'; +import * as VoiceMode from '../voice/voicemode.js?v=91'; const VOICE_STATE_LABELS = { idle: 'off', diff --git a/webapp/js/core/prefs.js b/webapp/js/core/prefs.js index 863c9e36..ab3d70ef 100644 --- a/webapp/js/core/prefs.js +++ b/webapp/js/core/prefs.js @@ -10,6 +10,7 @@ export const TRACKED = [ 'tts.enabled', 'tts.engine', 'tts.voice', + 'tts.voices', 'tts.lang', 'tts.speed', 'audio.volume', diff --git a/webapp/js/core/speech-card.js b/webapp/js/core/speech-card.js index 4e85d396..05060d1d 100644 --- a/webapp/js/core/speech-card.js +++ b/webapp/js/core/speech-card.js @@ -3,7 +3,7 @@ import { apiGet } from './api.js?v=1'; import { gaugeTier } from './mood-tier.js?v=2'; import * as Names from './names.js?v=1'; import * as MobileViewport from './viewport.js?v=1'; -import * as TTS from '../voice/tts.js?v=90'; +import * as TTS from '../voice/tts.js?v=91'; export let affection = 0; export let trust = 0; @@ -52,6 +52,12 @@ export function configureTts() { TTS.setEnabled(localStorage.getItem('tts.enabled') === '1'); TTS.setEngine(localStorage.getItem('tts.engine') || 'kokoro'); TTS.setVoice(localStorage.getItem('tts.voice') || 'af_heart'); + // Kokoro's language and per-language voices, as the settings left + // them. pocket-tts keeps its plain English here, as before. + if ((localStorage.getItem('tts.engine') || 'kokoro') === 'kokoro') { + TTS.setLang(localStorage.getItem('tts.lang') || 'english'); + try { TTS.setLangVoices(JSON.parse(localStorage.getItem('tts.voices') || '{}')); } catch { TTS.setLangVoices({}); } + } TTS.setSpeed(parseFloat(localStorage.getItem('tts.speed') || '1') || 1); return TTS.isEnabled(); } diff --git a/webapp/js/date.js b/webapp/js/date.js index 124bb213..fd78588f 100644 --- a/webapp/js/date.js +++ b/webapp/js/date.js @@ -13,11 +13,11 @@ import * as Auth from './core/auth.js?v=1'; import { api, apiJson } from './core/api.js?v=1'; import * as ChatAPI from './core/chat-api.js?v=7'; import * as Names from './core/names.js?v=1'; -import * as Prefs from './core/prefs.js?v=6'; -import { say } from './core/speech-card.js?v=18'; +import * as Prefs from './core/prefs.js?v=7'; +import { say } from './core/speech-card.js?v=19'; import { localTimeString, mealNow } from './core/util.js?v=1'; import * as Live2D from './live2d/live2d.js?v=90'; -import * as Outfit from './outfit/outfit.js?v=98'; +import * as Outfit from './outfit/outfit.js?v=99'; import * as Scene from './trip/scene.js?v=1'; import { startSkybox } from './trip/skybox.js?v=2'; import * as TripLoader from './trip/trip-loader.js?v=1'; diff --git a/webapp/js/karaoke-page.js b/webapp/js/karaoke-page.js index 182e2604..76e45d9c 100644 --- a/webapp/js/karaoke-page.js +++ b/webapp/js/karaoke-page.js @@ -1,9 +1,9 @@ import * as Auth from './core/auth.js?v=1'; import { api } from './core/api.js?v=1'; -import * as Prefs from './core/prefs.js?v=6'; +import * as Prefs from './core/prefs.js?v=7'; import * as Karaoke from './karaoke/karaoke.js?v=27'; import * as Live2D from './live2d/live2d.js?v=90'; -import * as Outfit from './outfit/outfit.js?v=98'; +import * as Outfit from './outfit/outfit.js?v=99'; import * as Scene from './trip/scene.js?v=1'; import { startSkybox } from './trip/skybox.js?v=2'; import * as TripLoader from './trip/trip-loader.js?v=1'; diff --git a/webapp/js/live2d/actions.js b/webapp/js/live2d/actions.js index d940b0d3..c9cb52d2 100644 --- a/webapp/js/live2d/actions.js +++ b/webapp/js/live2d/actions.js @@ -1,5 +1,5 @@ import * as Live2D from './live2d.js?v=90'; -import * as Outfit from '../outfit/outfit.js?v=98'; +import * as Outfit from '../outfit/outfit.js?v=99'; // Jun still fires off old [ACTION:...] tags sometimes, // so both forms stay. forever probably. diff --git a/webapp/js/live2d/touch.js b/webapp/js/live2d/touch.js index e621f9c7..f71d245f 100644 --- a/webapp/js/live2d/touch.js +++ b/webapp/js/live2d/touch.js @@ -1,4 +1,4 @@ -import * as Actions from './actions.js?v=98'; +import * as Actions from './actions.js?v=99'; import * as Live2D from './live2d.js?v=90'; let sendEvent = null; diff --git a/webapp/js/mods/engine.js b/webapp/js/mods/engine.js index d3497608..96a1ff85 100644 --- a/webapp/js/mods/engine.js +++ b/webapp/js/mods/engine.js @@ -1,8 +1,8 @@ import * as Live2D from '../live2d/live2d.js?v=90'; -import * as Prefs from '../core/prefs.js?v=6'; +import * as Prefs from '../core/prefs.js?v=7'; import { idbStore } from '../core/idb.js?v=1'; import { hexToRgb01 } from '../outfit/colors.js?v=1'; -import { applyDrawOrder, applyFootCover, hiddenItemDrawables, refreshVisibility, setModdedDrawables } from '../outfit/apply.js?v=28'; +import { applyDrawOrder, applyFootCover, hiddenItemDrawables, refreshVisibility, setModdedDrawables } from '../outfit/apply.js?v=29'; import { unzip } from './zip.js?v=1'; import { itemDrawables, parseMod } from './parse.js?v=18'; import { MOD_SLOT, bakeDrawable, fileUrl } from './layers.js?v=18'; diff --git a/webapp/js/mods/mods.js b/webapp/js/mods/mods.js index 240f92c9..7ed392e2 100644 --- a/webapp/js/mods/mods.js +++ b/webapp/js/mods/mods.js @@ -1 +1 @@ -export { applyAll, describe, holds, setDecalHooks, itemNames, owns, refreshTints, wearByName } from './engine.js?v=28'; +export { applyAll, describe, holds, setDecalHooks, itemNames, owns, refreshTints, wearByName } from './engine.js?v=29'; diff --git a/webapp/js/mods/section.js b/webapp/js/mods/section.js index 00e2c836..f56f9e65 100644 --- a/webapp/js/mods/section.js +++ b/webapp/js/mods/section.js @@ -1,8 +1,8 @@ -import { makeItemColorButton } from '../wardrobe/color-picker.js?v=31'; -import { bindTileDrag } from '../wardrobe/tiles.js?v=31'; -import { modItemKey } from '../wardrobe/worn.js?v=29'; +import { makeItemColorButton } from '../wardrobe/color-picker.js?v=33'; +import { bindTileDrag } from '../wardrobe/tiles.js?v=33'; +import { modItemKey } from '../wardrobe/worn.js?v=30'; import { itemThumbUrl } from './layers.js?v=18'; -import { ensureLoaded, followsHerColors, importZip, isEquipped, modState, mods, removeMod, setColor, setEquipped, setFollowsHerColors } from './engine.js?v=28'; +import { ensureLoaded, followsHerColors, importZip, isEquipped, modState, mods, removeMod, setColor, setEquipped, setFollowsHerColors } from './engine.js?v=29'; let uiBody = null; @@ -23,23 +23,39 @@ export function buildWardrobeSection(body) { bar.style.cssText = 'display:flex;gap:8px;align-items:center;margin:4px 0 8px'; bar.innerHTML = ` Mods stay in your browser - nothing is uploaded - `; + `; const input = bar.querySelector('input'); const msg = bar.querySelector('[data-mod-msg]'); bar.querySelector('[data-mod-load]').addEventListener('click', () => input.click()); input.addEventListener('change', async () => { - const f = input.files[0]; + // copy first: clearing the value also empties the live FileList + const files = [...input.files]; input.value = ''; - if (!f) return; - msg.textContent = 'Loading…'; - try { - const mod = await importZip(await f.arrayBuffer()); - msg.textContent = `Loaded "${mod.name}" - click an item to equip it`; - renderMods(); - } catch (e) { - console.error(e); - msg.textContent = 'Failed: ' + e.message; + if (!files.length) return; + const loaded = [], failed = []; + for (const [n, f] of files.entries()) { + msg.textContent = files.length > 1 ? `Loading ${n + 1}/${files.length}…` : 'Loading…'; + // one at a time: importZip rewrites the shared mod list and state, + // and one bad zip must not stop the rest + try { + const mod = await importZip(await f.arrayBuffer()); + loaded.push(mod.name); + renderMods(); + } catch (e) { + console.error(e); + failed.push({ file: f.name, error: e.message }); + } + } + if (files.length === 1) { + msg.textContent = loaded.length + ? `Loaded "${loaded[0]}" - click an item to equip it` + : 'Failed: ' + failed[0].error; + return; } + const parts = []; + if (loaded.length) parts.push(`Loaded ${loaded.length} mod${loaded.length > 1 ? 's' : ''} - click an item to equip it`); + if (failed.length) parts.push('Failed: ' + failed.map(x => `${x.file} (${x.error})`).join(', ')); + msg.textContent = parts.join('. '); }); const list = document.createElement('div'); list.dataset.modList = '1'; diff --git a/webapp/js/outfit/apply.js b/webapp/js/outfit/apply.js index 94bd4a16..5799fec5 100644 --- a/webapp/js/outfit/apply.js +++ b/webapp/js/outfit/apply.js @@ -1,13 +1,13 @@ import * as Live2D from '../live2d/live2d.js?v=90'; -import * as Mods from '../mods/mods.js?v=99'; +import * as Mods from '../mods/mods.js?v=100'; import { ALWAYS_HIDDEN, COLOR_GROUPS, ITEMS, VARIANTS, VARIANT_OWNER } from './catalog.js?v=4'; -import { applyGlassesTexture, applyStockingTexture, cancelStockingTexture, stockingColorMode } from './composite.js?v=28'; +import { applyGlassesTexture, applyStockingTexture, cancelStockingTexture, stockingColorMode } from './composite.js?v=29'; import { colors, state, variantState } from './current.js?v=4'; import { hexToRgb01, normalizeHex } from './colors.js?v=1'; -import { queueWardrobe, saveColors, textureAvailable } from './sync.js?v=28'; -import { snapshot } from './chat-tools.js?v=28'; +import { queueWardrobe, saveColors, textureAvailable } from './sync.js?v=29'; +import { snapshot } from './chat-tools.js?v=29'; import { hooks } from './hooks.js?v=1'; -import { applyDecals, forgetTints } from './decals.js?v=5'; +import { applyDecals, forgetTints } from './decals.js?v=6'; export const itemPatterns = (it) => it.colorPatterns || it.visibilityPatterns || []; diff --git a/webapp/js/outfit/chat-tools.js b/webapp/js/outfit/chat-tools.js index 8df4e106..3405f8a9 100644 --- a/webapp/js/outfit/chat-tools.js +++ b/webapp/js/outfit/chat-tools.js @@ -1,9 +1,9 @@ import { ITEMS, VARIANTS } from './catalog.js?v=4'; -import { Presets, applyPreset } from './presets.js?v=28'; -import { queueWardrobe } from './sync.js?v=28'; -import { setDraftItem } from './apply.js?v=28'; +import { Presets, applyPreset } from './presets.js?v=29'; +import { queueWardrobe } from './sync.js?v=29'; +import { setDraftItem } from './apply.js?v=29'; import { state, variantState } from './current.js?v=4'; -import * as Mods from '../mods/mods.js?v=99'; +import * as Mods from '../mods/mods.js?v=100'; export function describe() { const clothes = ITEMS.filter(it => !it.section); diff --git a/webapp/js/outfit/composite.js b/webapp/js/outfit/composite.js index d512537a..58597ffe 100644 --- a/webapp/js/outfit/composite.js +++ b/webapp/js/outfit/composite.js @@ -1,6 +1,6 @@ import * as Live2D from '../live2d/live2d.js?v=90'; import { GLASSES_STYLES, VARIANTS } from './catalog.js?v=4'; -import { availableAssets, textureAvailable } from './sync.js?v=28'; +import { availableAssets, textureAvailable } from './sync.js?v=29'; import { colors, variantState } from './current.js?v=4'; import { hexToRgb01 } from './colors.js?v=1'; diff --git a/webapp/js/outfit/decals.js b/webapp/js/outfit/decals.js index 9b2c7494..166d826f 100644 --- a/webapp/js/outfit/decals.js +++ b/webapp/js/outfit/decals.js @@ -1,5 +1,5 @@ import * as Live2D from '../live2d/live2d.js?v=90'; -import * as Mods from '../mods/mods.js?v=99'; +import * as Mods from '../mods/mods.js?v=100'; import { bakeDrawable } from '../mods/layers.js?v=18'; import { ITEMS, VARIANTS, VARIANT_OWNER, colorGroup } from './catalog.js?v=4'; import { hexToRgb01 } from './colors.js?v=1'; diff --git a/webapp/js/outfit/outfit.js b/webapp/js/outfit/outfit.js index 44c486b2..97397a09 100644 --- a/webapp/js/outfit/outfit.js +++ b/webapp/js/outfit/outfit.js @@ -1,3 +1,3 @@ -export { applyAll, hiddenItemDrawables, refreshVisibility, setModdedDrawables } from './apply.js?v=28'; -export { applyToolChange, describe, syncFromAction, wearLook } from './chat-tools.js?v=28'; -export { load } from './sync.js?v=28'; +export { applyAll, hiddenItemDrawables, refreshVisibility, setModdedDrawables } from './apply.js?v=29'; +export { applyToolChange, describe, syncFromAction, wearLook } from './chat-tools.js?v=29'; +export { load } from './sync.js?v=29'; diff --git a/webapp/js/outfit/presets.js b/webapp/js/outfit/presets.js index 83cf5e18..524ab0ec 100644 --- a/webapp/js/outfit/presets.js +++ b/webapp/js/outfit/presets.js @@ -1,9 +1,9 @@ import { api, apiJson } from '../core/api.js?v=1'; import { COLOR_GROUPS, ITEMS, VARIANTS, VARIANT_OWNER } from './catalog.js?v=4'; -import { activeAssets, authorizedAssets, queueWardrobe, saveColors } from './sync.js?v=28'; -import { applyColors, applyItems, applyVariants, itemDrawableIds } from './apply.js?v=28'; +import { activeAssets, authorizedAssets, queueWardrobe, saveColors } from './sync.js?v=29'; +import { applyColors, applyItems, applyVariants, itemDrawableIds } from './apply.js?v=29'; import { colors, state, variantState } from './current.js?v=4'; -import * as Mods from '../mods/mods.js?v=99'; +import * as Mods from '../mods/mods.js?v=100'; import { hooks } from './hooks.js?v=1'; export function exportPreset() { diff --git a/webapp/js/outfit/sync.js b/webapp/js/outfit/sync.js index f3f3bf4f..e866a17f 100644 --- a/webapp/js/outfit/sync.js +++ b/webapp/js/outfit/sync.js @@ -1,13 +1,13 @@ -import * as Prefs from '../core/prefs.js?v=6'; +import * as Prefs from '../core/prefs.js?v=7'; import { api, apiJson } from '../core/api.js?v=1'; import { COLOR_GROUPS, GLASSES_STYLES, ITEMS, VARIANTS, VARIANT_OWNER } from './catalog.js?v=4'; -import { Presets, applyChanged, loadPresetState } from './presets.js?v=28'; -import { applyGlassesTexture } from './composite.js?v=28'; -import { applyItems, applyVariants } from './apply.js?v=28'; +import { Presets, applyChanged, loadPresetState } from './presets.js?v=29'; +import { applyGlassesTexture } from './composite.js?v=29'; +import { applyItems, applyVariants } from './apply.js?v=29'; import { colors, state, variantState } from './current.js?v=4'; -import { loadBakedTiles } from '../wardrobe/tile-bake.js?v=28'; +import { loadBakedTiles } from '../wardrobe/tile-bake.js?v=29'; import { normalizeHex } from './colors.js?v=1'; -import * as Mods from '../mods/mods.js?v=99'; +import * as Mods from '../mods/mods.js?v=100'; import { hooks } from './hooks.js?v=1'; const STORAGE_KEY = 'omega.outfit.v1'; diff --git a/webapp/js/voice/kokoro-lang.js b/webapp/js/voice/kokoro-lang.js new file mode 100644 index 00000000..6f5596d2 --- /dev/null +++ b/webapp/js/voice/kokoro-lang.js @@ -0,0 +1,33 @@ +// Kokoro's auto-detect. Kokoro has one model for every language, +// switching costs nothing, so each chunk is routed on its own, +// unlike pocket-tts which locks a whole reply to one checkpoint. +// that also means a language here is told apart by the characters +// it writes, not by stopwords: a chunk either has hanzi in it or +// it doesn't. +// +// one rule per KOKORO_LANGUAGES id in tts/sidecar/config.py, first +// match wins, no match is english. a rule for a language the +// sidecar can't speak is harmless, it puts an id it doesn't offer +// back to english. + +// kana means Japanese. its kanji sit in the same range as hanzi, +// and reading them as Mandarin is worse than English's "Chinese +// letter", so any kana keeps the chunk off Mandarin. +const KANA_RE = /[぀-ヿㇰ-ㇿヲ-゚]/; +const HANZI_RE = /[㐀-鿿豈-﫿]/; + +const RULES = [ + { id: 'mandarin', test: (s) => HANZI_RE.test(s) && !KANA_RE.test(s) }, +]; + +export function detectKokoroLang(text) { + const s = text || ''; + for (const r of RULES) if (r.test(s)) return r.id; + return 'english'; +} + +// what the Play button says, so a voice is previewed in the +// language it's for. no entry falls back to the English line. +export const PREVIEW_LINES = { + mandarin: '你好,你喜歡這個聲音嗎?', +}; diff --git a/webapp/js/voice/tts.js b/webapp/js/voice/tts.js index f970d8cc..56f64ba7 100644 --- a/webapp/js/voice/tts.js +++ b/webapp/js/voice/tts.js @@ -1,13 +1,20 @@ import * as Live2D from '../live2d/live2d.js?v=90'; import { api, apiJson } from '../core/api.js?v=1'; +import { detectKokoroLang } from './kokoro-lang.js?v=1'; let enabled = false; let engine = 'kokoro'; let voice = 'af_heart'; -// lang is a pocket-tts thing. Kokoro ignores it completely, and -// autoLang just lets the reply text pick one. +// lang picks the pocket-tts checkpoint, or Kokoro's G2P. autoLang +// lets the text pick it instead: once per reply for pocket-tts, per +// chunk for Kokoro (js/voice/kokoro-lang.js). let lang = 'english'; let autoLang = false; +// Kokoro only: the voice picked for each language, so an auto +// routed chunk gets a voice that can speak it. anything missing +// falls back to `voice`, and the sidecar swaps a voice that isn't +// that language's for its default. +let langVoices = {}; let speed = 1.0; let volume = 1.0; let duckLevel = 1.0; @@ -60,6 +67,10 @@ export function isEnabled() { return enabled; } export function setEngine(e) { if (e) engine = e; } export function setVoice(v) { if (v) voice = v; } +export function setLangVoices(m) { langVoices = (m && typeof m === 'object') ? { ...m } : {}; } +function voiceFor(l) { + return (engine === 'kokoro' && langVoices[l]) || voice; +} // 'auto' turns detection on per reply, a real id pins that // language. the detector only ever hands back ids the sidecar // knows, so 'auto' never leaves this file. @@ -97,20 +108,26 @@ export async function listVoices() { // playful "~" is always a break, so each bit between tildes // becomes its own utterance and the voice can fall at the end // instead of running into the next one -const HARD_BREAK_RE = /[.!?~\n]/; +// the fullwidth ones are how Chinese (and Japanese) end a +// sentence. without them a Chinese reply was one chunk, nothing +// played until she finished writing, and a long one went over the +// 2000 byte cap tts.php puts on a chunk. +const HARD_BREAK_RE = /[.!?~\n\u3002\uff01\uff1f\uff5e]/; // only real punctuation breaks a chunk. keep dashes out, an ASCII // hyphen lands mid-word ("co-op", Jun's "H-hey" stutters) and // putting it in the class between the colon and an en dash // silently made a range over every letter, which cut the first // chunk at whatever character followed the third word. took a // while to find that one -const SOFT_BREAK_RE = /[,;:]/g; +const SOFT_BREAK_RE = /[,;:\uff0c\uff1b\uff1a]/g; // "Oh," on its own reads as a whole falling utterance. wait for // three words before the first soft split. const MIN_FIRST_WORDS = 3; +// Chinese has no spaces, so each hanzi counts as a word. close +// enough for "wait for a few words before a soft split". function wordCount(s) { - const m = s.match(/\S+/g); + const m = s.match(/[\u3400-\u9fff\uf900-\ufaff]|[^\s\u3400-\u9fff\uf900-\ufaff]+/g); return m ? m.length : 0; } @@ -137,7 +154,9 @@ function hardBreak(buf) { return -1; } -function nextChunk(buf, first) { +// exported for .github/scripts/js/tts-text.test.mjs, nothing else +// imports it +export function nextChunk(buf, first) { const hard = hardBreak(buf); if (!first) return hard < 0 ? null : cutAt(buf, hard); @@ -222,13 +241,28 @@ let replyLangLocked = false; // guess off your message so the caller can start loading the right // pocket-tts model while Jun is still writing. can't tell? keep the // language the conversation is already in. not English. +// +// Kokoro has nothing to predict a reply-wide language for, it +// routes per chunk. this only names a non-English language worth +// warming (its G2P and voice, a second or so) before the first +// chunk asks for it. export function predictLang(text) { - if (!enabled || !autoLang || engine !== 'pockettts') return null; + if (!enabled) return null; + if (engine === 'kokoro') { + const l = autoLang ? detectKokoroLang(text || '') : lang; + return l === 'english' ? null : l; + } + if (!autoLang || engine !== 'pockettts') return null; return detectLang(text || '') || lastLang; } export function warmLang(l) { - if (!enabled || !autoLang || engine !== 'pockettts' || !l) return; + if (!enabled || !l) return; + if (engine === 'kokoro') { + apiJson('tts.php?action=warm', { lang: l, voice: voiceFor(l), engine }).catch(() => {}); + return; + } + if (!autoLang || engine !== 'pockettts') return; apiJson('tts.php?action=warm', { lang: l, voice, engine }).catch(() => {}); } @@ -252,7 +286,8 @@ function updateDetect(text) { } } -function effectiveLang() { +function effectiveLang(text) { + if (engine === 'kokoro') return autoLang ? detectKokoroLang(text) : lang; if (!autoLang || engine !== 'pockettts') return lang; return replyLang || lastLang; } @@ -301,11 +336,13 @@ function enqueue(text, hooks) { updateDetect(text); const id = nextId++; + // each chunk keeps the language it started with. a later guess + // must not rewrite one already in flight + const jobLang = effectiveLang(text); const job = { id, text, - // each chunk keeps the language it started with. a later guess - // must not rewrite one already in flight - lang: effectiveLang(), + lang: jobLang, + voice: voiceFor(jobLang), abort: new AbortController(), status: 'queued', audioBuffer: null, @@ -330,7 +367,7 @@ function startJob(job) { job.blobPromise = (async () => { try { - const res = await apiJson('tts.php?action=tts', { text: job.text, voice, speed, engine, lang: job.lang }, + const res = await apiJson('tts.php?action=tts', { text: job.text, voice: job.voice, speed, engine, lang: job.lang }, { signal: job.abort.signal }); // 204 (nothing to say) is still a 2xx, so !res.ok won't catch it if (res.status === 204) { job.status = 'done'; return; } diff --git a/webapp/js/voice/voice.js b/webapp/js/voice/voice.js index 295ab4ad..51f51244 100644 --- a/webapp/js/voice/voice.js +++ b/webapp/js/voice/voice.js @@ -4,7 +4,7 @@ import { api } from '../core/api.js?v=1'; import * as Names from '../core/names.js?v=1'; import { base64Of, encodeWav } from '../core/wav.js?v=1'; -import * as TTS from './tts.js?v=90'; +import * as TTS from './tts.js?v=91'; // whisper wants 16 kHz. browser resamples for us, free real estate const SAMPLE_RATE = 16000; diff --git a/webapp/js/voice/voicemode.js b/webapp/js/voice/voicemode.js index c2bba23c..435a3326 100644 --- a/webapp/js/voice/voicemode.js +++ b/webapp/js/voice/voicemode.js @@ -1,7 +1,7 @@ import * as ui from '../core/ui.js?v=1'; import * as Live2D from '../live2d/live2d.js?v=90'; -import * as TTS from './tts.js?v=90'; -import * as Voice from './voice.js?v=91'; +import * as TTS from './tts.js?v=91'; +import * as Voice from './voice.js?v=92'; let active = false; let muted = false; diff --git a/webapp/js/wardrobe.js b/webapp/js/wardrobe.js index 52811a88..39f77f55 100644 --- a/webapp/js/wardrobe.js +++ b/webapp/js/wardrobe.js @@ -2,12 +2,12 @@ import * as Live2D from './live2d/live2d.js?v=90'; import * as MobileViewport from './core/viewport.js?v=1'; import * as Names from './core/names.js?v=1'; import { api } from './core/api.js?v=1'; -import * as Outfit from './outfit/outfit.js?v=98'; +import * as Outfit from './outfit/outfit.js?v=99'; import * as Scene from './trip/scene.js?v=1'; import { startSkybox } from './trip/skybox.js?v=2'; import * as TripLoader from './trip/trip-loader.js?v=1'; import { armCurtains } from './wardrobe/curtains.js?v=17'; -import { openWardrobe } from './wardrobe/panel.js?v=31'; +import { openWardrobe } from './wardrobe/panel.js?v=33'; const status = document.getElementById('stageStatus'); const HEAD_HOME_ICON = ''; diff --git a/webapp/js/wardrobe/color-picker.js b/webapp/js/wardrobe/color-picker.js index c36c0029..db736eec 100644 --- a/webapp/js/wardrobe/color-picker.js +++ b/webapp/js/wardrobe/color-picker.js @@ -3,8 +3,8 @@ import * as MobileViewport from '../core/viewport.js?v=1'; import { colorGroup } from '../outfit/catalog.js?v=4'; import { colors } from '../outfit/current.js?v=4'; import { hexToHsv, hsvToHex, normalizeHex } from '../outfit/colors.js?v=1'; -import { optPopEl, positionOptionsPopup } from './tiles.js?v=31'; -import { setColor } from '../outfit/apply.js?v=28'; +import { optPopEl, positionOptionsPopup } from './tiles.js?v=33'; +import { setColor } from '../outfit/apply.js?v=29'; function paintColorButton(button) { const keys = (button.dataset.colorKeys || '').split(',').filter(Boolean); diff --git a/webapp/js/wardrobe/looks.js b/webapp/js/wardrobe/looks.js index f8e7ec8f..db9ae684 100644 --- a/webapp/js/wardrobe/looks.js +++ b/webapp/js/wardrobe/looks.js @@ -1,5 +1,5 @@ -import { Presets, applyPreset, decodeOutfitCode, encodeOutfitCode, endPreview, schedulePreview } from '../outfit/presets.js?v=28'; -import { clearWornHover, wdOverlay } from './panel.js?v=31'; +import { Presets, applyPreset, decodeOutfitCode, encodeOutfitCode, endPreview, schedulePreview } from '../outfit/presets.js?v=29'; +import { clearWornHover, wdOverlay } from './panel.js?v=33'; export const looksOpen = () => document.body.classList.contains('looks-open'); diff --git a/webapp/js/wardrobe/panel.js b/webapp/js/wardrobe/panel.js index 920d0eb5..e13050c4 100644 --- a/webapp/js/wardrobe/panel.js +++ b/webapp/js/wardrobe/panel.js @@ -1,14 +1,14 @@ import * as Live2D from '../live2d/live2d.js?v=90'; import { BODY_VARIANTS, CLOTHING_VARIANTS, COLOR_GROUPS, ITEMS, ITEM_COLOR_GROUPS, ITEM_VARIANTS, VARIANTS } from '../outfit/catalog.js?v=4'; -import { fillTile, makeOptOrb, makeSwatch, makeTile, openTilePopup, syncOptionsPopup, wdMoveGhost, wdShowGhost } from './tiles.js?v=31'; +import { fillTile, makeOptOrb, makeSwatch, makeTile, openTilePopup, syncOptionsPopup, wdMoveGhost, wdShowGhost } from './tiles.js?v=33'; import { state, variantState } from '../outfit/current.js?v=4'; -import { itemThumb, variantThumb } from './tile-bake.js?v=28'; -import { looksOpen, toggleLooks } from './looks.js?v=31'; -import { reset, setItem } from '../outfit/apply.js?v=28'; -import { wornDrawableMap, wornHitAt, wornLabel, wornRemove, wornWear } from './worn.js?v=29'; -import { buildWardrobeSection, syncModTiles } from '../mods/section.js?v=31'; -import { activate, playOutro } from './reactions.js?v=88'; -import { refreshColorButtons } from './color-picker.js?v=31'; +import { itemThumb, variantThumb } from './tile-bake.js?v=29'; +import { looksOpen, toggleLooks } from './looks.js?v=33'; +import { reset, setItem } from '../outfit/apply.js?v=29'; +import { wornDrawableMap, wornHitAt, wornLabel, wornRemove, wornWear } from './worn.js?v=30'; +import { buildWardrobeSection, syncModTiles } from '../mods/section.js?v=33'; +import { activate, playOutro } from './reactions.js?v=89'; +import { refreshColorButtons } from './color-picker.js?v=33'; import { hooks } from '../outfit/hooks.js?v=1'; export let clearWornHover = null; diff --git a/webapp/js/wardrobe/reactions.js b/webapp/js/wardrobe/reactions.js index 988fbdf2..3f6dd97c 100644 --- a/webapp/js/wardrobe/reactions.js +++ b/webapp/js/wardrobe/reactions.js @@ -1,4 +1,4 @@ -import { affection, buildCard, configureTts, fetchGauges, openMood, present, refreshGauges, speakLine, trust, tension } from '../core/speech-card.js?v=18'; +import { affection, buildCard, configureTts, fetchGauges, openMood, present, refreshGauges, speakLine, trust, tension } from '../core/speech-card.js?v=19'; import { gaugeTier } from '../core/mood-tier.js?v=2'; import { hooks } from '../outfit/hooks.js?v=1'; import { OPEN_LINES } from './open-lines.js?v=1'; diff --git a/webapp/js/wardrobe/tile-bake.js b/webapp/js/wardrobe/tile-bake.js index f12dd8dc..9bef1aa5 100644 --- a/webapp/js/wardrobe/tile-bake.js +++ b/webapp/js/wardrobe/tile-bake.js @@ -1,7 +1,7 @@ import * as Live2D from '../live2d/live2d.js?v=90'; import { ITEMS, VARIANTS, VARIANT_OWNER } from '../outfit/catalog.js?v=4'; -import { applyChanged, exportPreset, loadPresetState } from '../outfit/presets.js?v=28'; -import { itemPatterns } from '../outfit/apply.js?v=28'; +import { applyChanged, exportPreset, loadPresetState } from '../outfit/presets.js?v=29'; +import { itemPatterns } from '../outfit/apply.js?v=29'; import { state, variantState } from '../outfit/current.js?v=4'; // tiles baked by tools/bake-items.html, sitting in the gitignored diff --git a/webapp/js/wardrobe/tiles.js b/webapp/js/wardrobe/tiles.js index 5fca7144..8ced2c46 100644 --- a/webapp/js/wardrobe/tiles.js +++ b/webapp/js/wardrobe/tiles.js @@ -1,10 +1,10 @@ import * as Live2D from '../live2d/live2d.js?v=90'; -import { closeColorPicker, colorPickerEl, makeColorButton, phonePopupMode, pickerEmbedded, schedulePopupPosition, showColorPicker, visualViewportRect, watchPopupViewport } from './color-picker.js?v=31'; +import { closeColorPicker, colorPickerEl, makeColorButton, phonePopupMode, pickerEmbedded, schedulePopupPosition, showColorPicker, visualViewportRect, watchPopupViewport } from './color-picker.js?v=33'; import { colorGroup } from '../outfit/catalog.js?v=4'; import { colors, state, variantState } from '../outfit/current.js?v=4'; -import { setItem, setVariant } from '../outfit/apply.js?v=28'; -import { variantThumb } from './tile-bake.js?v=28'; -import { wdGhost } from './panel.js?v=31'; +import { setItem, setVariant } from '../outfit/apply.js?v=29'; +import { variantThumb } from './tile-bake.js?v=29'; +import { wdGhost } from './panel.js?v=33'; export function wdMoveGhost(x, y) { wdGhost.style.left = (x + 10) + 'px'; diff --git a/webapp/js/wardrobe/worn.js b/webapp/js/wardrobe/worn.js index ffcaf41e..9fa4750c 100644 --- a/webapp/js/wardrobe/worn.js +++ b/webapp/js/wardrobe/worn.js @@ -1,8 +1,8 @@ import * as Live2D from '../live2d/live2d.js?v=90'; import { ITEMS, VARIANTS } from '../outfit/catalog.js?v=4'; -import { itemDrawableIds, setItem, setVariant } from '../outfit/apply.js?v=28'; +import { itemDrawableIds, setItem, setVariant } from '../outfit/apply.js?v=29'; import { state, variantState } from '../outfit/current.js?v=4'; -import { isEquipped, mods, setEquipped } from '../mods/engine.js?v=28'; +import { isEquipped, mods, setEquipped } from '../mods/engine.js?v=29'; import { itemDrawables } from '../mods/parse.js?v=18'; import { hooks } from '../outfit/hooks.js?v=1'; diff --git a/webapp/js/watch.js b/webapp/js/watch.js index 0ecf917a..c0fd9718 100644 --- a/webapp/js/watch.js +++ b/webapp/js/watch.js @@ -6,11 +6,11 @@ import * as Auth from './core/auth.js?v=1'; import { api, apiJson } from './core/api.js?v=1'; import * as ChatAPI from './core/chat-api.js?v=7'; import * as Names from './core/names.js?v=1'; -import * as Prefs from './core/prefs.js?v=6'; -import { say } from './core/speech-card.js?v=18'; +import * as Prefs from './core/prefs.js?v=7'; +import { say } from './core/speech-card.js?v=19'; import { localTimeString } from './core/util.js?v=1'; import * as Live2D from './live2d/live2d.js?v=90'; -import * as Outfit from './outfit/outfit.js?v=98'; +import * as Outfit from './outfit/outfit.js?v=99'; import * as Scene from './trip/scene.js?v=1'; import * as TripLoader from './trip/trip-loader.js?v=1'; import { startCapture } from './watch/grab.js?v=2'; diff --git a/webapp/karaoke.html b/webapp/karaoke.html index 25704435..c157bdde 100644 --- a/webapp/karaoke.html +++ b/webapp/karaoke.html @@ -131,6 +131,6 @@

Who sings each line?

- + diff --git a/webapp/wardrobe.html b/webapp/wardrobe.html index ecf2bb33..ed6458c5 100644 --- a/webapp/wardrobe.html +++ b/webapp/wardrobe.html @@ -21,6 +21,6 @@ - + diff --git a/webapp/watch.html b/webapp/watch.html index 35b46f99..d189a855 100644 --- a/webapp/watch.html +++ b/webapp/watch.html @@ -38,6 +38,6 @@ - +