diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a569d66..eade7962 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,6 +78,7 @@ jobs: gettext help2man texinfo texlive texlive-plain-generic nlohmann-json3-dev librubberband-dev pulseaudio ${{ matrix.deps }} + libonnxruntime-dev libxml2-dev libcurl4-gnutls-dev - name: install piper run: | wget https://github.com/OHF-Voice/piper1-gpl/archive/refs/tags/v1.7.0.tar.gz @@ -144,6 +145,7 @@ jobs: libdotconf-dev libglib2.0-dev libltdl-dev- libsystemd-dev systemd gettext help2man texinfo texlive texlive-plain-generic pulseaudio + libonnxruntime-dev libxml2-dev libcurl4-gnutls-dev - name: autoconf run: ./build.sh - name: configure diff --git a/configure.ac b/configure.ac index 2396b9b5..f728c0c8 100644 --- a/configure.ac +++ b/configure.ac @@ -56,6 +56,7 @@ if test "$enable_libs_only" = "yes"; then with_baratinoo=no with_kali=no with_piper=no + with_kitten=no with_pulse=no with_alsa=no @@ -425,6 +426,35 @@ AM_CONDITIONAL([kali_support], [test $with_kali != no]) AM_CONDITIONAL([kali_shim], [test $with_kali = shim]) AS_IF([test $with_kali != no], [output_modules="${output_modules} kali"]) +# check for kitten support +AC_ARG_WITH([kitten], + [AS_HELP_STRING([--with-kitten], [Compile with KittenTTS support])], + [], + [with_kitten=check]) +AS_IF([test $with_kitten != "no"], + [PKG_CHECK_MODULES([LIBONNXRUNTIME], [libonnxruntime], + [kitten_libonnxruntime="yes"], + [AS_IF([test $with_kitten = "yes"], + [AC_MSG_FAILURE([libonnxruntime is not available])])]) + PKG_CHECK_MODULES([LIBXML], [libxml-2.0], + [kitten_libxml="yes"], + [AS_IF([test $with_kitten = "yes"], + [AC_MSG_FAILURE([libxml-2.0 is not available])])]) + PKG_CHECK_MODULES([LIBCURL], [libcurl], + [kitten_libcurl="yes"], + [AS_IF([test $with_kitten = "yes"], + [AC_MSG_FAILURE([libcurl is not available])])]) + AS_IF([test "x$kitten_libonnxruntime" = "xyes" -a "x$kitten_libxml" = "xyes" -a "x$kitten_libcurl" = "xyes"], + [with_kitten="yes"])]) +AM_CONDITIONAL([kitten_support], [test $with_kitten = "yes"]) +AC_SUBST([LIBONNXRUNTIME_CFLAGS]) +AC_SUBST([LIBONNXRUNTIME_LIBS]) +AC_SUBST([LIBXML_CFLAGS]) +AC_SUBST([LIBXML_LIBS]) +AC_SUBST([LIBCURL_CFLAGS]) +AC_SUBST([LIBCURL_LIBS]) +AS_IF([test $with_kitten = "yes"], [output_modules="${output_modules} kitten"]) + # check for piper support AC_ARG_WITH([piper], [AS_HELP_STRING([--with-piper[[=yes|no|check|/path]]], [include Piper TTS support])], diff --git a/src/modules/.gitignore b/src/modules/.gitignore index e7715f58..59be7e78 100644 --- a/src/modules/.gitignore +++ b/src/modules/.gitignore @@ -42,6 +42,7 @@ /sd_kali /sd_openjtalk /sd_pico +/sd_kittentts /sd_skeleton0 /sd_skeleton0_espeak-ng-async /sd_skeleton0_espeak-ng-async-server diff --git a/src/modules/Makefile.am b/src/modules/Makefile.am index 56b75e28..43d7b327 100644 --- a/src/modules/Makefile.am +++ b/src/modules/Makefile.am @@ -24,6 +24,7 @@ EXTRA_DIST += voxin_shim.c voxin.h EXTRA_DIST += module_utils.h EXTRA_DIST += dummy-message-default.wav EXTRA_DIST += dummy-message.txt +EXTRA_DIST += kitten.h CLEANFILES = dummy-message.wav if HAVE_PYTHON @@ -133,6 +134,16 @@ sd_flite_LDADD = $(top_builddir)/src/common/libcommon.la \ $(common_LDADD) endif +# +# kitten tts module +# +if kitten_support +modulebin_PROGRAMS += sd_kittentts +sd_kittentts_SOURCES = kitten_server.c kitten_downloader.c kitten_model.c kitten_worker.c +sd_kittentts_CFLAGS = $(ESPEAK_NG_CFLAGS) $(GLIB_CFLAGS) $(LIBONNXRUNTIME_CFLAGS) $(LIBXML_CFLAGS) $(LIBCURL_CFLAGS) $(DOTCONF_CFLAGS) +sd_kittentts_LDADD = libspeechd_module.la $(ESPEAK_NG_LIBS) $(EXTRA_ESPEAK_LIBS) $(GLIB_LIBS) $(LIBONNXRUNTIME_LIBS) $(LIBXML_LIBS) $(LIBCURL_LIBS) $(DOTCONF_LIBS) +endif + # # ibmtts module # diff --git a/src/modules/kitten.h b/src/modules/kitten.h new file mode 100644 index 00000000..8b31b148 --- /dev/null +++ b/src/modules/kitten.h @@ -0,0 +1,154 @@ +/* +The MIT License (MIT) + +Copyright © 2026 John Settlemyer + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "module_utils.h" + +#include +#include "spd_module_main.h" + +#define CHECK_STATUS(expr) \ + do { \ + OrtStatus* status = (expr); \ + if (status != NULL) { \ + const char* msg = g_ort->GetErrorMessage(status); \ + fprintf(stderr, "ONNX Runtime Error: %s\n", msg); \ + g_ort->ReleaseStatus(status); \ + exit(1); \ + } \ + } while (0) + +extern int model_type; +extern int ROWS; +#define COLS 256 + +#define PAD "$" +#define PUNCTUATION ";:,.!?¡¿—…\"«»\"\" " +#define LETTERS "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +#define LETTERS_IPA "ɑɐɒæɓʙβɔɕçɗɖðʤəɘɚɛɜɝɞɟʄɡɠɢʛɦɧħɥʜɨɪʝɭɬɫɮʟɱɯɰŋɳɲɴøɵɸθœɶʘɹɺɾɻʀʁɽʂʃʈʧʉʊʋⱱʌɣɤʍχʎʏʑʐʒʔʡʕʢǀǁǂǃˈˌːˑʼʴʰʱʲʷˠˤ˞↓↑→↗↘'̩'ᵻ" + +#define SYMBOLS PAD PUNCTUATION LETTERS LETTERS_IPA + +// setting var's +extern float speed; +//['Leo','Kiki','Hugo','Rosie','Bruno','Luna','Jasper','Bella'] +extern GString *voice; +extern GString *voice_setting; +// paths var's +extern GString *model_path; +extern GString *voices_path; +extern const char *home_dir; +extern GString *model_dir; + +extern bool stop_generation; + +// holds and array of values that must be passed to the model based off the requested voice and length of the text. +extern float *voice_styles; + +extern const OrtApi* g_ort; +extern OrtEnv* env; +extern OrtSessionOptions* session_options; +extern OrtSession* session; + +extern GHashTable *files_hash_table; + +#define TARGET_SUBDIR ".cache/speech-dispatcher/kitten" +#define DISTRO_TARGET_SUBDIR "/usr/share/speech-dispatcher/models/kitten" + +typedef struct { + const char *quality; + const char *model_url; + const char *model_filename; + const char *model_expected_size; + const char *model_expected_sha256; + const char *voice_url; + const char *voice_filename; + const char *voice_expected_size; + const char *voice_expected_sha256; +} FileInfo; + +#define VOICE_LIST(X) \ + X(Leo) \ + X(Kiki) \ + X(Hugo) \ + X(Rosie) \ + X(Bruno) \ + X(Luna) \ + X(Jasper) \ + X(Bella) \ + X(Leo_Low) \ + X(Kiki_Low) \ + X(Hugo_Low) \ + X(Rosie_Low) \ + X(Bruno_Low) \ + X(Luna_Low) \ + X(Jasper_Low) \ + X(Bella_Low) \ + X(Leo_High) \ + X(Kiki_High) \ + X(Hugo_High) \ + X(Rosie_High) \ + X(Bruno_High) \ + X(Luna_High) \ + X(Jasper_High) \ + X(Bella_High) + +#define DEFINE_VOICE(name_token) static SPDVoice voice_##name_token = { .name = #name_token, .language = "en" }; +#define VOICE_PTR_ITEM(name_token) &voice_##name_token, + +// kitten_downloader.c +int download_models(void); +bool use_distro_path(void); +void init_file_hashtable(void); +void build_hash_from_defaults(GHashTable *ht); +void file_hash_add_sub_key(GHashTable *ht, const char* quality, const char* key, const char* value); +char* file_hash_get_value(GHashTable *ht, const char* quality, const char* key); + +// kitten_model.c +int init_voice_style(const char* voices_path); +void cleanup_voice_style(); +GArray *get_style(const char *text, const char *voice); +GArray *get_char_indices(const gchar *locate, const gchar *index_str); +GString *get_phonemes(const char *text); +int init_model(const char* model_path); +void cleanup_model(); +GArray* run_model(GArray *inputs_array, GArray *styles_array, float speed); +void convert_float_to_short(const float* in_buffer, GArray* out_buffer, size_t num_samples); +GArray* kitten_speak(const char* data); +int reload_models_and_voices(const char *model_filename, const char* voice_filename); + +// kitten_worker.c +int init_model_thread_pool(); +int cleanup_threads(); +int model_change_voice(const char *var, const char *val); +int model_change_speed(const char *var, const char *val); +int add_generate_speech_task(const char* data, size_t bytes); +int model_stop_generation(); \ No newline at end of file diff --git a/src/modules/kitten_downloader.c b/src/modules/kitten_downloader.c new file mode 100644 index 00000000..30086f79 --- /dev/null +++ b/src/modules/kitten_downloader.c @@ -0,0 +1,358 @@ +/* +The MIT License (MIT) + +Copyright © 2026 John Settlemyer + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#include "kitten.h" + +const FileInfo DEFAULT_FILES[] = { + { + "Normal", + "https://huggingface.co/KittenML/kitten-tts-micro-0.8/resolve/main/kitten_tts_micro_v0_8.onnx", + "kitten_tts_micro_v0_8.onnx", + "41384970", + "95481626fee1ba70ce683e69c534fc7cb38433c46ce42d3abbeafb4b9f1a4123", + "https://github.com/jsett/kittenvoices/raw/refs/heads/main/voices_micro.bin", + "voices_micro.bin", + "3276800", + "12ad10f1fcce8a458b5cf79769b8edd4ba0e11e9fb6532fd192c3500b2b37a5d" + }, + { + "High", + "https://huggingface.co/KittenML/kitten-tts-mini-0.8/resolve/main/kitten_tts_mini_v0_8.onnx", + "kitten_tts_mini_v0_8.onnx", + "78268016", + "0f5bbae4fc4800c98dbc544a87ecfa79510de2fb8222db30d12e5bfe9177df91", + "https://github.com/jsett/kittenvoices/raw/refs/heads/main/voices_mini.bin", + "voices_mini.bin", + "3276800", + "0e4965b46333db53ce09c73842623bf7055ea62c67f78803cb7ea9c16da6ac2b" + }, + { + "Low", + "https://huggingface.co/KittenML/kitten-tts-nano-0.2/resolve/main/kitten_tts_nano_v0_2.onnx", + "kitten_tts_nano_v0_2.onnx", + "23804156", + "42fa8809db319cd7c4c83b3c501e2313bf90edf610235291cad605e4adcb242d", + "https://github.com/jsett/kittenvoices/raw/refs/heads/main/voices_nano.bin", + "voices_nano.bin", + "8192", + "42a40a24a352a38657d6cb86ceee51bbc2b7780b29e04fb60bcdf959adccea01" + } +}; + +#define NUM_FILES (sizeof(DEFAULT_FILES) / sizeof(DEFAULT_FILES[0])) + +GHashTable *files_hash_table; + +void file_hash_create_sub_table(GHashTable *ht, const char* quality){ + GHashTable *tmp = g_hash_table_new_full( + g_str_hash, + g_str_equal, + g_free, + g_free + ); + + g_hash_table_insert(ht, g_strdup(quality), tmp); +} + +void file_hash_add_sub_key(GHashTable *ht, const char* quality, const char* key, const char* value){ + GHashTable *sub = g_hash_table_lookup(ht, quality); + g_hash_table_insert(sub, g_strdup(key), g_strdup(value)); +} + +int file_hash_get_value_as_num(GHashTable *ht, const char* quality, const char* key){ + GHashTable *sub = g_hash_table_lookup(ht, quality); + char* value = g_hash_table_lookup(sub, key); + gchar *endptr; + int num = g_ascii_strtoll(value, &endptr, 10); // convert str to int + return num; +} + +char* file_hash_get_value(GHashTable *ht, const char* quality, const char* key){ + GHashTable *sub = g_hash_table_lookup(ht, quality); + char* value = g_hash_table_lookup(sub, key); + return value; +} + +void build_hash_from_defaults(GHashTable *ht){ + for (size_t i = 0; i < NUM_FILES; i++) { + file_hash_add_sub_key(ht, DEFAULT_FILES[i].quality, "model_url", DEFAULT_FILES[i].model_url); + file_hash_add_sub_key(ht, DEFAULT_FILES[i].quality, "model_filename", DEFAULT_FILES[i].model_filename); + file_hash_add_sub_key(ht, DEFAULT_FILES[i].quality, "model_size", DEFAULT_FILES[i].model_expected_size); + file_hash_add_sub_key(ht, DEFAULT_FILES[i].quality, "model_sha256", DEFAULT_FILES[i].model_expected_sha256); + file_hash_add_sub_key(ht, DEFAULT_FILES[i].quality, "voices_url", DEFAULT_FILES[i].voice_url); + file_hash_add_sub_key(ht, DEFAULT_FILES[i].quality, "voices_filename", DEFAULT_FILES[i].voice_filename); + file_hash_add_sub_key(ht, DEFAULT_FILES[i].quality, "voices_size", DEFAULT_FILES[i].voice_expected_size); + file_hash_add_sub_key(ht, DEFAULT_FILES[i].quality, "voices_sha256", DEFAULT_FILES[i].voice_expected_sha256); + } +} + +void init_file_hashtable(void){ + files_hash_table = g_hash_table_new_full( + g_str_hash, + g_str_equal, + g_free, + (GDestroyNotify) g_hash_table_destroy + ); + + file_hash_create_sub_table(files_hash_table, "Low"); + file_hash_create_sub_table(files_hash_table, "High"); + file_hash_create_sub_table(files_hash_table, "Normal"); +} + +// Helper function to recursively create directories +static int ensure_directory_exists(const char *path) { + if (g_mkdir_with_parents(path, 0755) != 0) { + fprintf(stderr, "Error: Failed to create directory '%s': %s\n", path, g_strerror(errno)); + return -1; + } + return 0; +} + +// Helper function to check if file exists +static int file_exists(const char *path) { + struct stat buffer; + return (stat(path, &buffer) == 0); +} + +// Callback for curl to write HTTP data to disk +static size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) { + return fwrite(ptr, size, nmemb, stream); +} + +// Download file with libcurl +static int download_file(CURL *curl, const char *url, const char *dest_path) { + FILE *fp = fopen(dest_path, "wb"); + if (!fp) { + fprintf(stderr, "Error: Cannot open destination file '%s' for writing: %s\n", dest_path, g_strerror(errno)); + return -1; + } + + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); // Follow redirects + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); // Fail on HTTP errors (>=400) + + CURLcode res = curl_easy_perform(curl); + fclose(fp); + + if (res != CURLE_OK) { + fprintf(stderr, "Error: Download failed for '%s': %s\n", url, curl_easy_strerror(res)); + remove(dest_path); // Clean up partial download + return -1; + } + + return 0; +} + +// Verify file SHA256 using GLib GChecksum +static int verify_sha256(const char *filepath, const char *expected_sha256) { + GMappedFile *mfile = g_mapped_file_new(filepath, FALSE, NULL); + if (!mfile) { + fprintf(stderr, "Error: Failed to memory-map file '%s' for SHA256 calculation.\n", filepath); + return -1; + } + + gsize length = g_mapped_file_get_length(mfile); + const gchar *contents = g_mapped_file_get_contents(mfile); + + GChecksum *checksum = g_checksum_new(G_CHECKSUM_SHA256); + g_checksum_update(checksum, (const guchar *)contents, length); + const gchar *computed_sha256 = g_checksum_get_string(checksum); + + int match = (g_ascii_strcasecmp(computed_sha256, expected_sha256) == 0); + + if (!match) { + fprintf(stderr, "Error: Checksum mismatch for '%s'!\n Expected: %s\n Computed: %s\n", + filepath, expected_sha256, computed_sha256); + } + + g_checksum_free(checksum); + g_mapped_file_unref(mfile); + + return match ? 0 : -1; +} + +bool distro_file_integrity_check(char *target_dir_distro, char* filename, char* sha){ + bool output = true; + char *full_path = g_build_filename(target_dir_distro, filename, NULL); + // check if the file exists. + if (!file_exists(full_path)) { + output = false; + } else { + // verify the sha + if (verify_sha256(full_path, sha) != 0) { + fprintf(stderr, "Integrity check failed for '%s' in '%s'.\n", filename, full_path); + fprintf(stderr, "Cannot use distro directory\n"); + output = false; + } + } + + g_free(full_path); + return output; +} + +// determine if we should use DISTRO_TARGET_SUBDIR or TARGET_SUBDIR when looking +// for models. The rules are to use the distro path if all the files +// are in that path, and pass the verify_sha256. If not then fall back to TARGET_SUBDIR +// and download the files if needed. +// we should only check this once even if its called again. +bool use_distro_path_cache = false; +bool use_distro_path_return = false; +bool use_distro_path(void){ + if (use_distro_path_cache){ + return use_distro_path_return; + } + char *target_dir_distro = DISTRO_TARGET_SUBDIR; + bool use_distro_dir=true; + + GHashTableIter iter; + gpointer quality, value; + + g_hash_table_iter_init(&iter, files_hash_table); + while (g_hash_table_iter_next(&iter, &quality, &value)) { + char *model_filename = file_hash_get_value(files_hash_table, quality, "model_filename"); + char *model_sha = file_hash_get_value(files_hash_table, quality, "model_sha256"); + if (!distro_file_integrity_check(target_dir_distro,model_filename,model_sha)){ + use_distro_dir = false; + } + + char* voice_filename = file_hash_get_value(files_hash_table, quality, "voices_filename"); + char* voice_sha = file_hash_get_value(files_hash_table, quality, "voices_sha256"); + if (!distro_file_integrity_check(target_dir_distro, voice_filename,voice_sha)){ + use_distro_dir = false; + } + } + + use_distro_path_cache = true; + use_distro_path_return = use_distro_dir; + return use_distro_dir; +} + +int download_and_verify(char *target_dir, CURL *curl, char* url, char* filename, char* sha, int size){ + char *full_path = g_build_filename(target_dir, filename, NULL); + + // Check if file already exists + if (file_exists(full_path)) { + + // Verify SHA256 even for files that have already been downloaded. + if (verify_sha256(full_path, sha) != 0) { + fprintf(stderr, "Error: Integrity check failed for '%s'.\n", filename); + g_free(full_path); + return -1; + } else { + fprintf(stderr, "Info: File '%s' already exists. Skipping download.\n", filename); + g_free(full_path); + return 0; + } + } + + fprintf(stderr, "Info: Downloading '%s'...\n", filename); + if (download_file(curl, url, full_path) != 0) { + fprintf(stderr, "Error: Aborting process due to download error.\n"); + g_free(full_path); + return -1; + } + + // Verify File Size + struct stat st; + if (stat(full_path, &st) != 0) { + fprintf(stderr, "Error: Could not stat downloaded file '%s'.\n", full_path); + g_free(full_path); + return -1; + } + + if ((curl_off_t)st.st_size != (curl_off_t)size) { + fprintf(stderr, "Error: File size mismatch for '%s'! Expected: %ld bytes, Got: %ld bytes.\n", + filename, (long)size, (long)st.st_size); + g_free(full_path); + return -1; + } + + // Verify SHA256 using GLib + if (verify_sha256(full_path, sha) != 0) { + fprintf(stderr, "Error: Integrity check failed for '%s'.\n", filename); + g_free(full_path); + return -1; + } + + fprintf(stderr, "Info: Successfully downloaded and verified '%s'.\n", filename); + g_free(full_path); + return 0; +} + +// downloads the models and voices if they do not already exist. +// also verifys using sha256 and checks file size. +int download_models(void) { + + char *target_dir; + if (!use_distro_path()){ + // Build absolute destination directory path: ~/.cache/speech-dispatcher/kitten + target_dir = g_build_filename(home_dir, TARGET_SUBDIR, NULL); + } else { + return 0; //no need to download if the models are already in the distro path. + } + + if (ensure_directory_exists(target_dir) != 0) { + g_free(target_dir); + return -1; + } + + if (curl_global_init(CURL_GLOBAL_ALL) != 0) { + fprintf(stderr, "Error: Failed to initialize libcurl.\n"); + g_free(target_dir); + return -1; + } + + CURL *curl = curl_easy_init(); + if (!curl) { + fprintf(stderr, "Error: Failed to create libcurl handle.\n"); + curl_global_cleanup(); + g_free(target_dir); + return -1; + } + + int overall_status = 0; + + GHashTableIter iter; + gpointer quality, value; + + g_hash_table_iter_init(&iter, files_hash_table); + while (g_hash_table_iter_next(&iter, &quality, &value)) { + char *model_url = file_hash_get_value(files_hash_table, quality, "model_url"); + char *model_filename = file_hash_get_value(files_hash_table, quality, "model_filename"); + char *model_sha = file_hash_get_value(files_hash_table, quality, "model_sha256"); + int model_size = file_hash_get_value_as_num(files_hash_table, quality, "model_size"); + + if (download_and_verify(target_dir, curl, model_url, model_filename, model_sha, model_size) == -1){ + overall_status = -1; + break; + } + + char* voice_filename = file_hash_get_value(files_hash_table, quality, "voices_filename"); + char* voice_sha = file_hash_get_value(files_hash_table, quality, "voices_sha256"); + char* voice_url = file_hash_get_value(files_hash_table, quality, "voices_url"); + int voice_size = file_hash_get_value_as_num(files_hash_table, quality, "voices_size"); + + if (download_and_verify(target_dir, curl, voice_url, voice_filename, voice_sha, voice_size) == -1){ + overall_status = -1; + break; + } + } + + // Cleanup resources + curl_easy_cleanup(curl); + curl_global_cleanup(); + g_free(target_dir); + + return overall_status; +} \ No newline at end of file diff --git a/src/modules/kitten_model.c b/src/modules/kitten_model.c new file mode 100644 index 00000000..0d101ae6 --- /dev/null +++ b/src/modules/kitten_model.c @@ -0,0 +1,446 @@ +/* +The MIT License (MIT) + +Copyright © 2026 John Settlemyer + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#include "kitten.h" + +int model_type=0; +int ROWS = 0; + +// setting var's +float speed = 1.0f; +//['Leo','Kiki','Hugo','Rosie','Bruno','Luna','Jasper','Bella'] +GString *voice; +GString *voice_setting; +// paths var's +GString *model_path; +GString *voices_path; +const char *home_dir; +GString *model_dir; + +bool stop_generation=false; + +// holds and array of values that must be passed to the model based off the requested voice and length of the text. +float *voice_styles; + +// model var's +const OrtApi* g_ort = NULL; +OrtEnv* env = NULL; +OrtSessionOptions* session_options = NULL; +OrtSession* session = NULL; + +/* +The original voices bins come from the kitten tts repo but were in the formate +of a npz which is difficult to process in c. They are really only named tensors. +So I unpacked them and turned them into arrays of floats that can be easly +loaded in c. For reference here is the python script to generate them from the +original npz files. + +wget https://huggingface.co/KittenML/kitten-tts-nano-0.2/resolve/main/voices.npz +wget https://huggingface.co/KittenML/kitten-tts-micro-0.8/resolve/main/voices.npz +wget https://huggingface.co/KittenML/kitten-tts-mini-0.8/resolve/main/voices.npz + +import numpy as np +voices = np.load("voices.npz") +output = b"" +for k in voices.keys(): + print(k) + output = output+voices[k].tobytes() + +with open("voice.bin",'wb') as op: + op.write(output) +*/ +int init_voice_style(const char* voices_path){ + // Load "voices.bin" + gchar *file_contents = NULL; + gsize file_length = 0; + GError *error = NULL; + + if (!g_file_get_contents(voices_path, &file_contents, &file_length, &error)) { + g_printerr("Error reading file: %s\n", error->message); + g_clear_error(&error); + return -1; + } + + //2048 for nano + if (file_length == 2048 * sizeof(float)) { + model_type = 1; + ROWS = 8; + } + //819200 for mini + else if (file_length == 819200 * sizeof(float)) { + model_type = 2; + ROWS = 3200; + } + else { + g_printerr("Error: File size is incorrect must be 3200x256 floats or 8x256 floats.\n"); + g_free(file_contents); + return -1; + } + + voice_styles = (float *)file_contents; + return 0; +} + +void cleanup_voice_style(){ + // Clean up memory + g_free(voice_styles); +} + +// voices style loads and array of 256 floats that will get passed to the model +// to determine how it styles the voice. The style will change based of what +// voice you have selected and how long your text lenght is. But on the nano model +// It is simply a change in voice, the length is not used. +GArray *get_style(const char *text, const char *voice) { + const char *voices[] = { + "Leo", "Kiki", "Hugo", "Rosie", + "Bruno", "Luna", "Jasper", "Bella" + }; + gsize num_voices = G_N_ELEMENTS(voices); + + // Calculate ref_id + gsize text_len = g_utf8_strlen(text, -1); + int ref_id = (text_len < 399) ? (int)text_len : 399; + if (model_type == 1){ + ref_id = 0; //nano model does not change style based off text length + } + + // Find index of 'voice' in 'voices' + int voice_index = -1; + for (gsize i = 0; i < num_voices; i++) { + if (g_strcmp0(voice, voices[i]) == 0) { + voice_index = (int)i; // 0-based index + break; + } + } + + if (voice_index == -1) { + g_printerr("Error: Voice '%s' not found in voices list.\n", voice); + return NULL; + } + + // Calculate target index in voice_styles + // Formula: (voice_index * 400) + ref_id + int target_row; + if (model_type == 1){ + target_row = voice_index; + } + else { + target_row = (voice_index * 400) + ref_id; + } + + if (target_row < 0 || target_row >= ROWS) { + g_printerr("Error: Calculated row index %d out of bounds.\n", target_row); + return NULL; + } + + // Pointer to the specific 256-float block + float *selected_voice_style = &voice_styles[target_row * COLS]; + + // create our output array from the selection. + GArray *output_array = g_array_new(FALSE, FALSE, sizeof(gfloat)); + g_array_append_vals(output_array, selected_voice_style, COLS);//should always take 256 values + + return output_array; +} + +// The model will take your text in as char indices based off a fixed +// index string, the index string has been set above as SYMBOLS +// Which will get passed to this function in as the index_str +// the out will be an array of int's each being an indice. +GArray *get_char_indices(const gchar *locate, const gchar *index_str) { + if (locate == NULL || index_str == NULL) { + return NULL; + } + + gsize char_length = g_utf8_strlen(locate, -1); + if (char_length >= 400){ + fprintf(stderr, "Error: string is over 400 char's long, this indice will not run\n"); + } + + GArray *output_array = g_array_sized_new(FALSE, FALSE, sizeof(int64_t), char_length); + + // Starting value should be a 0. + g_array_append_val(output_array, (int64_t){0}); + + const gchar *curr = locate; + for (gsize i = 0; i < char_length; i++) { + // Extract the 32-bit Unicode code point (gunichar) at the current position + gunichar ch = g_utf8_get_char(curr); + + // Search for the character in index_str + gchar *match = g_utf8_strchr(index_str, -1, ch); + + int64_t index_val; + if (match != NULL) { + // Calculate character offset + index_val = (int64_t)g_utf8_pointer_to_offset(index_str, match); + } else { + index_val = 16; // Character not found, so change it to a space character. + } + + // Append the index value to the GArray + g_array_append_val(output_array, index_val); + + // Advance to the next UTF-8 character. + curr = g_utf8_next_char(curr); + } + + // Ending values should always be 10 and 0 + g_array_append_val(output_array, (int64_t){10}); + g_array_append_val(output_array, (int64_t){0}); + + return output_array; +} + +// we use espeak's phoneme function to generate phonemes for the model. +// the phonemes don't get passed directly to the model but turned into +// indices later on. +GString *get_phonemes(const char *text){ + // Initialize espeak. + int samplerate = espeak_Initialize(AUDIO_OUTPUT_SYNCHRONOUS, 0, NULL, 0); + if (samplerate < 0) { + g_printerr("Failed to initialize eSpeak.\n"); + return NULL; + } + + // Set the voice to US English + if (espeak_SetVoiceByName("en-us") != EE_OK) { + g_printerr("Failed to set voice to en-us.\n"); + return NULL; + } + + const void *text_ptr = (const void *)text; + + // Loop through the text to translate it into phonemes. + int textmode = espeakCHARS_AUTO; + + // phonememode = 0x02 triggers IPA (equivalent to how Python's phonemizer acts by default) + int phonememode = 0x02; + + // Create a new empty GString wrapper + GString *combined_phonemes = g_string_new(""); + + while (text_ptr != NULL) { + const char *phonemes = espeak_TextToPhonemes(&text_ptr, textmode, phonememode); + // phonemes is a single char we will have to combine them for the full string + if (phonemes != NULL && *phonemes != '\0') { + g_string_append(combined_phonemes, phonemes); + } + } + + return combined_phonemes; +} + +// we create the tensor that onnx use as input here. +// this tensor will map to our phoneme text inputs indices. +OrtValue* create_inputs_tensor(OrtMemoryInfo* memory_info, GArray *inputs_array){ + // set up the input ids tensor. + int64_t *input_ids_data = (int64_t *)inputs_array->data; + guint elt_size = g_array_get_element_size(inputs_array); + gsize total_bytes = (gsize)inputs_array->len * elt_size; + int64_t input_ids_dims[] = {1, inputs_array->len}; + size_t input_ids_dims_len = 2; + OrtValue* input_ids_tensor = NULL; + CHECK_STATUS(g_ort->CreateTensorWithDataAsOrtValue( + memory_info, input_ids_data, total_bytes, + input_ids_dims, input_ids_dims_len, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, &input_ids_tensor)); + return input_ids_tensor; +} + +// this tensor maps to the choose voice style. +OrtValue* create_styles_tensor(OrtMemoryInfo* memory_info, GArray *styles_array){ + // set up the styles tensor. + float *style_data = (float *)styles_array->data; + guint elt_size2 = g_array_get_element_size(styles_array); + gsize total_bytes2 = (gsize)styles_array->len * elt_size2; + int64_t style_dims[] = {1, 256}; + size_t style_dims_len = 2; + OrtValue* style_tensor = NULL; + CHECK_STATUS(g_ort->CreateTensorWithDataAsOrtValue( + memory_info, style_data, total_bytes2, + style_dims, style_dims_len, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &style_tensor)); + return style_tensor; +} + +OrtValue* create_speed_tensor(OrtMemoryInfo* memory_info, float* speed){ + float *speed_data = speed; + int64_t speed_dims[] = {1}; + size_t speed_dims_len = 1; + OrtValue* speed_tensor = NULL; + CHECK_STATUS(g_ort->CreateTensorWithDataAsOrtValue( + memory_info, speed_data, 4, + speed_dims, speed_dims_len, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &speed_tensor)); + return speed_tensor; +} + +int init_model(const char* model_path){ + + // Initialize the ONNX Runtime API table + g_ort = OrtGetApiBase()->GetApi(ORT_API_VERSION); + if (!g_ort) { + fprintf(stderr, "Failed to initialize ONNX Runtime API.\n"); + return -1; + } + + // Environment and Session Setup + CHECK_STATUS(g_ort->CreateEnv(ORT_LOGGING_LEVEL_WARNING, "kitten_tts_inference", &env)); + + CHECK_STATUS(g_ort->CreateSessionOptions(&session_options)); + // Default provider is CPU when no specific provider is added + CHECK_STATUS(g_ort->CreateSession(env, model_path, session_options, &session)); + + return 0; +} + +void cleanup_model(){ + g_ort->ReleaseSession(session); + g_ort->ReleaseSessionOptions(session_options); + g_ort->ReleaseEnv(env); +} + +// this is where the model is ran. +// the output is a sound wav in the form of a float array +GArray* run_model(GArray *inputs_array, GArray *styles_array, float speed){ + // Prepare Input Data & Allocators + OrtMemoryInfo* memory_info = NULL; + CHECK_STATUS(g_ort->CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &memory_info)); + + OrtValue* input_ids_tensor = create_inputs_tensor(memory_info, inputs_array); + OrtValue* style_tensor = create_styles_tensor(memory_info, styles_array); + float speed_data[] = {speed}; + OrtValue* speed_tensor = create_speed_tensor(memory_info, speed_data); + + // Group Input Names and Tensors + const char* input_names[] = {"input_ids", "style", "speed"}; + const OrtValue* input_tensors[] = {input_ids_tensor, style_tensor, speed_tensor}; + + // models output tensor. + size_t output_count = 1; + const char* output_names[] = {"waveform"}; // Match your specific model definition if needed + OrtValue* output_tensor = NULL; + + // Run Inference + CHECK_STATUS(g_ort->Run( + session, + NULL, + input_names, + input_tensors, + 3, // Total number of input elements + output_names, // Array of target output layer names + output_count, // Number of target outputs + &output_tensor // Target tensor + )); + + // Get the Tensor Type and Shape Info + OrtTensorTypeAndShapeInfo* tensor_info = NULL; + CHECK_STATUS(g_ort->GetTensorTypeAndShape(output_tensor, &tensor_info)); + + // Extract Total Element Count + size_t total_elements = 0; + CHECK_STATUS(g_ort->GetTensorShapeElementCount(tensor_info, &total_elements)); + + // Access the Underlying Raw Data Buffer + float* float_array = NULL; + // float_array need not be freed its simply a pointer to onnx data, it will be + // freed when output_tensor is. + CHECK_STATUS(g_ort->GetTensorMutableData(output_tensor, (void**)&float_array)); + + GArray *copy = g_array_sized_new(FALSE, FALSE, sizeof(float), total_elements); + g_array_append_vals(copy, float_array, total_elements); + + // Free Allocated System Memory Structures + g_ort->ReleaseTensorTypeAndShapeInfo(tensor_info); + g_ort->ReleaseValue(input_ids_tensor); + g_ort->ReleaseValue(style_tensor); + g_ort->ReleaseValue(speed_tensor); + g_ort->ReleaseValue(output_tensor); + g_ort->ReleaseMemoryInfo(memory_info); + + return copy; +} + +// speechd expects a wav of shorts but our model creates a wave of floats +// here we convert the floats wav to a shorts wav. +// based off my testing the floats wav values are always between -1 and 1 +// thus we can convert to shorts using that range. +void convert_float_to_short(const float* in_buffer, GArray* out_buffer, size_t num_samples) { + for (size_t i = 0; i < num_samples; i++) { + // Scale standard floating point [-1.0, 1.0] to 16-bit range + float scaled = in_buffer[i] * 32767.0f; + + // Clamp values to avoid nasty integer overflow wrap-around + if (scaled > 32767.0f) { + scaled = 32767.0f; + } else if (scaled < -32768.0f) { + scaled = -32768.0f; + } + + // Cast to short + g_array_index(out_buffer, gshort, i) = (short)scaled; + } +} + +/* +Here we put all the steps togeather. +The input is the text we wish to generate for. +The output is a GArray of shorts with our audio. +*/ +GArray* kitten_speak(const char* data){ + // turn the text into phonemes + GString *phonemes = get_phonemes(data); + const char* phonemes1 = phonemes->str; + + //The model needs the text to be converted to indices based off SYMBOLS + GArray *inputs_array = get_char_indices(phonemes1, SYMBOLS); + + // we have to get the voice styles. + GArray *styles_array = get_style(data, voice->str); + + // Run the model using onnx. + GArray* output = run_model(inputs_array, styles_array, speed); + + // output comes out as an array of floats representing the wav. + // but we need an array of shorts for speechd to use it. + GArray *output_s = g_array_sized_new(TRUE, TRUE, sizeof(gshort), output->len); + g_array_set_size(output_s, output->len); + convert_float_to_short((float*)output->data, output_s, output->len); + + //clean up. + g_array_unref(output); + g_array_unref(inputs_array); + g_array_unref(styles_array); + g_string_free(phonemes, TRUE); + + return output_s; +} + +// change the model and voice bin to the given model_filename and voice_filename. +int reload_models_and_voices(const char *model_filename, const char* voice_filename){ + //first clean up the old voice model and voice bin. + cleanup_voice_style(); + cleanup_model(); + + //change to the new paths + char *tmp; + tmp = g_build_filename(model_dir->str, model_filename, NULL); + g_string_assign(model_path, tmp); + g_free(tmp); + + tmp = g_build_filename(model_dir->str, voice_filename, NULL); + g_string_assign(voices_path, tmp); + g_free(tmp); + + //reload the model. + init_voice_style(voices_path->str); + init_model(model_path->str); + return 0; +} \ No newline at end of file diff --git a/src/modules/kitten_server.c b/src/modules/kitten_server.c new file mode 100644 index 00000000..fcea84d9 --- /dev/null +++ b/src/modules/kitten_server.c @@ -0,0 +1,218 @@ +/* +The MIT License (MIT) + +Copyright © 2026 John Settlemyer + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#include "kitten.h" + +#include + +static DOTCONF_CB(cb_voicefiles); + +static const configoption_t options[] = { + {"AddVoiceFile", ARG_LIST, cb_voicefiles, NULL, CTX_ALL}, + LAST_OPTION +}; + +bool built_voice_table = false; + +/* +the models .conf can be used to configure the model/voice files/downloads. +add a line to the .conf like below this to do so. +AddVoiceFile "voice_quality" "model url" "model filename" "model size bytes" "model sha256" "voices url" "voices filename" "voices size bytes" "voices sha256" + +here is an example using the same configs as the default. +https://gist.github.com/jsett/6146eb2803f8780a1830ed816bdc94ef +*/ +int module_config(const char *configfilepath) +{ + fprintf(stderr, "opening .conf file %s\n", configfilepath); + + init_file_hashtable(); + + configfile_t *configfile; + + configfile = dotconf_create((char* )configfilepath, + options, NULL, CASE_INSENSITIVE); + + if (!configfile) { + fprintf(stderr, "Error opening config file\n"); + return -1; + } + + if (dotconf_command_loop(configfile) == 0) + fprintf(stderr, "Nothing in config file\n"); + + dotconf_cleanup(configfile); + + if (!built_voice_table){ + fprintf(stderr, "building voice table from defaults. dot conf was not set\n"); + build_hash_from_defaults(files_hash_table); + } + + fprintf(stderr, "finished parsing config file\n"); + + return 0; +} + +DOTCONF_CB(cb_voicefiles) +{ + int i; + if (cmd->arg_count != 9){ + fprintf(stderr, "arg_count: %d\n", cmd->arg_count); + fprintf(stderr, "Incorrectly formated .conf line for AddVoiceFile\n"); + fprintf(stderr, "AddVoiceFile should be formated like\n"); + fprintf(stderr, "AddVoiceFile \"voice_quality\" \"model url\" \"model filename\" \"model size bytes\" \"model sha256\" \"voices url\" \"voices filename\" \"voices size bytes\" \"voices sha256\"\n"); + } else { + + char* quality = cmd->data.list[0]; + + file_hash_add_sub_key(files_hash_table, quality, "model_url", cmd->data.list[1]); + file_hash_add_sub_key(files_hash_table, quality, "model_filename", cmd->data.list[2]); + file_hash_add_sub_key(files_hash_table, quality, "model_size", cmd->data.list[3]); + file_hash_add_sub_key(files_hash_table, quality, "model_sha256", cmd->data.list[4]); + file_hash_add_sub_key(files_hash_table, quality, "voices_url", cmd->data.list[5]); + file_hash_add_sub_key(files_hash_table, quality, "voices_filename", cmd->data.list[6]); + file_hash_add_sub_key(files_hash_table, quality, "voices_size", cmd->data.list[7]); + file_hash_add_sub_key(files_hash_table, quality, "voices_sha256", cmd->data.list[8]); + + built_voice_table = true; + fprintf(stderr, "Added voice file for %s\n", cmd->data.list[2]); + } + return NULL; +} + +int module_init(char **msg) +{ + fprintf(stderr, "initializing\n"); + + int ret = init_model_thread_pool(); + if (ret == 0){ + *msg = strdup("ok!"); + } else{ + *msg = strdup("Failed Init"); + } + + return ret; +} + +SPDVoice **module_list_voices(void) +{ + VOICE_LIST(DEFINE_VOICE) + + static SPDVoice* voices[] = { + VOICE_LIST(VOICE_PTR_ITEM) + NULL + }; + + return voices; +} + +int module_set(const char *var, const char *val) +{ + fprintf(stderr,"got var '%s' to be set to '%s'\n", var, val); + + if (!strcmp(var, "synthesis_voice")) { + // change the voice. + model_change_voice(var, val); + return 0; + } else if (!strcmp(var, "rate")) { + // change the speed. + model_change_speed(var, val); + return 0; + } + + return 0; +} + +int module_audio_set(const char *var, const char *val) +{ + /* Optional: interpret audio parameter */ + if (!strcmp(var, "audio_output_method")) { + /* Only server-side audio supported */ + if (strcmp(val, "server") != 0) + return -1; + return 0; + } + return -1; +} + +int module_audio_init(char **status) +{ + /* Optional: open audio */ + return 0; +} + +int module_loglevel_set(const char *var, const char *val) +{ + /* Optional: accept loglevel change */ + return 0; +} + +int module_debug(int enable, const char *file) +{ + /* Optional: if enable == 1, open file to dump debugging */ + /* Otherwise close it */ + return 0; +} + +int module_loop(void) +{ + /* Main loop */ + fprintf(stderr, "main loop\n"); + + /* Let module_process run the protocol */ + int ret = module_process(STDIN_FILENO, 1); + + if (ret != 0) + fprintf(stderr, "Broken pipe, exiting...\n"); + + return ret; +} + +int module_speak(char *data, size_t bytes, SPDMessageType msgtype) +{ + add_generate_speech_task(data, bytes); + return 1; // delivery to the synthesizer is successful +} + +size_t module_pause(void) +{ + /* Pause playing */ + fprintf(stderr, "pausing\n"); + + // does not seem there is a resume function so + // pause will be handled the same as stop. + model_stop_generation(); + module_report_event_stop(); + + return 0; +} + +int module_stop(void) +{ + /* Stop any current synth */ + fprintf(stderr, "stopping\n"); + + model_stop_generation(); + module_report_event_stop(); + + return 0; +} + +int module_close(void) +{ + /* Deinitialize synthesizer */ + fprintf(stderr, "closing\n"); + + cleanup_threads(); + + return 0; +} diff --git a/src/modules/kitten_worker.c b/src/modules/kitten_worker.c new file mode 100644 index 00000000..a2c9da4d --- /dev/null +++ b/src/modules/kitten_worker.c @@ -0,0 +1,608 @@ +/* +The MIT License (MIT) + +Copyright © 2026 John Settlemyer + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#include "kitten.h" + +typedef struct { + char *data; + size_t size; +} GeneratePayload; + +static GMutex model_mutex; +static GMutex stop_mutex; +static GMutex ahead_mutex; +GAsyncQueue *message_queue; +GAsyncQueue *wav_queue; + +float ahead_by; + +enum wavCommands { BEGIN, STOP, DATA }; + +typedef struct { + enum wavCommands cmd; + GArray *op; + GString *mark; +} WavPayload; + +typedef struct { + GString *text; + GString *mark; +} SSMLPayload; + +static pthread_t kitten_generation_thread; +static pthread_t kitten_play_wav_thread; + +void free_garray(gpointer data) { + g_array_unref((GArray *)data); +} +void free_string(gpointer data) { + g_string_free((GString *)data, TRUE); +} + +void free_SSMLPayload(gpointer data) { + SSMLPayload *tmp = (SSMLPayload*) data; + g_string_free(tmp->mark, TRUE); + g_string_free(tmp->text, TRUE); + g_free(tmp); +} + +void free_WavPayload(gpointer data){ + WavPayload *wp = (WavPayload*) data; + if (wp->cmd == DATA){ + g_string_free(wp->mark, TRUE); + g_array_unref(wp->op); + } + g_free(data); +} + +void free_GeneratePayload(gpointer data){ + GeneratePayload *tmp = (GeneratePayload *) data; + g_free(tmp->data); + g_free(tmp); +} + +// changes the voice, the inputs are the same as module_set +// and this should be called from module_set +int model_change_voice(const char *var, const char *val){ + g_mutex_lock(&model_mutex); + + GString *voice_name = g_string_new(""); + GString *voice_quality = g_string_new(""); + const char *allowed_voices[] = { + "Leo", "Kiki", "Hugo", "Rosie", + "Bruno", "Luna", "Jasper", "Bella", + NULL // Required by g_strv_contains + }; + + // Split the string by '_', limit the split to a maximum of 2 tokens + gchar **tokens = g_strsplit(val, "_", 2); + + if (tokens[0] != NULL && tokens[1] != NULL) { + // Substring contains '_' + g_string_assign(voice_name, tokens[0]); + g_string_assign(voice_quality, tokens[1]); + } else if (tokens[0] != NULL) { + // No '_' present in string + g_string_assign(voice_name, tokens[0]); + g_string_assign(voice_quality, "Normal"); + } + + // Check if voice_name is in the allowed list if not set to Hugo. + if (!g_strv_contains(allowed_voices, voice_name->str)) { + g_string_assign(voice_name, "Hugo"); + } + + g_string_assign(voice, voice_name->str); + + // if the voice quality has changed then we have to reload the correct model and voices bin. + if (!g_string_equal(voice_quality, voice_setting)) { + g_string_assign(voice_setting, voice_quality->str); + if (g_strcmp0(voice_quality->str, "Low") == 0) { + char *model_filename = file_hash_get_value(files_hash_table, "Low", "model_filename"); + char *voice_filename = file_hash_get_value(files_hash_table, "Low", "voices_filename"); + reload_models_and_voices(model_filename, voice_filename); + } else if (g_strcmp0(voice_quality->str, "High") == 0) { + char *model_filename = file_hash_get_value(files_hash_table, "High", "model_filename"); + char *voice_filename = file_hash_get_value(files_hash_table, "High", "voices_filename"); + reload_models_and_voices(model_filename, voice_filename); + } else { + g_string_assign(voice_setting, "Normal"); + char *model_filename = file_hash_get_value(files_hash_table, "Normal", "model_filename"); + char *voice_filename = file_hash_get_value(files_hash_table, "Normal", "voices_filename"); + reload_models_and_voices(model_filename, voice_filename); + } + } + + // Free allocated memory + g_strfreev(tokens); + g_string_free(voice_name, true); + g_string_free(voice_quality, true); + + g_mutex_unlock(&model_mutex); +} + +// changes the speed, the inputs are the same as module_set +// and this should be called from module_set +int model_change_speed(const char *var, const char *val){ + g_mutex_lock(&model_mutex); + + // parse the string into a int + char *endptr = NULL; + gint64 int_val = g_ascii_strtoll(val, &endptr, 10); + int result = (int)int_val; + + // the speed is a multiplier for kitten not a value between -100 and 100 + // so we will map where the lowest value is half speed and the highest is 3X. + if (result < 0) { + // Map [-100, 0] -> [0.5, 1.0] + speed = 1.0f + ((float)result / 100.0f) * 0.5f; + } else { + // Map [0, 100] -> [1.0, 3.0] + speed = 1.0f + ((float)result / 100.0f) * 2.0f; + } + g_mutex_unlock(&model_mutex); +} + +// adds a wave to our wav_queue +void send_wav(GArray *wav, char *mark){ + WavPayload *wp = g_new(WavPayload, 1); + wp->cmd = DATA; + wp->op = wav; + if (mark == NULL){ + wp->mark = g_string_new(""); + } else { + wp->mark = g_string_new(mark); + } + // everything pushed to the wav_queue should get freed by + // _play_wav_thread or the free_WavPayload on destroy. + g_async_queue_push(wav_queue, wp); +} + +// add a begin command to the wav_queue +// this will be processed into a module_report_event_begin(); +void send_wav_start(){ + WavPayload *wp = g_new(WavPayload, 1); + wp->cmd = BEGIN; + // everything pushed to the wav_queue should get freed by + // _play_wav_thread or the free_WavPayload on destroy. + g_async_queue_push(wav_queue, wp); +} + +// add a begin command to the wav_queue +// this will be processed into a module_report_event_end(); +void send_wav_end(){ + WavPayload *wp = g_new(WavPayload, 1); + wp->cmd = STOP; + // everything pushed to the wav_queue should get freed by + // _play_wav_thread or the free_WavPayload on destroy. + g_async_queue_push(wav_queue, wp); +} + +// this gets called on our audio play thread to send out our audio. +void send_samples(short *wav, int len, int rate) +{ + if (!len) + return; + + AudioTrack track = { + .bits = 16, + .num_channels = 1, + .sample_rate = rate, + .num_samples = len, + .samples = wav, + }; + module_tts_output_server(&track, SPD_AUDIO_LE); +} + +void ahead_add(float val){ + g_mutex_lock(&ahead_mutex); + ahead_by += val; + g_mutex_unlock(&ahead_mutex); +} + +void ahead_set(float val){ + g_mutex_lock(&ahead_mutex); + ahead_by = val; + g_mutex_unlock(&ahead_mutex); +} + +float ahead_get(){ + float tmp; + g_mutex_lock(&ahead_mutex); + tmp = ahead_by; + g_mutex_unlock(&ahead_mutex); + return tmp; +} + +void ahead_print(){ + fprintf(stderr, "Ahead by %f seconds of audio\n", ahead_get()); +} + +void stop_set(bool val){ + g_mutex_lock(&stop_mutex); + stop_generation=val; + g_mutex_unlock(&stop_mutex); +} + +bool stop_get(){ + bool tmp; + g_mutex_lock(&stop_mutex); + tmp = stop_generation; + g_mutex_unlock(&stop_mutex); + return tmp; +} + +/* +we use lib xml to parse the ssml string into chunks containing +the text(usally a sentence) and the mark. These then get passed +into a gqueue for future use and returned by the function. +*/ +GQueue* parse_ssml_to_gqueue(const char *data, size_t bytes){ + GQueue *output = g_queue_new(); + + // use libxml to parse the ssml data. + xmlDocPtr doc = xmlReadMemory(data, bytes, "noname.xml", NULL, 0); + if (doc == NULL) { + fprintf(stderr, "Failed to parse XML\n"); + } + + xmlNodePtr root = xmlDocGetRootElement(doc); // node + xmlBufferPtr buffer = xmlBufferCreate(); + + // Iterate through child nodes inside + for (xmlNodePtr cur = root->children; cur != NULL; cur = cur->next) { + if (cur->type == XML_TEXT_NODE) { + // Append text content to buffer + xmlBufferCat(buffer, cur->content); + } + else if (cur->type == XML_ELEMENT_NODE && xmlStrcmp(cur->name, (const xmlChar *)"mark") == 0) { + + xmlChar *mark_name = xmlGetProp(cur, (const xmlChar *)"name"); + + SSMLPayload *pl = g_new(SSMLPayload,1); + + pl->mark = g_string_new(mark_name); + pl->text = g_string_new(buffer->content); + + g_queue_push_tail(output,pl); + + // Clean up the attribute memory and reset the buffer for the next segment + if (mark_name) xmlFree(mark_name); + xmlBufferEmpty(buffer); + + } + } + // add any trailing text after the final mark. + if (buffer->use > 0) { + SSMLPayload *pl = g_new(SSMLPayload,1); + + pl->mark = g_string_new(""); + pl->text = g_string_new(buffer->content); + + g_queue_push_tail(output, pl); + } + + xmlBufferFree(buffer); + xmlFreeDoc(doc); + xmlCleanupParser(); + + return output; +} + +/** + * Splits a GString into chunks containing at most max_utf8_chars UTF-8 characters. + * Returns a GPtrArray of GString pointers containing valid UTF-8 strings. + */ +GPtrArray* split_gstring_utf8(const GString *input_str, glong max_utf8_chars) { + GPtrArray *chunks = g_ptr_array_new_with_free_func((GDestroyNotify)g_string_free); + + if (!input_str || input_str->len == 0 || max_utf8_chars <= 0) { + return chunks; + } + + const gchar *start = input_str->str; + const gchar *end = input_str->str + input_str->len; + + while (start < end) { + // Find the pointer position 'max_utf8_chars' ahead (or stop at 'end' if fewer remain) + const gchar *next_boundary = g_utf8_offset_to_pointer(start, max_utf8_chars); + + // Ensure we don't go past the end of the string buffer + if (next_boundary > end) { + next_boundary = end; + } + + // Calculate exact byte length for this UTF-8 safe slice + gsize byte_len = next_boundary - start; + + // Create the chunk + GString *chunk = g_string_new_len(start, byte_len); + g_ptr_array_add(chunks, chunk); + + // Move start pointer forward + start = next_boundary; + } + + return chunks; +} + +int model_generate_speech(const char *data, size_t bytes){ + g_mutex_lock(&model_mutex); + + //The ssml is parsed using libxml and then returned in a GQueue. + GQueue *ssml_queue = parse_ssml_to_gqueue(data, bytes); + + fprintf(stderr, "speaking '%s'\n", data); + fprintf(stderr, "using voice %s and speed %f\n", voice->str, speed); + + stop_set(false); // make sure that stop generation has been reset to false. + + ahead_set(0.0); // number of seconds the generation is ahead of the played audio. + + send_wav_start(); + + while (!g_queue_is_empty(ssml_queue)) { + SSMLPayload *item = g_queue_pop_head(ssml_queue); + + // Split into chunks of max 399 UTF-8 characters + // our model can not handle anything that is larger. + const glong MAX_CHARS = 399; + GPtrArray *chunks = split_gstring_utf8(item->text, MAX_CHARS); + + for (guint i = 0; i < chunks->len; i++) { + GString *chunk = g_ptr_array_index(chunks, i); + + clock_t gen = clock(); + GArray *op = kitten_speak(chunk->str); + clock_t end = clock(); + double seconds = (double)(end - gen) / CLOCKS_PER_SEC; + fprintf(stderr, "Generated %f seconds of audio in %f seconds\n", op->len/24000.0, seconds); + + //The op array will be freed by the _generation_thread or free_GeneratePayload on destroy. + if (i == chunks->len){ + send_wav(op, item->mark->str); // on the last chunk send the mark also. + } else { + send_wav(op, NULL); + } + + ahead_add(op->len/24000.0); + ahead_print(); + } + + g_ptr_array_free(chunks, TRUE); + free_SSMLPayload(item); + + // if we get more then a 90 seconds ahead then block the thread, + // no need to blow up the cpu on super long texts. + float last_ahead = ahead_get(); + while (last_ahead >= 90.0){ + fprintf(stderr, "ahead 90sec, blocking\n"); + sleep(20);// 20 seconds. + last_ahead = ahead_get(); + + if (stop_get()) + break; + } + + // break the loop if we get a stop event. + if (stop_get()) + break; + } + + send_wav_end(); + + // Cleanup + g_queue_free_full(ssml_queue, free_SSMLPayload); + + g_mutex_unlock(&model_mutex); + return 0; +} + +// this thread loop handles the generation of audio. +void *_generation_thread(void *nothing) +{ + while (1) { + // use a async queue to handle signalling. + GeneratePayload *message_payload = (GeneratePayload *) g_async_queue_pop(message_queue);// this is thread safe and will block if nothing in the queue. + fprintf(stderr,"thread loop: got message: size %d\n",message_payload->size); + + model_generate_speech(message_payload->data, message_payload->size); + + free_GeneratePayload(message_payload); + } + + pthread_exit(NULL); +} + +// this thread loop handles sendding the audio out +// since sending the samples can block, +// it needs to be on its own thread so we don't block +// the model generation thread. +void *_play_wav_thread(void *nothing) +{ + while (1) { + // use a async queue to handle signalling. + WavPayload *wp = (WavPayload*) g_async_queue_pop(wav_queue);// this is thread safe and will block if nothing in the queue. + fprintf(stderr,"wav_thread: got payload\n"); + + if (wp->cmd == BEGIN){ + fprintf(stderr,"wav_thread: module_report_event_begin\n"); + module_report_event_begin(); + } else if (wp->cmd == STOP){ + fprintf(stderr,"wav_thread: module_report_event_end\n"); + module_report_event_end(); + } else if (wp->cmd == DATA){ + fprintf(stderr,"wav_thread: send_samples\n"); + GArray *op = (GArray *) wp->op; + GString *mark = wp->mark; + send_samples((short*)op->data, op->len, 24000); + ahead_add(-1 * ((float)op->len/24000.0)); + if (mark->len != 0){ + module_report_index_mark(mark->str); + } + } + free_WavPayload(wp); + } + + pthread_exit(NULL); +} + +// this pushs a message that will be spoken +// this should be called from module_speak +int add_generate_speech_task(const char* data, size_t bytes) { + GeneratePayload *message_payload; + message_payload = g_new(GeneratePayload, 1); + + message_payload->data = g_memdup2(data, bytes); + message_payload->size = bytes; + // everything pushed to the message_queue should get freed by + // _generation_thread or the free_GeneratePayload on destroy. + g_async_queue_push(message_queue, message_payload); + + return 0; +} + +// this will stop our generation +// its called from module_pause and module_stop +int model_stop_generation(){ + stop_set(true); + return 0; +} + +// This is the same as spd_pthread_create, I move it in here because including $(common_SOURCES) was creating circular dependences for me. +int kitty_pthread_create(pthread_t *thread, const pthread_attr_t *attr, + void *(*start_routine) (void *), void *arg) +{ + int retsig, ret; + sigset_t all_signals; + sigset_t old_signals; + + retsig = sigfillset(&all_signals); + if (retsig != 0) + fprintf(stderr, "Can't fill signal set (%d), expect problems when terminating!\n", retsig); + else { + retsig = pthread_sigmask(SIG_BLOCK, &all_signals, &old_signals); + if (retsig != 0) + fprintf(stderr, "Can't set signal set (%d), expect problems when terminating!\n", retsig); + } + + ret = pthread_create(thread, attr, start_routine, arg); + + if (retsig == 0) + pthread_sigmask(SIG_SETMASK, &old_signals, NULL); + + return ret; +} + +int init_model_thread_pool(){ + fprintf(stderr, "init_model_thread_pool();\n"); + GError *pool_error = NULL; + + g_mutex_init(&model_mutex); + g_mutex_init(&stop_mutex); + g_mutex_init(&ahead_mutex); + + message_queue = g_async_queue_new_full((GDestroyNotify) free_GeneratePayload); + wav_queue = g_async_queue_new_full((GDestroyNotify) free_WavPayload); + + // lock the model mutex while loading + g_mutex_lock(&model_mutex); + + // default the voice to hugo + voice = g_string_new("Hugo"); + // default to the micro model it seems to be the best combo of quality and speed for me. + voice_setting = g_string_new("Normal"); + + home_dir = g_get_home_dir(); + if (!home_dir) { + fprintf(stderr, "Error: Could not determine home directory.\n"); + g_mutex_unlock(&model_mutex); + return -1; + } + + if (use_distro_path()){ + // use disto path. + fprintf(stderr, "INFO: using distro path for loading models.\n"); + model_dir = g_string_new(DISTRO_TARGET_SUBDIR); + } else { + // Build absolute destination directory path: ~/.cache/speech-dispatcher/kitten + fprintf(stderr, "INFO: using user path for loading models.\n"); + model_dir = g_string_new_take(g_build_filename(home_dir, TARGET_SUBDIR, NULL)); + } + char *model_filename = file_hash_get_value(files_hash_table, "Normal", "model_filename"); + char *voice_filename = file_hash_get_value(files_hash_table, "Normal", "voices_filename"); + model_path = g_string_new_take(g_build_filename(model_dir->str, model_filename, NULL)); + voices_path = g_string_new_take(g_build_filename(model_dir->str, voice_filename, NULL)); + + if (kitty_pthread_create(&kitten_generation_thread, NULL, _generation_thread, NULL) != 0){ + fprintf(stderr, "Error: Creating _generation_thread()\n"); + g_mutex_unlock(&model_mutex); + return -1; + } + + if (kitty_pthread_create(&kitten_play_wav_thread, NULL, _play_wav_thread, NULL) != 0){ + fprintf(stderr, "Error: Creating _play_wav_thread()\n"); + g_mutex_unlock(&model_mutex); + return -1; + } + + // download models and voices if they have not been download already. + if (download_models() == -1){ + fprintf(stderr, "Error: Downloading/Verification of models failed\n"); + g_mutex_unlock(&model_mutex); + return -1; + }; + + if (init_voice_style(voices_path->str) == -1){ + fprintf(stderr, "Error: Initializing voice style\n"); + g_mutex_unlock(&model_mutex); + return -1; + }; + + if (init_model(model_path->str) == -1){ + fprintf(stderr, "Error Initializing model/onnx\n"); + g_mutex_unlock(&model_mutex); + return -1; + } + + g_mutex_unlock(&model_mutex); + + return 0; +} + +int cleanup_threads(){ + + // another thread might be using this data + // wait tell its finished before cleaning up. + g_mutex_lock(&model_mutex); + + cleanup_voice_style(); + cleanup_model(); + + g_string_free(model_dir, TRUE); + g_string_free(model_path, TRUE); + g_string_free(voices_path, TRUE); + g_string_free(voice_setting, TRUE); + g_string_free(voice, TRUE); + + g_mutex_unlock(&model_mutex); + + //free the mutex's and queues. + g_mutex_clear(&model_mutex); + g_mutex_clear(&stop_mutex); + g_mutex_clear(&ahead_mutex); + + g_async_queue_unref(wav_queue); + g_async_queue_unref(message_queue); + + g_hash_table_destroy(files_hash_table); +} \ No newline at end of file