From 911386f9dc7efa8678fecd160614339657af459c Mon Sep 17 00:00:00 2001 From: Sola Date: Sat, 15 Aug 2026 12:54:00 +0800 Subject: [PATCH] libpiper migration Migrate now deprecated/archived piper-phonemize[1] based cxxpiper implementation to libpiper[2] which was recently added to the upstream piper repository. Because we are now using the upstream libpiper C library directly, I figured we should probably also remove the cxx prefix from the module name. Changes made: - Removed conf options that new libpiper no longer support - Added peak normalization for parity with Python CLI - Simplified voice handling and configuration [1]: https://github.com/rhasspy/piper-phonemize [2]: https://github.com/OHF-Voice/piper1-gpl/tree/main/libpiper --- INSTALL | 30 +- config/modules/Makefile.am | 4 +- config/modules/cxxpiper.conf | 75 -- config/modules/piper.conf | 56 ++ config/speechd.conf | 1 + configure.ac | 99 ++- src/modules/Makefile.am | 14 +- src/modules/cxxpiper.cpp | 1247 ---------------------------------- src/modules/piper.cpp | 492 ++++++++++++++ 9 files changed, 618 insertions(+), 1400 deletions(-) delete mode 100644 config/modules/cxxpiper.conf create mode 100644 config/modules/piper.conf delete mode 100644 src/modules/cxxpiper.cpp create mode 100644 src/modules/piper.cpp diff --git a/INSTALL b/INSTALL index 0ee1c466e..7ed618e8f 100644 --- a/INSTALL +++ b/INSTALL @@ -152,22 +152,26 @@ chmod +x autogen.sh make make install -Notes about cxxpiper support -============================ +Notes about piper support +========================= -The cxxpiper module uses the Piper synthesis -https://github.com/rhasspy/piper/ +The piper module uses the Piper neural TTS engine +https://github.com/OHF-Voice/piper1-gpl -Upstream piper however does not provide an API that can really be used nicely, -so the build is quite convoluted. The following should be working: +Speech Dispatcher links against the `libpiper` C shared library (and `librubberband`). +To build and install `libpiper` from source: -wget https://github.com/rhasspy/piper-phonemize/releases/download/2023.11.14-4/piper-phonemize_linux_x86_64.tar.gz -sudo tar -C /opt -xvf piper-phonemize_linux_x86_64.tar.gz -wget https://github.com/rhasspy/piper/archive/refs/tags/2023.11.14-2.tar.gz -sudo tar -C /opt -xvf 2023.11.14-2.tar.gz -CXXFLAGS=-I/opt/piper_phonemize/include LDFLAGS=-L/opt/piper_phonemize/lib ./configure --with-piper=/opt/piper-2023.11.14-2 -make -make install + $ git clone https://github.com/OHF-Voice/piper1-gpl.git + $ cd piper1-gpl/libpiper + $ cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr + $ cmake --build build -j$(nproc) + $ sudo cmake --install build + +Then configure Speech Dispatcher with: + $ ./configure --with-piper + +(If libpiper is installed in a non-standard prefix like /opt/piper, use: + $ ./configure --with-piper=/opt/piper) Continue building Speech Dispatcher =================================== diff --git a/config/modules/Makefile.am b/config/modules/Makefile.am index 30cf2900b..6e885f70e 100644 --- a/config/modules/Makefile.am +++ b/config/modules/Makefile.am @@ -54,8 +54,8 @@ dist_moduleconforig_DATA += kali.conf endif if piper_support -dist_moduleconf_DATA += cxxpiper.conf -dist_moduleconforig_DATA += cxxpiper.conf +dist_moduleconf_DATA += piper.conf +dist_moduleconforig_DATA += piper.conf endif if ibmtts_support diff --git a/config/modules/cxxpiper.conf b/config/modules/cxxpiper.conf deleted file mode 100644 index f37d6accd..000000000 --- a/config/modules/cxxpiper.conf +++ /dev/null @@ -1,75 +0,0 @@ -# -# Configuration for cxxpiper speech dispatcher output module. -# - -Debug 0 - -# Piper doesn't have voices ala speech- dispatcher. Piper has a -# "model" and the model's "configuration". A model/config may be -# single speaker or multi-speaker. Single speaker models produce a -# single speech dispatcher voice. The speech dispatcher voice name -# can be listed with 'spd-say -o cxxpiper -L', but it is -# not needed as the voice will be the default, and only, voice available. -# -# Piper multi-speaker models produce a discrete speech dispatcher -# voice for each speaker the model suports. The configuration -# file for a multi-speaker model contains a "speaker_id_map" object -# that lists an integer speaker id and string mneumonic for each -# speaker supported by the model. Since speech dispatcher has no -# notion of speaker id, speaker selection details are hidden from the -# user by instead exposing voices of the form -# ›~~mneumonic with the output module -# mapping between "voice names" and the current model's speakers. - -# NB: Unsure if onnx models may allow different " languages within -# the same multi-speaker model. REgardless, if there's sufficient memory: it might be possible to load multiple -# models and have cxxpiper select between them, while presenting the union of -# the speakers and languages of each model. -# For now all directives with language fields require the language code, but ignore it. - -# ModelPath and ConfigPath are required. There should be exactly one of each of them. -ModelPath "/usr/share/piper/voices/clean100.onnx" -ConfigPath "/usr/share/piper/voices/clean100.onnx.json" - -# For single-speaker models, DefaultVoice is ignored, and logged as such, with -# a warning. For multi-speaker models, DefaultVoice is optional. If it is -# not specified, the first speaker of the multi-speaker model becomes -# the default speaker for the lifetime of the cxxpiper output module -# and also for future runtimes unless this configuration is changed. -# When specified along with a multi-speaker model, the argument is a string that matches one of the -# "voices" listed by spd-say -o cxxpiper -L . Note that piper's -# notion of "speaker" appears to the user as the "voice" concept of -# speech dispatcher. This is pretty much invisible to the user, but -# note that it means that the strings listed in the .json -# configuration file in the speaker_id_map object are not the same as -# the voices listed by spd-say (i.e. the speaker ids are substrings of -# the listed voices). We could also match the substrings, but we -# don't at least for now, only the "voice" string string is -# recognized. If the voice string can't be matched against the voices -# found when the model is loaded, then the first speaker becomes the -# default for the lifetime of the output module. A warning is logged -# if the string can't be matched and the voice name of the first -# speaker (index 0) is included in the warning message. -DefaultVoice "clean100~2~5393" - -# AddVoice (optional) reused from the generic output module. This maps types to voice names within a language code. -# It does not do anything useful for single speaker models and is ignored. For -# multi-speaker models the language code is required, but ignored, at least for now. -AddVoice "en_US" "MALE1" "clean100~33~8419" -AddVoice "en_US" "FEMALE1" "clean100~25~4137" - -# Sound Icons are configured and work like espeak. -SoundIconFolder "/usr/share/sounds/sound-icons/" -SoundIconVolume 0 - -# Piper uses ESpeak NG sometimes, depending on the model. Piper distributes this and distros -# may provide it. Default is "/usr/share/espeak-ng-data/". -# It should probably be considered required, but if a model doesn't use espeak -# it might work to omit it. -ESpeakNGDataDirPath "/usr/share/espeak-ng-data/" - -# IF you have a CUDA GPU, you can try to set this to 1 to benefit from its -# much better speed that your CPU. -UseCUDA 0 - -# End of cxxpiper.conf diff --git a/config/modules/piper.conf b/config/modules/piper.conf new file mode 100644 index 000000000..035d34756 --- /dev/null +++ b/config/modules/piper.conf @@ -0,0 +1,56 @@ +# +# Configuration for piper Speech Dispatcher output module. +# + +# Debugging level (0 = disabled, 1 = errors only, 5 = full debug) +Debug 0 + +# -- Model Configuration -- +# +# ModelPath is required. On Arch Linux (e.g. using the piper-voices-en-us package from AUR), +# voices are installed to /usr/share/piper-voices/////. +# +# en_US-lessac-medium is the recommended standard default voice. +ModelPath "/usr/share/piper-voices/en/en_US/lessac/medium/en_US-lessac-medium.onnx" + +# ConfigPath (optional): +# Path to the voice's JSON configuration file. If omitted, piper automatically +# looks for .json. +# ConfigPath "/usr/share/piper-voices/en/en_US/lessac/medium/en_US-lessac-medium.onnx.json" + +# -- Multi-Speaker Voice Selection (Optional) -- +# +# For single-speaker models (like lessac), a single voice named after the model stem is exposed. +# For multi-speaker models (e.g., libritts_r, vctk, thorsten_emotional), each speaker defined in the +# model's "speaker_id_map" is exposed as an individual voice. +# +# Listing available speaker names: +# 1. Using the Speech Dispatcher CLI: +# spd-say -o piper -L +# 2. Inspecting the model's JSON configuration file directly: +# jq '.speaker_id_map | keys' /path/to/model.onnx.json +# +# DefaultVoice (optional): +# Sets the initial speaker for multi-speaker models using a speaker name from speaker_id_map. +# DefaultVoice "8419" +# +# AddVoice (optional): +# Maps standard Speech Dispatcher voice types (MALE1, FEMALE1, etc.) to specific speakers +# within a language code. +# +# Examples for multi-speaker models: +# AddVoice "en_US" "MALE1" "8419" +# AddVoice "en_US" "FEMALE1" "4137" +# AddVoice "de_DE" "MALE1" "neutral" +# AddVoice "de_DE" "MALE2" "amused" + +# -- Sound Icons -- +# Folder containing sound icon audio files (path must end with a trailing slash '/'). +SoundIconFolder "/usr/share/sounds/sound-icons/" + +# -- ESpeak NG Data Directory -- +# Path to the espeak-ng-data directory used by Piper for phonemization. +# On Arch Linux, this is installed by the 'espeak-ng' package to /usr/share/espeak-ng-data/. +ESpeakNGDataDirPath "/usr/share/espeak-ng-data/" + +# End of piper.conf diff --git a/config/speechd.conf b/config/speechd.conf index 39aa5462b..ca069c29e 100644 --- a/config/speechd.conf +++ b/config/speechd.conf @@ -287,6 +287,7 @@ SymbolsPreprocFile "orca-math.dic" #AddModule "baratinoo" "sd_baratinoo" "baratinoo.conf" #AddModule "rhvoice" "sd_rhvoice" "rhvoice.conf" #AddModule "voxin" "sd_voxin" "voxin.conf" +#AddModule "piper" "sd_piper" "piper.conf" # The output module testing doesn't actually connect to anything. It # outputs the requested commands to standard output and reads diff --git a/configure.ac b/configure.ac index 7858a10b0..be8c2c643 100644 --- a/configure.ac +++ b/configure.ac @@ -427,70 +427,59 @@ AS_IF([test $with_kali != no], [output_modules="${output_modules} kali"]) # check for piper support AC_ARG_WITH([piper], - [AS_HELP_STRING([--with-piper=/path], [include Piper support])], + [AS_HELP_STRING([--with-piper[[=yes|no|check|/path]]], [include Piper TTS support])], [], - [with_piper=no]) -AS_IF([test $with_espeak = yes -a $with_piper != no], [ - # We currently need various headers of piper - PIPER_SRC_DIR="$with_piper/src/cpp" + [with_piper=check]) + +AS_IF([test "$with_piper" != "no"], [ + PIPER_CFLAGS="" + PIPER_LIBS="" + AS_IF([test "$with_piper" != "yes" -a "$with_piper" != "check"], [ + PIPER_CFLAGS="-I$with_piper/include" + PIPER_LIBS="-L$with_piper/lib -lpiper" + ]) OLDCXXFLAGS="$CXXFLAGS" OLDLDFLAGS="$LDFLAGS" - CXXFLAGS="$CXXFLAGS -I$PIPER_SRC_DIR" - - AS_IF([test $with_piper != no], - [PKG_CHECK_MODULES([ONNXRUNTIME], [libonnxruntime], [ - CXXFLAGS="$CXXFLAGS $ONNXRUNTIME_CFLAGS" - LDFLAGS="$LDFLAGS $ONNXRUNTIME_LIBS" - ], [AC_CHECK_LIB([onnxruntime], [OrtGetApiBase], - [ONNXRUNTIME_LIBS="-lonnxruntime"], - [with_piper=no]) - ])]) - - AS_IF([test $with_piper != no], - [PKG_CHECK_MODULES([RUBBERBAND], [rubberband], [ - CXXFLAGS="$CXXFLAGS $RUBBERBAND_CFLAGS" - LDFLAGS="$LDFLAGS $RUBBERBAND_LIBS" - ], [with_piper=no])]) + OLDLIBS="$LIBS" + CXXFLAGS="$CXXFLAGS $PIPER_CFLAGS" + LDFLAGS="$LDFLAGS $PIPER_LIBS" AC_LANG_PUSH(C++) - AS_IF([test $with_piper != no], - [AC_CHECK_LIB([piper_phonemize], [_ZN5piper19DEFAULT_PHONEME_MAPB5cxx11E], - [:], - [with_piper=no])]) - - AS_IF([test $with_piper != no], - [AC_CHECK_HEADER([json.hpp], - [], - [with_piper=no])]) - AS_IF([test $with_piper != no], - [AC_CHECK_HEADER([piper.hpp], - [], - [with_piper=no])]) - AS_IF([test $with_piper != no], - [AC_CHECK_HEADER([utf8.h], - [], - [with_piper=no])]) - AS_IF([test $with_piper != no], - [AC_CHECK_HEADER([wavfile.hpp], - [], - [with_piper=no])]) - - AS_IF([test $with_piper != no], - # We need espeak with https://github.com/espeak-ng/espeak-ng/pull/2127 applied - [AC_CHECK_LIB([espeak-ng], [espeak_TextToPhonemesWithTerminator], - [:], - [with_piper=no], - [$ESPEAK_NG_LIBS])]) - + # Check for piper.h header + AC_CHECK_HEADER([piper.h], [have_piper_h=yes], [have_piper_h=no]) + # Check for libpiper library + AC_CHECK_LIB([piper], [piper_create], [have_libpiper=yes], [have_libpiper=no]) + # Check for nlohmann json + AC_CHECK_HEADER([nlohmann/json.hpp], [have_json_h=yes], [ + AC_CHECK_HEADER([json.hpp], [have_json_h=yes], [have_json_h=no]) + ]) AC_LANG_POP(C++) + CXXFLAGS="$OLDCXXFLAGS" LDFLAGS="$OLDLDFLAGS" + LIBS="$OLDLIBS" + + AS_IF([test "$have_piper_h" = "yes" -a "$have_libpiper" = "yes" -a "$have_json_h" = "yes"], [ + PKG_CHECK_MODULES([RUBBERBAND], [rubberband], [ + AS_IF([test -z "$PIPER_LIBS"], [PIPER_LIBS="-lpiper"]) + with_piper=yes + ], [ + AS_IF([test "$with_piper" = "yes"], [ + AC_MSG_FAILURE([rubberband is required for piper]) + ]) + with_piper=no + ]) + ], [ + AS_IF([test "$with_piper" = "yes"], [ + AC_MSG_FAILURE([libpiper (piper.h, libpiper, and json.hpp) is not available]) + ]) + with_piper=no + ]) ]) -AM_CONDITIONAL([piper_support], [test $with_piper != no]) -AS_IF([test $with_piper != no], [output_modules="${output_modules} cxxpiper"]) -AC_SUBST([PIPER_SRC_DIR]) -AC_SUBST([ONNXRUNTIME_CFLAGS]) -AC_SUBST([ONNXRUNTIME_LIBS]) +AM_CONDITIONAL([piper_support], [test "$with_piper" = "yes"]) +AS_IF([test "$with_piper" = "yes"], [output_modules="${output_modules} piper"]) +AC_SUBST([PIPER_CFLAGS]) +AC_SUBST([PIPER_LIBS]) AC_SUBST([RUBBERBAND_CFLAGS]) AC_SUBST([RUBBERBAND_LIBS]) diff --git a/src/modules/Makefile.am b/src/modules/Makefile.am index 5cc53d537..56b75e284 100644 --- a/src/modules/Makefile.am +++ b/src/modules/Makefile.am @@ -295,16 +295,14 @@ endif endif # -# cxxpiper +# piper # if piper_support -modulebin_PROGRAMS += sd_cxxpiper -sd_cxxpiper_SOURCES = cxxpiper.cpp module_utils_addvoice.c module_utils_play.c $(common_SOURCES) -sd_cxxpiper_CPPFLAGS = -I$(PIPER_SRC_DIR) $(ONNXRUNTIME_CFLAGS) $(RUBBERBAND_CFLAGS) $(AM_CPPFLAGS) -sd_cxxpiper_LDADD = $(top_builddir)/src/common/libcommon.la \ - -lpiper_phonemize \ - $(ONNXRUNTIME_LIBS) \ - -lespeak-ng \ +modulebin_PROGRAMS += sd_piper +sd_piper_SOURCES = piper.cpp module_utils_addvoice.c module_utils_play.c $(common_SOURCES) +sd_piper_CPPFLAGS = $(PIPER_CFLAGS) $(RUBBERBAND_CFLAGS) $(AM_CPPFLAGS) +sd_piper_LDADD = $(top_builddir)/src/common/libcommon.la \ + $(PIPER_LIBS) \ $(RUBBERBAND_LIBS) \ $(SNDFILE_LIBS) \ $(common_LDADD) diff --git a/src/modules/cxxpiper.cpp b/src/modules/cxxpiper.cpp deleted file mode 100644 index ec7da2dd4..000000000 --- a/src/modules/cxxpiper.cpp +++ /dev/null @@ -1,1247 +0,0 @@ -/* - MIT License - -Copyright (c) 2022 Michael Hansen, 2025 Derek L Davies - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "spd_audio.h" -#include -#include "module_utils.h" - -#define MODULE_NAME "Cxxpiper" -#define DBG_MODNAME "Cxxpiper" -#define MODULE_VERSION "0.0" -#define DEBUG_MODULE 5 -DECLARE_DEBUG(); -//} -namespace cxxpiper { - using namespace std; - using json = nlohmann::json; - // rubberband stretcher. "Realtime" mode, streaming (single pass, no "study"), clipping be damned. - // Used for real time pitch, rate and volume adjustment. - using RubberBand::RubberBandStretcher; - using namespace piper; - -#ifdef _CXXPIPER_VERSION -#define _STR(x) #x -#define STR(x) _STR(x) - const std::string VERSION = STR(_CXXPIPER_VERSION); -#else - const std::string VERSION = ""; -#endif - std::string getVersion() { return VERSION; } - const std::string instanceName{"cxxpiper"}; - - const float MIN_WAV_VALUE = -32766.0f; - const float MAX_WAV_VALUE = 32767.0f; - - static const int bs = 1024; - static float **cbuf; - static piper::PiperConfig piperConfig; - static char *cmdInp; - static int stop_requested; - - static size_t cbCnt = 0; // DEBUG only - static size_t cbTot = 0; - static size_t prCnt = 0; - static size_t prTot = 0; - static size_t izCnt = 0; - static size_t izTot = 0; - static size_t ezCnt = 0; - static size_t ezTot = 0; - - extern "C" { - static int cxxpiper_free_cbuf(); - static void cxxpiper_free_voice_list(); - } - -// Copied from piper distribution. -// True if the string is a single UTF-8 codepoint - bool isSingleCodepoint(std::string s) - { - return utf8::distance(s.begin(), s.end()) == 1; - } - -// Copied from piper distribution. -// Get the first UTF-8 codepoint of a string - Phoneme getCodepoint(std::string s) - { - utf8::iterator character_iter(s.begin(), s.begin(), s.end()); - return *character_iter; - } - -// Copied from piper distribution. - void parsePhonemizeConfig(json &configRoot, PhonemizeConfig &phonemizeConfig) - { - // { - // "espeak": { - // "voice": "" - // }, - // "phoneme_type": "", - // "phoneme_map": { - // "": ["", "", ...] - // }, - // "phoneme_id_map": { - // "": [, , ...] - // } - // } - if (configRoot.contains("espeak")) { - auto espeakValue = configRoot["espeak"]; - if (espeakValue.contains("voice")) { - phonemizeConfig.eSpeak.voice = espeakValue["voice"].get(); - } - } - if (configRoot.contains("phoneme_type")) { - auto phonemeTypeStr = configRoot["phoneme_type"].get(); - if (phonemeTypeStr == "text") { - phonemizeConfig.phonemeType = TextPhonemes; - } - } - // phoneme to [id] map - // Maps phonemes to one or more phoneme ids (required). - if (configRoot.contains("phoneme_id_map")) { - auto phonemeIdMapValue = configRoot["phoneme_id_map"]; - for (auto &fromPhonemeItem : phonemeIdMapValue.items()) { - std::string fromPhoneme = fromPhonemeItem.key(); - if (!isSingleCodepoint(fromPhoneme)) { - std::stringstream idsStr; - for (auto &toIdValue : fromPhonemeItem.value()) { - PhonemeId toId = toIdValue.get(); - idsStr << toId << ","; - } - //error("\"{}\" is not a single codepoint (ids={})", fromPhoneme, - //idsStr.str()); - throw std::runtime_error( - "Phonemes must be one codepoint (phoneme id map)"); - } - auto fromCodepoint = getCodepoint(fromPhoneme); - for (auto &toIdValue : fromPhonemeItem.value()) { - PhonemeId toId = toIdValue.get(); - phonemizeConfig.phonemeIdMap[fromCodepoint].push_back(toId); - } - } - } - // phoneme to [phoneme] map - // Maps phonemes to one or more other phonemes (not normally used). - if (configRoot.contains("phoneme_map")) { - if (!phonemizeConfig.phonemeMap) { - phonemizeConfig.phonemeMap.emplace(); - } - auto phonemeMapValue = configRoot["phoneme_map"]; - for (auto &fromPhonemeItem : phonemeMapValue.items()) { - std::string fromPhoneme = fromPhonemeItem.key(); - if (!isSingleCodepoint(fromPhoneme)) { - //error("\"{}\" is not a single codepoint", fromPhoneme); - throw std::runtime_error( - "Phonemes must be one codepoint (phoneme map)"); - } - auto fromCodepoint = getCodepoint(fromPhoneme); - for (auto &toPhonemeValue : fromPhonemeItem.value()) { - std::string toPhoneme = toPhonemeValue.get(); - if (!isSingleCodepoint(toPhoneme)) { - throw std::runtime_error( - "Phonemes must be one codepoint (phoneme map)"); - } - auto toCodepoint = getCodepoint(toPhoneme); - (*phonemizeConfig.phonemeMap)[fromCodepoint].push_back(toCodepoint); - } - } - } - } - -// Copied from piper distribution. - void parseSynthesisConfig(json &configRoot, SynthesisConfig &synthesisConfig) - { - if (configRoot.contains("audio")) { - auto audioValue = configRoot["audio"]; - if (audioValue.contains("sample_rate")) { - // Default sample rate is 22050 Hz - synthesisConfig.sampleRate = audioValue.value("sample_rate", 22050); - } - } - if (configRoot.contains("inference")) { - // Overrides default inference settings - auto inferenceValue = configRoot["inference"]; - if (inferenceValue.contains("noise_scale")) { - synthesisConfig.noiseScale = inferenceValue.value("noise_scale", 0.667f); - } - if (inferenceValue.contains("length_scale")) { - synthesisConfig.lengthScale = inferenceValue.value("length_scale", 1.0f); - } - if (inferenceValue.contains("noise_w")) { - synthesisConfig.noiseW = inferenceValue.value("noise_w", 0.8f); - } - if (inferenceValue.contains("phoneme_silence")) { - // phoneme -> seconds of silence to add after - synthesisConfig.phonemeSilenceSeconds.emplace(); - auto phonemeSilenceValue = inferenceValue["phoneme_silence"]; - for (auto &phonemeItem : phonemeSilenceValue.items()) { - std::string phonemeStr = phonemeItem.key(); - if (!isSingleCodepoint(phonemeStr)) { - //error("\"{}\" is not a single codepoint", phonemeStr); - throw std::runtime_error( - "Phonemes must be one codepoint (phoneme silence)"); - } - auto phoneme = getCodepoint(phonemeStr); - (*synthesisConfig.phonemeSilenceSeconds)[phoneme] = - phonemeItem.value().get(); - } - } // if phoneme_silence - } // if inference - } - -// Copied from piper distribution. - void parseModelConfig(json &configRoot, ModelConfig &modelConfig) - { - modelConfig.numSpeakers = configRoot["num_speakers"].get(); - if (configRoot.contains("speaker_id_map")) { - if (!modelConfig.speakerIdMap) { - modelConfig.speakerIdMap.emplace(); - } - auto speakerIdMapValue = configRoot["speaker_id_map"]; - for (auto &speakerItem : speakerIdMapValue.items()) { - std::string speakerName = speakerItem.key(); - (*modelConfig.speakerIdMap)[speakerName] = - speakerItem.value().get(); - //fprintf(stderr, "DLD| %lu speakerName %s\n", - //speakerItem.value().get(), speakerName); - } - } - } - -// Copied from piper distribution. - void initialize(PiperConfig &config) - { - if (config.useESpeak) { - // Set up espeak-ng for calling espeak_TextToPhonemesWithTerminator - // See: https://github.com/rhasspy/espeak-ng - //debug("Initializing eSpeak"); - int result = espeak_Initialize(AUDIO_OUTPUT_SYNCHRONOUS, 0, config.eSpeakDataPath.c_str(), 0); - if (result < 0) { - throw std::runtime_error("Failed to initialize eSpeak-ng"); - } - } - // Load onnx model for libtashkeel - // https://github.com/mush42/libtashkeel/ - if (config.useTashkeel) { - //debug("Using libtashkeel for diacritization"); - if (!config.tashkeelModelPath) { - throw std::runtime_error("No path to libtashkeel model"); - } - //debug("Loading libtashkeel model from {}", - //config.tashkeelModelPath.value()); - config.tashkeelState = std::make_unique(); - tashkeel::tashkeel_load(config.tashkeelModelPath.value(), - *config.tashkeelState); - } - //info("Initialized piper"); - } - - void terminate(PiperConfig &config) - { - (void)cxxpiper_free_cbuf(); - (void)cxxpiper_free_voice_list(); - if (config.useESpeak) { - espeak_Terminate(); - } - //piper::terminate(piperConfig); - } - -// Copied from piper distribution. - void loadModel(std::string modelPath, ModelSession &session, bool useCuda) - { - //info("Loading onnx model from {}", modelPath); - session.env = Ort::Env(OrtLoggingLevel::ORT_LOGGING_LEVEL_WARNING, - instanceName.c_str()); - session.env.DisableTelemetryEvents(); - if (useCuda) { - // Use CUDA provider - OrtCUDAProviderOptions cuda_options{}; - cuda_options.cudnn_conv_algo_search = OrtCudnnConvAlgoSearchHeuristic; - session.options.AppendExecutionProvider_CUDA(cuda_options); - } - // Slows down performance by ~2x - // session.options.SetIntraOpNumThreads(1); - // Roughly doubles load time for no visible inference benefit - // session.options.SetGraphOptimizationLevel( - // GraphOptimizationLevel::ORT_ENABLE_EXTENDED); - session.options.SetGraphOptimizationLevel( - GraphOptimizationLevel::ORT_DISABLE_ALL); - // Slows down performance very slightly - // session.options.SetExecutionMode(ExecutionMode::ORT_PARALLEL); - session.options.DisableCpuMemArena(); - session.options.DisableMemPattern(); - session.options.DisableProfiling(); - //auto startTime = std::chrono::steady_clock::now(); - auto modelPathStr = modelPath.c_str(); - session.onnx =Ort::Session(session.env, modelPathStr, session.options); - //auto endTime = std::chrono::steady_clock::now(); - //info("Loaded onnx model in {} second(s)", - //std::chrono::duration(endTime - startTime).count()); - } - -// Copied from piper distribution. - void loadVoice(PiperConfig &config, std::string modelPath, - std::string modelConfigPath, Voice &voice, - std::optional &speakerId, bool useCuda) - { - //debug("loadVoice: Parsing voice config from {}", modelConfigPath); - std::ifstream modelConfigFile(modelConfigPath); - voice.configRoot = json::parse(modelConfigFile); - parsePhonemizeConfig(voice.configRoot, voice.phonemizeConfig); - parseSynthesisConfig(voice.configRoot, voice.synthesisConfig); - parseModelConfig(voice.configRoot, voice.modelConfig); - if (voice.modelConfig.numSpeakers > 1) { - if (speakerId) { - DBG(DBG_MODNAME "Using Multi-speaker model, found configured speaker id %ld", speakerId.value()); - voice.synthesisConfig.speakerId = speakerId; - } else { - DBG(DBG_MODNAME "Using Multi-speaker model with default speaker id 0"); - voice.synthesisConfig.speakerId = 0; - } - } - DBG(DBG_MODNAME "Model contains %d speaker(s)", voice.modelConfig.numSpeakers); - loadModel(modelPath, voice.session, useCuda); - } - -// Copied from piper distribution. - void synthesize(std::vector &phonemeIds, - SynthesisConfig &synthesisConfig, ModelSession &session, ModelConfig &modelConfig, - std::vector &audioBuffer, SynthesisResult &result) - { - //debug("Synthesizing audio for {} phoneme id(s)", phonemeIds.size()); - auto memoryInfo = Ort::MemoryInfo::CreateCpu( - OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault); - // Allocate - std::vector phonemeIdLengths{(int64_t)phonemeIds.size()}; - std::vector scales{synthesisConfig.noiseScale, - synthesisConfig.lengthScale, - synthesisConfig.noiseW}; - std::vector inputTensors; - std::vector phonemeIdsShape{1, (int64_t)phonemeIds.size()}; - inputTensors.push_back(Ort::Value::CreateTensor( - memoryInfo, phonemeIds.data(), phonemeIds.size(), phonemeIdsShape.data(), - phonemeIdsShape.size())); - std::vector phomemeIdLengthsShape{(int64_t)phonemeIdLengths.size()}; - inputTensors.push_back(Ort::Value::CreateTensor( - memoryInfo, phonemeIdLengths.data(), phonemeIdLengths.size(), - phomemeIdLengthsShape.data(), phomemeIdLengthsShape.size())); - std::vector scalesShape{(int64_t)scales.size()}; - inputTensors.push_back( - Ort::Value::CreateTensor(memoryInfo, scales.data(), scales.size(), - scalesShape.data(), scalesShape.size())); - // Add speaker id. - // NOTE: These must be kept outside the "if" below to avoid being deallocated. - std::vector speakerId{ - (int64_t)synthesisConfig.speakerId.value_or(0)}; - std::vector speakerIdShape{(int64_t)speakerId.size()}; - if (modelConfig.numSpeakers > 1) { - inputTensors.push_back(Ort::Value::CreateTensor( - memoryInfo, speakerId.data(), speakerId.size(), speakerIdShape.data(), - speakerIdShape.size())); - } - // From export_onnx.py - std::array inputNames_multi = {"input", "input_lengths", "scales", "sid"}; - std::array inputNames = {"input", "input_lengths", "scales"}; - std::array outputNames = {"output"}; - // Infer - auto startTime = std::chrono::steady_clock::now(); - auto outputTensors = session.onnx.Run( - Ort::RunOptions{nullptr}, modelConfig.numSpeakers > 1 ? inputNames_multi.data() : inputNames.data(), inputTensors.data(), - inputTensors.size(), outputNames.data(), outputNames.size()); - auto endTime = std::chrono::steady_clock::now(); - if ((outputTensors.size() != 1) || (!outputTensors.front().IsTensor())) { - throw std::runtime_error("Invalid output tensors"); - } - auto inferDuration = std::chrono::duration(endTime - startTime); - result.inferSeconds = inferDuration.count(); - const float *audio = outputTensors.front().GetTensorData(); - auto audioShape = - outputTensors.front().GetTensorTypeAndShapeInfo().GetShape(); - int64_t audioCount = audioShape[audioShape.size() - 1]; - result.audioSeconds = (double)audioCount / (double)synthesisConfig.sampleRate; - result.realTimeFactor = 0.0; - if (result.audioSeconds > 0) { - result.realTimeFactor = result.inferSeconds / result.audioSeconds; - } - //fprintf(stderr, "Synthesized %f second(s) of audio in %f second(s)\n", - //result.audioSeconds, result.inferSeconds); - // Get max audio value for scaling - float maxAudioValue = 0.01f; - for (int64_t i = 0; i < audioCount; i++) { - float audioValue = abs(audio[i]); - if (audioValue > maxAudioValue) { - maxAudioValue = audioValue; - } - } - // We know the size up front - audioBuffer.reserve(audioCount); - // Scale audio to fill range and convert to int16 - float audioScale = (MAX_WAV_VALUE / std::max(0.01f, maxAudioValue)); - for (int64_t i = 0; i < audioCount; i++) { - int16_t intAudioValue = static_cast( - std::clamp(audio[i] * audioScale, - static_cast(std::numeric_limits::min()), - static_cast(std::numeric_limits::max()))); - audioBuffer.push_back(intAudioValue); - } - // Clean up - for (std::size_t i = 0; i < outputTensors.size(); i++) { - Ort::detail::OrtRelease(outputTensors[i].release()); - } - for (std::size_t i = 0; i < inputTensors.size(); i++) { - Ort::detail::OrtRelease(inputTensors[i].release()); - } - } - -// Copied from piper . distribution. - void textToAudio(PiperConfig &config, Voice &voice, std::string text, - std::vector &audioBuffer, SynthesisResult &result, - const std::function &audioCallback) - { - std::size_t sentenceSilenceSamples = 0; - if (voice.synthesisConfig.sentenceSilenceSeconds > 0) { - sentenceSilenceSamples = (std::size_t)( - voice.synthesisConfig.sentenceSilenceSeconds * - voice.synthesisConfig.sampleRate * voice.synthesisConfig.channels); - } - if (config.useTashkeel) { - if (!config.tashkeelState) { - throw std::runtime_error("Tashkeel model is not loaded"); - } - //debug("Diacritizing text with libtashkeel: {}", text); - text = tashkeel::tashkeel_run(text, *config.tashkeelState); - } - // Phonemes for each sentence - std::vector> phonemes; - if (voice.phonemizeConfig.phonemeType == eSpeakPhonemes) { - // Use espeak-ng for phonemization - eSpeakPhonemeConfig eSpeakConfig; - eSpeakConfig.voice = voice.phonemizeConfig.eSpeak.voice; - phonemize_eSpeak(text, eSpeakConfig, phonemes); - } else { - // Use UTF-8 codepoints as "phonemes" - CodepointsPhonemeConfig codepointsConfig; - phonemize_codepoints(text, codepointsConfig, phonemes); - } - // Synthesize each sentence independently. - std::vector phonemeIds; - std::map missingPhonemes; - for (auto phonemesIter = phonemes.begin(); phonemesIter != phonemes.end(); - ++phonemesIter) { - std::vector &sentencePhonemes = *phonemesIter; - std::vector>> phrasePhonemes; - std::vector phraseResults; - std::vector phraseSilenceSamples; - // Use phoneme/id map from config - PhonemeIdConfig idConfig; - idConfig.phonemeIdMap = - std::make_shared(voice.phonemizeConfig.phonemeIdMap); - if (voice.synthesisConfig.phonemeSilenceSeconds) { - // Split into phrases - std::map &phonemeSilenceSeconds = - *voice.synthesisConfig.phonemeSilenceSeconds; - auto currentPhrasePhonemes = std::make_shared>(); - phrasePhonemes.push_back(currentPhrasePhonemes); - for (auto sentencePhonemesIter = sentencePhonemes.begin(); - sentencePhonemesIter != sentencePhonemes.end(); - sentencePhonemesIter++) { - Phoneme ¤tPhoneme = *sentencePhonemesIter; - currentPhrasePhonemes->push_back(currentPhoneme); - if (phonemeSilenceSeconds.count(currentPhoneme) > 0) { - // Split at phrase boundary - phraseSilenceSamples.push_back( - (std::size_t)(phonemeSilenceSeconds[currentPhoneme] * - voice.synthesisConfig.sampleRate * - voice.synthesisConfig.channels)); - currentPhrasePhonemes = std::make_shared>(); - phrasePhonemes.push_back(currentPhrasePhonemes); - } - } - } else { - // Use all phonemes - phrasePhonemes.push_back( - std::make_shared>(sentencePhonemes)); - } - // Ensure results/samples are the same size - while (phraseResults.size() < phrasePhonemes.size()) { - phraseResults.emplace_back(); - } - while (phraseSilenceSamples.size() < phrasePhonemes.size()) { - phraseSilenceSamples.push_back(0); - } - // phonemes -> ids -> audio - for (size_t phraseIdx = 0; phraseIdx < phrasePhonemes.size(); phraseIdx++) { - if (phrasePhonemes[phraseIdx]->size() <= 0) { - continue; - } - // phonemes -> ids - phonemes_to_ids(*(phrasePhonemes[phraseIdx]), idConfig, phonemeIds, - missingPhonemes); - // ids -> audio - synthesize(phonemeIds, voice.synthesisConfig, voice.session, voice.modelConfig, audioBuffer, - phraseResults[phraseIdx]); - // Add end of phrase silence - for (std::size_t i = 0; i < phraseSilenceSamples[phraseIdx]; i++) { - audioBuffer.push_back(0); - } - result.audioSeconds += phraseResults[phraseIdx].audioSeconds; - result.inferSeconds += phraseResults[phraseIdx].inferSeconds; - phonemeIds.clear(); - } - // Add end of sentence silence - if (sentenceSilenceSamples > 0) { - for (std::size_t i = 0; i < sentenceSilenceSamples; i++) { - audioBuffer.push_back(0); - } - } - if (audioCallback) { - // Call back must copy audio since it is cleared afterwards. - audioCallback(); - audioBuffer.clear(); - } - phonemeIds.clear(); - } - if (missingPhonemes.size() > 0) { - //warn("Missing {} phoneme(s) from phoneme/id map!", - //missingPhonemes.size()); - for (auto phonemeCount : missingPhonemes) { - std::string phonemeStr; - utf8::append(phonemeCount.first, std::back_inserter(phonemeStr)); - //warn("Missing \"{}\" (\\u{:04X}): {} time(s)", phonemeStr, - //(uint32_t)phonemeCount.first, phonemeCount.second); - } - } - if (result.audioSeconds > 0) { - result.realTimeFactor = result.inferSeconds / result.audioSeconds; - } - } - - RubberBandStretcher::Options initOptions() - { - RubberBandStretcher::Options options = 0; - options |= RubberBandStretcher::OptionProcessRealTime; - options |= RubberBandStretcher::OptionPhaseIndependent; - switch (0) { - case 0: - options |= RubberBandStretcher::OptionThreadingAuto; - break; - case 1: - options |= RubberBandStretcher::OptionThreadingNever; - break; - case 2: - options |= RubberBandStretcher::OptionThreadingAlways; - break; - } - options |= RubberBandStretcher::OptionTransientsCrisp; - options |= RubberBandStretcher::OptionDetectorCompound; - - return options; - } - - void internalizeSamples(vector ibuf, float** cbuf, const int channels, const int start, const int count) - { - //debug("Internalizing: ibuf.size: [], start: [], count: []", ibuf.size(), start, count); - for (int c = 0; c < channels; ++c) { - for (int i = 0; i < count; ++i) { - cbuf[c][i] = (float)(ibuf[start + (i * channels) + c]); - } - if (count < bs) { - for (size_t i = count; i < bs; ++i) { - cbuf[c][i] = MIN_WAV_VALUE; - } - } - } - ++izCnt; - izTot += count; - } - - void externalizeSamples(float **cbuf, vector& ibuf, const float& gain, const int channels, const int start, const int count) - { - //debug("Externalizing: ibuf.size: [], start: [], count: []", ibuf.size(), start, count); - for (int c = 0; c < channels; ++c) { - for (int i = 0; i < count; ++i) { - float value = gain * cbuf[c][i]; - if (value < MIN_WAV_VALUE) value = MIN_WAV_VALUE; - if (value > MAX_WAV_VALUE) value = MAX_WAV_VALUE; - ibuf.push_back((int16_t)value); - } - } - ++ezCnt; - ezTot += count; - } - - int adjust(const int samplerate, const int channels, double ratio, double pitchshift, float gain, - vector& audioBuffer, vector& sharedAudioBuffer) - { - DBG("adjust, ab size: %lu ratio: %f pitchshift %f gain %f", audioBuffer.size(), ratio, pitchshift, gain); - const size_t abSize = audioBuffer.size(); - assert(abSize > 0); - double frequencyshift = 1.0; - if (pitchshift != 0.0) frequencyshift *= pow(2.0, pitchshift / 12.0); - RubberBandStretcher ts(samplerate, channels, initOptions(), ratio, frequencyshift); - ts.setExpectedInputDuration(audioBuffer.size()); - ts.setMaxProcessSize(bs); - size_t countIn = 0; - size_t countOut = 0; - // "ib" stands for "input block", "ob" for "output block". - size_t ibSize = bs; - size_t ibLastFullIdx = abSize / bs; - assert(ibLastFullIdx >= 0); - for (size_t ibIdx = 0; ibIdx < ibLastFullIdx; ++ibIdx) { - assert(countIn % bs == 0); - ibSize = (ibIdx == ibLastFullIdx) ? abSize - countIn : bs; - assert(ibSize >= 0); - assert(ibSize > 0); - assert(ibSize <= bs); - internalizeSamples(audioBuffer, cbuf, channels, countIn, ibSize); - countIn += ibSize; - //DBG("About to ts.process, countIn is now %lu, final? %d", countIn, ibIdx == ibLastFullIdx); - ts.process(cbuf, ibSize, ibIdx == ibLastFullIdx); - ++prCnt; - prTot += ibSize; - int avail; - while ((avail = ts.available()) > 0) { - // All data processed for this block. - if (avail == -1) break; - size_t obSize = bs; - const size_t obLastFullIdx = avail / bs; - // Do all the full blocks. - for (size_t obIdx = 0; obIdx < obLastFullIdx; ++obIdx) { - assert(obSize <= bs); - ts.retrieve(cbuf, obSize); - (void)externalizeSamples(cbuf, sharedAudioBuffer, gain, channels, obIdx * bs, obSize); - countOut += obSize; - } - // do any remaining partial block. - size_t remaining = avail - (obLastFullIdx * bs); - assert(remaining >= 0); - if (remaining > 0) { - //DBG("Retrieving remaining: %lu", remaining); - assert(remaining <= bs); - ts.retrieve(cbuf, remaining); - (void)externalizeSamples(cbuf, sharedAudioBuffer, gain, channels, obLastFullIdx* bs, remaining); - countOut += remaining; - } - } - // All data processed, all done. - if (avail == -1) break; - } - return 0; - } - - extern "C" { - - // OutputType and RunConfig are copied from piper and are , needed for synthesis - // and populated from the arguments to piper's command line demo - // program 'piper'. We, as an SD oOM, populate this mostly from - // the output module config file, cxxpiper.conf . But some stuff - // like speaker id may be updated/overridden per-query from SSIP data. - enum OutputType { OUTPUT_FILE, OUTPUT_DIRECTORY, OUTPUT_STDOUT, OUTPUT_RAW }; - struct RunConfig { - // Path to .onnx voice file - filesystem::path modelPath; - // Path to JSON voice config file - filesystem::path modelConfigPath; - char* defaultVoiceName; - OutputType outputType = OUTPUT_STDOUT; - optional outputPath = filesystem::path("."); - // Numerical id of the default speaker (multi-speaker voices) - optional speakerId; - optional noiseScale; - // Speed of speaking (1 = normal, < 1 is faster, > 1 is slower) - optional lengthScale; - optional noiseW; - // Seconds of silence to add after each sentence - optional sentenceSilenceSeconds; - // Path to espeak-ng data directory (default is next to piper executable) - optional eSpeakDataPath; - // Path to libtashkeel ort model - optional tashkeelModelPath; - bool jsonInput = false; - // Seconds of extra silence to insert after a single phoneme - optional> phonemeSilenceSeconds; - // true to use CUDA execution provider - bool useCuda = false; - }; - static RunConfig runConfig; - static piper::Voice voice; - static SPDVoice **cxxpiper_voice_list = NULL; - - static int cxxpiper_alloc_cbuf(); - static SPDVoice **cxxpiper_allocate_voice_list(); - static void cxxpiper_free_voice_list(); - static void cxxpiper_handle_text(const char *); - static void cxxpiper_handle_sound_icon(const char *); - static void cxxpiper_set_language(char *); - static void cxxpiper_set_synthesis_voice(char *voice_name); - static void cxxpiper_set_voice_type(SPDVoiceType); - static int cxxpiper_voice_name_to_speaker_id(const char *); - - MOD_OPTION_1_INT(UseCUDA) - MOD_OPTION_1_STR(ModelPath) - MOD_OPTION_1_STR(ConfigPath) - MOD_OPTION_1_STR(SoundIconFolder) - MOD_OPTION_1_INT(SoundIconVolume) - // SIC: see below regarding NONE and ALL - MOD_OPTION_1_STR(PunctSome) - MOD_OPTION_1_STR(PunctMost) - MOD_OPTION_1_STR(ESpeakNGDataDirPath) - - int module_load(void) - { - INIT_SETTINGS_TABLES(); - REGISTER_DEBUG(); - MOD_OPTION_1_INT_REG(UseCUDA, 0); - MOD_OPTION_1_STR_REG(ModelPath, ""); - MOD_OPTION_1_STR_REG(ConfigPath, ""); - MOD_OPTION_1_STR_REG(SoundIconFolder, "/usr/share/sounds/sound-icons/"); - MOD_OPTION_1_INT_REG(SoundIconVolume, 0); - // NB: NONE and ALL are constant so no config. NONE means omit as much as possible - // while ALL means omit as little as possible. - // NB: Don't provide defaults, since empty string means fallback to SD's definitions. - MOD_OPTION_1_STR_REG(PunctSome, ""); - MOD_OPTION_1_STR_REG(PunctMost, ""); - MOD_OPTION_1_STR_REG(ESpeakNGDataDirPath, "/usr/share/espeak-ng-data/"); - module_register_available_voices(); - module_register_settings_voices(); - return 0; - } - - int module_init(char **status_info) - { - module_audio_set_server(); - module_audio_init(status_info); - runConfig.useCuda = UseCUDA; - runConfig.modelPath = filesystem::path(ModelPath); - runConfig.modelConfigPath = filesystem::path(ConfigPath); - runConfig.eSpeakDataPath = filesystem::path(ESpeakNGDataDirPath); - runConfig.outputPath = nullopt; - char *default_voice = module_getdefaultvoice(); - runConfig.defaultVoiceName = default_voice; - runConfig.speakerId = cxxpiper_voice_name_to_speaker_id(runConfig.defaultVoiceName); - DBG("Default Voice is %s", runConfig.defaultVoiceName); - try { - cxxpiper::loadVoice(piperConfig, runConfig.modelPath.string(), runConfig.modelConfigPath.string(), - voice, runConfig.speakerId, runConfig.useCuda); - if (voice.phonemizeConfig.phonemeType == piper::eSpeakPhonemes) { - piperConfig.eSpeakDataPath = runConfig.eSpeakDataPath.value().string(); - } - else { - piperConfig.useESpeak = false; - } - cxxpiper::initialize(piperConfig); - cxxpiper_voice_list = cxxpiper_allocate_voice_list(); - cxxpiper_alloc_cbuf(); - *status_info = g_strdup(DBG_MODNAME " Initialized successfully."); - } - catch (const Ort::Exception& e) { - DBG(DBG_MODNAME " Could not initialize,caught onnx runtime (model) exception: %s", e.what()); - *status_info = g_strdup(e.what()); - return -1; - } - catch (const json::parse_error& e) { - DBG(DBG_MODNAME " Could not initialize,caught JSON exception: %s", e.what()); - *status_info = g_strdup(e.what()); - return -1; - } - catch (const std::runtime_error& e) { - DBG(DBG_MODNAME " Could not initialize,caught runtime_error exception: %s", e.what()); - *status_info = g_strdup(e.what()); - return -1; - } - return 0; - } - - SPDVoice **module_list_voices(void) - { - return cxxpiper_voice_list; - } - - void module_speak_sync(const char *data, size_t bytes, SPDMessageType msgtype) - { - stop_requested = 0; - UPDATE_STRING_PARAMETER(voice.language, cxxpiper_set_language); - UPDATE_PARAMETER(voice_type, cxxpiper_set_voice_type); - UPDATE_STRING_PARAMETER(voice.name, cxxpiper_set_synthesis_voice); - module_speak_ok(); - switch (msgtype) { - case SPD_MSGTYPE_CHAR: - case SPD_MSGTYPE_KEY: - case SPD_MSGTYPE_SPELL: - case SPD_MSGTYPE_TEXT: - cxxpiper_handle_text(data); - break; - case SPD_MSGTYPE_SOUND_ICON: - cxxpiper_handle_sound_icon(data); - break; - } - } - - int module_stop(void) - { - stop_requested = 1; - return 0; - } - - size_t module_pause(void) - { - stop_requested = 1; - return 0; - } - - int module_close() - { - if (piperConfig.useESpeak) espeak_Terminate(); - (void)cxxpiper::terminate(piperConfig); - return 0; - } - - static int cxxpiper_alloc_cbuf() - { - const int channels = voice.synthesisConfig.channels; - cbuf = g_new0(float *, channels); - if ( ! cbuf) { - throw std::runtime_error("Failure allocating dynamic memory for adjustment buffer"); - } - for (int c = 0; c < channels; ++c) { - cbuf[c] = g_new0(float, bs); - if ( ! cbuf[c]) { - throw std::runtime_error("Failure allocating dynamic memory for adjustment buffer values"); - } - } - return 0; - } - - static int cxxpiper_free_cbuf() - { - const int channels = voice.synthesisConfig.channels; - if (cbuf) { - for (int c = 0; c < channels; ++c) g_free(cbuf[c]); - g_free(cbuf); - cbuf = NULL; - } - return 0; - } - - static SPDVoice **cxxpiper_allocate_voice_list() - { - SPDVoice **result = NULL; - int num_spkrs = voice.configRoot["num_speakers"]; - DBG("Model config reports num spkrs %d", num_spkrs); - // result consistes of an array of SPDVoice* addresses, - // one address per voice and one more NULL address as - // delimiter. The non-NULL elements point to SPDVoice - // structs and we'll dynamically allocate memory for them, - // below. First the "root" array: - result = g_new0(SPDVoice *, num_spkrs + 1); - if ( ! result) { - throw std::runtime_error("Failure allocating dynamic memory for voice list"); - } - if (num_spkrs > 1) { - if ( ! voice.configRoot.contains("speaker_id_map")) { - //warn("ERROR: Using multispeaker model but could not find speaker_id_map in json model configureation file"); - //throw std::runtime_error("Using multispeaker model but could not find speaker_id_map in json model configureation file"); - } - auto speakerIdMapValue = voice.configRoot["speaker_id_map"]; - DBG("Model config speaker_id_map has %lu entries", speakerIdMapValue.size()); - SPDVoice **reg_voices = module_list_registered_voices(); - int i = 0; - for (auto &speakerItem : speakerIdMapValue.items()) { - std::string spkr_name = speakerItem.key(); - int spkr_id = speakerItem.value().get(); - // Dynamically allocate an SPDVoice and point to it from result[[], the "root" array. - SPDVoice *v = g_new0(SPDVoice, 1); - v->name = g_strdup_printf("%s~%d~%s", runConfig.modelPath.stem().string().c_str(), spkr_id, spkr_name.c_str()); - v->language = g_strdup(voice.configRoot["language"]["code"].get().c_str()); - char *var_str = NULL; - for (int j = 0; reg_voices && reg_voices[j]; ++j) { - SPDVoice* r_v = reg_voices[j]; - if (strncasecmp(v->name, r_v->name, strlen(v->name)) == 0) { - var_str = r_v->variant; - DBG("matched configured voice type %s with voice %s", var_str, r_v->name); - break; - } - } - if (strncasecmp(v->name, runConfig.defaultVoiceName, strlen(v->name)) == 0) { - v->variant = g_strdup((var_str == NULL) ? "(default)" : strcat(var_str, " (default)")); - } - else { - v->variant = g_strdup((var_str == NULL) ? "" : var_str); - } - result[i] = v; - ++i; - } - } - else { - // Theoretically, there is no need to copy voice type string into varient b/c there's only one voice - // and it will be used. However, voice list should show the type string, assuming proper AddVoice configuration. - DBG("Running with single-speaker model"); - SPDVoice *v = g_new0(SPDVoice, 1); - v->name = g_strdup(runConfig.modelPath.stem().string().c_str()); - v->language = g_strdup(voice.configRoot["language"]["code"].get().c_str()); - v->variant = g_strdup(""); - result[0] = v; - } - // Delimit the "root array. - result[num_spkrs] = NULL; - return result; - } - - static void cxxpiper_free_voice_list() - { -#ifdef ESPEAK_NG_INCLUDE - free(espeak_variants_array); -#endif - if (cxxpiper_voice_list != NULL) { - for (int i = 0; cxxpiper_voice_list[i] != NULL; i++) { - g_free(cxxpiper_voice_list[i]->name); - g_free(cxxpiper_voice_list[i]->language); - g_free(cxxpiper_voice_list[i]->variant); - g_free(cxxpiper_voice_list[i]); - } - g_free(cxxpiper_voice_list); - cxxpiper_voice_list = NULL; - } - } - - static char *cxxpiper_voice_enum_to_str(SPDVoiceType voice_type) - { - char *voicename; - switch (voice_type) { - case SPD_MALE1: - voicename = g_strdup("MALE1"); - break; - case SPD_MALE2: - voicename = g_strdup("MALE2"); - break; - case SPD_MALE3: - voicename = g_strdup("MALE3"); - break; - case SPD_FEMALE1: - voicename = g_strdup("FEMALE1"); - break; - case SPD_FEMALE2: - voicename = g_strdup("FEMALE2"); - break; - case SPD_FEMALE3: - voicename = g_strdup("FEMALE3"); - break; - case SPD_CHILD_MALE: - voicename = g_strdup("CHILD_MALE"); - break; - case SPD_CHILD_FEMALE: - voicename = g_strdup("CHILD_FEMALE"); - break; - case SPD_UNSPECIFIED: - // SIC: fall through to default - default: - voicename = NULL; - break; - } - return voicename; - } - - static const char *cxxpiper_search_for_sound_icon(const char *icon_name) - { - char *fn = NULL; - if (strlen(SoundIconFolder) == 0) return fn; - GString *filename = g_string_new(SoundIconFolder); - filename = g_string_append(filename, icon_name); - if (g_file_test(filename->str, G_FILE_TEST_EXISTS)) fn = filename->str; - /* if the file was found, the pointer *fn points to the character data - of the string filename. In this situation the string filename must be - freed but its character data must be preserved. - If the file is not found, the pointer *fn contains NULL. In this - situation the string filename must be freed, including its character - data. - */ - return g_string_free(filename, (fn == NULL)); - } - - void cxxpiper_stretch_and_copy(const int samplerate, const int channels, vector& audioBuffer, vector& sharedAudioBuffer) - { - // In rubberband stretcher units, 1.0 is no duration change. - // Map -100 .. 100 onto 0.0 .. 2.0 . - double ratio = (((double)msg_settings.rate) + 100.0) / 100.0; - // Pitch is converted to frequency internally, as per 'piper' - // command line utility. The 'adjust' function uses the same - // conversion. I think -10.0 to 10.0 - // provides a usable range plus a bit of margin. - // . Map -100 .. 100 onto -10.0 .. 10.0 - // Also clamp input values otherwise we will crash! - double mS = ((double)msg_settings.pitch); - if (mS < -100.0) mS = -100.0; - if (mS > 100.0) mS = 100.0; - double pitchshift = mS / 10.0; - // In rubberband stretcher units, 1.0 is no gain/attenuation. - // Map -100 .. 100 onto 0.0 .. 1.0 - float gain = (((float)msg_settings.volume) + 100.0f) / 200.0f; - (void)adjust(samplerate, channels, ratio, pitchshift, gain, audioBuffer, sharedAudioBuffer); - } - - static void cxxpiper_handle_text(const char *data) - { - DBG("Input data before strip XML: %s", data); - cmdInp = (char *) module_strip_ssml(data); - DBG("Input after strip XML: %s", cmdInp); - mutex mutAudio; - condition_variable cvAudio; - bool audioReady = false; - cbCnt = 0; - cbTot = 0; - prCnt = 0; - prTot = 0; - izCnt =0; - izTot =0; - ezCnt =0; - ezTot =0; - runConfig.lengthScale = msg_settings.rate / 100.0; - vector audioBuffer; - vector sharedAudioBuffer; - piper::SynthesisResult result; - auto audioCallback = [&audioBuffer, &sharedAudioBuffer, &mutAudio, - &cvAudio, &audioReady]() - { - unique_lock lockAudio(mutAudio); -#if 0 - copy(audioBuffer.begin(), audioBuffer.end(), back_inserter(sharedAudioBuffer)); -#else - cxxpiper_stretch_and_copy(voice.synthesisConfig.sampleRate, voice.synthesisConfig.channels, audioBuffer, sharedAudioBuffer); -#endif - audioReady = true; - ++cbCnt; - cbTot += audioBuffer.size(); //cerr << "CVaudio notivy 1" << endl; - cvAudio.notify_one(); - }; - DBG("Sending begin event"); - module_report_event_begin(); - (void)cxxpiper::textToAudio(piperConfig, voice, cmdInp, - audioBuffer, result, audioCallback); - DBG("Did synthesis, ab size is %lu sab size is %lu", - audioBuffer.size(), sharedAudioBuffer.size()); - AudioFormat format = SPD_AUDIO_LE; - AudioTrack track; - track.bits = voice.synthesisConfig.sampleWidth * 8; - track.num_samples = sharedAudioBuffer.size(); - track.samples = &sharedAudioBuffer[0]; // outbuf; - track.num_channels = voice.synthesisConfig.channels; - track.sample_rate = voice.synthesisConfig.sampleRate; - DBG("callback called %lu times, total: %lu", cbCnt, cbTot); - DBG("process called %lu times, total: %lu", prCnt, prTot); - DBG("internalize called %lu times, total: %lu", izCnt, izTot); - DBG("externalize called %lu times, total: %lu", ezCnt, ezTot); - DBG("sending track to audio output"); - module_tts_output_server(&track, format); - DBG("Sending end event"); - module_report_event_end(); - } - - static void cxxpiper_handle_sound_icon(const char *icon_name) - { - int fallback_to_speech = 0; - const char *icon_path = cxxpiper_search_for_sound_icon(icon_name); - if (icon_path != NULL) { - module_report_event_begin(); - (void)module_play_file(icon_path); - module_report_icon(icon_path); - module_report_event_end(); - } - else { - fallback_to_speech = 1; - } - if (fallback_to_speech) { - MSG(3, "Warning: Speaking sound icon name, %s, as a fallback, since audio file can not be found.", icon_name); - cxxpiper_handle_text(icon_name); - } - } - - static void cxxpiper_set_language_and_voice(char *lang, SPDVoiceType voice_type, char *name); - static void cxxpiper_set_language_and_voice(char *lang, SPDVoiceType voice_type, char *name) - { - int i = 0; - int index = -1; - char *tstr = cxxpiper_voice_enum_to_str(voice_type); - - DBG("%s, lang=%s, voice_type=%d, name=%s", - __FUNCTION__, lang, (int)voice_type, name ? name : ""); - // -1 is explicitly "unspecified", 0 is plain old undefined. - if (voice_type > 0) { - // The idea is that if the user submits a query with a - // type, we should honor that type even if we can't - // find a language match. We prefer language and type - // to match, of course. Name is insignificant when type - // is included except that if we find no type match at - // all we fall back to language/name matching, as if - // no type were given in the query. - SPDVoice **vs = module_list_registered_voices(); - SPDVoice *first_close_match = NULL; - for (int i = 0; vs && vs[i]; ++i) { - SPDVoice* v = vs[i]; - DBG("typematch voice name: %s lang: %s variant: %s", v->name, v->language, v->variant); - int type_match_p = strncasecmp(tstr, v->variant, strlen(tstr)); - if (type_match_p == 0 && - strncasecmp(lang, v->language, strlen(lang)) == 0) { - DBG("strong match on language and type, $voice is now %s.", v->name); - voice.synthesisConfig.speakerId = cxxpiper_voice_name_to_speaker_id(v->name); - runConfig.speakerId = voice.synthesisConfig.speakerId; - return; - } - else if (type_match_p) { - first_close_match = v; - } - } - if (first_close_match) { - DBG("Weak match on type only! Assuming configuration is sane, new voice is %s", first_close_match->name); - voice.synthesisConfig.speakerId = cxxpiper_voice_name_to_speaker_id(first_close_match->name); - runConfig.speakerId = voice.synthesisConfig.speakerId; - return; - } - } - SPDVoice **lst = cxxpiper_voice_list; - if (name && *name) { - for (i = 0; lst[i]; ++i) { - if (!strcasecmp(lst[i]->name, name)) { - voice.synthesisConfig.speakerId = cxxpiper_voice_name_to_speaker_id(lst[i]->name); - runConfig.speakerId = voice.synthesisConfig.speakerId; - index = i; - break; - } - } - } - if ((index == -1) && lang) { - char *langbase; // requested base language + '-' - char *dash = strchr(lang, '-'); - if (dash) - langbase = g_strndup(lang, dash-lang+1); - else - langbase = g_strdup_printf("%s-", lang); - for (i = 0; lst[i]; ++i) { - if (!strcasecmp(lst[i]->language, lang)) { - DBG("strong match on language new voice is %s", lst[i]->name); - voice.synthesisConfig.speakerId = cxxpiper_voice_name_to_speaker_id(lst[i]->name); - runConfig.speakerId = voice.synthesisConfig.speakerId; - index = i; - break; - } - if (index == -1) { - /* Try base language matching as fallback */ - if (!strncasecmp(lst[i]->language, langbase, strlen(langbase))) { - DBG("Base language match, new voice is %s", lst[i]->name); - voice.synthesisConfig.speakerId = cxxpiper_voice_name_to_speaker_id(lst[i]->name); - runConfig.speakerId = voice.synthesisConfig.speakerId; - index = i; - } - } - } - g_free(langbase); - } - if (index == -1) { // no matching voice: choose the first available voice - if (!cxxpiper_voice_list[0]) - voice.synthesisConfig.speakerId = cxxpiper_voice_name_to_speaker_id(cxxpiper_voice_list[0]->name); - runConfig.speakerId = voice.synthesisConfig.speakerId; - index = 0; - return; - } - } - - static void cxxpiper_set_voice_type(SPDVoiceType voice_type) - { - const char *vt_str = cxxpiper_voice_enum_to_str(voice_type); - // NB: spd-say -t female1 twice in a row results in voice_type 4 in the first request, then -1 in the second!! - SPDVoice **vs = module_list_registered_voices(); - if (vs != NULL) { - if (vt_str == NULL) { - MSG(3, "Warning: voice type from server is NULL!! Should not happen."); - return; - } - for (int i = 0; vs[i] != NULL; i++) { - const char *vari = vs[i]->variant; - if (strlen(vari) <= strlen(vt_str) && - strncasecmp(vt_str, vari, strlen(vt_str)) == 0) { - int spkr_id = cxxpiper_voice_name_to_speaker_id(vs[i]->name); - voice.synthesisConfig.speakerId = spkr_id; - runConfig.speakerId = spkr_id; - return; - } - } - MSG(3, "Warning: No definition of type %s, check 'AddVoice' directives in cxxpiper.conf file.", vt_str); - } - else { - MSG(3, "Warning: no configuredc voice types, please use AddVoice in cxxpiper.conf to define voice types needed at runtime."); - } - } - - static void cxxpiper_set_language(char *lang) - { - cxxpiper_set_language_and_voice(lang, msg_settings.voice_type, NULL); - } - - static void cxxpiper_set_synthesis_voice(char *voice_name) - { - if (voice_name == NULL) { - return; - } - int spkr_id = -1; - if (msg_settings.voice.name != NULL) { - spkr_id = cxxpiper_voice_name_to_speaker_id(msg_settings.voice.name); - } - else { - spkr_id = cxxpiper_voice_name_to_speaker_id(runConfig.defaultVoiceName); // cxxpiper_voice_name_to_speaker_id(default_voice); - } - voice.synthesisConfig.speakerId = spkr_id; - runConfig.speakerId = voice.synthesisConfig.speakerId; - return; - } - - static int cxxpiper_voice_name_to_speaker_id(const char *voice_name) - { - // The form of a voice name is: - // ~{}~ - int spkr_id; - int num_matches = sscanf(voice_name, "%*[^~]~%d~%*s", &spkr_id); - if (num_matches != 1) return -1; - return spkr_id; - } - - } // extern "C" -} // namespace cxxpiperb diff --git a/src/modules/piper.cpp b/src/modules/piper.cpp new file mode 100644 index 000000000..5d77a116b --- /dev/null +++ b/src/modules/piper.cpp @@ -0,0 +1,492 @@ +/* + * piper.cpp - Speech Dispatcher Piper output module + * + * Copyright (C) 2022 Michael Hansen + * Copyright (C) 2025 Derek L Davies + * Copyright (C) 2026 Sola + * + * This is free software; you can redistribute it and/or modify it under the + * terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 2.1, or (at your option) any later + * version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if __has_include() +#include +#elif __has_include() +#include +#endif + +#include + +#include "spd_audio.h" +#include +#include "module_utils.h" + +#define MODULE_NAME "piper" +#define DBG_MODNAME "piper" +#define MODULE_VERSION "0.1" +#define DEBUG_MODULE 5 +DECLARE_DEBUG(); + +namespace piper_module { + using json = nlohmann::json; + using RubberBand::RubberBandStretcher; + + MOD_OPTION_1_STR(ModelPath) + MOD_OPTION_1_STR(ConfigPath) + MOD_OPTION_1_STR(SoundIconFolder) + MOD_OPTION_1_STR(ESpeakNGDataDirPath) + + static const int bs = 1024; + static piper_synthesizer *synth = NULL; + static json model_config; + static int model_sample_rate = 22050; + static int model_num_speakers = 1; + static std::string model_language = "en_US"; + static std::atomic stop_requested{false}; + static int current_speaker_id = 0; + static const char *default_voice_name = NULL; + static SPDVoice **piper_voice_list = NULL; + + static RubberBandStretcher::Options initOptions() + { + RubberBandStretcher::Options options = 0; + options |= RubberBandStretcher::OptionProcessRealTime; + options |= RubberBandStretcher::OptionPhaseIndependent; + options |= RubberBandStretcher::OptionThreadingAuto; + options |= RubberBandStretcher::OptionTransientsCrisp; + options |= RubberBandStretcher::OptionDetectorCompound; + return options; + } + + static void piper_stretch_and_copy(const int samplerate, const int channels, const float *in_samples, const size_t num_samples, std::vector& sharedAudioBuffer) + { + if (num_samples == 0 || in_samples == NULL) return; + + // Peak normalization: Piper neural vocoders output raw float amplitudes + // peaking around 0.4 - 0.6. Normalize to 100% full scale (1.0 / max_sample). + float max_sample = 0.01f; + for (size_t i = 0; i < num_samples; ++i) { + float abs_s = std::abs(in_samples[i]); + if (abs_s > max_sample) { + max_sample = abs_s; + } + } + float norm_factor = (max_sample > 0.01f) ? (1.0f / max_sample) : 1.0f; + + // Speed multiplier: map -100..100 onto duration ratio + double speed = 1.0; + if (msg_settings.rate > 0) { + speed = 1.0 + ((double)msg_settings.rate / 50.0); + } else if (msg_settings.rate < 0) { + speed = 1.0 + ((double)msg_settings.rate / 150.0); + } + double ratio = 1.0 / std::max(0.1, speed); + if (ratio < 0.01) ratio = 0.01; + + // Pitch shift: map -100 .. 100 onto -10.0 .. 10.0 semitones + double mS = std::clamp((double)msg_settings.pitch, -100.0, 100.0); + double pitchshift = mS / 10.0; + double frequencyshift = 1.0; + if (pitchshift != 0.0) frequencyshift *= std::pow(2.0, pitchshift / 12.0); + + // Volume / gain: map SSIP -100..100 onto gain factor (0 is nominal 1.0f gain) + float gain = 1.0f; + if (msg_settings.volume < 0) { + gain = ((float)msg_settings.volume + 100.0f) / 100.0f; + } else if (msg_settings.volume > 0) { + gain = 1.0f + ((float)msg_settings.volume / 100.0f); + } + gain = std::max(0.0f, gain); + + // Pre-allocate buffer capacity to avoid multiple dynamic reallocations during stretching + sharedAudioBuffer.reserve(static_cast(num_samples * std::max(1.0, ratio) + 1024)); + + RubberBandStretcher ts(samplerate, channels, initOptions(), ratio, frequencyshift); + ts.setExpectedInputDuration(num_samples); + ts.setMaxProcessSize(bs); + + const float sample_scale = norm_factor * 32767.0f; + float cbuf_arr[bs]; + float *cbuf_ptrs[1] = { cbuf_arr }; + + size_t countIn = 0; + while (countIn < num_samples) { + size_t ibSize = std::min((size_t)bs, num_samples - countIn); + for (size_t i = 0; i < ibSize; ++i) { + // Apply peak normalization and scale to 16-bit amplitude range [-32767, 32767] + cbuf_arr[i] = in_samples[countIn + i] * sample_scale; + } + countIn += ibSize; + ts.process(cbuf_ptrs, ibSize, countIn >= num_samples); + + int avail; + while ((avail = ts.available()) > 0) { + size_t obSize = std::min((size_t)avail, (size_t)bs); + ts.retrieve(cbuf_ptrs, obSize); + for (size_t i = 0; i < obSize; ++i) { + float value = gain * cbuf_arr[i]; + value = std::clamp(value, -32768.0f, 32767.0f); + sharedAudioBuffer.push_back(static_cast(value)); + } + } + } + } + + static int piper_voice_name_to_speaker_id(const char *voice_name) + { + if (voice_name && model_config.contains("speaker_id_map")) { + auto &spkr_map = model_config["speaker_id_map"]; + if (spkr_map.contains(voice_name)) { + return spkr_map[voice_name].get(); + } + } + return 0; + } + + static SPDVoice **piper_allocate_voice_list() + { + std::filesystem::path modelPath(ModelPath ? ModelPath : "piper"); + std::string model_stem = modelPath.stem().string(); + + if (model_num_speakers > 1 && model_config.contains("speaker_id_map") && !model_config["speaker_id_map"].empty()) { + auto &speakerIdMap = model_config["speaker_id_map"]; + int count = speakerIdMap.size(); + SPDVoice **result = g_new0(SPDVoice *, count + 1); + + SPDVoice **reg_voices = module_list_registered_voices(); + int i = 0; + for (auto &speakerItem : speakerIdMap.items()) { + std::string spkr_name = speakerItem.key(); + SPDVoice *v = g_new0(SPDVoice, 1); + v->name = g_strdup(spkr_name.c_str()); + v->language = g_strdup(model_language.c_str()); + const char *var_str = "none"; + for (int j = 0; reg_voices && reg_voices[j]; ++j) { + SPDVoice *r_v = reg_voices[j]; + if (r_v->name && strcasecmp(v->name, r_v->name) == 0) { + if (r_v->variant && strlen(r_v->variant) > 0) { + var_str = r_v->variant; + } + break; + } + } + v->variant = g_strdup(var_str); + result[i++] = v; + } + result[count] = NULL; + return result; + } else { + SPDVoice **result = g_new0(SPDVoice *, 2); + SPDVoice *v = g_new0(SPDVoice, 1); + v->name = g_strdup(model_stem.c_str()); + v->language = g_strdup(model_language.c_str()); + v->variant = g_strdup("none"); + result[0] = v; + result[1] = NULL; + return result; + } + } + + static void piper_free_voice_list() + { + if (piper_voice_list != NULL) { + for (int i = 0; piper_voice_list[i] != NULL; i++) { + g_free(piper_voice_list[i]->name); + g_free(piper_voice_list[i]->language); + g_free(piper_voice_list[i]->variant); + g_free(piper_voice_list[i]); + } + g_free(piper_voice_list); + piper_voice_list = NULL; + } + } + + static const char *piper_search_for_sound_icon(const char *icon_name) + { + char *fn = NULL; + if (!SoundIconFolder || strlen(SoundIconFolder) == 0) return fn; + GString *filename = g_string_new(SoundIconFolder); + filename = g_string_append(filename, icon_name); + if (g_file_test(filename->str, G_FILE_TEST_EXISTS)) fn = filename->str; + return g_string_free(filename, (fn == NULL)); + } + + static void piper_handle_text(const char *data) + { + if (!synth) { + DBG("Synth not initialized"); + return; + } + char *cmdInp = (char *) module_strip_ssml(data); + DBG("Input after strip XML: %s", cmdInp); + + piper_synthesize_options options = piper_default_synthesize_options(synth); + options.speaker_id = current_speaker_id; + + int ret = piper_synthesize_start(synth, cmdInp, &options); + g_free(cmdInp); + if (ret != PIPER_OK) { + DBG("Failed to start synthesis with piper_synthesize_start (error code: %d)", ret); + return; + } + + module_report_event_begin(); + + piper_audio_chunk chunk; + do { + int status = piper_synthesize_next(synth, &chunk); + if (status == PIPER_ERR_GENERIC) { + DBG("piper_synthesize_next failed with PIPER_ERR_GENERIC"); + break; + } + if (stop_requested) { + DBG("Stop requested during synthesis"); + break; + } + if (chunk.num_samples > 0 && chunk.samples != NULL) { + std::vector chunkAudioBuffer; + piper_stretch_and_copy(chunk.sample_rate, 1, chunk.samples, chunk.num_samples, chunkAudioBuffer); + if (!chunkAudioBuffer.empty() && !stop_requested) { + AudioFormat format = SPD_AUDIO_LE; + AudioTrack track; + track.bits = 16; + track.num_samples = chunkAudioBuffer.size(); + track.samples = chunkAudioBuffer.data(); + track.num_channels = 1; + track.sample_rate = chunk.sample_rate; + module_tts_output_server(&track, format); + } + } + } while (!chunk.is_last); + + module_report_event_end(); + } + + static void piper_handle_sound_icon(const char *icon_name) + { + char *icon_path = (char *) piper_search_for_sound_icon(icon_name); + if (icon_path != NULL) { + module_report_event_begin(); + (void)module_play_file(icon_path); + module_report_icon(icon_path); + module_report_event_end(); + g_free(icon_path); + } else { + MSG(3, "Warning: Speaking sound icon name, %s, as a fallback, since audio file can not be found.", icon_name); + piper_handle_text(icon_name); + } + } + + static void piper_set_language(char *lang) + { + DBG("%s, lang=%s, voice_type=%d", __FUNCTION__, lang ? lang : "", (int)msg_settings.voice_type); + + if (msg_settings.voice_type > 0) { + const char *vname = module_getvoice(lang ? lang : model_language.c_str(), msg_settings.voice_type); + if (vname) { + current_speaker_id = piper_voice_name_to_speaker_id(vname); + return; + } + } + + SPDVoice **lst = piper_voice_list; + int index = -1; + if (lang && lst) { + char *dash = strchr(lang, '-'); + char *langbase = dash ? g_strndup(lang, dash - lang + 1) : g_strdup_printf("%s-", lang); + for (int i = 0; lst[i]; ++i) { + if (lst[i]->language && !strcasecmp(lst[i]->language, lang)) { + current_speaker_id = piper_voice_name_to_speaker_id(lst[i]->name); + index = i; + break; + } + if (index == -1 && lst[i]->language) { + if (!strncasecmp(lst[i]->language, langbase, strlen(langbase))) { + current_speaker_id = piper_voice_name_to_speaker_id(lst[i]->name); + index = i; + } + } + } + g_free(langbase); + } + if (index == -1) { + if (piper_voice_list && piper_voice_list[0]) + current_speaker_id = piper_voice_name_to_speaker_id(piper_voice_list[0]->name); + } + } + + static void piper_set_voice_type(SPDVoiceType voice_type) + { + const char *lang = msg_settings.voice.language ? msg_settings.voice.language : model_language.c_str(); + const char *vname = module_getvoice(lang, voice_type); + if (vname) { + current_speaker_id = piper_voice_name_to_speaker_id(vname); + } else { + MSG(3, "Warning: No definition of voice type %d, check 'AddVoice' directives in piper.conf file.", (int)voice_type); + } + } + + static void piper_set_synthesis_voice(char *voice_name) + { + const char *vname = voice_name ? voice_name : (msg_settings.voice.name ? msg_settings.voice.name : default_voice_name); + current_speaker_id = piper_voice_name_to_speaker_id(vname); + if (current_speaker_id < 0) current_speaker_id = 0; + } + + extern "C" { + + int module_load(void) + { + INIT_SETTINGS_TABLES(); + REGISTER_DEBUG(); + MOD_OPTION_1_STR_REG(ModelPath, ""); + MOD_OPTION_1_STR_REG(ConfigPath, ""); + MOD_OPTION_1_STR_REG(SoundIconFolder, "/usr/share/sounds/sound-icons/"); + MOD_OPTION_1_STR_REG(ESpeakNGDataDirPath, "/usr/share/espeak-ng-data/"); + module_register_available_voices(); + module_register_settings_voices(); + return 0; + } + + int module_init(char **status_info) + { + module_audio_set_server(); + module_audio_init(status_info); + + if (!ModelPath || strlen(ModelPath) == 0) { + *status_info = g_strdup("piper: ModelPath is not configured in piper.conf"); + return -1; + } + + std::string config_path_str; + if (ConfigPath && strlen(ConfigPath) > 0) { + config_path_str = ConfigPath; + } else { + config_path_str = std::string(ModelPath) + ".json"; + } + + try { + std::ifstream config_file(config_path_str); + if (config_file.is_open()) { + model_config = json::parse(config_file); + if (model_config.contains("audio") && model_config["audio"].contains("sample_rate")) { + model_sample_rate = model_config["audio"]["sample_rate"].get(); + } + if (model_config.contains("num_speakers")) { + model_num_speakers = model_config["num_speakers"].get(); + } + if (model_config.contains("language") && model_config["language"].contains("code")) { + model_language = model_config["language"]["code"].get(); + } + } else { + DBG(DBG_MODNAME " Warning: Could not open JSON config file at '%s'", config_path_str.c_str()); + } + + piper_create_options opts; + piper_init_create_options(&opts); + opts.model_path = ModelPath; + opts.config_path = config_path_str.c_str(); + opts.espeak_data_path = (ESpeakNGDataDirPath && strlen(ESpeakNGDataDirPath) > 0) ? ESpeakNGDataDirPath : NULL; + + synth = piper_create_with_options(&opts); + if (!synth) { + *status_info = g_strdup("piper: Failed to create piper synthesizer with piper_create_with_options"); + return -1; + } + + default_voice_name = module_getdefaultvoice(); + current_speaker_id = piper_voice_name_to_speaker_id(default_voice_name); + if (current_speaker_id < 0) current_speaker_id = 0; + + piper_voice_list = piper_allocate_voice_list(); + *status_info = g_strdup(DBG_MODNAME " Initialized successfully."); + } + catch (const json::parse_error& e) { + DBG(DBG_MODNAME " Could not initialize, caught JSON exception: %s", e.what()); + *status_info = g_strdup(e.what()); + return -1; + } + catch (const std::runtime_error& e) { + DBG(DBG_MODNAME " Could not initialize, caught runtime_error exception: %s", e.what()); + *status_info = g_strdup(e.what()); + return -1; + } + catch (const std::exception& e) { + DBG(DBG_MODNAME " Could not initialize, caught exception: %s", e.what()); + *status_info = g_strdup(e.what()); + return -1; + } + return 0; + } + + SPDVoice **module_list_voices(void) + { + return piper_voice_list; + } + + void module_speak_sync(const char *data, size_t bytes, SPDMessageType msgtype) + { + stop_requested = false; + UPDATE_STRING_PARAMETER(voice.language, piper_set_language); + UPDATE_PARAMETER(voice_type, piper_set_voice_type); + UPDATE_STRING_PARAMETER(voice.name, piper_set_synthesis_voice); + module_speak_ok(); + switch (msgtype) { + case SPD_MSGTYPE_CHAR: + case SPD_MSGTYPE_KEY: + case SPD_MSGTYPE_SPELL: + case SPD_MSGTYPE_TEXT: + piper_handle_text(data); + break; + case SPD_MSGTYPE_SOUND_ICON: + piper_handle_sound_icon(data); + break; + } + } + + int module_stop(void) + { + stop_requested = true; + return 0; + } + + size_t module_pause(void) + { + stop_requested = true; + return 0; + } + + int module_close(void) + { + if (synth) { + piper_free(synth); + synth = NULL; + } + piper_free_voice_list(); + return 0; + } + + } // extern "C" + +} // namespace piper_module