diff --git a/include/engine/models/qwen3_tts/talker.h b/include/engine/models/qwen3_tts/talker.h index 70ca372b..b82d4a0d 100644 --- a/include/engine/models/qwen3_tts/talker.h +++ b/include/engine/models/qwen3_tts/talker.h @@ -37,6 +37,10 @@ struct Qwen3TalkerCodes { Qwen3SpeechCodes decoder_input_codes; }; +void validate_qwen3_talker_voice_clone_prefill( + const Qwen3TalkerPrefill & prefill, + int64_t hidden_size); + class Qwen3TalkerWeightsRuntime; class Qwen3TalkerStepRuntime; diff --git a/src/models/qwen3_tts/session.cpp b/src/models/qwen3_tts/session.cpp index 970aed69..d23686b9 100644 --- a/src/models/qwen3_tts/session.cpp +++ b/src/models/qwen3_tts/session.cpp @@ -26,7 +26,7 @@ std::shared_ptr require_assets(std::shared_ptr 1.0F) { + throw std::runtime_error("Qwen3 TTS top_p must be in [0, 1]"); + } + if (options.repetition_penalty <= 0.0F) { + throw std::runtime_error("Qwen3 TTS repetition_penalty must be positive"); + } + if (options.subtalker_do_sample && options.subtalker_temperature <= 0.0F) { + throw std::runtime_error("Qwen3 TTS subtalker_temperature must be positive when sampling"); + } + if (options.subtalker_top_k < 0) { + throw std::runtime_error("Qwen3 TTS subtalker_top_k must be non-negative"); + } + if (options.subtalker_top_p < 0.0F || options.subtalker_top_p > 1.0F) { + throw std::runtime_error("Qwen3 TTS subtalker_top_p must be in [0, 1]"); + } options.seed = runtime::parse_u32_option(request.options, {"seed"}) .value_or(runtime::random_u32_seed()); return options; @@ -179,8 +190,46 @@ void validate_conv_weight_storage(engine::assets::TensorStorageType storage_type throw std::runtime_error(std::string(option_name) + " currently supports only native, f32, and f16"); } +class TalkerCachedStepReleaseGuard { +public: + TalkerCachedStepReleaseGuard(Qwen3TalkerStepRuntime * runtime, bool enabled) + : runtime_(runtime), enabled_(enabled) {} + + ~TalkerCachedStepReleaseGuard() noexcept { + try { + release(); + } catch (...) { + // Cleanup must not replace the request exception during stack unwinding. + } + } + + void release() { + if (!enabled_ || released_ || runtime_ == nullptr) { + return; + } + const auto release_start = Clock::now(); + const int64_t released_steps = runtime_->release_cached_step_graph(); + debug::timing_log_scalar( + "qwen3_tts.talker.cached_step_release_ms", + engine::debug::elapsed_ms(release_start, Clock::now())); + debug::timing_log_scalar("qwen3_tts.talker.cached_step_released_steps", released_steps); + released_ = true; + } + +private: + Qwen3TalkerStepRuntime * runtime_ = nullptr; + bool enabled_ = false; + bool released_ = false; +}; + } // namespace +Qwen3TTSGenerationOptions qwen3_tts_generation_options_from_request( + const runtime::TaskRequest & request, + const Qwen3TTSConfig & config) { + return generation_options_from_request_impl(request, config); +} + bool Qwen3TTSSession::VoicePromptCacheKeyEqual::operator()( const VoicePromptCacheKey & lhs, const VoicePromptCacheKey & rhs) const noexcept { @@ -336,16 +385,7 @@ void Qwen3TTSSession::prepare(const runtime::SessionPreparationRequest & request runtime::TaskResult Qwen3TTSSession::run(const runtime::TaskRequest & request) { require_prepared("Qwen3 TTS run"); const auto wall_start = Clock::now(); - auto release_talker_cached_step_graph = [&]() { - if (mem_saver_) { - const auto release_start = Clock::now(); - const int64_t released_steps = talker_step_->release_cached_step_graph(); - debug::timing_log_scalar( - "qwen3_tts.talker.cached_step_release_ms", - engine::debug::elapsed_ms(release_start, Clock::now())); - debug::timing_log_scalar("qwen3_tts.talker.cached_step_released_steps", released_steps); - } - }; + TalkerCachedStepReleaseGuard release_talker_cached_step_graph(talker_step_.get(), mem_saver_); const int64_t text_chunk_size = engine::text::parse_text_chunk_size_override(request.options).value_or(kDefaultTextChunkSize); const auto chunk_requests = runtime::chunk_text_request(request, text_chunk_size); @@ -375,7 +415,7 @@ runtime::TaskResult Qwen3TTSSession::run(const runtime::TaskRequest & request) { speech_decoder_->decode(codes.generated_codes)); decoder_ms += engine::debug::elapsed_ms(decoder_start, Clock::now()); } - release_talker_cached_step_graph(); + release_talker_cached_step_graph.release(); runtime::TaskResult result; result.audio_output = std::move(merged_audio); debug::timing_log_scalar("qwen3_tts.voice_design_prefill_build_ms", prefill_ms); @@ -410,7 +450,7 @@ runtime::TaskResult Qwen3TTSSession::run(const runtime::TaskRequest & request) { speech_decoder_->decode(codes.generated_codes)); decoder_ms += engine::debug::elapsed_ms(decoder_start, Clock::now()); } - release_talker_cached_step_graph(); + release_talker_cached_step_graph.release(); runtime::TaskResult result; result.audio_output = std::move(merged_audio); debug::timing_log_scalar("qwen3_tts.custom_voice_prefill_build_ms", prefill_ms); @@ -444,9 +484,6 @@ runtime::TaskResult Qwen3TTSSession::run(const runtime::TaskRequest & request) { const auto prefill_start = Clock::now(); const auto prefill = prompt_builder.build_prefill(qwen_request, voice_prompt); prefill_ms += engine::debug::elapsed_ms(prefill_start, Clock::now()); - if (!voice_prompt.reference_codes.has_value()) { - throw std::runtime_error("Qwen3 base TTS talker currently requires ICL reference codes"); - } const auto talker_start = Clock::now(); const auto codes = talker_step_->generate( prefill, @@ -454,14 +491,15 @@ runtime::TaskResult Qwen3TTSSession::run(const runtime::TaskRequest & request) { qwen_request.generation.repetition_penalty); talker_ms += engine::debug::elapsed_ms(talker_start, Clock::now()); const auto decoder_start = Clock::now(); - runtime::append_audio_buffer( - merged_audio, - speech_decoder_->decode_and_trim_reference( - *voice_prompt.reference_codes, - codes.generated_codes)); + runtime::AudioBuffer decoded = voice_prompt.reference_codes.has_value() + ? speech_decoder_->decode_and_trim_reference( + *voice_prompt.reference_codes, + codes.generated_codes) + : speech_decoder_->decode(codes.generated_codes); + runtime::append_audio_buffer(merged_audio, decoded); decoder_ms += engine::debug::elapsed_ms(decoder_start, Clock::now()); } - release_talker_cached_step_graph(); + release_talker_cached_step_graph.release(); runtime::TaskResult result; result.audio_output = std::move(merged_audio); debug::timing_log_scalar("qwen3_tts.voice_prompt_ms", prompt_ms); @@ -529,7 +567,7 @@ Qwen3TTSRequest Qwen3TTSSession::make_request(const runtime::TaskRequest & reque Qwen3TTSRequest out; out.text = request.text_input->text; out.language = !request.text_input->language.empty() ? request.text_input->language : "Auto"; - out.generation = generation_options_from_request(request, assets_->config); + out.generation = qwen3_tts_generation_options_from_request(request, assets_->config); if (assets_->config.variant == Qwen3TTSVariant::Base) { const runtime::AudioBuffer * reference_audio = nullptr; if (request.voice.has_value() diff --git a/src/models/qwen3_tts/talker.cpp b/src/models/qwen3_tts/talker.cpp index 5bad35ca..b2ea438d 100644 --- a/src/models/qwen3_tts/talker.cpp +++ b/src/models/qwen3_tts/talker.cpp @@ -34,6 +34,21 @@ #include namespace engine::models::qwen3_tts { + +void validate_qwen3_talker_voice_clone_prefill( + const Qwen3TalkerPrefill & prefill, + int64_t hidden_size) { + if (!prefill.speaker_embedding.has_value() || prefill.speaker_embedding->dims != hidden_size) { + throw std::runtime_error("Qwen3 talker voice clone prefill requires speaker embedding"); + } + if (prefill.icl_mode == prefill.x_vector_only_mode) { + throw std::runtime_error("Qwen3 talker voice clone prefill requires exactly one prompt mode"); + } + if (prefill.icl_mode && (!prefill.reference_codes.has_value() || prefill.reference_ids.empty())) { + throw std::runtime_error("Qwen3 talker ICL prefill requires reference ids and codes"); + } +} + namespace { using Clock = std::chrono::steady_clock; @@ -521,12 +536,7 @@ PromptEmbeddingState build_prompt_state( return state; } - if (!prefill.speaker_embedding.has_value() || prefill.speaker_embedding->dims != config.hidden_size) { - throw std::runtime_error("Qwen3 talker voice clone prefill requires speaker embedding"); - } - if (!prefill.reference_codes.has_value() || prefill.reference_ids.empty()) { - throw std::runtime_error("Qwen3 talker ICL prefill requires reference ids and codes"); - } + validate_qwen3_talker_voice_clone_prefill(prefill, config.hidden_size); PromptEmbeddingState state; state.tts_pad = tts_pad; @@ -549,6 +559,26 @@ PromptEmbeddingState build_prompt_state( append_row(state.prompt, add_rows(tts_bos, row_at(codec_embed, codec_rows - 2, config.hidden_size))); const std::vector text_ids(prefill.input_ids.begin() + 3, prefill.input_ids.end() - 5); + if (prefill.x_vector_only_mode) { + auto text_embed = text_project_host( + lookup_rows(weights.text_embedding, config.text_hidden_size, text_ids), + static_cast(text_ids.size()), + weights, + config); + append_row(text_embed, tts_eos); + append_rows( + state.prompt, + add_rows( + text_embed, + lookup_rows( + weights.codec_embedding, + config.hidden_size, + std::vector(text_ids.size() + 1, static_cast(config.codec_pad_id))))); + append_row(state.prompt, add_rows(tts_pad, row_at(codec_embed, codec_rows - 1, config.hidden_size))); + state.trailing_text = tts_pad; + return state; + } + const std::vector ref_text_ids(prefill.reference_ids.begin() + 3, prefill.reference_ids.end() - 2); std::vector combined_text = ref_text_ids; combined_text.insert(combined_text.end(), text_ids.begin(), text_ids.end()); @@ -968,8 +998,9 @@ class TalkerPrefillGraph { } ggml_backend_tensor_set(input_, embeddings.data(), 0, embeddings.size() * sizeof(float)); core::set_backend_threads(weights_->backend(), weights_->threads()); + // compute_backend_graph is synchronous; an additional backend-wide wait + // before the blocking output reads only duplicates that synchronization. const ggml_status status = engine::core::compute_backend_graph(weights_->backend(), graph_); - ggml_backend_synchronize(weights_->backend()); if (status != GGML_STATUS_SUCCESS) { throw std::runtime_error("Qwen3 talker prefill graph compute failed"); } @@ -1120,7 +1151,6 @@ class TalkerCachedStepGraph { core::set_backend_threads(weights_->backend(), weights_->threads()); timing_start = Clock::now(); const ggml_status status = engine::core::compute_backend_graph(weights_->backend(), graph_); - ggml_backend_synchronize(weights_->backend()); last_timing_.graph_compute_ms = engine::debug::elapsed_ms(timing_start, Clock::now()); if (status != GGML_STATUS_SUCCESS) { throw std::runtime_error("Qwen3 talker cached step graph compute failed"); @@ -1555,7 +1585,6 @@ class CodePredictorGraph { core::set_backend_threads(weights_->backend(), weights_->threads()); timing_start = Clock::now(); const ggml_status status = engine::core::compute_backend_graph(weights_->backend(), prefill_graph_); - ggml_backend_synchronize(weights_->backend()); timing_.graph_compute_ms += engine::debug::elapsed_ms(timing_start, Clock::now()); if (status != GGML_STATUS_SUCCESS) { throw std::runtime_error("Qwen3 code predictor prefill graph compute failed"); @@ -1597,7 +1626,6 @@ class CodePredictorGraph { core::set_backend_threads(weights_->backend(), weights_->threads()); timing_start = Clock::now(); const ggml_status status = engine::core::compute_backend_graph(weights_->backend(), step_graph.graph); - ggml_backend_synchronize(weights_->backend()); timing_.graph_compute_ms += engine::debug::elapsed_ms(timing_start, Clock::now()); if (status != GGML_STATUS_SUCCESS) { throw std::runtime_error("Qwen3 code predictor step graph compute failed");