Skip to content

Support Qwen3-TTS prompt modes without ICL reference codes #202

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
fedeizzo wants to merge 1 commit into
base: main
Choose a base branch
from
+120 −50
Open

Support Qwen3-TTS prompt modes without ICL reference codes #202

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions include/engine/models/qwen3_tts/talker.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ struct Qwen3TalkerCodes {
Qwen3SpeechCodes decoder_input_codes;
};

void validate_qwen3_talker_voice_clone_prefill(
const Qwen3TalkerPrefill & prefill,
int64_t hidden_size);

class Qwen3TalkerWeightsRuntime;
class Qwen3TalkerStepRuntime;

Expand Down
118 changes: 78 additions & 40 deletions src/models/qwen3_tts/session.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ std::shared_ptr<const Qwen3TTSAssets> require_assets(std::shared_ptr<const Qwen3
return assets;
}

Qwen3TTSGenerationOptions generation_options_from_request(
Qwen3TTSGenerationOptions generation_options_from_request_impl(
const runtime::TaskRequest & request,
const Qwen3TTSConfig & config) {
Qwen3TTSGenerationOptions options;
Expand All @@ -40,40 +40,51 @@ Qwen3TTSGenerationOptions generation_options_from_request(
if (const auto value = runtime::find_option(request.options, {"do_sample"})) {
options.do_sample = runtime::parse_bool_option(*value, "do_sample");
}
if (const auto value = runtime::find_option(
request.options,
{"subtalker_do_sample"})) {
if (const auto value = runtime::find_option(request.options, {"subtalker_do_sample"})) {
options.subtalker_do_sample = runtime::parse_bool_option(*value, "subtalker_do_sample");
}
if (const auto value = runtime::parse_float_option(request.options, {"temperature"})) {
if (const auto value = runtime::parse_finite_float_option(request.options, {"temperature"})) {
options.temperature = *value;
}
if (const auto value = runtime::parse_int_option(request.options, {"top_k"})) {
options.top_k = *value;
}
if (const auto value = runtime::parse_float_option(request.options, {"top_p"})) {
if (const auto value = runtime::parse_finite_float_option(request.options, {"top_p"})) {
options.top_p = *value;
}
if (const auto value = runtime::parse_float_option(
request.options,
{"repetition_penalty"})) {
if (const auto value = runtime::parse_finite_float_option(request.options, {"repetition_penalty"})) {
options.repetition_penalty = *value;
}
if (const auto value = runtime::parse_float_option(
request.options,
{"subtalker_temperature"})) {
if (const auto value = runtime::parse_finite_float_option(request.options, {"subtalker_temperature"})) {
options.subtalker_temperature = *value;
}
if (const auto value = runtime::parse_int_option(
request.options,
{"subtalker_top_k"})) {
if (const auto value = runtime::parse_int_option(request.options, {"subtalker_top_k"})) {
options.subtalker_top_k = *value;
}
if (const auto value = runtime::parse_float_option(
request.options,
{"subtalker_top_p"})) {
if (const auto value = runtime::parse_finite_float_option(request.options, {"subtalker_top_p"})) {
options.subtalker_top_p = *value;
}
if (options.do_sample && options.temperature <= 0.0F) {
throw std::runtime_error("Qwen3 TTS temperature must be positive when sampling");
}
if (options.top_k < 0) {
throw std::runtime_error("Qwen3 TTS top_k must be non-negative");
}
if (options.top_p < 0.0F || options.top_p > 1.0F) {
throw std::runtime_error("Qwen3 TTS top_p must be in [0, 1]");
}
if (options.repetition_penalty <= 0.0F) {
throw std::runtime_error("Qwen3 TTS repetition_penalty must be positive");
}
if (options.subtalker_do_sample && options.subtalker_temperature <= 0.0F) {
throw std::runtime_error("Qwen3 TTS subtalker_temperature must be positive when sampling");
}
if (options.subtalker_top_k < 0) {
throw std::runtime_error("Qwen3 TTS subtalker_top_k must be non-negative");
}
if (options.subtalker_top_p < 0.0F || options.subtalker_top_p > 1.0F) {
throw std::runtime_error("Qwen3 TTS subtalker_top_p must be in [0, 1]");
}
options.seed = runtime::parse_u32_option(request.options, {"seed"})
.value_or(runtime::random_u32_seed());
return options;
Expand Down Expand Up @@ -179,8 +190,46 @@ void validate_conv_weight_storage(engine::assets::TensorStorageType storage_type
throw std::runtime_error(std::string(option_name) + " currently supports only native, f32, and f16");
}

class TalkerCachedStepReleaseGuard {
public:
TalkerCachedStepReleaseGuard(Qwen3TalkerStepRuntime * runtime, bool enabled)
: runtime_(runtime), enabled_(enabled) {}

~TalkerCachedStepReleaseGuard() noexcept {
try {
release();
} catch (...) {
// Cleanup must not replace the request exception during stack unwinding.
}
}

void release() {
if (!enabled_ || released_ || runtime_ == nullptr) {
return;
}
const auto release_start = Clock::now();
const int64_t released_steps = runtime_->release_cached_step_graph();
debug::timing_log_scalar(
"qwen3_tts.talker.cached_step_release_ms",
engine::debug::elapsed_ms(release_start, Clock::now()));
debug::timing_log_scalar("qwen3_tts.talker.cached_step_released_steps", released_steps);
released_ = true;
}

private:
Qwen3TalkerStepRuntime * runtime_ = nullptr;
bool enabled_ = false;
bool released_ = false;
};

} // namespace

Qwen3TTSGenerationOptions qwen3_tts_generation_options_from_request(
const runtime::TaskRequest & request,
const Qwen3TTSConfig & config) {
return generation_options_from_request_impl(request, config);
}

bool Qwen3TTSSession::VoicePromptCacheKeyEqual::operator()(
const VoicePromptCacheKey & lhs,
const VoicePromptCacheKey & rhs) const noexcept {
Expand Down Expand Up @@ -336,16 +385,7 @@ void Qwen3TTSSession::prepare(const runtime::SessionPreparationRequest & request
runtime::TaskResult Qwen3TTSSession::run(const runtime::TaskRequest & request) {
require_prepared("Qwen3 TTS run");
const auto wall_start = Clock::now();
auto release_talker_cached_step_graph = [&]() {
if (mem_saver_) {
const auto release_start = Clock::now();
const int64_t released_steps = talker_step_->release_cached_step_graph();
debug::timing_log_scalar(
"qwen3_tts.talker.cached_step_release_ms",
engine::debug::elapsed_ms(release_start, Clock::now()));
debug::timing_log_scalar("qwen3_tts.talker.cached_step_released_steps", released_steps);
}
};
TalkerCachedStepReleaseGuard release_talker_cached_step_graph(talker_step_.get(), mem_saver_);
const int64_t text_chunk_size =
engine::text::parse_text_chunk_size_override(request.options).value_or(kDefaultTextChunkSize);
const auto chunk_requests = runtime::chunk_text_request(request, text_chunk_size);
Expand Down Expand Up @@ -375,7 +415,7 @@ runtime::TaskResult Qwen3TTSSession::run(const runtime::TaskRequest & request) {
speech_decoder_->decode(codes.generated_codes));
decoder_ms += engine::debug::elapsed_ms(decoder_start, Clock::now());
}
release_talker_cached_step_graph();
release_talker_cached_step_graph.release();
runtime::TaskResult result;
result.audio_output = std::move(merged_audio);
debug::timing_log_scalar("qwen3_tts.voice_design_prefill_build_ms", prefill_ms);
Expand Down Expand Up @@ -410,7 +450,7 @@ runtime::TaskResult Qwen3TTSSession::run(const runtime::TaskRequest & request) {
speech_decoder_->decode(codes.generated_codes));
decoder_ms += engine::debug::elapsed_ms(decoder_start, Clock::now());
}
release_talker_cached_step_graph();
release_talker_cached_step_graph.release();
runtime::TaskResult result;
result.audio_output = std::move(merged_audio);
debug::timing_log_scalar("qwen3_tts.custom_voice_prefill_build_ms", prefill_ms);
Expand Down Expand Up @@ -444,24 +484,22 @@ runtime::TaskResult Qwen3TTSSession::run(const runtime::TaskRequest & request) {
const auto prefill_start = Clock::now();
const auto prefill = prompt_builder.build_prefill(qwen_request, voice_prompt);
prefill_ms += engine::debug::elapsed_ms(prefill_start, Clock::now());
if (!voice_prompt.reference_codes.has_value()) {
throw std::runtime_error("Qwen3 base TTS talker currently requires ICL reference codes");
}
const auto talker_start = Clock::now();
const auto codes = talker_step_->generate(
prefill,
qwen_request.generation,
qwen_request.generation.repetition_penalty);
talker_ms += engine::debug::elapsed_ms(talker_start, Clock::now());
const auto decoder_start = Clock::now();
runtime::append_audio_buffer(
merged_audio,
speech_decoder_->decode_and_trim_reference(
*voice_prompt.reference_codes,
codes.generated_codes));
runtime::AudioBuffer decoded = voice_prompt.reference_codes.has_value()
? speech_decoder_->decode_and_trim_reference(
*voice_prompt.reference_codes,
codes.generated_codes)
: speech_decoder_->decode(codes.generated_codes);
runtime::append_audio_buffer(merged_audio, decoded);
decoder_ms += engine::debug::elapsed_ms(decoder_start, Clock::now());
}
release_talker_cached_step_graph();
release_talker_cached_step_graph.release();
runtime::TaskResult result;
result.audio_output = std::move(merged_audio);
debug::timing_log_scalar("qwen3_tts.voice_prompt_ms", prompt_ms);
Expand Down Expand Up @@ -529,7 +567,7 @@ Qwen3TTSRequest Qwen3TTSSession::make_request(const runtime::TaskRequest & reque
Qwen3TTSRequest out;
out.text = request.text_input->text;
out.language = !request.text_input->language.empty() ? request.text_input->language : "Auto";
out.generation = generation_options_from_request(request, assets_->config);
out.generation = qwen3_tts_generation_options_from_request(request, assets_->config);
if (assets_->config.variant == Qwen3TTSVariant::Base) {
const runtime::AudioBuffer * reference_audio = nullptr;
if (request.voice.has_value()
Expand Down
48 changes: 38 additions & 10 deletions src/models/qwen3_tts/talker.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,21 @@
#include <vector>

namespace engine::models::qwen3_tts {

void validate_qwen3_talker_voice_clone_prefill(
const Qwen3TalkerPrefill & prefill,
int64_t hidden_size) {
if (!prefill.speaker_embedding.has_value() || prefill.speaker_embedding->dims != hidden_size) {
throw std::runtime_error("Qwen3 talker voice clone prefill requires speaker embedding");
}
if (prefill.icl_mode == prefill.x_vector_only_mode) {
throw std::runtime_error("Qwen3 talker voice clone prefill requires exactly one prompt mode");
}
if (prefill.icl_mode && (!prefill.reference_codes.has_value() || prefill.reference_ids.empty())) {
throw std::runtime_error("Qwen3 talker ICL prefill requires reference ids and codes");
}
}

namespace {

using Clock = std::chrono::steady_clock;
Expand Down Expand Up @@ -521,12 +536,7 @@ PromptEmbeddingState build_prompt_state(
return state;
}

if (!prefill.speaker_embedding.has_value() || prefill.speaker_embedding->dims != config.hidden_size) {
throw std::runtime_error("Qwen3 talker voice clone prefill requires speaker embedding");
}
if (!prefill.reference_codes.has_value() || prefill.reference_ids.empty()) {
throw std::runtime_error("Qwen3 talker ICL prefill requires reference ids and codes");
}
validate_qwen3_talker_voice_clone_prefill(prefill, config.hidden_size);

PromptEmbeddingState state;
state.tts_pad = tts_pad;
Expand All @@ -549,6 +559,26 @@ PromptEmbeddingState build_prompt_state(
append_row(state.prompt, add_rows(tts_bos, row_at(codec_embed, codec_rows - 2, config.hidden_size)));

const std::vector<int32_t> text_ids(prefill.input_ids.begin() + 3, prefill.input_ids.end() - 5);
if (prefill.x_vector_only_mode) {
auto text_embed = text_project_host(
lookup_rows(weights.text_embedding, config.text_hidden_size, text_ids),
static_cast<int64_t>(text_ids.size()),
weights,
config);
append_row(text_embed, tts_eos);
append_rows(
state.prompt,
add_rows(
text_embed,
lookup_rows(
weights.codec_embedding,
config.hidden_size,
std::vector<int32_t>(text_ids.size() + 1, static_cast<int32_t>(config.codec_pad_id)))));
append_row(state.prompt, add_rows(tts_pad, row_at(codec_embed, codec_rows - 1, config.hidden_size)));
state.trailing_text = tts_pad;
return state;
}

const std::vector<int32_t> ref_text_ids(prefill.reference_ids.begin() + 3, prefill.reference_ids.end() - 2);
std::vector<int32_t> combined_text = ref_text_ids;
combined_text.insert(combined_text.end(), text_ids.begin(), text_ids.end());
Expand Down Expand Up @@ -968,8 +998,9 @@ class TalkerPrefillGraph {
}
ggml_backend_tensor_set(input_, embeddings.data(), 0, embeddings.size() * sizeof(float));
core::set_backend_threads(weights_->backend(), weights_->threads());
// compute_backend_graph is synchronous; an additional backend-wide wait
// before the blocking output reads only duplicates that synchronization.
const ggml_status status = engine::core::compute_backend_graph(weights_->backend(), graph_);
ggml_backend_synchronize(weights_->backend());
if (status != GGML_STATUS_SUCCESS) {
throw std::runtime_error("Qwen3 talker prefill graph compute failed");
}
Expand Down Expand Up @@ -1120,7 +1151,6 @@ class TalkerCachedStepGraph {
core::set_backend_threads(weights_->backend(), weights_->threads());
timing_start = Clock::now();
const ggml_status status = engine::core::compute_backend_graph(weights_->backend(), graph_);
ggml_backend_synchronize(weights_->backend());
last_timing_.graph_compute_ms = engine::debug::elapsed_ms(timing_start, Clock::now());
if (status != GGML_STATUS_SUCCESS) {
throw std::runtime_error("Qwen3 talker cached step graph compute failed");
Expand Down Expand Up @@ -1555,7 +1585,6 @@ class CodePredictorGraph {
core::set_backend_threads(weights_->backend(), weights_->threads());
timing_start = Clock::now();
const ggml_status status = engine::core::compute_backend_graph(weights_->backend(), prefill_graph_);
ggml_backend_synchronize(weights_->backend());
timing_.graph_compute_ms += engine::debug::elapsed_ms(timing_start, Clock::now());
if (status != GGML_STATUS_SUCCESS) {
throw std::runtime_error("Qwen3 code predictor prefill graph compute failed");
Expand Down Expand Up @@ -1597,7 +1626,6 @@ class CodePredictorGraph {
core::set_backend_threads(weights_->backend(), weights_->threads());
timing_start = Clock::now();
const ggml_status status = engine::core::compute_backend_graph(weights_->backend(), step_graph.graph);
ggml_backend_synchronize(weights_->backend());
timing_.graph_compute_ms += engine::debug::elapsed_ms(timing_start, Clock::now());
if (status != GGML_STATUS_SUCCESS) {
throw std::runtime_error("Qwen3 code predictor step graph compute failed");
Expand Down
Loading