From 95e6526a21b1067fe9745538b0c41811a317fc08 Mon Sep 17 00:00:00 2001 From: Hans Date: Tue, 8 Sep 2026 20:54:09 +0800 Subject: [PATCH 1/3] mtmd : add Soprano audio generation Assisted-by: Codex --- conversion/__init__.py | 1 + conversion/qwen.py | 5 ++ conversion/soprano.py | 50 +++++++++++ gguf-py/gguf/constants.py | 13 +++ gguf-py/gguf/tensor_mapping.py | 9 ++ src/llama-vocab.cpp | 38 +++++++- src/llama-vocab.h | 1 + tools/mtmd/CMakeLists.txt | 1 + tools/mtmd/clip-impl.h | 2 + tools/mtmd/clip-model.h | 13 +++ tools/mtmd/clip.cpp | 67 +++++++++++++- tools/mtmd/models/models.h | 7 ++ tools/mtmd/models/soprano.cpp | 68 ++++++++++++++ tools/mtmd/mtmd-helper-gen.cpp | 158 +++++++++++++++++++++++++++++++++ tools/mtmd/mtmd.cpp | 18 +++- tools/mtmd/mtmd.h | 1 + tools/tts/README.md | 14 +++ 17 files changed, 460 insertions(+), 6 deletions(-) create mode 100644 conversion/soprano.py create mode 100644 tools/mtmd/models/soprano.cpp diff --git a/conversion/__init__.py b/conversion/__init__.py index 4d58bcd1060e..5ef422506e59 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -285,6 +285,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { + "Qwen3ForCausalLM": "soprano", "AudioFlamingo3ForConditionalGeneration": "ultravox", "CogVLMForCausalLM": "cogvlm", "DeepseekOCR2ForCausalLM": "deepseek", diff --git a/conversion/qwen.py b/conversion/qwen.py index c7e0809f38c4..39864b49a705 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -208,6 +208,11 @@ def set_vocab(self): super().set_vocab() + def get_vocab_base_pre(self, tokenizer) -> str: + if tokenizer.convert_tokens_to_ids("[TEXT]") == 1 and tokenizer.convert_tokens_to_ids("[START]") == 2 and tokenizer.convert_tokens_to_ids("[STOP]") == 3: + return "soprano" + return super().get_vocab_base_pre(tokenizer) + def _find_rerank_config(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model) diff --git a/conversion/soprano.py b/conversion/soprano.py new file mode 100644 index 000000000000..5494f7338ff5 --- /dev/null +++ b/conversion/soprano.py @@ -0,0 +1,50 @@ +# Copyright (c) 2026 codec.cpp contributors +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +import torch + +from .base import ModelBase, MmprojModel, gguf + + +@ModelBase.register("Qwen3ForCausalLM") +@ModelBase.example("ekwek/Soprano-1.1-80M") +class SopranoModel(MmprojModel): + has_vision_encoder = False + has_audio_encoder = False + + def get_audio_config(self): + return {"num_hidden_layers": 8} + + def set_gguf_parameters(self): + self.gguf_writer.add_file_type(self.ftype) + self.gguf_writer.add_clip_has_gen_audio_encoder(True) + self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.SOPRANO) + self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text) + self.gguf_writer.add_gen_audio_embedding_length(768) + self.gguf_writer.add_gen_audio_feed_forward_length(2304) + self.gguf_writer.add_gen_audio_block_count(8) + self.gguf_writer.add_gen_audio_head_count(1) + self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-6) + + def get_tensors(self): + state = torch.load(self.dir_model / "decoder.pth", map_location="cpu", weights_only=True) + if "state_dict" in state: + state = state["state_dict"] + yield from state.items() + + def tensor_force_quant(self, name, new_name, bid, n_dims): + if "dwconv.weight" in new_name: + return gguf.GGMLQuantizationType.F16 + return super().tensor_force_quant(name, new_name, bid, n_dims) + + def modify_tensors(self, data_torch, name, bid): + if name == "head.istft.window": + expected = torch.hann_window(2048) + if not torch.equal(data_torch.float(), expected): + raise ValueError("Soprano requires the periodic 2048-sample Hann window") + return + if name == "decoder.embed.weight": + data_torch = data_torch.squeeze(-1) + yield self.map_tensor_name(name), data_torch diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index d3a639f374c0..3b8c32f394a4 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -1084,6 +1084,10 @@ class MODEL_TENSOR(IntEnum): A_GEN_CODE_FFN_GATE = auto() A_GEN_CODE_FFN_UP = auto() A_GEN_CODE_FFN_DOWN = auto() + A_GEN_WAV_INPUT = auto() + A_GEN_WAV_NORM = auto() + A_GEN_WAV_OUTPUT_NORM = auto() + A_GEN_WAV_OUTPUT = auto() A_GEN_CODE_OUTPUT_NORM = auto() # qwen3tts code2wav: RVQ codes -> raw PCM A_GEN_WAV_QUANT_FIRST_IN = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D) @@ -1835,6 +1839,10 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.A_GEN_CODE_FFN_GATE: "a.gen.code.blk.{bid}.ffn_gate", MODEL_TENSOR.A_GEN_CODE_FFN_UP: "a.gen.code.blk.{bid}.ffn_up", MODEL_TENSOR.A_GEN_CODE_FFN_DOWN: "a.gen.code.blk.{bid}.ffn_down", + MODEL_TENSOR.A_GEN_WAV_INPUT: "a.gen.wav.input", + MODEL_TENSOR.A_GEN_WAV_NORM: "a.gen.wav.norm", + MODEL_TENSOR.A_GEN_WAV_OUTPUT_NORM: "a.gen.wav.output_norm", + MODEL_TENSOR.A_GEN_WAV_OUTPUT: "a.gen.wav.output", MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM: "a.gen.code.output_norm", MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN: "a.gen.wav.quant.first.in_proj", MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT: "a.gen.wav.quant.first.out_proj", @@ -2192,6 +2200,10 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.A_GEN_CODE_FFN_GATE, MODEL_TENSOR.A_GEN_CODE_FFN_UP, MODEL_TENSOR.A_GEN_CODE_FFN_DOWN, + MODEL_TENSOR.A_GEN_WAV_INPUT, + MODEL_TENSOR.A_GEN_WAV_NORM, + MODEL_TENSOR.A_GEN_WAV_OUTPUT_NORM, + MODEL_TENSOR.A_GEN_WAV_OUTPUT, MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM, MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN, MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT, @@ -5813,6 +5825,7 @@ class VisionProjectorType: QWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictor POCKETTTS_SPKENC = "pockettts_spkenc" # audio: mimi encoder as voice-prompt encoder POCKETTTS_GEN = "pockettts_gen" # audio generation: flow-matching decoder + mimi decoder + SOPRANO = "soprano" # audio generation: Vocos decoder HUNYUANVL = "hunyuanvl" PARAKEET = "parakeet" # audio MINIMAXM3 = "minimax_m3" diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index d2dfeece5952..eaac63154075 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -7,6 +7,10 @@ class TensorNameMap: mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = { + MODEL_TENSOR.A_GEN_WAV_INPUT: ("decoder.embed",), + MODEL_TENSOR.A_GEN_WAV_NORM: ("decoder.norm",), + MODEL_TENSOR.A_GEN_WAV_OUTPUT_NORM: ("decoder.final_layer_norm",), + MODEL_TENSOR.A_GEN_WAV_OUTPUT: ("head.out",), # Token embeddings MODEL_TENSOR.TOKEN_EMBD: ( "gpt_neox.embed_in", # gptneox @@ -182,6 +186,11 @@ class TensorNameMap: } block_mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = { + MODEL_TENSOR.A_GEN_WAV_UP_DWCONV: ("decoder.convnext.{bid}.dwconv",), + MODEL_TENSOR.A_GEN_WAV_UP_NORM: ("decoder.convnext.{bid}.norm",), + MODEL_TENSOR.A_GEN_WAV_UP_PW1: ("decoder.convnext.{bid}.pwconv1",), + MODEL_TENSOR.A_GEN_WAV_UP_PW2: ("decoder.convnext.{bid}.pwconv2",), + MODEL_TENSOR.A_GEN_WAV_UP_GAMMA: ("decoder.convnext.{bid}.gamma",), # Attention norm MODEL_TENSOR.ATTN_NORM: ( "gpt_neox.layers.{bid}.input_layernorm", # gptneox diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index ee65faf23e7f..d9f16182a1c7 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -545,6 +545,10 @@ struct llm_tokenizer_bpe : llm_tokenizer { "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}+| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", }; break; + case LLAMA_VOCAB_PRE_TYPE_SOPRANO: + regex_exprs = { "\\p{N}", "\\s+|[\\p{L}\\p{N}_]+|[^\\p{L}\\p{N}_\\s]+" }; + byte_encode = false; + break; case LLAMA_VOCAB_PRE_TYPE_WHITESPACE: // whitespace pre-tokenizer (jinaai/jina-embeddings-v2-base-zh) regex_exprs = { @@ -612,7 +616,22 @@ struct llm_tokenizer_bpe_session { virtual void tokenize(const std::string & text, std::vector & output) { int final_prev_index = -1; - const auto word_collection = unicode_regex_split(text, tokenizer.regex_exprs, tokenizer.byte_encode); + std::string normalized; + if (vocab.get_pre_type() == LLAMA_VOCAB_PRE_TYPE_SOPRANO) { + bool space = false; + for (uint32_t cpt : unicode_cpts_from_utf8(text)) { + if (unicode_cpt_flags_from_cpt(cpt).is_whitespace) { + if (!space) { normalized += ' '; } + space = true; + } else { + // Unicode full lowercase expands LATIN CAPITAL LETTER I WITH DOT ABOVE. + normalized += cpt == 0x0130 ? "i\xcc\x87" : unicode_cpt_to_utf8(unicode_tolower(cpt)); + space = false; + } + } + } + const auto & input_text = vocab.get_pre_type() == LLAMA_VOCAB_PRE_TYPE_SOPRANO ? normalized : text; + const auto word_collection = unicode_regex_split(input_text, tokenizer.regex_exprs, tokenizer.byte_encode); symbols_final.clear(); auto tok_pre = vocab.get_pre_type(); @@ -708,7 +727,9 @@ struct llm_tokenizer_bpe_session { const std::string str = std::string(symbol.text, symbol.n); const auto token = vocab.text_to_token(str); - if (token == LLAMA_TOKEN_NULL) { + if (token == LLAMA_TOKEN_NULL && tok_pre == LLAMA_VOCAB_PRE_TYPE_SOPRANO) { + output.push_back(vocab.token_unk()); + } else if (token == LLAMA_TOKEN_NULL) { for (auto j = str.begin(); j != str.end(); ++j) { llama_token token_multibyte = LLAMA_TOKEN_NULL; if (tokenizer.byte_encode) { @@ -2209,6 +2230,12 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { tokenizer_pre == "mellum" || tokenizer_pre == "modern-bert") { pre_type = LLAMA_VOCAB_PRE_TYPE_GPT2; + } else if ( + tokenizer_pre == "soprano") { + pre_type = LLAMA_VOCAB_PRE_TYPE_SOPRANO; + special_unk_id = 0; + special_eos_id = 3; + add_bos = false; } else if ( tokenizer_pre == "jais-2") { pre_type = LLAMA_VOCAB_PRE_TYPE_JAIS2; @@ -2916,6 +2943,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { } } + if (pre_type == LLAMA_VOCAB_PRE_TYPE_SOPRANO) { + special_eog_ids.insert(special_eos_id); + } + // sanity checks if (special_eos_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eos_id) == 0) { special_eog_ids.insert(special_eos_id); @@ -3657,6 +3688,9 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t return _try_copy(token_text.data(), token_text.size()); } if (attr & LLAMA_TOKEN_ATTR_NORMAL) { + if (pre_type == LLAMA_VOCAB_PRE_TYPE_SOPRANO) { + return _try_copy(token_text.data(), token_text.size()); + } if (escape_whitespaces) { // SPM-style BPE: tokens contain ▁ for spaces std::string result = token_text; diff --git a/src/llama-vocab.h b/src/llama-vocab.h index 65293c026173..b9ea4b39c6f2 100644 --- a/src/llama-vocab.h +++ b/src/llama-vocab.h @@ -67,6 +67,7 @@ enum llama_vocab_pre_type { LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56, LLAMA_VOCAB_PRE_TYPE_HY_V4 = 57, LLAMA_VOCAB_PRE_TYPE_SPARK2_5 = 58, + LLAMA_VOCAB_PRE_TYPE_SOPRANO = 59, }; struct LLM_KV; diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 907468e87ec7..4d57690b8979 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -63,6 +63,7 @@ add_library(mtmd models/pockettts-seanet.cpp models/pockettts-spkenc.cpp models/pockettts-gen.cpp + models/soprano.cpp models/step3vl.cpp models/siglip.cpp models/whisper-enc.cpp diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 72148a4d9a9b..4c2f5d5a8725 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -503,6 +503,7 @@ enum projector_type { PROJECTOR_TYPE_QWEN3TTS_GEN, PROJECTOR_TYPE_POCKETTTS_SPKENC, PROJECTOR_TYPE_POCKETTTS_GEN, + PROJECTOR_TYPE_SOPRANO, PROJECTOR_TYPE_MUSE_GLIMMER, PROJECTOR_TYPE_UNKNOWN, }; @@ -568,6 +569,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_QWEN3TTS_GEN, "qwen3tts_gen"}, { PROJECTOR_TYPE_POCKETTTS_SPKENC, "pockettts_spkenc"}, { PROJECTOR_TYPE_POCKETTTS_GEN, "pockettts_gen"}, + { PROJECTOR_TYPE_SOPRANO, "soprano"}, { PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"}, }; diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index f737ccc24527..0bef763620b1 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -552,6 +552,18 @@ struct clip_code2wav { ggml_tensor * dac_post_conv_b = nullptr; }; +struct clip_vocos { + ggml_tensor * input_w = nullptr; + ggml_tensor * input_b = nullptr; + ggml_tensor * norm_w = nullptr; + ggml_tensor * norm_b = nullptr; + ggml_tensor * output_norm_w = nullptr; + ggml_tensor * output_norm_b = nullptr; + ggml_tensor * output_w = nullptr; + ggml_tensor * output_b = nullptr; + std::vector blocks; +}; + struct clip_model { clip_modality modality = CLIP_MODALITY_VISION; projector_type proj_type = PROJECTOR_TYPE_MLP; @@ -786,6 +798,7 @@ struct clip_model { // qwen3tts code2wav: RVQ codes -> raw PCM clip_code2wav c2w; + clip_vocos vocos; // pocket-tts: SEANet stack, shared by the encoder (speaker path) and the decoder (gen path) clip_seanet seanet; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index cd6421def528..cc5f36bd2f24 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1116,6 +1116,11 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const const int n_frames = params && params->feats ? (int) (params->feats->size() / n_latent) : 1; builder = std::make_unique(ctx, img, gen_process, n_step, n_frames); } break; + case PROJECTOR_TYPE_SOPRANO: + { + const int n_frames = params && params->feats ? (int) (params->feats->size() / 512) : 2; + builder = std::make_unique(ctx, img, n_frames); + } break; case PROJECTOR_TYPE_QWEN3TTS_GEN: { const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE; @@ -1823,6 +1828,11 @@ struct clip_model_loader { "%s: mimo_audio: %s must be > 0\n", __func__, KEY_A_LOCAL_GROUP_SIZE)); } } break; + case PROJECTOR_TYPE_SOPRANO: + if (hparams.n_layer != 8 || hparams.n_embd != 768 || hparams.n_ff != 2304 || hparams.projection_dim != 512) { + throw std::runtime_error("unsupported Soprano decoder configuration"); + } + break; case PROJECTOR_TYPE_QWEN3TTS_SPKENC: { // ECAPA-TDNN speaker encoder, mel front-end uses the Slaney default (fmin=0, fmax=sr/2) @@ -2239,7 +2249,8 @@ struct clip_model_loader { const bool has_standard_layers = ( model.proj_type != PROJECTOR_TYPE_GEMMA3NV && model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC && - model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN); + model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN && + model.proj_type != PROJECTOR_TYPE_SOPRANO); // layers const int n_layers_to_load = has_standard_layers ? hparams.n_layer : 0; @@ -2910,6 +2921,46 @@ struct clip_model_loader { model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight")); model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight")); } break; + case PROJECTOR_TYPE_SOPRANO: + { + auto get_vocos = [&](const std::string & name, std::initializer_list shape) { + auto * t = get_tensor(name); + int i = 0; + for (int64_t n : shape) { + if (t->ne[i++] != n) { + throw std::runtime_error("invalid Soprano tensor shape: " + name); + } + } + for (; i < GGML_MAX_DIMS; ++i) { + if (t->ne[i] != 1) { + throw std::runtime_error("invalid Soprano tensor shape: " + name); + } + } + return t; + }; + auto & v = model.vocos; + v.input_w = get_vocos("a.gen.wav.input.weight", {512, 768}); + v.input_b = get_vocos("a.gen.wav.input.bias", {768}); + v.norm_w = get_vocos("a.gen.wav.norm.weight", {768}); + v.norm_b = get_vocos("a.gen.wav.norm.bias", {768}); + v.output_norm_w = get_vocos("a.gen.wav.output_norm.weight", {768}); + v.output_norm_b = get_vocos("a.gen.wav.output_norm.bias", {768}); + v.output_w = get_vocos("a.gen.wav.output.weight", {768, 2050}); + v.output_b = get_vocos("a.gen.wav.output.bias", {2050}); + v.blocks.resize(hparams.n_layer); + for (int il = 0; il < hparams.n_layer; ++il) { + auto & b = v.blocks[il]; + b.dwconv_w = get_vocos(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight"), {3, 1, 768}); + b.dwconv_b = get_vocos(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias"), {768}); + b.norm_w = get_vocos(string_format(TN_A_GEN_WAV_UP_NORM, il, "weight"), {768}); + b.norm_b = get_vocos(string_format(TN_A_GEN_WAV_UP_NORM, il, "bias"), {768}); + b.pw1_w = get_vocos(string_format(TN_A_GEN_WAV_UP_PW1, il, "weight"), {768, 2304}); + b.pw1_b = get_vocos(string_format(TN_A_GEN_WAV_UP_PW1, il, "bias"), {2304}); + b.pw2_w = get_vocos(string_format(TN_A_GEN_WAV_UP_PW2, il, "weight"), {2304, 768}); + b.pw2_b = get_vocos(string_format(TN_A_GEN_WAV_UP_PW2, il, "bias"), {768}); + b.gamma = get_vocos(string_format(TN_A_GEN_WAV_UP_GAMMA, il), {768}); + } + } break; case PROJECTOR_TYPE_QWEN3TTS_SPKENC: { // stem TDNN (block 0) @@ -4345,6 +4396,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { // pooling gives one speaker embedding, whatever the clip length is n_patches = 1; } break; + case PROJECTOR_TYPE_SOPRANO: case PROJECTOR_TYPE_QWEN3TTS_GEN: { // one hidden-state vector fed back to the talker per call @@ -5155,6 +5207,10 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { { set_pockettts_tfm_inputs(); } break; + case PROJECTOR_TYPE_SOPRANO: + { + set_input_f32("inp_feats", *params->feats); + } break; case PROJECTOR_TYPE_POCKETTTS_GEN: { if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) { @@ -5812,7 +5868,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { } } if (params->out_audio != nullptr) { - ggml_tensor * audio = ggml_graph_get_tensor(gf, "out_audio"); + ggml_tensor * audio = ggml_graph_get_tensor(gf, model.proj_type == PROJECTOR_TYPE_SOPRANO ? "out_spectrum" : "out_audio"); if (audio == nullptr) { GGML_ABORT("out_audio requested but graph has no \"out_audio\" tensor"); } @@ -5820,6 +5876,11 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { out_audio.resize(ggml_nelements(audio)); ggml_backend_tensor_get(audio, out_audio.data(), 0, ggml_nbytes(audio)); + if (model.proj_type == PROJECTOR_TYPE_SOPRANO) { + std::vector spectrum = std::move(out_audio); + clip_graph_soprano::decode_spectrum(spectrum, out_audio); + } + // drop the tail audio that comes from the code-0 rear padding const int64_t n_codes = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0; const int64_t n_frames_w = hparams.wav_tfm_swa; @@ -6003,6 +6064,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { return ctx->model.gen_code_out_embd_w->ne[0]; case PROJECTOR_TYPE_POCKETTTS_SPKENC: return ctx->model.spk_proj_w->ne[1]; + case PROJECTOR_TYPE_SOPRANO: + return ctx->model.vocos.input_w->ne[0]; case PROJECTOR_TYPE_POCKETTTS_GEN: return ctx->model.gen_input_lin_w->ne[1]; case PROJECTOR_TYPE_PARAKEET: diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 5945c6d92cb7..c5034e80ac50 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -12,6 +12,13 @@ * We encourage human contributors to ensure the quality and reliability of the codebase. */ +struct clip_graph_soprano : clip_graph { + clip_graph_soprano(clip_ctx * ctx, const clip_image_f32 & img, int n_frames) : clip_graph(ctx, img), n_frames(n_frames) {} + ggml_cgraph * build() override; + static void decode_spectrum(const std::vector & spectrum, std::vector & pcm); + int n_frames; +}; + struct clip_graph_siglip : clip_graph { clip_graph_siglip(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/models/soprano.cpp b/tools/mtmd/models/soprano.cpp new file mode 100644 index 000000000000..92fe60fe4f9f --- /dev/null +++ b/tools/mtmd/models/soprano.cpp @@ -0,0 +1,68 @@ +// Copyright (c) 2026 codec.cpp contributors +// SPDX-License-Identifier: MIT + +#include "models.h" +#include "../mtmd-audio.h" + +// Soprano-1.1 Vocos: normalized Qwen3 hidden states -> STFT coefficients. +ggml_cgraph * clip_graph_soprano::build() { + const auto & v = model.vocos; + const int n_up = 4 * (n_frames - 1) + 1; + ggml_tensor * cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 512, n_frames); + ggml_set_name(cur, "inp_feats"); + ggml_set_input(cur); + + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + cur = ggml_interpolate(ctx0, cur, n_up, 512, 1, 1, GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS); + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + cur = ggml_add(ctx0, build_mm(v.input_w, cur), v.input_b); + cur = build_norm(cur, v.norm_w, v.norm_b, NORM_TYPE_NORMAL, 1e-6f, -1); + + for (size_t il = 0; il < v.blocks.size(); ++il) { + const auto & b = v.blocks[il]; + ggml_tensor * residual = cur; + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + cur = ggml_conv_1d_dw(ctx0, b.dwconv_w, cur, 1, 1, 1); + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + cur = ggml_add(ctx0, cur, b.dwconv_b); + cur = build_norm(cur, b.norm_w, b.norm_b, NORM_TYPE_NORMAL, 1e-6f, il); + cur = ggml_add(ctx0, build_mm(b.pw1_w, cur), b.pw1_b); + cur = ggml_gelu_erf(ctx0, cur); + cur = ggml_add(ctx0, build_mm(b.pw2_w, cur), b.pw2_b); + cur = ggml_add(ctx0, residual, ggml_mul(ctx0, cur, b.gamma)); + cb(cur, "vocos_block", il); + } + cur = build_norm(cur, v.output_norm_w, v.output_norm_b, NORM_TYPE_NORMAL, 1e-6f, -1); + cur = ggml_add(ctx0, build_mm(v.output_w, cur), v.output_b); + cb(cur, "vocos_head", -1); + + const int n_bins = 1025; + ggml_tensor * mag = ggml_view_2d(ctx0, cur, n_bins, n_up, cur->nb[1], 0); + ggml_tensor * phase = ggml_view_2d(ctx0, cur, n_bins, n_up, cur->nb[1], n_bins * sizeof(float)); + mag = ggml_clamp(ctx0, ggml_exp(ctx0, ggml_cont(ctx0, mag)), 0.0f, 100.0f); + phase = ggml_cont(ctx0, phase); + ggml_tensor * re = ggml_mul(ctx0, mag, ggml_cos(ctx0, phase)); + ggml_tensor * im = ggml_mul(ctx0, mag, ggml_sin(ctx0, phase)); + re = ggml_reshape_3d(ctx0, re, 1, n_bins, n_up); + im = ggml_reshape_3d(ctx0, im, 1, n_bins, n_up); + cur = ggml_concat(ctx0, re, im, 0); + ggml_set_name(cur, "out_spectrum"); + ggml_set_output(cur); + ggml_build_forward_expand(gf, cur); + return gf; +} + +void clip_graph_soprano::decode_spectrum(const std::vector & spectrum, std::vector & pcm) { + const size_t n_frames = spectrum.size() / 2050; + mtmd_audio_streaming_istft istft(2048, 512); + pcm.clear(); + for (size_t i = 0; i < n_frames; ++i) { + auto frame = istft.process_frame(spectrum.data() + i * 2050); + pcm.insert(pcm.end(), frame.begin(), frame.end()); + } + auto tail = istft.flush(); + pcm.insert(pcm.end(), tail.begin(), tail.end()); + // The shared ISTFT removes 768 samples; torch.istft(center=True) removes 1024. + pcm.erase(pcm.begin(), pcm.begin() + 256); + pcm.resize((n_frames - 1) * 512); +} diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index 1c58d3ae1959..eda5bedc2865 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -993,8 +993,166 @@ class pockettts_gen_audio_pipeline : public mtmd_gen_audio_pipeline { std::vector out_buf; }; +// Soprano uses the hidden state that predicts each non-EOS token, including the prompt's last state. +class soprano_gen_audio_pipeline : public mtmd_gen_audio_pipeline { +public: + using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; + + void reset() override { + prompt.clear(); + features.clear(); + h_next.clear(); + audio.clear(); + wav.clear(); + pos = 0; + prompt_pos = 0; + } + + int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { + reset(); + if (n_embd != 512 || inp->speaker_ref || !inp->prompt || inp->prompt_len == 0 || inp->prompt_len > INT32_MAX || + inp->seq_id < 0 || (uint32_t) inp->seq_id >= llama_n_seq_max(lctx)) { + LOG_ERR("mtmd_helper_gen_audio: soprano requires text and a 512-d backbone, without a speaker reference\n"); + return 1; + } + seq_id = inp->seq_id; + out_type = inp->out_type; + const llama_token text = find_special_token(vocab, "[TEXT]"); + const llama_token start = find_special_token(vocab, "[START]"); + const llama_token stop = find_special_token(vocab, "[STOP]"); + if (text != 1 || start != 2 || stop != 3) { + LOG_ERR("mtmd_helper_gen_audio: incompatible soprano vocabulary\n"); + return 1; + } + const int required = llama_tokenize(vocab, inp->prompt, inp->prompt_len, nullptr, 0, false, false); + if (required >= 0 || required < -509 || (uint32_t) (-required + 3) >= llama_n_ctx_seq(lctx)) { + LOG_ERR("mtmd_helper_gen_audio: soprano prompt must fit within 512 tokens and the context\n"); + return 1; + } + const int n = -required; + prompt.resize(n + 3); + prompt[0] = stop; + prompt[1] = text; + if (llama_tokenize(vocab, inp->prompt, inp->prompt_len, prompt.data() + 2, n, false, false) != n) { + return 1; + } + prompt.back() = start; + if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) { + LOG_ERR("mtmd_helper_gen_audio: cannot reset soprano sequence\n"); + return 1; + } + return 0; + } + + int32_t step_prompt(int32_t n_batch) override { + if (n_batch <= 0 || prompt.empty()) { + return -1; + } + const int n = std::min(n_batch, (int) prompt.size() - prompt_pos); + if (n == 0) { + return 0; + } + if (!decode(prompt.data() + prompt_pos, n)) { + return -1; + } + prompt_pos += n; + return (int) prompt.size() - prompt_pos; + } + + int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override { + *h_state_out = nullptr; + *out_stop = false; + if (sampled < 0 || sampled >= llama_vocab_n_tokens(vocab) || !h_state_in || prompt_pos != (int) prompt.size() || prompt.empty()) { + return 1; + } + if (llama_vocab_is_eog(vocab, sampled) || features.size() / n_embd >= 512) { + *out_stop = true; + return 0; + } + if ((uint32_t) pos >= llama_n_ctx_seq(lctx)) { + LOG_ERR("mtmd_helper_gen_audio: soprano context exhausted\n"); + return 1; + } + features.insert(features.end(), h_state_in, h_state_in + n_embd); + if (!decode(&sampled, 1)) { + features.resize(features.size() - n_embd); + return 1; + } + const float * h = llama_get_embeddings_ith(lctx, -1); + if (!h) { + return 1; + } + h_next.assign(h, h + n_embd); + *h_state_out = h_next.data(); + return 0; + } + + int32_t get_output(int32_t * sample_rate, const char ** data, size_t * data_len, int64_t * n_samples) override { + if (features.size() < 2 * (size_t) n_embd) { + LOG_ERR("mtmd_helper_gen_audio: soprano needs at least two generated frames\n"); + return 1; + } + mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; + inp.feats = features.data(); + inp.n_feats = features.size(); + mtmd_gen_out out{}; + if (mtmd_gen_audio_process(mctx, &inp, &out) != 0) { + return 1; + } + audio.assign(out.audio, out.audio + out.n_samples); + *sample_rate = info.sample_rate; + if (n_samples) { *n_samples = audio.size(); } + if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { + *data = (const char *) audio.data(); + *data_len = audio.size() * sizeof(float); + } else { + wav.clear(); + if (!write_wav16(wav, audio, info.sample_rate)) { + return 1; + } + *data = wav.data(); + *data_len = wav.size(); + } + return 0; + } + +private: + bool decode(llama_token * tokens, int n) { + llama_batch batch = llama_batch_init(n, 0, 1); + batch.n_tokens = n; + for (int i = 0; i < n; ++i) { + batch.token[i] = tokens[i]; + batch.pos[i] = pos + i; + batch.n_seq_id[i] = 1; + batch.seq_id[i][0] = seq_id; + batch.logits[i] = i == n - 1; + } + const int ret = llama_decode(lctx, batch); + llama_batch_free(batch); + if (ret != 0) { + LOG_ERR("mtmd_helper_gen_audio: soprano backbone decode failed\n"); + return false; + } + pos += n; + return true; + } + + llama_seq_id seq_id = 0; + llama_pos pos = 0; + int prompt_pos = 0; + std::vector prompt; + std::vector features; + std::vector h_next; + std::vector audio; + std::vector wav; + mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; +}; + static std::unique_ptr make_pipeline(llama_context * lctx, mtmd_context * mctx) { switch (mtmd_gen_audio_get_info(mctx).type) { + case MTMD_GEN_AUDIO_TYPE_SOPRANO: + return std::unique_ptr(new soprano_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_QWEN3TTS: return std::unique_ptr(new qwen3tts_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_POCKETTTS: diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 00ecadcf4dfe..581a04d2ec57 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -583,7 +583,7 @@ struct mtmd_context { ctx_v = res.ctx_v; ctx_a = res.ctx_a; ctx_gen_a = res.ctx_gen_a; - if (!ctx_v && !ctx_a) { + if (!ctx_v && !ctx_a && !ctx_gen_a) { throw std::runtime_error(string_format("Failed to load CLIP model from %s\n", mmproj_fname)); } @@ -600,7 +600,7 @@ struct mtmd_context { // since we already validate n_embd of vision and audio mmproj, // we can safely assume that they are the same - int n_embd_clip = clip_n_mmproj_embd(ctx_v ? ctx_v : ctx_a); + int n_embd_clip = clip_n_mmproj_embd(ctx_v ? ctx_v : (ctx_a ? ctx_a : ctx_gen_a)); if (n_embd_text > 0 && n_embd_text != n_embd_clip) { throw std::runtime_error(string_format( "mismatch between text model (n_embd = %d) and mmproj (n_embd = %d)\n" @@ -1881,6 +1881,10 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { info.type = MTMD_GEN_AUDIO_TYPE_QWEN3TTS; info.sample_rate = 24000; break; + case PROJECTOR_TYPE_SOPRANO: + info.type = MTMD_GEN_AUDIO_TYPE_SOPRANO; + info.sample_rate = 32000; + break; case PROJECTOR_TYPE_POCKETTTS_GEN: info.type = MTMD_GEN_AUDIO_TYPE_POCKETTTS; info.sample_rate = 24000; @@ -1901,6 +1905,9 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { } switch (clip_get_projector_type(ctx->ctx_gen_a)) { + case PROJECTOR_TYPE_SOPRANO: + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; + break; case PROJECTOR_TYPE_QWEN3TTS_GEN: // https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base/blob/main/generation_config.json inp.top_k = 50; @@ -1928,6 +1935,13 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_in *out = {}; + if (clip_get_projector_type(ctx_clip) == PROJECTOR_TYPE_SOPRANO && + (inp->type != MTMD_GEN_PROCESS_TYPE_GEN_WAV || !inp->feats || inp->n_feats < 1024 || + inp->n_feats % 512 != 0 || inp->n_feats / 512 > 512 || inp->codes || inp->state_size)) { + LOG_ERR("%s: soprano requires 2 to 512 frames of 512 features, without codes or state\n", __func__); + return 1; + } + if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_CODE) { const size_t n_embd = (size_t) clip_n_mmproj_embd(ctx_clip); diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index c2de26eeee2c..70944bdc53a1 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -371,6 +371,7 @@ enum mtmd_gen_audio_type { MTMD_GEN_AUDIO_TYPE_NONE, // not supported MTMD_GEN_AUDIO_TYPE_QWEN3TTS, MTMD_GEN_AUDIO_TYPE_POCKETTTS, + MTMD_GEN_AUDIO_TYPE_SOPRANO, }; struct mtmd_gen_audio_info { diff --git a/tools/tts/README.md b/tools/tts/README.md index 1b08d5ef3218..3dbff947557c 100644 --- a/tools/tts/README.md +++ b/tools/tts/README.md @@ -57,3 +57,17 @@ The [upstream repository](https://huggingface.co/kyutai/pocket-tts) holds one co python convert_hf_to_gguf.py path/to/pocket-tts/languages/english --outfile pocket-tts.gguf python convert_hf_to_gguf.py path/to/pocket-tts/languages/english --mmproj --outfile mmproj-pocket-tts.gguf ``` + +## Soprano + +Soprano-1.1-80M uses a Qwen3 backbone and a Vocos decoder. Convert both files from the same local [model directory](https://huggingface.co/ekwek/Soprano-1.1-80M), including `decoder.pth`: + +```sh +python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --outfile soprano.gguf --outtype f16 +python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --mmproj --outfile mmproj-soprano.gguf --outtype f16 +llama-tts -m soprano.gguf -mm mmproj-soprano.gguf -p "Hello world!" --temp 0 --output out.wav +``` + +This pipeline generates mono audio at 32 kHz with the model's fixed voice. It does not accept `--tts-speaker-file`; `--tts-lang` is unused. The helper adds the `[STOP][TEXT]...[START]` prompt format and accumulates hidden states before reconstructing the waveform. The core `GEN_WAV` call accepts 2 to 512 frames of 512 continuous features in frame-major order; it does not use codes or persistent state. + +Supply normalized English text. The tokenizer lowercases text and collapses whitespace, but the Python reference's number/abbreviation expansion, transliteration and sentence splitting are not included. Split long text into separate requests; each prompt must fit within 512 tokens including the three control tokens. Audio is returned after generation finishes; incremental audio output is not supported in this initial implementation. From 14537dd35a60d1d6ec0b802e75f41ff9bf59d77d Mon Sep 17 00:00:00 2001 From: Hans Date: Tue, 8 Sep 2026 21:05:44 +0800 Subject: [PATCH 2/3] convert : select Soprano mmproj explicitly Assisted-by: Codex --- conversion/__init__.py | 2 +- conversion/soprano.py | 10 +++++++++- convert_hf_to_gguf.py | 9 ++++++++- tools/tts/README.md | 4 +++- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/conversion/__init__.py b/conversion/__init__.py index 5ef422506e59..73170ec43798 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -285,7 +285,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = { - "Qwen3ForCausalLM": "soprano", "AudioFlamingo3ForConditionalGeneration": "ultravox", "CogVLMForCausalLM": "cogvlm", "DeepseekOCR2ForCausalLM": "deepseek", @@ -335,6 +334,7 @@ "Qwen3ASRForConditionalGeneration": "qwen3vl", "Qwen3OmniMoeForConditionalGeneration": "qwen3vl", "PocketTTSModel": "pockettts", + "SopranoModel": "soprano", "Qwen3TTSForConditionalGeneration": "qwen3tts", "Qwen3VLForConditionalGeneration": "qwen3vl", "Qwen3VLMoeForConditionalGeneration": "qwen3vl", diff --git a/conversion/soprano.py b/conversion/soprano.py index 5494f7338ff5..a4344510be35 100644 --- a/conversion/soprano.py +++ b/conversion/soprano.py @@ -4,17 +4,25 @@ from __future__ import annotations import torch +from transformers import AutoTokenizer from .base import ModelBase, MmprojModel, gguf -@ModelBase.register("Qwen3ForCausalLM") +@ModelBase.register("SopranoModel") @ModelBase.example("ekwek/Soprano-1.1-80M") class SopranoModel(MmprojModel): has_vision_encoder = False has_audio_encoder = False def get_audio_config(self): + if self.hparams.get("model_type") != "qwen3" or self.hparams.get("hidden_size") != 512 or self.hparams.get("vocab_size") != 8192: + raise ValueError("Soprano requires a Qwen3 backbone with hidden_size=512 and vocab_size=8192") + tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=False) + if tokenizer.convert_tokens_to_ids(["[UNK]", "[TEXT]", "[START]", "[STOP]"]) != [0, 1, 2, 3]: + raise ValueError("Soprano requires its [UNK], [TEXT], [START] and [STOP] control tokens") + if not (self.dir_model / "decoder.pth").is_file(): + raise ValueError("Soprano requires decoder.pth in the model directory") return {"num_hidden_layers": 8} def set_gguf_parameters(self): diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index e09616b190cf..e0501b5a2cd4 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -16,6 +16,7 @@ import gguf from conversion import ( + MMPROJ_MODEL_MAP, ModelBase, ModelType, get_model_architecture, @@ -117,6 +118,10 @@ def parse_args() -> argparse.Namespace: "--mmproj", action="store_true", help="Export multimodal projector (mmproj) for vision models. This will only work on some vision models. An 'mmproj-' prefix will be added to the output file name.", ) + parser.add_argument( + "--mmproj-architecture", choices=sorted(MMPROJ_MODEL_MAP), + help="Select the mmproj architecture explicitly when config.json only describes the text backbone. Requires --mmproj.", + ) parser.add_argument( "--mtp", action="store_true", help="Export only the multi-token prediction (MTP) head as a separate GGUF, suitable for use as a speculative draft. An 'mtp-' prefix will be added to the output file name.", @@ -171,6 +176,8 @@ def parse_args() -> argparse.Namespace: ) args = parser.parse_args() + if args.mmproj_architecture and (not args.mmproj or args.mistral_format): + parser.error("--mmproj-architecture requires --mmproj and does not support --mistral-format") if not args.print_supported_models and args.model is None: parser.error("the following arguments are required: model") return args @@ -244,7 +251,7 @@ def main() -> None: model_type = ModelType.MMPROJ if args.mmproj else ModelType.TEXT hparams = ModelBase.load_hparams(dir_model, is_mistral_format) if not is_mistral_format: - model_architecture = get_model_architecture(hparams, model_type) + model_architecture = args.mmproj_architecture or get_model_architecture(hparams, model_type) logger.info(f"Model architecture: {model_architecture}") try: model_class = get_model_class(model_architecture, mmproj=(model_type == ModelType.MMPROJ)) diff --git a/tools/tts/README.md b/tools/tts/README.md index 3dbff947557c..a1c7f82485bf 100644 --- a/tools/tts/README.md +++ b/tools/tts/README.md @@ -64,10 +64,12 @@ Soprano-1.1-80M uses a Qwen3 backbone and a Vocos decoder. Convert both files fr ```sh python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --outfile soprano.gguf --outtype f16 -python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --mmproj --outfile mmproj-soprano.gguf --outtype f16 +python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --mmproj --mmproj-architecture SopranoModel --outfile mmproj-soprano.gguf --outtype f16 llama-tts -m soprano.gguf -mm mmproj-soprano.gguf -p "Hello world!" --temp 0 --output out.wav ``` +The explicit mmproj architecture is required because the model config only identifies the Qwen3 text backbone, which does not identify an audio decoder. + This pipeline generates mono audio at 32 kHz with the model's fixed voice. It does not accept `--tts-speaker-file`; `--tts-lang` is unused. The helper adds the `[STOP][TEXT]...[START]` prompt format and accumulates hidden states before reconstructing the waveform. The core `GEN_WAV` call accepts 2 to 512 frames of 512 continuous features in frame-major order; it does not use codes or persistent state. Supply normalized English text. The tokenizer lowercases text and collapses whitespace, but the Python reference's number/abbreviation expansion, transliteration and sentence splitting are not included. Split long text into separate requests; each prompt must fit within 512 tokens including the three control tokens. Audio is returned after generation finishes; incremental audio output is not supported in this initial implementation. From 90697e004f9b52833d7e62dbe8e04e6d72f91802 Mon Sep 17 00:00:00 2001 From: Hans Date: Tue, 8 Sep 2026 23:32:33 +0800 Subject: [PATCH 3/3] docs : match Soprano usage to existing TTS sections Assisted-by: Codex --- tools/tts/README.md | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tools/tts/README.md b/tools/tts/README.md index a1c7f82485bf..151cb9f459ed 100644 --- a/tools/tts/README.md +++ b/tools/tts/README.md @@ -60,16 +60,25 @@ python convert_hf_to_gguf.py path/to/pocket-tts/languages/english --mmproj --out ## Soprano -Soprano-1.1-80M uses a Qwen3 backbone and a Vocos decoder. Convert both files from the same local [model directory](https://huggingface.co/ekwek/Soprano-1.1-80M), including `decoder.pth`: +Available params: +- `--tts-speaker-file` is not supported; the model uses a fixed voice +- Note: `lang` is not used. Supply normalized English text, with at most 509 text tokens per request + +Example usage: ```sh -python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --outfile soprano.gguf --outtype f16 -python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --mmproj --mmproj-architecture SopranoModel --outfile mmproj-soprano.gguf --outtype f16 -llama-tts -m soprano.gguf -mm mmproj-soprano.gguf -p "Hello world!" --temp 0 --output out.wav +llama-tts -m soprano.gguf \ + -mm mmproj-soprano.gguf \ + -p "Hello world" \ + --temp 0 \ + --output out.wav ``` -The explicit mmproj architecture is required because the model config only identifies the Qwen3 text backbone, which does not identify an audio decoder. +**Note for GGUF conversion:** -This pipeline generates mono audio at 32 kHz with the model's fixed voice. It does not accept `--tts-speaker-file`; `--tts-lang` is unused. The helper adds the `[STOP][TEXT]...[START]` prompt format and accumulates hidden states before reconstructing the waveform. The core `GEN_WAV` call accepts 2 to 512 frames of 512 continuous features in frame-major order; it does not use codes or persistent state. +Convert both files from the same [Soprano-1.1-80M directory](https://huggingface.co/ekwek/Soprano-1.1-80M), including `decoder.pth`. The decoder requires `--mmproj-architecture SopranoModel`: -Supply normalized English text. The tokenizer lowercases text and collapses whitespace, but the Python reference's number/abbreviation expansion, transliteration and sentence splitting are not included. Split long text into separate requests; each prompt must fit within 512 tokens including the three control tokens. Audio is returned after generation finishes; incremental audio output is not supported in this initial implementation. +```sh +python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --outtype f16 --outfile soprano.gguf +python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --mmproj --mmproj-architecture SopranoModel --outtype f16 --outfile mmproj-soprano.gguf +```