mtmd : add Soprano audio generation

conversion/__init__.py

@@ -334,6 +334,7 @@
     "Qwen3ASRForConditionalGeneration": "qwen3vl",
     "Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
     "PocketTTSModel": "pockettts",
+    "SopranoModel": "soprano",
     "Qwen3TTSForConditionalGeneration": "qwen3tts",
     "Qwen3VLForConditionalGeneration": "qwen3vl",
     "Qwen3VLMoeForConditionalGeneration": "qwen3vl",

conversion/qwen.py

@@ -208,6 +208,11 @@ def set_vocab(self):
 
         super().set_vocab()
 
+    def get_vocab_base_pre(self, tokenizer) -> str:
+        if tokenizer.convert_tokens_to_ids("[TEXT]") == 1 and tokenizer.convert_tokens_to_ids("[START]") == 2 and tokenizer.convert_tokens_to_ids("[STOP]") == 3:
+            return "soprano"
+        return super().get_vocab_base_pre(tokenizer)
+
     def _find_rerank_config(self):
         from transformers import AutoTokenizer
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model)

conversion/soprano.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2026 codec.cpp contributors
+# SPDX-License-Identifier: MIT
+
+from __future__ import annotations
+
+import torch
+from transformers import AutoTokenizer
+
+from .base import ModelBase, MmprojModel, gguf
+
+
+@ModelBase.register("SopranoModel")
+@ModelBase.example("ekwek/Soprano-1.1-80M")
+class SopranoModel(MmprojModel):
+    has_vision_encoder = False
+    has_audio_encoder = False
+
+    def get_audio_config(self):
+        if self.hparams.get("model_type") != "qwen3" or self.hparams.get("hidden_size") != 512 or self.hparams.get("vocab_size") != 8192:
+            raise ValueError("Soprano requires a Qwen3 backbone with hidden_size=512 and vocab_size=8192")
+        tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=False)
+        if tokenizer.convert_tokens_to_ids(["[UNK]", "[TEXT]", "[START]", "[STOP]"]) != [0, 1, 2, 3]:
+            raise ValueError("Soprano requires its [UNK], [TEXT], [START] and [STOP] control tokens")
+        if not (self.dir_model / "decoder.pth").is_file():
+            raise ValueError("Soprano requires decoder.pth in the model directory")
+        return {"num_hidden_layers": 8}
+
+    def set_gguf_parameters(self):
+        self.gguf_writer.add_file_type(self.ftype)
+        self.gguf_writer.add_clip_has_gen_audio_encoder(True)
+        self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.SOPRANO)
+        self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
+        self.gguf_writer.add_gen_audio_embedding_length(768)
+        self.gguf_writer.add_gen_audio_feed_forward_length(2304)
+        self.gguf_writer.add_gen_audio_block_count(8)
+        self.gguf_writer.add_gen_audio_head_count(1)
+        self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-6)
+
+    def get_tensors(self):
+        state = torch.load(self.dir_model / "decoder.pth", map_location="cpu", weights_only=True)
+        if "state_dict" in state:
+            state = state["state_dict"]
+        yield from state.items()
+
+    def tensor_force_quant(self, name, new_name, bid, n_dims):
+        if "dwconv.weight" in new_name:
+            return gguf.GGMLQuantizationType.F16
+        return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+    def modify_tensors(self, data_torch, name, bid):
+        if name == "head.istft.window":
+            expected = torch.hann_window(2048)
+            if not torch.equal(data_torch.float(), expected):
+                raise ValueError("Soprano requires the periodic 2048-sample Hann window")
+            return
+        if name == "decoder.embed.weight":
+            data_torch = data_torch.squeeze(-1)
+        yield self.map_tensor_name(name), data_torch

convert_hf_to_gguf.py

@@ -16,6 +16,7 @@
 import gguf
 
 from conversion import (
+    MMPROJ_MODEL_MAP,
     ModelBase,
     ModelType,
     get_model_architecture,
@@ -117,6 +118,10 @@ def parse_args() -> argparse.Namespace:
         "--mmproj", action="store_true",
         help="Export multimodal projector (mmproj) for vision models. This will only work on some vision models. An 'mmproj-' prefix will be added to the output file name.",
     )
+    parser.add_argument(
+        "--mmproj-architecture", choices=sorted(MMPROJ_MODEL_MAP),
+        help="Select the mmproj architecture explicitly when config.json only describes the text backbone. Requires --mmproj.",
+    )
     parser.add_argument(
         "--mtp", action="store_true",
         help="Export only the multi-token prediction (MTP) head as a separate GGUF, suitable for use as a speculative draft. An 'mtp-' prefix will be added to the output file name.",
@@ -171,6 +176,8 @@ def parse_args() -> argparse.Namespace:
     )
 
     args = parser.parse_args()
+    if args.mmproj_architecture and (not args.mmproj or args.mistral_format):
+        parser.error("--mmproj-architecture requires --mmproj and does not support --mistral-format")
     if not args.print_supported_models and args.model is None:
         parser.error("the following arguments are required: model")
     return args
@@ -244,7 +251,7 @@ def main() -> None:
         model_type = ModelType.MMPROJ if args.mmproj else ModelType.TEXT
         hparams = ModelBase.load_hparams(dir_model, is_mistral_format)
         if not is_mistral_format:
-            model_architecture = get_model_architecture(hparams, model_type)
+            model_architecture = args.mmproj_architecture or get_model_architecture(hparams, model_type)
             logger.info(f"Model architecture: {model_architecture}")
             try:
                 model_class = get_model_class(model_architecture, mmproj=(model_type == ModelType.MMPROJ))

gguf-py/gguf/constants.py

@@ -1084,6 +1084,10 @@ class MODEL_TENSOR(IntEnum):
     A_GEN_CODE_FFN_GATE    = auto()
     A_GEN_CODE_FFN_UP      = auto()
     A_GEN_CODE_FFN_DOWN    = auto()
+    A_GEN_WAV_INPUT               = auto()
+    A_GEN_WAV_NORM                = auto()
+    A_GEN_WAV_OUTPUT_NORM         = auto()
+    A_GEN_WAV_OUTPUT              = auto()
     A_GEN_CODE_OUTPUT_NORM = auto()
     # qwen3tts code2wav: RVQ codes -> raw PCM
     A_GEN_WAV_QUANT_FIRST_IN       = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D)
@@ -1835,6 +1839,10 @@ class MODEL_TENSOR(IntEnum):
     MODEL_TENSOR.A_GEN_CODE_FFN_GATE:       "a.gen.code.blk.{bid}.ffn_gate",
     MODEL_TENSOR.A_GEN_CODE_FFN_UP:         "a.gen.code.blk.{bid}.ffn_up",
     MODEL_TENSOR.A_GEN_CODE_FFN_DOWN:       "a.gen.code.blk.{bid}.ffn_down",
+    MODEL_TENSOR.A_GEN_WAV_INPUT:         "a.gen.wav.input",
+    MODEL_TENSOR.A_GEN_WAV_NORM:          "a.gen.wav.norm",
+    MODEL_TENSOR.A_GEN_WAV_OUTPUT_NORM:   "a.gen.wav.output_norm",
+    MODEL_TENSOR.A_GEN_WAV_OUTPUT:        "a.gen.wav.output",
     MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM:    "a.gen.code.output_norm",
     MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN:  "a.gen.wav.quant.first.in_proj",
     MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT: "a.gen.wav.quant.first.out_proj",
@@ -2192,6 +2200,10 @@ class MODEL_TENSOR(IntEnum):
         MODEL_TENSOR.A_GEN_CODE_FFN_GATE,
         MODEL_TENSOR.A_GEN_CODE_FFN_UP,
         MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,
+        MODEL_TENSOR.A_GEN_WAV_INPUT,
+        MODEL_TENSOR.A_GEN_WAV_NORM,
+        MODEL_TENSOR.A_GEN_WAV_OUTPUT_NORM,
+        MODEL_TENSOR.A_GEN_WAV_OUTPUT,
         MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM,
         MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN,
         MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT,
@@ -5813,6 +5825,7 @@ class VisionProjectorType:
     QWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictor
     POCKETTTS_SPKENC = "pockettts_spkenc" # audio: mimi encoder as voice-prompt encoder
     POCKETTTS_GEN = "pockettts_gen" # audio generation: flow-matching decoder + mimi decoder
+    SOPRANO = "soprano" # audio generation: Vocos decoder
     HUNYUANVL      = "hunyuanvl"
     PARAKEET       = "parakeet"  # audio
     MINIMAXM3      = "minimax_m3"

gguf-py/gguf/tensor_mapping.py

@@ -7,6 +7,10 @@
 
 class TensorNameMap:
     mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = {
+        MODEL_TENSOR.A_GEN_WAV_INPUT:       ("decoder.embed",),
+        MODEL_TENSOR.A_GEN_WAV_NORM:        ("decoder.norm",),
+        MODEL_TENSOR.A_GEN_WAV_OUTPUT_NORM: ("decoder.final_layer_norm",),
+        MODEL_TENSOR.A_GEN_WAV_OUTPUT:      ("head.out",),
         # Token embeddings
         MODEL_TENSOR.TOKEN_EMBD: (
             "gpt_neox.embed_in",                         # gptneox
@@ -182,6 +186,11 @@ class TensorNameMap:
     }
 
     block_mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = {
+        MODEL_TENSOR.A_GEN_WAV_UP_DWCONV: ("decoder.convnext.{bid}.dwconv",),
+        MODEL_TENSOR.A_GEN_WAV_UP_NORM:   ("decoder.convnext.{bid}.norm",),
+        MODEL_TENSOR.A_GEN_WAV_UP_PW1:    ("decoder.convnext.{bid}.pwconv1",),
+        MODEL_TENSOR.A_GEN_WAV_UP_PW2:    ("decoder.convnext.{bid}.pwconv2",),
+        MODEL_TENSOR.A_GEN_WAV_UP_GAMMA:  ("decoder.convnext.{bid}.gamma",),
         # Attention norm
         MODEL_TENSOR.ATTN_NORM: (
             "gpt_neox.layers.{bid}.input_layernorm",                # gptneox

src/llama-vocab.cpp

@@ -545,6 +545,10 @@ struct llm_tokenizer_bpe : llm_tokenizer {
                     "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}+| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
                 };
                 break;
+            case LLAMA_VOCAB_PRE_TYPE_SOPRANO:
+                regex_exprs = { "\\p{N}", "\\s+|[\\p{L}\\p{N}_]+|[^\\p{L}\\p{N}_\\s]+" };
+                byte_encode = false;
+                break;
             case LLAMA_VOCAB_PRE_TYPE_WHITESPACE:
                 // whitespace pre-tokenizer (jinaai/jina-embeddings-v2-base-zh)
                 regex_exprs = {
@@ -612,7 +616,22 @@ struct llm_tokenizer_bpe_session {
 
     virtual void tokenize(const std::string & text, std::vector<llama_token> & output) {
         int final_prev_index = -1;
-        const auto word_collection = unicode_regex_split(text, tokenizer.regex_exprs, tokenizer.byte_encode);
+        std::string normalized;
+        if (vocab.get_pre_type() == LLAMA_VOCAB_PRE_TYPE_SOPRANO) {
+            bool space = false;
+            for (uint32_t cpt : unicode_cpts_from_utf8(text)) {
+                if (unicode_cpt_flags_from_cpt(cpt).is_whitespace) {
+                    if (!space) { normalized += ' '; }
+                    space = true;
+                } else {
+                    // Unicode full lowercase expands LATIN CAPITAL LETTER I WITH DOT ABOVE.
+                    normalized += cpt == 0x0130 ? "i\xcc\x87" : unicode_cpt_to_utf8(unicode_tolower(cpt));
+                    space = false;
+                }
+            }
+        }
+        const auto & input_text = vocab.get_pre_type() == LLAMA_VOCAB_PRE_TYPE_SOPRANO ? normalized : text;
+        const auto word_collection = unicode_regex_split(input_text, tokenizer.regex_exprs, tokenizer.byte_encode);
 
         symbols_final.clear();
         auto tok_pre = vocab.get_pre_type();
@@ -708,7 +727,9 @@ struct llm_tokenizer_bpe_session {
                 const std::string str = std::string(symbol.text, symbol.n);
                 const auto token = vocab.text_to_token(str);
 
-                if (token == LLAMA_TOKEN_NULL) {
+                if (token == LLAMA_TOKEN_NULL && tok_pre == LLAMA_VOCAB_PRE_TYPE_SOPRANO) {
+                    output.push_back(vocab.token_unk());
+                } else if (token == LLAMA_TOKEN_NULL) {
                     for (auto j = str.begin(); j != str.end(); ++j) {
                         llama_token token_multibyte = LLAMA_TOKEN_NULL;
                         if (tokenizer.byte_encode) {
@@ -2209,6 +2230,12 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                     tokenizer_pre == "mellum"  ||
                     tokenizer_pre == "modern-bert") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_GPT2;
+            } else if (
+                    tokenizer_pre == "soprano") {
+                pre_type = LLAMA_VOCAB_PRE_TYPE_SOPRANO;
+                special_unk_id = 0;
+                special_eos_id = 3;
+                add_bos = false;
             } else if (
                     tokenizer_pre == "jais-2") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_JAIS2;
@@ -2916,6 +2943,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
             }
         }
 
+        if (pre_type == LLAMA_VOCAB_PRE_TYPE_SOPRANO) {
+            special_eog_ids.insert(special_eos_id);
+        }
+
         // sanity checks
         if (special_eos_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eos_id) == 0) {
             special_eog_ids.insert(special_eos_id);
@@ -3657,6 +3688,9 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t
                     return _try_copy(token_text.data(), token_text.size());
                 }
                 if (attr & LLAMA_TOKEN_ATTR_NORMAL) {
+                    if (pre_type == LLAMA_VOCAB_PRE_TYPE_SOPRANO) {
+                        return _try_copy(token_text.data(), token_text.size());
+                    }
                     if (escape_whitespaces) {
                         // SPM-style BPE: tokens contain ▁ for spaces
                         std::string result = token_text;

src/llama-vocab.h

@@ -67,6 +67,7 @@ enum llama_vocab_pre_type {
     LLAMA_VOCAB_PRE_TYPE_LAGUNA            = 56,
     LLAMA_VOCAB_PRE_TYPE_HY_V4             = 57,
     LLAMA_VOCAB_PRE_TYPE_SPARK2_5          = 58,
+    LLAMA_VOCAB_PRE_TYPE_SOPRANO           = 59,
 };
 
 struct LLM_KV;

tools/mtmd/CMakeLists.txt

@@ -63,6 +63,7 @@ add_library(mtmd
             models/pockettts-seanet.cpp
             models/pockettts-spkenc.cpp
             models/pockettts-gen.cpp
+            models/soprano.cpp
             models/step3vl.cpp
             models/siglip.cpp
             models/whisper-enc.cpp

tools/mtmd/clip-impl.h

@@ -503,6 +503,7 @@ enum projector_type {
     PROJECTOR_TYPE_QWEN3TTS_GEN,
     PROJECTOR_TYPE_POCKETTTS_SPKENC,
     PROJECTOR_TYPE_POCKETTTS_GEN,
+    PROJECTOR_TYPE_SOPRANO,
     PROJECTOR_TYPE_MUSE_GLIMMER,
     PROJECTOR_TYPE_UNKNOWN,
 };
@@ -568,6 +569,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
     { PROJECTOR_TYPE_QWEN3TTS_GEN,      "qwen3tts_gen"},
     { PROJECTOR_TYPE_POCKETTTS_SPKENC,  "pockettts_spkenc"},
     { PROJECTOR_TYPE_POCKETTTS_GEN,     "pockettts_gen"},
+    { PROJECTOR_TYPE_SOPRANO,           "soprano"},
     { PROJECTOR_TYPE_MUSE_GLIMMER,      "muse-glimmer"},
 };
 

tools/mtmd/clip-model.h

@@ -552,6 +552,18 @@ struct clip_code2wav {
     ggml_tensor * dac_post_conv_b      = nullptr;
 };
 
+struct clip_vocos {
+    ggml_tensor * input_w        = nullptr;
+    ggml_tensor * input_b        = nullptr;
+    ggml_tensor * norm_w         = nullptr;
+    ggml_tensor * norm_b         = nullptr;
+    ggml_tensor * output_norm_w  = nullptr;
+    ggml_tensor * output_norm_b  = nullptr;
+    ggml_tensor * output_w       = nullptr;
+    ggml_tensor * output_b       = nullptr;
+    std::vector<clip_code2wav::upsample_block> blocks;
+};
+
 struct clip_model {
     clip_modality modality = CLIP_MODALITY_VISION;
     projector_type proj_type = PROJECTOR_TYPE_MLP;
@@ -786,6 +798,7 @@ struct clip_model {
 
     // qwen3tts code2wav: RVQ codes -> raw PCM
     clip_code2wav c2w;
+    clip_vocos vocos;
 
     // pocket-tts: SEANet stack, shared by the encoder (speaker path) and the decoder (gen path)
     clip_seanet seanet;