diff --git a/otherarch/sdcpp/src/name_conversion.cpp b/otherarch/sdcpp/src/name_conversion.cpp index 7f2d0006302..126a4ddbcc6 100644 --- a/otherarch/sdcpp/src/name_conversion.cpp +++ b/otherarch/sdcpp/src/name_conversion.cpp @@ -185,6 +185,20 @@ std::string convert_cond_stage_model_name(std::string name, std::string prefix) } std::string convert_qwen3_vl_vision_name(std::string name) { + static const std::vector> qwen3_vl_deepstack_name_map{ + {"v.deepstack_merger_list.", "deepstack_merger_list."}, + {"v.deepstack.5.", "deepstack_merger_list.0."}, + {"v.deepstack.8.", "deepstack_merger_list.0."}, + {"v.deepstack.11.", "deepstack_merger_list.1."}, + {"v.deepstack.16.", "deepstack_merger_list.1."}, + {"v.deepstack.17.", "deepstack_merger_list.2."}, + {"v.deepstack.24.", "deepstack_merger_list.2."}, + {"fc1.", "linear_fc1."}, + {"fc2.", "linear_fc2."}, + {"ffn_up.", "linear_fc1."}, + {"ffn_down.", "linear_fc2."}, + {"ffn_norm.", "norm."}, + }; static const std::vector> qwen3_vl_vision_name_map{ {"mm.0.", "merger.linear_fc1."}, {"mm.2.", "merger.linear_fc2."}, @@ -201,6 +215,10 @@ std::string convert_qwen3_vl_vision_name(std::string name) { {"ln1.", "norm1."}, {"ln2.", "norm2."}, }; + if (contains(name, "v.deepstack_merger_list.") || contains(name, "v.deepstack.")) { + replace_with_name_map(name, qwen3_vl_deepstack_name_map); + return name; + } replace_with_name_map(name, qwen3_vl_vision_name_map); return name; } @@ -1181,6 +1199,7 @@ std::string convert_sep_to_dot(std::string name) { "x_embedder", "cross_attn", "output_proj", + "token_refiner", }; // record the positions of underscores that should NOT be replaced diff --git a/otherarch/sdcpp/src/runtime/latent-preview.h b/otherarch/sdcpp/src/runtime/latent-preview.h index 15239908199..c19a0eebcf0 100644 --- a/otherarch/sdcpp/src/runtime/latent-preview.h +++ b/otherarch/sdcpp/src/runtime/latent-preview.h @@ -4,6 +4,33 @@ #include "core/tensor.hpp" #include "ggml.h" +const float minimax_latent_rgb_proj[24][3] = { + {0.19819857f, 0.11584999f, 0.07929777f}, + {-0.16047224f, -0.10601170f, -0.15996324f}, + {0.47391951f, 0.37602475f, 0.20267826f}, + {-0.09857441f, -0.27435449f, -0.51681751f}, + {-0.18930605f, -0.10512278f, -0.28571478f}, + {-0.15639569f, -0.18000929f, -0.25432852f}, + {-0.07176921f, -0.10901598f, -0.06654253f}, + {-0.05014077f, -0.05839826f, -0.05516087f}, + {-0.05201424f, -0.04351913f, -0.01507579f}, + {0.24750438f, 0.13307422f, 0.17684120f}, + {0.07377446f, 0.10235858f, 0.11707827f}, + {0.02908304f, 0.06587022f, 0.10643690f}, + {-0.00670531f, -0.03857879f, 0.01750151f}, + {-0.07119107f, -0.03083323f, -0.01995450f}, + {-0.08612627f, -0.07253841f, -0.01442890f}, + {0.08793202f, 0.08681750f, 0.02994647f}, + {0.00876893f, 0.02721868f, 0.00091178f}, + {-0.03484412f, -0.02711262f, -0.00110101f}, + {-0.00679772f, -0.01844275f, -0.01683359f}, + {0.04287028f, 0.01601068f, 0.04037397f}, + {-0.00493432f, -0.00230528f, 0.00353911f}, + {0.01495088f, 0.00292306f, 0.00416671f}, + {0.00495307f, 0.05066542f, 0.05210543f}, + {-0.02154842f, -0.01518524f, 0.00442402f}}; +float minimax_latent_rgb_bias[3] = {0.07776964f, -0.01580954f, -0.06561434f}; + const float ltxav_latent_rgb_proj[128][3] = { {-0.0293802f, -0.0362516f, -0.0291386f}, {0.0117735f, 0.0223435f, 0.018856f}, diff --git a/otherarch/sdcpp/src/stable-diffusion.cpp b/otherarch/sdcpp/src/stable-diffusion.cpp index 8204470f358..eff54afe0b5 100644 --- a/otherarch/sdcpp/src/stable-diffusion.cpp +++ b/otherarch/sdcpp/src/stable-diffusion.cpp @@ -2572,6 +2572,14 @@ class StableDiffusionGGML { LOG_WARN("No latent to RGB projection known for this model"); return; } + } else if (channels == 24) { + if(sd_version_is_minimax_h3(version)){ + latent_rgb_proj = minimax_latent_rgb_proj; + latent_rgb_bias = minimax_latent_rgb_bias; + } else { + LOG_WARN("No latent to RGB projection known for this model"); + return; + } } else if (channels == 16) { if (sd_version_is_sd3(version)) { latent_rgb_proj = sd3_latent_rgb_proj; @@ -5925,6 +5933,18 @@ SD_API bool generate_image(sd_ctx_t* sd_ctx, return false; } + // MiniMax-H3 is video-only. Its denoiser always splits the packed latent into a video and an + // audio half, and only generate_video ever computes the audio length, so reaching this + // function with an H3 checkpoint is guaranteed to die on + // GGML_ASSERT(!audio_input_cache.empty()) with a core dump, after the several minutes it + // takes to load the weights, and with nothing in the output pointing at the missing --mode. + // (The AnimateDiff path below routes vid_gen back through here, but that is SD1.5 plus a + // motion module, never H3.) + if (sd_version_is_minimax_h3(sd_ctx->sd->version)) { + LOG_ERROR("MiniMax-H3 is a video model and cannot be run in img_gen mode; use --mode vid_gen"); + return false; + } + sd_ctx->sd->reset_cancel_flag(); int64_t t0 = ggml_time_ms();