feat: dispatch MiniMax H3 ConvRot weights natively

.github/workflows/build.yml

@@ -369,7 +369,7 @@ jobs:
       matrix:
         include:
           - build: "cpu"
-            defines: "-DGGML_NATIVE=OFF -DSD_BUILD_SHARED_LIBS=ON -DSD_BUILD_SHARED_GGML_LIB=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON"
+            defines: "-DGGML_NATIVE=OFF -DSD_BUILD_TESTS=ON -DSD_BUILD_SHARED_LIBS=ON -DSD_BUILD_SHARED_GGML_LIB=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON"
           - build: "cuda12"
             defines: "-DSD_CUDA=ON -DSD_BUILD_SHARED_LIBS=ON -DCMAKE_CUDA_ARCHITECTURES='61;70;75;80;86;89;90;100;120' -DCMAKE_CUDA_FLAGS='-Xcudafe \"--diag_suppress=177\" -Xcudafe \"--diag_suppress=550\"' -DGGML_NATIVE=OFF -DSD_BUILD_SHARED_GGML_LIB=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON"
           - build: "vulkan"
@@ -421,6 +421,10 @@ jobs:
           cmake .. -DCMAKE_CXX_FLAGS='/bigobj' -G Ninja -DCMAKE_C_COMPILER=cl.exe -DCMAKE_CXX_COMPILER=cl.exe -DCMAKE_BUILD_TYPE=Release ${{ matrix.defines }}
           cmake --build .
 
+      - name: Test ConvRot safetensors loader
+        if: ${{ matrix.build == 'cpu' }}
+        run: ctest --test-dir build -R "^test-safetensors-convrot$" --output-on-failure
+
       - name: Get commit hash
         id: commit
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}

src/core/ggml_extend.hpp

@@ -5,8 +5,8 @@
 #include <inttypes.h>
 #include <stdarg.h>
 #include <algorithm>
-#include <atomic>
 #include <cstdlib>
+#include <atomic>
 #include <cstring>
 #include <fstream>
 #include <functional>
@@ -21,6 +21,7 @@
 #include <set>
 #include <sstream>
 #include <string>
+#include <stdexcept>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
@@ -43,6 +44,81 @@
 
 #define EPS 1e-05f
 
+// Construct only the operation metadata needed for the normal backend
+// supports_op query.  This stays private to the loader policy: backend
+// capabilities are expressed through the existing ggml interface, not a new
+// public ConvRot-specific API.
+inline bool ggml_backend_supports_convrot_op(ggml_backend_t backend) {
+    if (backend == nullptr) {
+        return false;
+    }
+    std::vector<uint8_t> storage(4 * ggml_tensor_overhead() + 1024);
+    ggml_init_params params = {
+        /*.mem_size   =*/ storage.size(),
+        /*.mem_buffer =*/ storage.data(),
+        /*.no_alloc   =*/ true,
+    };
+    ggml_context* ctx = ggml_init(params);
+    if (ctx == nullptr) {
+        return false;
+    }
+    ggml_tensor* activations = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 256, 1);
+    ggml_tensor* weights     = ggml_new_tensor_2d(ctx, GGML_TYPE_I8, 256, 1);
+    ggml_tensor* scales      = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
+    ggml_tensor* op          = ggml_mul_mat_convrot(ctx, activations, weights, scales, 256);
+    const bool supported      = ggml_backend_supports_op(backend, op);
+    ggml_free(ctx);
+    return supported;
+}
+
+// Select the compact representation before any model parameter tensor is
+// created.  The default is deliberately native: an unsupported backend is a
+// configuration error rather than a silent CPU reroute or full F16 expansion.
+// Set SD_CONVROT_MODE=compat to explicitly request the compatibility loader.
+inline String2TensorStorage select_convrot_tensor_storage(ggml_backend_t backend,
+                                                           const String2TensorStorage& source,
+                                                           const std::string& component,
+                                                           const std::string& prefix = "") {
+    // OrderedMap's default copy also copies its iterator index; rebuild it so
+    // this independent policy view owns a valid index into its own list.
+    String2TensorStorage selected;
+    bool has_convrot = false;
+    for (const auto& [name, storage] : source) {
+        selected.insert({name, storage});
+        has_convrot = has_convrot ||
+                      (name.rfind(prefix, 0) == 0 && storage.is_comfy_int8_convrot_weight());
+    }
+    if (!has_convrot) {
+        return selected;
+    }
+
+    const char* mode = std::getenv("SD_CONVROT_MODE");
+    const bool compatibility_mode = mode != nullptr && std::strcmp(mode, "compat") == 0;
+    if (mode != nullptr && !compatibility_mode && std::strcmp(mode, "native") != 0) {
+        throw std::runtime_error("invalid SD_CONVROT_MODE; expected 'native' or 'compat'");
+    }
+    const char* backend_name = backend != nullptr ? ggml_backend_name(backend) : "unknown";
+    if (compatibility_mode) {
+        LOG_INFO("ConvRot: using explicitly selected F16 compatibility path for %s on backend %s",
+                 component.c_str(), backend_name);
+        return selected;
+    }
+    if (!ggml_backend_supports_convrot_op(backend)) {
+        throw std::runtime_error("ConvRot native support is required for " + component +
+                                 " but backend '" + backend_name +
+                                 "' lacks the 256-wide I8/F32 ConvRot operation; use a capable backend or set "
+                                 "SD_CONVROT_MODE=compat to select the F16 compatibility path");
+    }
+    for (auto& [name, storage] : selected) {
+        if (name.rfind(prefix, 0) == 0 && storage.is_comfy_int8_convrot_weight()) {
+            storage.comfy_int8_native_enabled = true;
+        }
+    }
+    LOG_INFO("ConvRot: selected native compact I8/F32 path for %s on backend %s",
+             component.c_str(), backend_name);
+    return selected;
+}
+
 #ifndef __STATIC_INLINE__
 #define __STATIC_INLINE__ static inline
 #endif
@@ -1695,6 +1771,15 @@ struct WeightAdapter {
                                            ggml_tensor* b,
                                            const std::string& prefix,
                                            ForwardParams forward_params)                                                              = 0;
+    // Return only the adapter's output-space contribution.  Native operations
+    // such as compact ConvRot own their base-weight arithmetic and therefore
+    // cannot use forward_with_lora() without recomputing an incompatible base.
+    virtual ggml_tensor* lora_output_delta(ggml_context* ctx,
+                                            ggml_backend_t backend,
+                                            ggml_tensor* x,
+                                            ggml_tensor* w,
+                                            const std::string& prefix,
+                                            ForwardParams forward_params)                                                           = 0;
     virtual size_t get_extra_graph_size()                                                                                             = 0;
 };
 
@@ -3875,12 +3960,31 @@ class Linear : public UnaryBlock {
     bool force_prec_f32;
     bool allow_weight_scale;
     bool has_weight_scale = false;
+    // This is distinct from `weight_scale`: the latter is a regular
+    // post-linear model parameter, while ConvRot's F32 vector is a private
+    // sidecar input to GGML_OP_MUL_MAT_CONVROT.
+    bool has_convrot_weight = false;
+    bool use_convrot_f16_compat = false;
     float scale;
     std::string prefix;
 
     void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
         this->prefix         = prefix;
         has_weight_scale     = false;
+        has_convrot_weight   = false;
+        use_convrot_f16_compat = false;
+        const auto storage_it = tensor_storage_map.find(prefix + "weight");
+        if (storage_it != tensor_storage_map.end() && storage_it->second.is_comfy_int8_convrot_weight() &&
+            storage_it->second.comfy_int8_native_enabled) {
+            params["weight"]                 = ggml_new_tensor_2d(ctx, GGML_TYPE_I8, in_features, out_features);
+            params["weight.convrot_scale"]   = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, out_features);
+            has_convrot_weight                 = true;
+            use_convrot_f16_compat             = storage_it->second.name.rfind("text_encoders.llm.", 0) == 0;
+            if (bias) {
+                params["bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, out_features);
+            }
+            return;
+        }
         enum ggml_type wtype = get_type(prefix + "weight", tensor_storage_map, GGML_TYPE_F32);
         if (in_features % ggml_blck_size(wtype) != 0 || force_f32) {
             wtype = GGML_TYPE_F32;
@@ -3928,7 +4032,33 @@ class Linear : public UnaryBlock {
         }
         ggml_tensor* linear_bias = has_weight_scale ? nullptr : b;
         ggml_tensor* out         = nullptr;
-        if (ctx->weight_adapter) {
+        if (has_convrot_weight) {
+            // ConvRot weights and their tensor-wise scales remain compact at
+            // rest.  The operator owns the scale semantics; do not route it
+            // through the ordinary `weight_scale` post-multiply path.
+            out = ggml_mul_mat_convrot(ctx->ggml_ctx, x, w, params["weight.convrot_scale"], 256);
+            // MiniMax H3's ConvRot text encoder is calibrated for the F16
+            // compatibility arithmetic. CUDA reconstructs one F16 matrix at
+            // a time and uses its standard F16 GEMM without retaining an F16
+            // copy of the complete text encoder. Other backends may ignore
+            // this hint and keep their native compact implementation.
+            if (use_convrot_f16_compat) {
+                ggml_mul_mat_convrot_set_f16_compat(out, true);
+            }
+            if (b != nullptr) {
+                out = ggml_add_inplace(ctx->ggml_ctx, out, b);
+            }
+            if (ctx->weight_adapter) {
+                WeightAdapter::ForwardParams forward_params;
+                forward_params.op_type               = WeightAdapter::ForwardParams::op_type_t::OP_LINEAR;
+                forward_params.linear.force_prec_f32 = force_prec_f32;
+                forward_params.linear.scale          = scale;
+                if (ggml_tensor* delta = ctx->weight_adapter->lora_output_delta(
+                        ctx->ggml_ctx, ctx->backend, x, w, prefix, forward_params)) {
+                    out = ggml_add_inplace(ctx->ggml_ctx, out, delta);
+                }
+            }
+        } else if (ctx->weight_adapter) {
             WeightAdapter::ForwardParams forward_params;
             forward_params.op_type               = WeightAdapter::ForwardParams::op_type_t::OP_LINEAR;
             forward_params.linear.force_prec_f32 = force_prec_f32;

src/model/adapter/lora.hpp

@@ -1300,6 +1300,22 @@ struct MultiLoraAdapter : public WeightAdapter {
         return out;
     }
 
+    ggml_tensor* lora_output_delta(ggml_context* ctx,
+                                   ggml_backend_t backend,
+                                   ggml_tensor* x,
+                                   ggml_tensor* w,
+                                   const std::string& prefix,
+                                   WeightAdapter::ForwardParams forward_params) override {
+        ggml_tensor* delta = nullptr;
+        for (auto& lora_model : lora_models) {
+            ggml_tensor* current = lora_model->get_out_diff(ctx, backend, x, w, forward_params, prefix + "weight");
+            if (current != nullptr) {
+                delta = delta == nullptr ? current : ggml_add_inplace(ctx, delta, current);
+            }
+        }
+        return delta;
+    }
+
     size_t get_extra_graph_size() override {
         size_t lora_tensor_num = 0;
         for (auto& lora_model : lora_models) {

src/model/diffusion/minimax_h3.hpp

@@ -986,7 +986,9 @@ namespace MiniMaxH3 {
             : DiffusionModelRunner(backend, prefix, weight_manager),
               config(Config::detect_from_weights(tensors, prefix)),
               model(config) {
-            model.init(params_ctx, tensors, prefix);
+            model.init(params_ctx,
+                       select_convrot_tensor_storage(backend, tensors, "MiniMax-H3 diffusion model", prefix),
+                       prefix);
         }
 
         std::string get_desc() override {

src/model/te/llm.hpp

@@ -1771,7 +1771,9 @@ namespace LLM {
                 }
             }
             model = LLM(config, enable_vision, config.llama_cpp_style);
-            model.init(params_ctx, tensor_storage_map, prefix);
+            model.init(params_ctx,
+                       select_convrot_tensor_storage(backend, tensor_storage_map, "LLM", prefix),
+                       prefix);
         }
 
         std::string get_desc() override {