From 40298e35765ac685045819cca247834328f56c40 Mon Sep 17 00:00:00 2001
From: shoyofight <zyc20030415@126.com>
Date: Sat, 15 Aug 2026 15:39:12 +0800
Subject: [PATCH 1/2] bugfix: support minimax max-vram skill

---
 docs/minimax-h3.md                            |  9 ++++--
 src/core/backend/ggml/ggml_extend.hpp         |  2 +-
 .../pipelines/minimax_h3_pipeline.cpp         | 32 +++++++++++++++----
 3 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/docs/minimax-h3.md b/docs/minimax-h3.md
index f793ab8..c3f9da8 100644
--- a/docs/minimax-h3.md
+++ b/docs/minimax-h3.md
@@ -43,7 +43,7 @@ Supported reference inputs:
 | Reference image | `--ref-image <image>` | Repeatable. Presented as `<Picture N>` to Qwen3-VL. |
 | Reference video | `--ref-video <frame-dir>` | Repeatable. Directory of image frames sorted lexicographically; treated as 24 fps. |
 | Paired video audio | `--ref-video-audio <wav>` | Repeatable. The Nth WAV is paired with the Nth `--ref-video`. |
-| Standalone audio | `--ref-audio <wav>` | Repeatable. Independent audio reference, not attached to a video. |
+| Standalone audio | `--ref-audio <wav>` | Repeatable. Independent audio reference, not attached to a video. Must be paired with at least one `--ref-image` or `--ref-video` (audio-only Ref2VA is rejected). |
 
 The reference inputs can be combined freely within Ref2VA. Ref2VA cannot be used
 with `--image` or `--end-img` in the same request.
@@ -168,14 +168,19 @@ ed-cli --video \
 
 ### Standalone reference audio
 
+Standalone `--ref-audio` must be paired with at least one `--ref-image` or
+`--ref-video` (audio-only Ref2VA is rejected). The example below anchors the
+audio to a reference image.
+
 ```bash
 ed-cli --video \
   --diffusion-model minimax_h3_ref2va_pruned-Q4_K_M.gguf \
   --vae minimax_h3_video_vae_fp16.safetensors \
   --audio-vae minimax_h3_audio_vae_fp32.safetensors \
   --llm qwen3vl_32b_minimax_h3-Q4_K_M.gguf \
+  --ref-image reference.png \
   --ref-audio reference_audio.wav \
-  -p "Generate a video with audio inspired by <Audio 1>." \
+  -p "Use the scene from <Picture 1> and generate audio inspired by <Audio 1>." \
   --cfg-scale 1 -W 864 -H 480 --fps 24 --video-frames 56 --steps 20 \
   --diffusion-fa --rng cpu --video-format mp4 \
   -o minimax_h3_ref_audio.mp4
diff --git a/src/core/backend/ggml/ggml_extend.hpp b/src/core/backend/ggml/ggml_extend.hpp
index 90e9580..8922ae4 100644
--- a/src/core/backend/ggml/ggml_extend.hpp
+++ b/src/core/backend/ggml/ggml_extend.hpp
@@ -6775,7 +6775,7 @@ struct GGMLRunner {
 #endif
             auto segment_output             = execute_graph<T>(segment_graph,
                                        n_threads,
-                                       false,
+                                       true,
                                        runtime_param_tensors,
                                        true,
                                        !is_last_segment || no_return,
diff --git a/src/dit_models/pipelines/minimax_h3_pipeline.cpp b/src/dit_models/pipelines/minimax_h3_pipeline.cpp
index 5c807d1..8915e23 100644
--- a/src/dit_models/pipelines/minimax_h3_pipeline.cpp
+++ b/src/dit_models/pipelines/minimax_h3_pipeline.cpp
@@ -535,12 +535,15 @@ bool MiniMaxH3Pipeline::prepare(const ed_context_params_t& params,
         return set_minimax_error(error, "MiniMax-H3 diffusion transformer signature is incomplete");
     }
 
-    const bool diffusion_offload = runtime.dit_offload_params_to_cpu();
+    runtime.reset_auto_allocate_state();
+    const size_t eff_budget = runtime.effective_budget_bytes();
+    size_t remaining_free = eff_budget;
+    const bool diffusion_offload = runtime.dit_offload_params_to_cpu() ||
+                                   runtime.plan_component_offload(loader, "model.diffusion_model", remaining_free);
     diffusion_ = std::make_unique<MiniMaxH3::MiniMaxH3Runner>(runtime.backend(),
                                                               loader.get_tensor_storage_map(),
                                                               "model.diffusion_model",
                                                               diffusion_offload);
-    diffusion_->set_max_graph_vram_bytes(runtime.max_graph_vram_bytes());
     diffusion_->set_flash_attention_enabled(runtime.flash_attention());
     if (auto process_group = runtime.graph_process_group_ref()) {
         diffusion_->set_process_group(process_group);
@@ -564,7 +567,8 @@ bool MiniMaxH3Pipeline::prepare(const ed_context_params_t& params,
         LOG_INFO("MiniMax-H3 synthesized missing rope.inv_freq (%zu values)", values.size());
     }
 
-    const bool text_offload = runtime.clip_offload_params_to_cpu();
+    const bool text_offload = runtime.clip_offload_params_to_cpu() ||
+                             runtime.plan_component_offload(loader, "text_encoders", remaining_free);
     String2TensorStorage conditioner_tensors = loader.get_tensor_storage_map();
     const std::regex language_layer_pattern(R"(^text_encoders\.llm\.model\.layers\.(\d+)\.)");
     for (auto it = conditioner_tensors.begin(); it != conditioner_tensors.end();) {
@@ -589,11 +593,12 @@ bool MiniMaxH3Pipeline::prepare(const ed_context_params_t& params,
     conditioner_->alloc_params_buffer();
     conditioner_->get_param_tensors(registry.tensors(), "text_encoders.llm");
 
+    const bool vae_offload = runtime.vae_offload_params_to_cpu() ||
+                            runtime.plan_component_offload(loader, "first_stage_model", remaining_free);
     vae_ = std::make_unique<MiniMaxH3VAE::MiniMaxH3VideoVAERunner>(runtime.vae_backend(),
-                                                                    runtime.vae_offload_params_to_cpu(),
+                                                                    vae_offload,
                                                                     loader.get_tensor_storage_map(),
                                                                     "first_stage_model");
-    vae_->set_max_graph_vram_bytes(runtime.max_graph_vram_bytes());
     vae_->set_flash_attention_enabled(runtime.flash_attention());
     vae_->alloc_params_buffer();
     vae_->get_param_tensors(registry.tensors(), "first_stage_model");
@@ -602,11 +607,12 @@ bool MiniMaxH3Pipeline::prepare(const ed_context_params_t& params,
                                            loader.get_tensor_storage_map().end(),
                                            [](const auto& item) { return starts_with(item.first, "audio_vae."); });
     if (has_audio_vae) {
+        const bool audio_vae_offload = runtime.vae_offload_params_to_cpu() ||
+                                       runtime.plan_component_offload(loader, "audio_vae", remaining_free);
         audio_vae_ = std::make_unique<MiniMaxH3Audio::AudioVAERunner>(runtime.vae_backend(),
-                                                                        runtime.vae_offload_params_to_cpu(),
+                                                                        audio_vae_offload,
                                                                         loader.get_tensor_storage_map(),
                                                                         "audio_vae");
-        audio_vae_->set_max_graph_vram_bytes(runtime.max_graph_vram_bytes());
         audio_vae_->set_flash_attention_enabled(runtime.flash_attention());
         audio_vae_->alloc_params_buffer();
         audio_vae_->get_param_tensors(registry.tensors(), "audio_vae");
@@ -614,6 +620,17 @@ bool MiniMaxH3Pipeline::prepare(const ed_context_params_t& params,
         LOG_INFO("MiniMax-H3 audio VAE not provided; generated video will have no decoded audio track");
     }
 
+    runtime.finalize_auto_segment_budget(eff_budget);
+
+    // Apply finalized segment budget to all components (must come after finalize).
+    diffusion_->set_max_graph_vram_bytes(runtime.max_graph_vram_bytes());
+    vae_->set_max_graph_vram_bytes(runtime.max_graph_vram_bytes());
+    if (audio_vae_) {
+        audio_vae_->set_max_graph_vram_bytes(runtime.max_graph_vram_bytes());
+    }
+    conditioner_->model.set_max_graph_vram_bytes(
+        runtime.text_encoder_segment_budget(conditioner_->model.get_params_buffer_size()));
+
     registry.ignore_prefix("first_stage_model.encoder.");
     registry.ignore_prefix("text_encoders.llm.visual.");
     registry.ignore_prefix("text_encoders.llm.");
@@ -1380,6 +1397,7 @@ ed_status_t MiniMaxH3Pipeline::generate_video(const ed_video_generation_params_t
             h3_trace_tensor(("step_" + std::to_string(step) + "_velocity").c_str(), velocity);
         }
         packed += velocity * (sigma_next - sigma);
+        LOG_INFO("minimax-h3 step %d/%d sigma=%.6f next=%.6f", step + 1, steps, sigma, sigma_next);
         if (h3_trace_enabled()) {
             h3_trace_tensor(("step_" + std::to_string(step) + "_packed").c_str(), packed);
         }

From 2b920dc7f1daff5a1789c1912fe1503926ccf2c4 Mon Sep 17 00:00:00 2001
From: shoyofight <zyc20030415@126.com>
Date: Sat, 15 Aug 2026 23:25:44 +0800
Subject: [PATCH 2/2] docs: add RTX 4090 MiniMax-H3 performance data

---
 docs/minimax-h3.md | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/docs/minimax-h3.md b/docs/minimax-h3.md
index 3ffca88..f910fa6 100644
--- a/docs/minimax-h3.md
+++ b/docs/minimax-h3.md
@@ -198,6 +198,47 @@ MiniMax-H3 always uses its fixed `16x16` video-VAE tiling path. Generic
 `--vae-tiling` and `--vae-tile-size` values do not replace this model-specific
 layout.
 
+## RTX 4090 offloaded comparison
+
+Measured on one RTX 4090 (24 GB), `864x480`, 56 frames at 24 fps, 20 steps,
+CFG 1, seed `42`, every component offloaded to CPU (`--offload-to-cpu`),
+`--max-vram 22`, model-specific `16x16` VAE tiling. On a single 24 GB card
+neither the BF16 DiT nor the Qwen3-VL encoder fits resident, so both engines
+stream weights per segment. Q8_0 rows quantize the BF16 files on load
+(`--type q8_0`). Wall is the end-to-end process time (measured identically for
+both engines); TE / Diffusion / VAE are per-stage times. "Load" is not shown as
+a column because the engines load differently — edge-dit loads all components up
+front, sd.cpp loads each lazily on first use — so a single load figure would not
+be comparable. Peak VRAM is sampled over the whole process. Bold marks the
+better value in each edge-dit vs stable-diffusion.cpp pair.
+
+| Task | Precision | System | Wall (s) | TE (ms) | Diffusion (ms) | VAE (ms) | Peak VRAM (MiB) |
+|---|---|---|---:|---:|---:|---:|---:|
+| Text | q4_K_M | edge-dit.cpp | 117 | **1551** | **81466** | **4929** | **18630** |
+| | | stable-diffusion.cpp | **113** | 10370 | 82550 | 13450 | 20554 |
+| First frame | q4_K_M | edge-dit.cpp | 129 | **3099** | **91589** | **4804** | **18544** |
+| | | stable-diffusion.cpp | **123** | 11500 | 91670 | 11170 | 20632 |
+| First + last frames | q4_K_M | edge-dit.cpp | 140 | **4065** | **101269** | **4812** | **18258** |
+| | | stable-diffusion.cpp | **135** | 11910 | 101350 | 12590 | 20868 |
+| Text | q8_0 | edge-dit.cpp | **159** | **2154** | **83656** | **4755** | **18808** |
+| | | stable-diffusion.cpp | 185 | 32750 | 132220 | 12810 | 22472 |
+| First frame | q8_0 | edge-dit.cpp | **170** | **3366** | **93107** | **4514** | **19094** |
+| | | stable-diffusion.cpp | 195 | 33730 | 138440 | 12980 | 22554 |
+| Text | bf16 | edge-dit.cpp | 206 | 4060 | 134334 | 4962 | 21120 |
+| | | stable-diffusion.cpp | — | — | — | — | — |
+| First frame | bf16 | edge-dit.cpp | 218 | 5544 | 145351 | 4850 | 20986 |
+| | | stable-diffusion.cpp | — | — | — | — | — |
+
+At q4_K_M the two engines match on diffusion time; edge-dit encodes text
+**3–8x faster**, decodes the VAE **~2.5x faster**, and holds peak VRAM roughly
+**2 GB lower**. At q8_0 edge-dit leads across the board — **~25–30s lower wall
+time**, diffusion **~40% faster**, peak VRAM **~3.5 GB lower**.
+
+sd.cpp has no BF16 rows on this card: it stages the full-precision DiT weights
+to the GPU in one block (~38 GB), which overflows the 24 GB budget and aborts
+before sampling. edge-dit streams the BF16 weights per segment, so it runs
+within the 24 GB limit.
+
 ## H200 BF16 comparison
 
 The tables below use one H200, resident components, `864x480`, 124 frames at