feat(mm): recognize MiniMax H3 Ref2VA models

invokeai/app/invocations/minimax_h3_model_loader.py

@@ -15,7 +15,13 @@
     VAEField,
 )
 from invokeai.app.services.shared.invocation_context import InvocationContext
-from invokeai.backend.model_manager.taxonomy import BaseModelType, ModelFormat, ModelType, SubModelType
+from invokeai.backend.model_manager.taxonomy import (
+    BaseModelType,
+    MiniMaxH3VariantType,
+    ModelFormat,
+    ModelType,
+    SubModelType,
+)
 
 
 @invocation_output("minimax_h3_model_loader_output")
@@ -121,9 +127,21 @@ def invoke(self, context: InvocationContext) -> MiniMaxH3ModelLoaderOutput:
         if self.transformer_model is not None:
             if not context.models.exists(self.transformer_model.key):
                 raise ValueError(f"Unknown transformer model: {self.transformer_model.key}")
+            transformer_config = context.models.get_config(self.transformer_model.key)
             transformer = self.transformer_model.model_copy(update={"submodel_type": SubModelType.Transformer})
         else:
+            transformer_config = main_config
             transformer = self.model.model_copy(update={"submodel_type": SubModelType.Transformer})
+        # TODO(ref2va): replace this rejection with variant stamping when reference-conditioned
+        # generation lands. Until then a Ref2VA transformer (whose weights expect reference
+        # conditioning rows nothing here packs) must fail fast rather than silently produce
+        # degraded output - the webv2 picker filters it out, but hand-authored workflows and
+        # API clients do not go through that filter.
+        if getattr(transformer_config, "variant", None) is MiniMaxH3VariantType.REF2VA:
+            raise ValueError(
+                f"'{transformer_config.name}' is a Ref2VA (reference-to-video) transformer, which this "
+                "version cannot generate with yet. Select an FL2VA transformer instead."
+            )
         tokenizer = self.model.model_copy(update={"submodel_type": SubModelType.Tokenizer})
         processor = self.model.model_copy(update={"submodel_type": SubModelType.Processor})
         if self.text_encoder_model is not None:

invokeai/backend/model_manager/configs/main.py

@@ -1595,7 +1595,8 @@ def from_model_on_disk(cls, mod: ModelOnDisk, override_fields: dict[str, Any]) -
             {"AutoencoderKLMiniMaxH3Audio"},
         )
 
-        variant = override_fields.pop("variant", None) or cls._get_variant(mod)
+        # An override may arrive as the raw string; normalize so the folder lookup below compares enums.
+        variant = MiniMaxH3VariantType(override_fields.pop("variant", None) or cls._get_variant(mod))
 
         repo_variant = override_fields.pop("repo_variant", None) or cls._get_repo_variant_or_raise(mod)
 
@@ -1605,10 +1606,19 @@ def from_model_on_disk(cls, mod: ModelOnDisk, override_fields: dict[str, Any]) -
         # can require those selections up front rather than failing mid-generation.
         components_only = override_fields.pop("components_only", None)
         if components_only is None:
-            transformer_dir = mod.path / "transformer"
-            components_only = not any(
-                any(transformer_dir.glob(pattern)) for pattern in ("*.safetensors", "*.bin", "*.pth", "*.pt", "*.ckpt")
-            )
+            if variant is MiniMaxH3VariantType.REF2VA:
+                # Ref2VA folder weights are not folder-loadable in this version (`SubModelType`
+                # has no `transformer_ref` member), so a Ref2VA install always needs the
+                # single-file overrides - weight shards present or not. Marking it
+                # components-only makes the UI require them up front instead of failing
+                # minutes into a run.
+                components_only = True
+            else:
+                transformer_dir = mod.path / "transformer"
+                components_only = not any(
+                    any(transformer_dir.glob(pattern))
+                    for pattern in ("*.safetensors", "*.bin", "*.pth", "*.pt", "*.ckpt")
+                )
 
         return cls(
             **override_fields,
@@ -1623,16 +1633,21 @@ def _get_variant(cls, mod: ModelOnDisk) -> MiniMaxH3VariantType:
 
         H3's task checkpoints share every component except the transformer folder: ``transformer``
         (FL2VA: text / first/last-frame to audio-video) vs ``transformer_ref`` (Ref2VA: multi-
-        reference). Only FL2VA is supported so far. A Ref2VA-only download is a real H3 model this
-        version cannot run, so identification fails rather than mislabeling it as FL2VA.
+        reference). A folder holding both (the full official repo) identifies as FL2VA - that is
+        the folder-loadable variant. A ``transformer_ref``-only folder identifies as REF2VA, but
+        note its weights are NOT folder-loadable in this version (``SubModelType`` has no
+        ``transformer_ref`` member); the supported Ref2VA generation path is a components install
+        plus a single-file transformer override selected in the model loader.
         """
         transformer_config = mod.path / "transformer" / "config.json"
-        if not transformer_config.exists():
-            raise NotAMatchError(
-                "no FL2VA transformer folder (`transformer/`); Ref2VA-only installs are not supported yet"
-            )
-        raise_for_class_name(transformer_config, {"MiniMaxH3Transformer3DModel"})
-        return MiniMaxH3VariantType.FL2VA
+        if transformer_config.exists():
+            raise_for_class_name(transformer_config, {"MiniMaxH3Transformer3DModel"})
+            return MiniMaxH3VariantType.FL2VA
+        ref_transformer_config = mod.path / "transformer_ref" / "config.json"
+        if ref_transformer_config.exists():
+            raise_for_class_name(ref_transformer_config, {"MiniMaxH3Transformer3DModel"})
+            return MiniMaxH3VariantType.REF2VA
+        raise NotAMatchError("no transformer folder (`transformer/` or `transformer_ref/`)")
 
 
 def _has_minimax_h3_keys(state_dict: dict[str | int, Any]) -> bool:
@@ -1656,9 +1671,12 @@ class Main_Checkpoint_MiniMaxH3_Config(Checkpoint_Config_Base, Main_Config_Base,
     file holds ONLY the transformer - the text encoder, VAEs, tokenizer and processor must come
     from an installed H3 diffusers-layout folder.
 
-    A Ref2VA transformer single file is key-for-key indistinguishable from FL2VA, so Ref2VA is
-    excluded only by filename; a renamed Ref2VA file would load and run but produce degraded
-    output (it expects reference conditioning rows this integration never packs).
+    The FL2VA and Ref2VA task transformers are key-for-key (and, except the non-pruned
+    int8_convrot repacks, byte-size) indistinguishable, so the FILENAME is the variant
+    classifier and ``variant`` is the user-correctable override for renamed files (e.g.
+    re-uploads). A misclassified variant loads and runs but produces degraded output - FL2VA
+    expects no reference conditioning rows, Ref2VA expects them - which is why the override
+    exists rather than any attempt at content sniffing.
     """
 
     base: Literal[BaseModelType.MiniMaxH3] = Field(default=BaseModelType.MiniMaxH3)
@@ -1672,17 +1690,16 @@ def from_model_on_disk(cls, mod: ModelOnDisk, override_fields: dict[str, Any]) -
 
         raise_for_override_fields(cls, override_fields)
 
-        if "ref2va" in mod.path.name.lower():
-            raise NotAMatchError("Ref2VA single-file transformers are not supported yet")
-
         state_dict = mod.load_state_dict()
         if not _has_minimax_h3_keys(state_dict):
             raise NotAMatchError("state dict does not look like a MiniMax H3 transformer")
 
         if _has_ggml_tensors(state_dict):
             raise NotAMatchError("GGUF-quantized MiniMax H3 checkpoints are not supported yet")
 
-        variant = override_fields.pop("variant", None) or MiniMaxH3VariantType.FL2VA
+        variant = override_fields.pop("variant", None) or (
+            MiniMaxH3VariantType.REF2VA if "ref2va" in mod.path.name.lower() else MiniMaxH3VariantType.FL2VA
+        )
         pruned = "adaln_t_table" in state_dict
 
         return cls(**override_fields, variant=variant, pruned=pruned)

invokeai/backend/model_manager/starter_models.py

@@ -1993,6 +1993,20 @@ def _gemini_3_resolution_presets(
     dependencies=[minimax_h3_components, minimax_h3_int8_text_encoder],
 )
 
+minimax_h3_ref2va_int8_transformer = StarterModel(
+    name="MiniMax H3 Ref2VA Transformer (int8, pruned)",
+    base=BaseModelType.MiniMaxH3,
+    source="Comfy-Org/MiniMax-H3::diffusion_models/minimax_h3_ref2va_pruned_int8_convrot.safetensors",
+    description="MiniMax H3 reference-conditioned video+audio generation: up to 3 video and 9 image "
+    "references. AdaLN-pruned int8 single-file transformer (~21 GB); select it in the MiniMax H3 "
+    "Model Loader's transformer field. Total size with dependencies: ~59 GB. NOTE: This model is "
+    "distributed under a restrictive license that forbids its use in certain territories. Please "
+    "see https://huggingface.co/MiniMaxAI/MiniMax-H3 for details.",
+    type=ModelType.Main,
+    format=ModelFormat.Checkpoint,
+    dependencies=[minimax_h3_components, minimax_h3_int8_text_encoder],
+)
+
 minimax_h3_turbo_lora = StarterModel(
     name="MiniMax H3 Turbo LoRA",
     base=BaseModelType.MiniMaxH3,
@@ -2018,6 +2032,17 @@ def _gemini_3_resolution_presets(
     type=ModelType.LoRA,
     format=ModelFormat.LyCORIS,
 )
+
+minimax_h3_ref2v_turbo_lora = StarterModel(
+    name="MiniMax H3 Ref2V Turbo LoRA",
+    base=BaseModelType.MiniMaxH3,
+    source="Comfy-Org/MiniMax-H3::loras/minimax_h3_ref2v_turbo_4step_v0.1_comfyui_bf16.safetensors",
+    description="Step-distillation LoRA for the MiniMax H3 Ref2VA transformer (~2 GB): renders "
+    "reference-conditioned video+audio in ~4 denoising steps instead of ~50. Apply at strength 1.0 "
+    "and lower Steps to 4. Trained against the Ref2VA transformer; not intended for FL2VA.",
+    type=ModelType.LoRA,
+    format=ModelFormat.LyCORIS,
+)
 # endregion
 
 alibabacloud_wan26_t2i = StarterModel(
@@ -2539,10 +2564,12 @@ def _gemini_3_resolution_presets(
     wan_22_ti2v_5b_gguf_q4_k_m,
     wan_22_ti2v_5b_gguf_q8_0,
     minimax_h3_int8_transformer,
+    minimax_h3_ref2va_int8_transformer,
     minimax_h3_int8_text_encoder,
     minimax_h3_components,
     minimax_h3_turbo_lora,
     minimax_h3_lightx2v_turbo_lora,
+    minimax_h3_ref2v_turbo_lora,
     gemini_flash_image,
     gemini_pro_image_preview,
     gemini_3_1_flash_image_preview,
@@ -2714,16 +2741,18 @@ def _gemini_3_resolution_presets(
     ideogram_4_nf4,
 ]
 
-# The working set for MiniMax H3 video+audio generation (~62 GB): shared components from
-# the official repo plus Comfy-Org's int8 single-file transformer and text encoder, and the
-# two turbo (step-distillation) LoRAs for fast low-step rendering. See the license note in
-# the MiniMax H3 region above.
+# The working set for MiniMax H3 video+audio generation (~85 GB): shared components from
+# the official repo plus Comfy-Org's int8 single-file FL2VA and Ref2VA transformers and the
+# text encoder, and the three turbo (step-distillation) LoRAs for fast low-step rendering.
+# See the license note in the MiniMax H3 region above.
 minimax_h3_bundle: list[StarterModel] = [
     minimax_h3_components,
     minimax_h3_int8_text_encoder,
     minimax_h3_int8_transformer,
+    minimax_h3_ref2va_int8_transformer,
     minimax_h3_turbo_lora,
     minimax_h3_lightx2v_turbo_lora,
+    minimax_h3_ref2v_turbo_lora,
 ]
 
 STARTER_BUNDLES: dict[str, StarterModelBundle] = {

invokeai/backend/model_manager/taxonomy.py

@@ -264,6 +264,11 @@ class MiniMaxH3VariantType(str, Enum):
     """First/last-frame + text to audio-video: text-to-video and first/last-frame image-to-video
     (HF repo subfolder ``transformer``)."""
 
+    REF2VA = "ref2va"
+    """Multi-reference to audio-video: up to 3 video + 9 image references condition generation
+    (HF repo subfolder ``transformer_ref``). Same architecture and config as FL2VA, different
+    weights; supports only the reference task."""
+
 
 class MistralVariantType(str, Enum):
     """Mistral text encoder variants used by FLUX.2 [dev]."""

invokeai/frontend/web/openapi.json

@@ -60677,7 +60677,7 @@
           "pruned"
         ],
         "title": "Main_Checkpoint_MiniMaxH3_Config",
-        "description": "Model config for MiniMax H3 single-file transformer checkpoints (safetensors).\n\nCovers MiniMax's single-file FL2VA transformer repacks (Comfy-Org and mirrors): bf16 or\nComfy ``int8_tensorwise``(+convrot) quantized, full or AdaLN-pruned (\"adaln curves\"). The\nfile holds ONLY the transformer - the text encoder, VAEs, tokenizer and processor must come\nfrom an installed H3 diffusers-layout folder.\n\nA Ref2VA transformer single file is key-for-key indistinguishable from FL2VA, so Ref2VA is\nexcluded only by filename; a renamed Ref2VA file would load and run but produce degraded\noutput (it expects reference conditioning rows this integration never packs)."
+        "description": "Model config for MiniMax H3 single-file transformer checkpoints (safetensors).\n\nCovers MiniMax's single-file FL2VA transformer repacks (Comfy-Org and mirrors): bf16 or\nComfy ``int8_tensorwise``(+convrot) quantized, full or AdaLN-pruned (\"adaln curves\"). The\nfile holds ONLY the transformer - the text encoder, VAEs, tokenizer and processor must come\nfrom an installed H3 diffusers-layout folder.\n\nThe FL2VA and Ref2VA task transformers are key-for-key (and, except the non-pruned\nint8_convrot repacks, byte-size) indistinguishable, so the FILENAME is the variant\nclassifier and ``variant`` is the user-correctable override for renamed files (e.g.\nre-uploads). A misclassified variant loads and runs but produces degraded output - FL2VA\nexpects no reference conditioning rows, Ref2VA expects them - which is why the override\nexists rather than any attempt at content sniffing."
       },
       "Main_Checkpoint_QwenImage_Config": {
         "properties": {
@@ -71365,7 +71365,7 @@
       },
       "MiniMaxH3VariantType": {
         "type": "string",
-        "enum": ["fl2va"],
+        "enum": ["fl2va", "ref2va"],
         "title": "MiniMaxH3VariantType",
         "description": "MiniMax H3 model variants (task-specific transformer checkpoints sharing every other component)."
       },

invokeai/frontend/web/src/features/modelManagerV2/models.ts

@@ -306,6 +306,7 @@ export const MODEL_VARIANT_TO_LONG_NAME: Record<AnyModelVariant, string> = {
   generate: 'Qwen Image',
   edit: 'Qwen Image Edit',
   fl2va: 'MiniMax H3 FL2VA',
+  ref2va: 'MiniMax H3 Ref2VA',
   t2v_a14b: 'Wan 2.2 T2V A14B',
   i2v_a14b: 'Wan 2.2 I2V A14B',
   ti2v_5b: 'Wan 2.2 TI2V 5B',

invokeai/frontend/web/src/features/nodes/types/common.ts

@@ -206,7 +206,7 @@ const zWanVariantType = z.enum(['t2v_a14b', 'i2v_a14b', 'ti2v_5b']);
  *  targets. A14B = inner_dim 5120 (both T2V and I2V), 5B = inner_dim 3072. */
 const zWanLoRAVariantType = z.enum(['a14b', '5b']);
 export const zQwen3VariantType = z.enum(['qwen3_4b', 'qwen3_8b', 'qwen3_06b']);
-const zMiniMaxH3VariantType = z.enum(['fl2va']);
+const zMiniMaxH3VariantType = z.enum(['fl2va', 'ref2va']);
 const zMistralVariantType = z.enum(['cow_mistral3_small', 'mistral3_24b']);
 const zPiDDecoderVariantType = z.enum(['res2k_sr4x', 'res2kto4k_sr4x']);
 export const zAnyModelVariant = z.union([

invokeai/frontend/web/src/services/api/schema.ts

@@ -24843,9 +24843,12 @@ export type components = {
          *     file holds ONLY the transformer - the text encoder, VAEs, tokenizer and processor must come
          *     from an installed H3 diffusers-layout folder.
          *
-         *     A Ref2VA transformer single file is key-for-key indistinguishable from FL2VA, so Ref2VA is
-         *     excluded only by filename; a renamed Ref2VA file would load and run but produce degraded
-         *     output (it expects reference conditioning rows this integration never packs).
+         *     The FL2VA and Ref2VA task transformers are key-for-key (and, except the non-pruned
+         *     int8_convrot repacks, byte-size) indistinguishable, so the FILENAME is the variant
+         *     classifier and ``variant`` is the user-correctable override for renamed files (e.g.
+         *     re-uploads). A misclassified variant loads and runs but produces degraded output - FL2VA
+         *     expects no reference conditioning rows, Ref2VA expects them - which is why the override
+         *     exists rather than any attempt at content sniffing.
          */
         Main_Checkpoint_MiniMaxH3_Config: {
             /**
@@ -30428,7 +30431,7 @@ export type components = {
          * @description MiniMax H3 model variants (task-specific transformer checkpoints sharing every other component).
          * @enum {string}
          */
-        MiniMaxH3VariantType: "fl2va";
+        MiniMaxH3VariantType: "fl2va" | "ref2va";
         /**
          * Mistral3EncoderField
          * @description Field for Mistral3 text encoder used by ERNIE-Image models.

invokeai/frontend/webv2/src/features/models/core/taxonomy.test.ts

@@ -49,7 +49,7 @@ describe('variant options', () => {
     expect(getVariantOptionsFor('sd-2', 'main')).toEqual(['normal', 'inpaint', 'depth']);
     expect(getVariantOptionsFor('flux', 'main')).toEqual(['schnell', 'dev', 'dev_fill']);
     expect(getVariantOptionsFor('wan', 'main')).toEqual(['t2v_a14b', 'i2v_a14b', 'ti2v_5b']);
-    expect(getVariantOptionsFor('minimax-h3', 'main')).toEqual(['fl2va']);
+    expect(getVariantOptionsFor('minimax-h3', 'main')).toEqual(['fl2va', 'ref2va']);
   });
 
   it('distinguishes wan main and wan lora variants', () => {
@@ -70,6 +70,7 @@ describe('variant options', () => {
   it('labels known variants and title-cases unknown ones', () => {
     expect(getModelVariantLabel('dev_fill')).toBe('FLUX Dev - Fill');
     expect(getModelVariantLabel('fl2va')).toBe('MiniMax H3 FL2VA');
+    expect(getModelVariantLabel('ref2va')).toBe('MiniMax H3 Ref2VA');
     expect(getModelVariantLabel('some_new_variant')).toBe('Some New Variant');
   });
 

invokeai/frontend/webv2/src/features/models/core/taxonomy.ts

@@ -138,6 +138,7 @@ export const MODEL_VARIANT_LABELS: Record<string, string> = {
   qwen3_06b: 'Qwen3 0.6B',
   qwen3_4b: 'Qwen3 4B',
   qwen3_8b: 'Qwen3 8B',
+  ref2va: 'MiniMax H3 Ref2VA',
   res2k_sr4x: 'PiD 2K (4x SR)',
   res2kto4k_sr4x: 'PiD 4K (4x SR Upscale)',
   schnell: 'FLUX Schnell',
@@ -155,7 +156,7 @@ const MAIN_VARIANTS_BY_BASE: Record<string, readonly string[]> = {
   flux: ['schnell', 'dev', 'dev_fill'],
   flux2: ['klein_4b', 'klein_4b_base', 'klein_9b', 'klein_9b_base', 'dev'],
   'krea-2': ['krea2_turbo', 'krea2_base'],
-  'minimax-h3': ['fl2va'],
+  'minimax-h3': ['fl2va', 'ref2va'],
   'qwen-image': ['generate', 'edit'],
   'sd-1': ['normal', 'inpaint'],
   'sd-2': ['normal', 'inpaint', 'depth'],