feat(backend): MiniMax H3 Ref2VA generation — packing, conditioning, invocations

invokeai/app/invocations/fields.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Any, Callable, Optional, Tuple
+from typing import Any, Callable, Literal, Optional, Tuple
 
 from pydantic import BaseModel, ConfigDict, Field, RootModel, TypeAdapter
 from pydantic.fields import _Unset
@@ -191,6 +191,8 @@ class FieldDescriptions:
     minimax_h3_text_encoder = "Qwen3-VL-32B tokenizer, processor and text encoder for MiniMax H3"
     minimax_h3_frame_conditioning = "First/last-keyframe (VAE-latent) conditioning for MiniMax H3"
     minimax_h3_audio_vae = "Audio VAE (stereo, 32 kHz) for MiniMax H3"
+    minimax_h3_reference_media = "One ordered Ref2VA reference (image or video) for MiniMax H3"
+    minimax_h3_reference_conditioning = "Ordered, VAE-encoded Ref2VA reference conditioning for MiniMax H3"
     sdxl_main_model = "SDXL Main model (UNet, VAE, CLIP1, CLIP2) to load"
     sdxl_refiner_model = "SDXL Refiner Main Modde (UNet, VAE, CLIP2) to load"
     onnx_main_model = "ONNX Main model (UNet, VAE, CLIP) to load"
@@ -477,6 +479,67 @@ class MiniMaxH3FrameConditioningField(BaseModel):
     height: int = Field(description="Canvas height used during VAE encoding (matches denoise height).")
 
 
+class MiniMaxH3ReferenceMediaField(BaseModel):
+    """One ordered Ref2VA reference for MiniMax H3: the raw media plus its conditioning options.
+
+    Carries no tensors — both the Prompt node and the Reference Conditioning node normalize
+    the same media independently (the FL2VA keyframe precedent), and the denoise node
+    cross-checks the two sides via the signature embedded in each side's output. Exactly one
+    of ``image`` / ``video`` is set; ``video_conditioning`` selects which streams a video
+    reference conditions ("audio" maps to upstream's standalone audio-reference kind, sourced
+    from the video's soundtrack).
+    """
+
+    image: Optional[ImageField] = Field(default=None, description="The reference image, for an image reference.")
+    video: Optional[VideoField] = Field(default=None, description="The reference video, for a video/audio reference.")
+    video_conditioning: Literal["video_audio", "video", "audio"] = Field(
+        default="video_audio",
+        description="Which streams a video reference conditions: video + soundtrack, video only, or soundtrack only.",
+    )
+    image_detail: Literal["max", "match"] = Field(
+        default="max",
+        description="Image reference sizing: 'max' (2048 px short edge, highest fidelity) or 'match' "
+        "(scaled to the generation's pixel area, several times faster).",
+    )
+    start_frame: int = Field(
+        default=0, description="First source frame of a video reference (inclusive, 0-based; negative from the end)."
+    )
+    end_frame: int = Field(
+        default=-1, description="Last source frame of a video reference (inclusive; negative from the end)."
+    )
+
+
+class MiniMaxH3EncodedReferenceField(BaseModel):
+    """One VAE-encoded Ref2VA reference, in packed order."""
+
+    kind: Literal["image", "video", "audio"] = Field(description="The reference's packed-block kind.")
+    video_rows_name: Optional[str] = Field(
+        default=None, description="Name of the saved clean (N, 96) visual rows tensor. None for 'audio'."
+    )
+    latent_frames: Optional[int] = Field(default=None, description="Latent frame count of the visual rows.")
+    latent_height: Optional[int] = Field(default=None, description="Latent height of the visual rows.")
+    latent_width: Optional[int] = Field(default=None, description="Latent width of the visual rows.")
+    audio_rows_name: Optional[str] = Field(
+        default=None, description="Name of the saved clean (A, 32) soundtrack rows tensor, when the reference has one."
+    )
+
+
+class MiniMaxH3ReferenceConditioningField(BaseModel):
+    """Ordered, VAE-encoded Ref2VA reference conditioning for MiniMax H3.
+
+    Rows are CLEAN: the denoise node noise-augments the visual rows to t=0.999 with the
+    request seed's leading draws; audio rows are never noised. ``signature`` is the ordered
+    per-reference fingerprint the denoise node compares against the prompt conditioning's,
+    so the two sides cannot silently disagree about what was encoded.
+    """
+
+    references: list[MiniMaxH3EncodedReferenceField] = Field(description="The encoded references, in packed order.")
+    num_frames: int = Field(description="The generated frame count the references were truncated for.")
+    signature: list[str] = Field(description="Ordered structural fingerprint, one entry per reference.")
+    width: int = Field(default=1344, description="Target canvas width the references were prepared for.")
+    height: int = Field(default=768, description="Target canvas height the references were prepared for.")
+
+
 class ConditioningField(BaseModel):
     """A conditioning tensor primitive value"""
 

invokeai/app/invocations/minimax_h3_denoise.py

@@ -27,6 +27,7 @@
     LatentsField,
     MiniMaxH3ConditioningField,
     MiniMaxH3FrameConditioningField,
+    MiniMaxH3ReferenceConditioningField,
     OutputField,
 )
 from invokeai.app.invocations.model import MiniMaxH3TransformerField
@@ -54,7 +55,9 @@
     MINIMAX_H3_SPATIAL_COMPRESSION,
     MINIMAX_H3_STILL_NUM_FRAMES,
     MINIMAX_H3_VAE_LATENT_CHANNELS,
+    MiniMaxH3EncodedReference,
     build_denoise_state,
+    build_ref2va_denoise_state,
     validate_canvas,
     validate_num_frames,
 )
@@ -129,7 +132,7 @@ class MiniMaxH3DenoiseOutput(BaseInvocationOutput):
     title="Denoise - MiniMax H3",
     tags=["latents", "video", "audio", "minimax"],
     category="latents",
-    version="1.3.0",
+    version="1.4.0",
     classification=Classification.Prototype,
 )
 class MiniMaxH3DenoiseInvocation(BaseInvocation):
@@ -147,6 +150,12 @@ class MiniMaxH3DenoiseInvocation(BaseInvocation):
         input=Input.Connection,
         title="Frame Conditioning",
     )
+    reference_conditioning: MiniMaxH3ReferenceConditioningField | None = InputField(
+        default=None,
+        description=FieldDescriptions.minimax_h3_reference_conditioning,
+        input=Input.Connection,
+        title="Reference Conditioning",
+    )
     width: int = InputField(
         default=1344,
         gt=0,
@@ -223,6 +232,87 @@ def _load_preview_decoder(context: InvocationContext) -> LoadedModelWithoutConfi
             )
             return None
 
+    def _build_reference_state(
+        self,
+        context: InvocationContext,
+        cond_info: MiniMaxH3ConditioningInfo,
+        num_frames: int,
+        num_latent_frames: int,
+        latent_height: int,
+        latent_width: int,
+        num_audio_latents: int,
+        device: torch.device,
+    ):
+        """Load the encoded references and build the Ref2VA denoise state.
+
+        The reference list must agree with the prompt's vision context - same references,
+        same order, same options, same generated duration - so the signature both nodes
+        derived from their inputs is compared entry for entry, exactly as FL2VA cross-checks
+        its keyframe anchors.
+        """
+        assert self.reference_conditioning is not None
+        field = self.reference_conditioning
+        if num_frames == MINIMAX_H3_STILL_NUM_FRAMES:
+            raise ValueError("Reference-to-video cannot generate the 5-frame still-image clip.")
+        if not cond_info.reference_signature:
+            # Checked before any num_frames comparison so the remedy named is the right one:
+            # a prompt with no references at all needs them WIRED, not a frame count changed.
+            raise ValueError(
+                "The prompt was encoded without references, but reference conditioning is wired. Connect "
+                "the same ordered references to Prompt - MiniMax H3 as well."
+            )
+        if (field.width, field.height) != (self.width, self.height):
+            raise ValueError(
+                f"The references were prepared for a {field.width}x{field.height} canvas but this denoise "
+                f"runs at {self.width}x{self.height}. Re-run Reference Conditioning - MiniMax H3 with "
+                "matching width/height."
+            )
+        if field.num_frames != num_frames:
+            raise ValueError(
+                f"The references were prepared for {field.num_frames} frames but this denoise runs "
+                f"{num_frames}. Re-run Reference Conditioning - MiniMax H3 with a matching Number of Frames."
+            )
+        if cond_info.reference_num_frames != num_frames:
+            raise ValueError(
+                f"The prompt's references were prepared for {cond_info.reference_num_frames} frames but this "
+                f"denoise runs {num_frames}. Re-run Prompt - MiniMax H3 with a matching Number of Frames."
+            )
+        if tuple(field.signature) != tuple(cond_info.reference_signature):
+            raise ValueError(
+                "Reference mismatch: the prompt was encoded with different references (a different order, "
+                "trim, or conditioning choice - or, for a 'match'-detail image, different width/height on "
+                "the two nodes) than Reference Conditioning provides. Wire the SAME ordered references to "
+                "both Prompt - MiniMax H3 and Reference Conditioning - MiniMax H3, with the same settings."
+            )
+
+        references: list[MiniMaxH3EncodedReference] = []
+        for encoded in field.references:
+            video_rows = context.tensors.load(encoded.video_rows_name) if encoded.video_rows_name else None
+            audio_rows = context.tensors.load(encoded.audio_rows_name) if encoded.audio_rows_name else None
+            latent_shape = None
+            if (
+                encoded.latent_frames is not None
+                and encoded.latent_height is not None
+                and encoded.latent_width is not None
+            ):
+                latent_shape = (encoded.latent_frames, encoded.latent_height, encoded.latent_width)
+            references.append(
+                MiniMaxH3EncodedReference(
+                    kind=encoded.kind, video_rows=video_rows, latent_shape=latent_shape, audio_rows=audio_rows
+                )
+            )
+        return build_ref2va_denoise_state(
+            text_token_tags=cond_info.text_token_tags,
+            references=references,
+            num_latent_frames=num_latent_frames,
+            latent_height=latent_height,
+            latent_width=latent_width,
+            num_audio_latents=num_audio_latents,
+            num_inference_steps=self.steps,
+            seed=self.seed,
+            device=device,
+        )
+
     @torch.no_grad()
     def invoke(self, context: InvocationContext) -> MiniMaxH3DenoiseOutput:
         # The field is a choice list of grid-aligned strings; validate anyway so a hand-authored graph that
@@ -243,6 +333,23 @@ def invoke(self, context: InvocationContext) -> MiniMaxH3DenoiseOutput:
         num_latent_frames = video_latent_num_frames(num_frames)
         num_audio_latents = audio_latent_num_frames(num_frames)
 
+        if self.frame_conditioning is not None and self.reference_conditioning is not None:
+            raise ValueError(
+                "Frame Conditioning (first/last keyframes, FL2VA) and Reference Conditioning (Ref2VA) are "
+                "mutually exclusive - wire one or the other."
+            )
+        transformer_variant = self.transformer.variant
+        if self.reference_conditioning is not None and transformer_variant == "fl2va":
+            raise ValueError(
+                "This transformer is the FL2VA task checkpoint, which cannot consume references. Select the "
+                "Ref2VA transformer in the model loader, or remove the reference conditioning."
+            )
+        if self.reference_conditioning is None and transformer_variant == "ref2va":
+            raise ValueError(
+                "This transformer is the Ref2VA task checkpoint, which requires reference conditioning. Wire "
+                "Reference Conditioning - MiniMax H3, or select the FL2VA transformer in the model loader."
+            )
+
         keyframe_anchors: tuple[str, ...] = ()
         clean_condition_rows: torch.Tensor | None = None
         if self.frame_conditioning is not None:
@@ -271,18 +378,41 @@ def invoke(self, context: InvocationContext) -> MiniMaxH3DenoiseOutput:
                 "width/height."
             )
 
-        state = build_denoise_state(
-            text_token_tags=cond_info.text_token_tags,
-            num_latent_frames=num_latent_frames,
-            latent_height=latent_height,
-            latent_width=latent_width,
-            num_audio_latents=num_audio_latents,
-            num_inference_steps=self.steps,
-            seed=self.seed,
-            device=device,
-            keyframe_anchors=keyframe_anchors,
-            clean_condition_rows=clean_condition_rows,
-        )
+        if self.reference_conditioning is not None:
+            state = self._build_reference_state(
+                context,
+                cond_info,
+                num_frames,
+                num_latent_frames,
+                latent_height,
+                latent_width,
+                num_audio_latents,
+                device,
+            )
+        else:
+            if cond_info.reference_signature:
+                raise ValueError(
+                    "The prompt was encoded with Ref2VA references but no reference conditioning is wired. "
+                    "Connect the same references to Reference Conditioning - MiniMax H3 and to this node."
+                )
+            state = build_denoise_state(
+                text_token_tags=cond_info.text_token_tags,
+                num_latent_frames=num_latent_frames,
+                latent_height=latent_height,
+                latent_width=latent_width,
+                num_audio_latents=num_audio_latents,
+                num_inference_steps=self.steps,
+                seed=self.seed,
+                device=device,
+                keyframe_anchors=keyframe_anchors,
+                clean_condition_rows=clean_condition_rows,
+            )
+
+        if state.layout.sequence_length > 100_000:
+            context.logger.info(
+                f"MiniMax H3 packed sequence is {state.layout.sequence_length} rows (references included); "
+                "expect a long render."
+            )
 
         num_condition_video_rows = state.layout.num_condition_video_rows
 

invokeai/app/invocations/minimax_h3_frame_conditioning.py

@@ -21,6 +21,7 @@
 from invokeai.backend.minimax_h3.keyframe_conditioning import encode_keyframes, prepare_keyframes
 from invokeai.backend.minimax_h3.packing import MINIMAX_H3_CANVAS_MULTIPLE
 from invokeai.backend.model_manager.load.model_cache.utils import get_effective_device
+from invokeai.backend.util.vae_working_memory import estimate_vae_working_memory_minimax_h3
 
 
 @invocation_output("minimax_h3_frame_conditioning_output")
@@ -37,7 +38,7 @@ class MiniMaxH3FrameConditioningOutput(BaseInvocationOutput):
     title="Frame Conditioning - MiniMax H3",
     tags=["conditioning", "minimax", "video", "i2v"],
     category="conditioning",
-    version="1.0.0",
+    version="1.0.1",
     classification=Classification.Prototype,
 )
 class MiniMaxH3FrameConditioningInvocation(BaseInvocation):
@@ -76,7 +77,8 @@ def invoke(self, context: InvocationContext) -> MiniMaxH3FrameConditioningOutput
             raise TypeError(
                 f"Expected AutoencoderKLMiniMaxH3 for the MiniMax H3 video VAE, got {type(vae_info.model).__name__}."
             )
-        with vae_info.model_on_device() as (_, vae):
+        working_memory = estimate_vae_working_memory_minimax_h3("encode", vae_info.model, self.height, self.width, 1)
+        with vae_info.model_on_device(working_mem_bytes=working_memory) as (_, vae):
             assert isinstance(vae, AutoencoderKLMiniMaxH3)
             context.util.signal_progress("Encoding MiniMax H3 keyframes")
             rows = encode_keyframes(vae, keyframes, device=get_effective_device(vae))

invokeai/app/invocations/minimax_h3_model_loader.py

@@ -29,7 +29,7 @@ class MiniMaxH3ModelLoaderOutput(BaseInvocationOutput):
     """MiniMax H3 model loader output."""
 
     transformer: MiniMaxH3TransformerField = OutputField(
-        description="MiniMax H3 FL2VA transformer", title="Transformer"
+        description="MiniMax H3 task transformer (FL2VA or Ref2VA)", title="Transformer"
     )
     text_encoder: MiniMaxH3TextEncoderField = OutputField(
         description=FieldDescriptions.minimax_h3_text_encoder, title="Qwen3-VL Encoder"
@@ -57,17 +57,22 @@ class MiniMaxH3ModelLoaderOutput(BaseInvocationOutput):
     title="Main Model - MiniMax H3",
     tags=["model", "minimax", "video"],
     category="model",
-    version="1.3.0",
+    version="1.4.0",
     classification=Classification.Prototype,
 )
 class MiniMaxH3ModelLoaderInvocation(BaseInvocation):
-    """Loads a MiniMax H3 (FL2VA) model, outputting its submodels.
+    """Loads a MiniMax H3 model, outputting its submodels.
 
     All six submodels (transformer, text encoder, tokenizer, processor, video VAE, audio VAE)
     come from the one diffusers-layout install. Optionally, a single-file transformer checkpoint
     (e.g. the pruned int8 repack) replaces the folder's transformer, and/or a single-file
     truncated Qwen3-VL encoder (e.g. the int8 repack) replaces the folder's text encoder, while
     everything else keeps coming from the folder install.
+
+    A Ref2VA diffusers folder's ``transformer_ref/`` weights are NOT folder-loadable (the
+    submodel map has no entry for that folder), so Ref2VA generation always selects a
+    single-file Ref2VA transformer here; identification marks such folders components-only so
+    the UI requires it up front.
     """
 
     model: ModelIdentifierField = InputField(
@@ -132,16 +137,11 @@ def invoke(self, context: InvocationContext) -> MiniMaxH3ModelLoaderOutput:
         else:
             transformer_config = main_config
             transformer = self.model.model_copy(update={"submodel_type": SubModelType.Transformer})
-        # TODO(ref2va): replace this rejection with variant stamping when reference-conditioned
-        # generation lands. Until then a Ref2VA transformer (whose weights expect reference
-        # conditioning rows nothing here packs) must fail fast rather than silently produce
-        # degraded output - the webv2 picker filters it out, but hand-authored workflows and
-        # API clients do not go through that filter.
-        if getattr(transformer_config, "variant", None) is MiniMaxH3VariantType.REF2VA:
-            raise ValueError(
-                f"'{transformer_config.name}' is a Ref2VA (reference-to-video) transformer, which this "
-                "version cannot generate with yet. Select an FL2VA transformer instead."
-            )
+        # Stamp the transformer's task variant onto the field: the denoise node uses it to
+        # reject a task/conditioning mismatch (references on FL2VA weights, or Ref2VA weights
+        # without references) instead of silently producing degraded output.
+        variant = getattr(transformer_config, "variant", None)
+        transformer_variant = variant.value if isinstance(variant, MiniMaxH3VariantType) else None
         tokenizer = self.model.model_copy(update={"submodel_type": SubModelType.Tokenizer})
         processor = self.model.model_copy(update={"submodel_type": SubModelType.Processor})
         if self.text_encoder_model is not None:
@@ -154,7 +154,7 @@ def invoke(self, context: InvocationContext) -> MiniMaxH3ModelLoaderOutput:
         audio_vae = self.model.model_copy(update={"submodel_type": SubModelType.AudioVAE})
 
         return MiniMaxH3ModelLoaderOutput(
-            transformer=MiniMaxH3TransformerField(transformer=transformer),
+            transformer=MiniMaxH3TransformerField(transformer=transformer, variant=transformer_variant),
             text_encoder=MiniMaxH3TextEncoderField(tokenizer=tokenizer, processor=processor, text_encoder=text_encoder),
             vae=VAEField(vae=vae),
             audio_vae=VAEField(vae=audio_vae),