diff --git a/invokeai/app/invocations/minimax_h3_model_loader.py b/invokeai/app/invocations/minimax_h3_model_loader.py index 219e9f86265..9184882bf0c 100644 --- a/invokeai/app/invocations/minimax_h3_model_loader.py +++ b/invokeai/app/invocations/minimax_h3_model_loader.py @@ -15,7 +15,13 @@ VAEField, ) from invokeai.app.services.shared.invocation_context import InvocationContext -from invokeai.backend.model_manager.taxonomy import BaseModelType, ModelFormat, ModelType, SubModelType +from invokeai.backend.model_manager.taxonomy import ( + BaseModelType, + MiniMaxH3VariantType, + ModelFormat, + ModelType, + SubModelType, +) @invocation_output("minimax_h3_model_loader_output") @@ -121,9 +127,21 @@ def invoke(self, context: InvocationContext) -> MiniMaxH3ModelLoaderOutput: if self.transformer_model is not None: if not context.models.exists(self.transformer_model.key): raise ValueError(f"Unknown transformer model: {self.transformer_model.key}") + transformer_config = context.models.get_config(self.transformer_model.key) transformer = self.transformer_model.model_copy(update={"submodel_type": SubModelType.Transformer}) else: + transformer_config = main_config transformer = self.model.model_copy(update={"submodel_type": SubModelType.Transformer}) + # TODO(ref2va): replace this rejection with variant stamping when reference-conditioned + # generation lands. Until then a Ref2VA transformer (whose weights expect reference + # conditioning rows nothing here packs) must fail fast rather than silently produce + # degraded output - the webv2 picker filters it out, but hand-authored workflows and + # API clients do not go through that filter. + if getattr(transformer_config, "variant", None) is MiniMaxH3VariantType.REF2VA: + raise ValueError( + f"'{transformer_config.name}' is a Ref2VA (reference-to-video) transformer, which this " + "version cannot generate with yet. Select an FL2VA transformer instead." + ) tokenizer = self.model.model_copy(update={"submodel_type": SubModelType.Tokenizer}) processor = self.model.model_copy(update={"submodel_type": SubModelType.Processor}) if self.text_encoder_model is not None: diff --git a/invokeai/backend/model_manager/configs/main.py b/invokeai/backend/model_manager/configs/main.py index 7281beee7c7..fedda3c7815 100644 --- a/invokeai/backend/model_manager/configs/main.py +++ b/invokeai/backend/model_manager/configs/main.py @@ -1595,7 +1595,8 @@ def from_model_on_disk(cls, mod: ModelOnDisk, override_fields: dict[str, Any]) - {"AutoencoderKLMiniMaxH3Audio"}, ) - variant = override_fields.pop("variant", None) or cls._get_variant(mod) + # An override may arrive as the raw string; normalize so the folder lookup below compares enums. + variant = MiniMaxH3VariantType(override_fields.pop("variant", None) or cls._get_variant(mod)) repo_variant = override_fields.pop("repo_variant", None) or cls._get_repo_variant_or_raise(mod) @@ -1605,10 +1606,19 @@ def from_model_on_disk(cls, mod: ModelOnDisk, override_fields: dict[str, Any]) - # can require those selections up front rather than failing mid-generation. components_only = override_fields.pop("components_only", None) if components_only is None: - transformer_dir = mod.path / "transformer" - components_only = not any( - any(transformer_dir.glob(pattern)) for pattern in ("*.safetensors", "*.bin", "*.pth", "*.pt", "*.ckpt") - ) + if variant is MiniMaxH3VariantType.REF2VA: + # Ref2VA folder weights are not folder-loadable in this version (`SubModelType` + # has no `transformer_ref` member), so a Ref2VA install always needs the + # single-file overrides - weight shards present or not. Marking it + # components-only makes the UI require them up front instead of failing + # minutes into a run. + components_only = True + else: + transformer_dir = mod.path / "transformer" + components_only = not any( + any(transformer_dir.glob(pattern)) + for pattern in ("*.safetensors", "*.bin", "*.pth", "*.pt", "*.ckpt") + ) return cls( **override_fields, @@ -1623,16 +1633,21 @@ def _get_variant(cls, mod: ModelOnDisk) -> MiniMaxH3VariantType: H3's task checkpoints share every component except the transformer folder: ``transformer`` (FL2VA: text / first/last-frame to audio-video) vs ``transformer_ref`` (Ref2VA: multi- - reference). Only FL2VA is supported so far. A Ref2VA-only download is a real H3 model this - version cannot run, so identification fails rather than mislabeling it as FL2VA. + reference). A folder holding both (the full official repo) identifies as FL2VA - that is + the folder-loadable variant. A ``transformer_ref``-only folder identifies as REF2VA, but + note its weights are NOT folder-loadable in this version (``SubModelType`` has no + ``transformer_ref`` member); the supported Ref2VA generation path is a components install + plus a single-file transformer override selected in the model loader. """ transformer_config = mod.path / "transformer" / "config.json" - if not transformer_config.exists(): - raise NotAMatchError( - "no FL2VA transformer folder (`transformer/`); Ref2VA-only installs are not supported yet" - ) - raise_for_class_name(transformer_config, {"MiniMaxH3Transformer3DModel"}) - return MiniMaxH3VariantType.FL2VA + if transformer_config.exists(): + raise_for_class_name(transformer_config, {"MiniMaxH3Transformer3DModel"}) + return MiniMaxH3VariantType.FL2VA + ref_transformer_config = mod.path / "transformer_ref" / "config.json" + if ref_transformer_config.exists(): + raise_for_class_name(ref_transformer_config, {"MiniMaxH3Transformer3DModel"}) + return MiniMaxH3VariantType.REF2VA + raise NotAMatchError("no transformer folder (`transformer/` or `transformer_ref/`)") def _has_minimax_h3_keys(state_dict: dict[str | int, Any]) -> bool: @@ -1656,9 +1671,12 @@ class Main_Checkpoint_MiniMaxH3_Config(Checkpoint_Config_Base, Main_Config_Base, file holds ONLY the transformer - the text encoder, VAEs, tokenizer and processor must come from an installed H3 diffusers-layout folder. - A Ref2VA transformer single file is key-for-key indistinguishable from FL2VA, so Ref2VA is - excluded only by filename; a renamed Ref2VA file would load and run but produce degraded - output (it expects reference conditioning rows this integration never packs). + The FL2VA and Ref2VA task transformers are key-for-key (and, except the non-pruned + int8_convrot repacks, byte-size) indistinguishable, so the FILENAME is the variant + classifier and ``variant`` is the user-correctable override for renamed files (e.g. + re-uploads). A misclassified variant loads and runs but produces degraded output - FL2VA + expects no reference conditioning rows, Ref2VA expects them - which is why the override + exists rather than any attempt at content sniffing. """ base: Literal[BaseModelType.MiniMaxH3] = Field(default=BaseModelType.MiniMaxH3) @@ -1672,9 +1690,6 @@ def from_model_on_disk(cls, mod: ModelOnDisk, override_fields: dict[str, Any]) - raise_for_override_fields(cls, override_fields) - if "ref2va" in mod.path.name.lower(): - raise NotAMatchError("Ref2VA single-file transformers are not supported yet") - state_dict = mod.load_state_dict() if not _has_minimax_h3_keys(state_dict): raise NotAMatchError("state dict does not look like a MiniMax H3 transformer") @@ -1682,7 +1697,9 @@ def from_model_on_disk(cls, mod: ModelOnDisk, override_fields: dict[str, Any]) - if _has_ggml_tensors(state_dict): raise NotAMatchError("GGUF-quantized MiniMax H3 checkpoints are not supported yet") - variant = override_fields.pop("variant", None) or MiniMaxH3VariantType.FL2VA + variant = override_fields.pop("variant", None) or ( + MiniMaxH3VariantType.REF2VA if "ref2va" in mod.path.name.lower() else MiniMaxH3VariantType.FL2VA + ) pruned = "adaln_t_table" in state_dict return cls(**override_fields, variant=variant, pruned=pruned) diff --git a/invokeai/backend/model_manager/starter_models.py b/invokeai/backend/model_manager/starter_models.py index bc8cd64cca3..f6b8b51a26a 100644 --- a/invokeai/backend/model_manager/starter_models.py +++ b/invokeai/backend/model_manager/starter_models.py @@ -1993,6 +1993,20 @@ def _gemini_3_resolution_presets( dependencies=[minimax_h3_components, minimax_h3_int8_text_encoder], ) +minimax_h3_ref2va_int8_transformer = StarterModel( + name="MiniMax H3 Ref2VA Transformer (int8, pruned)", + base=BaseModelType.MiniMaxH3, + source="Comfy-Org/MiniMax-H3::diffusion_models/minimax_h3_ref2va_pruned_int8_convrot.safetensors", + description="MiniMax H3 reference-conditioned video+audio generation: up to 3 video and 9 image " + "references. AdaLN-pruned int8 single-file transformer (~21 GB); select it in the MiniMax H3 " + "Model Loader's transformer field. Total size with dependencies: ~59 GB. NOTE: This model is " + "distributed under a restrictive license that forbids its use in certain territories. Please " + "see https://huggingface.co/MiniMaxAI/MiniMax-H3 for details.", + type=ModelType.Main, + format=ModelFormat.Checkpoint, + dependencies=[minimax_h3_components, minimax_h3_int8_text_encoder], +) + minimax_h3_turbo_lora = StarterModel( name="MiniMax H3 Turbo LoRA", base=BaseModelType.MiniMaxH3, @@ -2018,6 +2032,17 @@ def _gemini_3_resolution_presets( type=ModelType.LoRA, format=ModelFormat.LyCORIS, ) + +minimax_h3_ref2v_turbo_lora = StarterModel( + name="MiniMax H3 Ref2V Turbo LoRA", + base=BaseModelType.MiniMaxH3, + source="Comfy-Org/MiniMax-H3::loras/minimax_h3_ref2v_turbo_4step_v0.1_comfyui_bf16.safetensors", + description="Step-distillation LoRA for the MiniMax H3 Ref2VA transformer (~2 GB): renders " + "reference-conditioned video+audio in ~4 denoising steps instead of ~50. Apply at strength 1.0 " + "and lower Steps to 4. Trained against the Ref2VA transformer; not intended for FL2VA.", + type=ModelType.LoRA, + format=ModelFormat.LyCORIS, +) # endregion alibabacloud_wan26_t2i = StarterModel( @@ -2539,10 +2564,12 @@ def _gemini_3_resolution_presets( wan_22_ti2v_5b_gguf_q4_k_m, wan_22_ti2v_5b_gguf_q8_0, minimax_h3_int8_transformer, + minimax_h3_ref2va_int8_transformer, minimax_h3_int8_text_encoder, minimax_h3_components, minimax_h3_turbo_lora, minimax_h3_lightx2v_turbo_lora, + minimax_h3_ref2v_turbo_lora, gemini_flash_image, gemini_pro_image_preview, gemini_3_1_flash_image_preview, @@ -2714,16 +2741,18 @@ def _gemini_3_resolution_presets( ideogram_4_nf4, ] -# The working set for MiniMax H3 video+audio generation (~62 GB): shared components from -# the official repo plus Comfy-Org's int8 single-file transformer and text encoder, and the -# two turbo (step-distillation) LoRAs for fast low-step rendering. See the license note in -# the MiniMax H3 region above. +# The working set for MiniMax H3 video+audio generation (~85 GB): shared components from +# the official repo plus Comfy-Org's int8 single-file FL2VA and Ref2VA transformers and the +# text encoder, and the three turbo (step-distillation) LoRAs for fast low-step rendering. +# See the license note in the MiniMax H3 region above. minimax_h3_bundle: list[StarterModel] = [ minimax_h3_components, minimax_h3_int8_text_encoder, minimax_h3_int8_transformer, + minimax_h3_ref2va_int8_transformer, minimax_h3_turbo_lora, minimax_h3_lightx2v_turbo_lora, + minimax_h3_ref2v_turbo_lora, ] STARTER_BUNDLES: dict[str, StarterModelBundle] = { diff --git a/invokeai/backend/model_manager/taxonomy.py b/invokeai/backend/model_manager/taxonomy.py index a25c7bcc7be..22ce899c86b 100644 --- a/invokeai/backend/model_manager/taxonomy.py +++ b/invokeai/backend/model_manager/taxonomy.py @@ -264,6 +264,11 @@ class MiniMaxH3VariantType(str, Enum): """First/last-frame + text to audio-video: text-to-video and first/last-frame image-to-video (HF repo subfolder ``transformer``).""" + REF2VA = "ref2va" + """Multi-reference to audio-video: up to 3 video + 9 image references condition generation + (HF repo subfolder ``transformer_ref``). Same architecture and config as FL2VA, different + weights; supports only the reference task.""" + class MistralVariantType(str, Enum): """Mistral text encoder variants used by FLUX.2 [dev].""" diff --git a/invokeai/frontend/web/openapi.json b/invokeai/frontend/web/openapi.json index 14a9684363e..ea5ab22d1cb 100644 --- a/invokeai/frontend/web/openapi.json +++ b/invokeai/frontend/web/openapi.json @@ -60677,7 +60677,7 @@ "pruned" ], "title": "Main_Checkpoint_MiniMaxH3_Config", - "description": "Model config for MiniMax H3 single-file transformer checkpoints (safetensors).\n\nCovers MiniMax's single-file FL2VA transformer repacks (Comfy-Org and mirrors): bf16 or\nComfy ``int8_tensorwise``(+convrot) quantized, full or AdaLN-pruned (\"adaln curves\"). The\nfile holds ONLY the transformer - the text encoder, VAEs, tokenizer and processor must come\nfrom an installed H3 diffusers-layout folder.\n\nA Ref2VA transformer single file is key-for-key indistinguishable from FL2VA, so Ref2VA is\nexcluded only by filename; a renamed Ref2VA file would load and run but produce degraded\noutput (it expects reference conditioning rows this integration never packs)." + "description": "Model config for MiniMax H3 single-file transformer checkpoints (safetensors).\n\nCovers MiniMax's single-file FL2VA transformer repacks (Comfy-Org and mirrors): bf16 or\nComfy ``int8_tensorwise``(+convrot) quantized, full or AdaLN-pruned (\"adaln curves\"). The\nfile holds ONLY the transformer - the text encoder, VAEs, tokenizer and processor must come\nfrom an installed H3 diffusers-layout folder.\n\nThe FL2VA and Ref2VA task transformers are key-for-key (and, except the non-pruned\nint8_convrot repacks, byte-size) indistinguishable, so the FILENAME is the variant\nclassifier and ``variant`` is the user-correctable override for renamed files (e.g.\nre-uploads). A misclassified variant loads and runs but produces degraded output - FL2VA\nexpects no reference conditioning rows, Ref2VA expects them - which is why the override\nexists rather than any attempt at content sniffing." }, "Main_Checkpoint_QwenImage_Config": { "properties": { @@ -71365,7 +71365,7 @@ }, "MiniMaxH3VariantType": { "type": "string", - "enum": ["fl2va"], + "enum": ["fl2va", "ref2va"], "title": "MiniMaxH3VariantType", "description": "MiniMax H3 model variants (task-specific transformer checkpoints sharing every other component)." }, diff --git a/invokeai/frontend/web/src/features/modelManagerV2/models.ts b/invokeai/frontend/web/src/features/modelManagerV2/models.ts index 392b494d9f8..4f20ea85625 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/models.ts +++ b/invokeai/frontend/web/src/features/modelManagerV2/models.ts @@ -306,6 +306,7 @@ export const MODEL_VARIANT_TO_LONG_NAME: Record = { generate: 'Qwen Image', edit: 'Qwen Image Edit', fl2va: 'MiniMax H3 FL2VA', + ref2va: 'MiniMax H3 Ref2VA', t2v_a14b: 'Wan 2.2 T2V A14B', i2v_a14b: 'Wan 2.2 I2V A14B', ti2v_5b: 'Wan 2.2 TI2V 5B', diff --git a/invokeai/frontend/web/src/features/nodes/types/common.ts b/invokeai/frontend/web/src/features/nodes/types/common.ts index 377ec1efb42..cfef7f72d24 100644 --- a/invokeai/frontend/web/src/features/nodes/types/common.ts +++ b/invokeai/frontend/web/src/features/nodes/types/common.ts @@ -206,7 +206,7 @@ const zWanVariantType = z.enum(['t2v_a14b', 'i2v_a14b', 'ti2v_5b']); * targets. A14B = inner_dim 5120 (both T2V and I2V), 5B = inner_dim 3072. */ const zWanLoRAVariantType = z.enum(['a14b', '5b']); export const zQwen3VariantType = z.enum(['qwen3_4b', 'qwen3_8b', 'qwen3_06b']); -const zMiniMaxH3VariantType = z.enum(['fl2va']); +const zMiniMaxH3VariantType = z.enum(['fl2va', 'ref2va']); const zMistralVariantType = z.enum(['cow_mistral3_small', 'mistral3_24b']); const zPiDDecoderVariantType = z.enum(['res2k_sr4x', 'res2kto4k_sr4x']); export const zAnyModelVariant = z.union([ diff --git a/invokeai/frontend/web/src/services/api/schema.ts b/invokeai/frontend/web/src/services/api/schema.ts index 12d3d46e6f1..aeb1b68e5d0 100644 --- a/invokeai/frontend/web/src/services/api/schema.ts +++ b/invokeai/frontend/web/src/services/api/schema.ts @@ -24843,9 +24843,12 @@ export type components = { * file holds ONLY the transformer - the text encoder, VAEs, tokenizer and processor must come * from an installed H3 diffusers-layout folder. * - * A Ref2VA transformer single file is key-for-key indistinguishable from FL2VA, so Ref2VA is - * excluded only by filename; a renamed Ref2VA file would load and run but produce degraded - * output (it expects reference conditioning rows this integration never packs). + * The FL2VA and Ref2VA task transformers are key-for-key (and, except the non-pruned + * int8_convrot repacks, byte-size) indistinguishable, so the FILENAME is the variant + * classifier and ``variant`` is the user-correctable override for renamed files (e.g. + * re-uploads). A misclassified variant loads and runs but produces degraded output - FL2VA + * expects no reference conditioning rows, Ref2VA expects them - which is why the override + * exists rather than any attempt at content sniffing. */ Main_Checkpoint_MiniMaxH3_Config: { /** @@ -30428,7 +30431,7 @@ export type components = { * @description MiniMax H3 model variants (task-specific transformer checkpoints sharing every other component). * @enum {string} */ - MiniMaxH3VariantType: "fl2va"; + MiniMaxH3VariantType: "fl2va" | "ref2va"; /** * Mistral3EncoderField * @description Field for Mistral3 text encoder used by ERNIE-Image models. diff --git a/invokeai/frontend/webv2/src/features/models/core/taxonomy.test.ts b/invokeai/frontend/webv2/src/features/models/core/taxonomy.test.ts index 288cc58d7ef..e9d7cb247d9 100644 --- a/invokeai/frontend/webv2/src/features/models/core/taxonomy.test.ts +++ b/invokeai/frontend/webv2/src/features/models/core/taxonomy.test.ts @@ -49,7 +49,7 @@ describe('variant options', () => { expect(getVariantOptionsFor('sd-2', 'main')).toEqual(['normal', 'inpaint', 'depth']); expect(getVariantOptionsFor('flux', 'main')).toEqual(['schnell', 'dev', 'dev_fill']); expect(getVariantOptionsFor('wan', 'main')).toEqual(['t2v_a14b', 'i2v_a14b', 'ti2v_5b']); - expect(getVariantOptionsFor('minimax-h3', 'main')).toEqual(['fl2va']); + expect(getVariantOptionsFor('minimax-h3', 'main')).toEqual(['fl2va', 'ref2va']); }); it('distinguishes wan main and wan lora variants', () => { @@ -70,6 +70,7 @@ describe('variant options', () => { it('labels known variants and title-cases unknown ones', () => { expect(getModelVariantLabel('dev_fill')).toBe('FLUX Dev - Fill'); expect(getModelVariantLabel('fl2va')).toBe('MiniMax H3 FL2VA'); + expect(getModelVariantLabel('ref2va')).toBe('MiniMax H3 Ref2VA'); expect(getModelVariantLabel('some_new_variant')).toBe('Some New Variant'); }); diff --git a/invokeai/frontend/webv2/src/features/models/core/taxonomy.ts b/invokeai/frontend/webv2/src/features/models/core/taxonomy.ts index b07cdbf3abf..bc973fb01ff 100644 --- a/invokeai/frontend/webv2/src/features/models/core/taxonomy.ts +++ b/invokeai/frontend/webv2/src/features/models/core/taxonomy.ts @@ -138,6 +138,7 @@ export const MODEL_VARIANT_LABELS: Record = { qwen3_06b: 'Qwen3 0.6B', qwen3_4b: 'Qwen3 4B', qwen3_8b: 'Qwen3 8B', + ref2va: 'MiniMax H3 Ref2VA', res2k_sr4x: 'PiD 2K (4x SR)', res2kto4k_sr4x: 'PiD 4K (4x SR Upscale)', schnell: 'FLUX Schnell', @@ -155,7 +156,7 @@ const MAIN_VARIANTS_BY_BASE: Record = { flux: ['schnell', 'dev', 'dev_fill'], flux2: ['klein_4b', 'klein_4b_base', 'klein_9b', 'klein_9b_base', 'dev'], 'krea-2': ['krea2_turbo', 'krea2_base'], - 'minimax-h3': ['fl2va'], + 'minimax-h3': ['fl2va', 'ref2va'], 'qwen-image': ['generate', 'edit'], 'sd-1': ['normal', 'inpaint'], 'sd-2': ['normal', 'inpaint', 'depth'], diff --git a/invokeai/frontend/webv2/src/features/video/core/videoPolicies.test.ts b/invokeai/frontend/webv2/src/features/video/core/videoPolicies.test.ts index 22d554d3caf..4a07015b1f2 100644 --- a/invokeai/frontend/webv2/src/features/video/core/videoPolicies.test.ts +++ b/invokeai/frontend/webv2/src/features/video/core/videoPolicies.test.ts @@ -391,6 +391,22 @@ describe('MiniMax H3 Turbo', () => { expect(findMiniMaxH3TurboLora([TURBO, turboRider])).toMatchObject({ key: 'turbo' }); }); + it('never auto-picks a Ref2VA-trained turbo LoRA for FL2VA generation', () => { + // The Ref2V Turbo repack is trained against the Ref2VA transformer only; despite sorting + // before "MiniMax H3 Turbo LoRA" and matching the family+turbo patterns, it must lose. + const ref2vTurbo = { base: 'minimax-h3', key: 'ref2v', name: 'MiniMax H3 Ref2V Turbo LoRA', type: 'lora' as const }; + const ref2vFile = { + base: 'minimax-h3', + key: 'ref2v-file', + name: 'minimax_h3_ref2v_turbo_4step_v0.1_comfyui_bf16', + type: 'lora' as const, + }; + + expect(findMiniMaxH3TurboLora([ref2vTurbo, TURBO])).toMatchObject({ key: 'turbo' }); + expect(findMiniMaxH3TurboLora([ref2vTurbo])).toBeNull(); + expect(findMiniMaxH3TurboLora([ref2vFile])).toBeNull(); + }); + it('never strips a user LoRA that merely shares an accelerator-style name', () => { const turboRider = { base: 'minimax-h3', key: 'rider', name: 'Turbo Rider', type: 'lora' as const }; const model = h3Model(); @@ -742,6 +758,16 @@ describe('component section policy', () => { expect(transformerSlot?.filter?.(h3Model('checkpoint'), ctx)).toBe(true); expect(transformerSlot?.filter?.(h3Model('diffusers'), ctx)).toBe(false); }); + + it('excludes Ref2VA transformers from the H3 transformer picker until the reference mode lands', () => { + // TODO(ref2va): invert this expectation when reference-conditioned generation ships. + const model = h3Model(); + const policy = getVideoComponentSectionPolicy(model, settingsFor(model)); + const transformerSlot = policy.slots[0]; + const ctx = { model, selectedComponents: settingsFor(model), settings: settingsFor(model) }; + + expect(transformerSlot?.filter?.({ ...h3Model('checkpoint'), variant: 'ref2va' }, ctx)).toBe(false); + }); }); describe('getWanExpertWiringWarning', () => { diff --git a/invokeai/frontend/webv2/src/features/video/core/videoPolicies.ts b/invokeai/frontend/webv2/src/features/video/core/videoPolicies.ts index fc4a476cdde..4473e7bfc8a 100644 --- a/invokeai/frontend/webv2/src/features/video/core/videoPolicies.ts +++ b/invokeai/frontend/webv2/src/features/video/core/videoPolicies.ts @@ -530,13 +530,18 @@ export const findWanLightningLoraPair = ( const TURBO_PATTERN = /(?:^|[^a-z0-9])turbo(?:[^a-z0-9]|$)/i; const MINIMAX_H3_NAME_PATTERN = /(?:^|[^a-z0-9])(?:minimax|h3)(?:[^a-z0-9]|$)/i; +// Ref2VA-trained distillation LoRAs (delimited "ref2v"/"ref2va" token). They must never +// auto-apply to an FL2VA generation - the Ref2V Turbo repack is trained against the Ref2VA +// transformer only. +const MINIMAX_H3_REF2V_PATTERN = /(?:^|[^a-z0-9])ref2va?(?:[^a-z0-9]|$)/i; /** * The installed MiniMax H3 Turbo distillation LoRA, if any. Distillation LoRAs * carry no dedicated taxonomy, so this is a name heuristic: a delimited * "turbo" token, preferring names that also name the model family, with a * deterministic tie-break — a user's own "Turbo …" style LoRA loses to the - * real repack whenever one is installed. + * real repack whenever one is installed. Ref2VA-trained turbo LoRAs are + * excluded: the FL2VA accelerator must not pick them. */ export const findMiniMaxH3TurboLora = ( models: readonly ModelConfig[], @@ -551,6 +556,7 @@ export const findMiniMaxH3TurboLora = ( isLoraModelConfig(model) && model.base === 'minimax-h3' && TURBO_PATTERN.test(model.name) && + !MINIMAX_H3_REF2V_PATTERN.test(model.name) && (!requireFamilyName || MINIMAX_H3_NAME_PATTERN.test(model.name)) ) .sort((a, b) => score(a) - score(b) || a.name.localeCompare(b.name))[0] ?? null @@ -1031,8 +1037,13 @@ export const getVideoComponentSectionPolicy = ( return createComponentPolicy(componentsOnly, [ { + // TODO(ref2va): remove the variant exclusion when the reference generation mode lands — + // until then a Ref2VA transformer must not silently generate under FL2VA's config. filter: (candidate) => - candidate.type === 'main' && candidate.base === 'minimax-h3' && candidate.format === 'checkpoint', + candidate.type === 'main' && + candidate.base === 'minimax-h3' && + candidate.format === 'checkpoint' && + candidate.variant !== 'ref2va', helpText: componentsOnly ? 'Required: this main model is a components-only install, so the transformer must come from a single-file checkpoint (e.g. pruned int8).' : 'Optional single-file transformer (e.g. pruned int8) used in place of the main model’s transformer.', diff --git a/tests/app/invocations/test_minimax_h3_model_loader.py b/tests/app/invocations/test_minimax_h3_model_loader.py index c2f49f493f4..738f9d5d073 100644 --- a/tests/app/invocations/test_minimax_h3_model_loader.py +++ b/tests/app/invocations/test_minimax_h3_model_loader.py @@ -13,7 +13,7 @@ from invokeai.app.invocations.minimax_h3_model_loader import MiniMaxH3ModelLoaderInvocation from invokeai.app.invocations.model import ModelIdentifierField -from invokeai.backend.model_manager.taxonomy import BaseModelType, ModelFormat, ModelType +from invokeai.backend.model_manager.taxonomy import BaseModelType, MiniMaxH3VariantType, ModelFormat, ModelType def _identifier(key: str = "h3-main") -> ModelIdentifierField: @@ -57,3 +57,39 @@ def test_accepts_diffusers_folder_main(): output = node.invoke(_context(_config())) assert output.transformer.transformer.key == "h3-main" assert output.vae.vae.key == "h3-main" + + +# TODO(ref2va): these rejections flip to acceptance (with variant stamping) when +# reference-conditioned generation lands. + + +def _context_with_override(main_config: MagicMock, override_config: MagicMock) -> MagicMock: + context = MagicMock() + context.models.exists.return_value = True + context.models.get_config.side_effect = lambda key: override_config if key == "h3-ckpt" else main_config + return context + + +def test_rejects_ref2va_transformer_override(): + ref2va = _config(format_=ModelFormat.Checkpoint) + ref2va.variant = MiniMaxH3VariantType.REF2VA + ref2va.name = "MiniMax H3 Ref2VA Transformer (int8, pruned)" + node = MiniMaxH3ModelLoaderInvocation(id="loader", model=_identifier(), transformer_model=_identifier("h3-ckpt")) + with pytest.raises(ValueError, match="Ref2VA.*cannot generate with yet"): + node.invoke(_context_with_override(_config(), ref2va)) + + +def test_rejects_ref2va_diffusers_main_without_override(): + main = _config() + main.variant = MiniMaxH3VariantType.REF2VA + node = MiniMaxH3ModelLoaderInvocation(id="loader", model=_identifier()) + with pytest.raises(ValueError, match="Ref2VA.*cannot generate with yet"): + node.invoke(_context(main)) + + +def test_accepts_fl2va_transformer_override(): + fl2va = _config(format_=ModelFormat.Checkpoint) + fl2va.variant = MiniMaxH3VariantType.FL2VA + node = MiniMaxH3ModelLoaderInvocation(id="loader", model=_identifier(), transformer_model=_identifier("h3-ckpt")) + output = node.invoke(_context_with_override(_config(), fl2va)) + assert output.transformer.transformer.key == "h3-ckpt" diff --git a/tests/backend/model_manager/test_starter_models.py b/tests/backend/model_manager/test_starter_models.py index d104320328f..a997a2831d4 100644 --- a/tests/backend/model_manager/test_starter_models.py +++ b/tests/backend/model_manager/test_starter_models.py @@ -83,15 +83,18 @@ def test_krea2_gguf_dependency_models_are_registered_in_starter_models() -> None def test_minimax_h3_bundle_contains_working_set_and_turbo_loras() -> None: bundle = STARTER_BUNDLES[BaseModelType.MiniMaxH3] by_source = {model.source: model for model in bundle.models} - # The minimal working set: shared components, text encoder, transformer. + # The minimal working set: shared components, text encoder, and both task transformers. assert any(s.startswith("MiniMaxAI/MiniMax-H3::") for s in by_source) assert any(m.type is ModelType.Qwen3VLEncoder for m in by_source.values()) assert any(m.type is ModelType.Main and m.format is ModelFormat.Checkpoint for m in by_source.values()) - # Both turbo (step-distillation) LoRAs. + assert "Comfy-Org/MiniMax-H3::diffusion_models/minimax_h3_fl2va_pruned_int8_convrot.safetensors" in by_source + assert "Comfy-Org/MiniMax-H3::diffusion_models/minimax_h3_ref2va_pruned_int8_convrot.safetensors" in by_source + # All three turbo (step-distillation) LoRAs. loras = [m for m in bundle.models if m.type is ModelType.LoRA] lora_sources = {m.source for m in loras} assert "larryvrh/MiniMax-H3-Turbo-Lora::minimax_h3_turbo_v4_step600_ema.safetensors" in lora_sources assert "lightx2v/Minimax-h3-Turbo::minimax_h3_fl2v_turbo_8step_v1.0_comfyui_bf16.safetensors" in lora_sources + assert "Comfy-Org/MiniMax-H3::loras/minimax_h3_ref2v_turbo_4step_v0.1_comfyui_bf16.safetensors" in lora_sources def test_minimax_h3_bundle_models_are_registered_in_starter_models() -> None: diff --git a/tests/model_identification/stripped_models/01e489b0-ba16-4db8-9f36-df86ce2887ac/__test_metadata__.json b/tests/model_identification/stripped_models/01e489b0-ba16-4db8-9f36-df86ce2887ac/__test_metadata__.json new file mode 100644 index 00000000000..5506e9db24b --- /dev/null +++ b/tests/model_identification/stripped_models/01e489b0-ba16-4db8-9f36-df86ce2887ac/__test_metadata__.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3f904ac7453fd3b4ecc57464b9609052a08bf7e78e4cc81cc3840c3b3f06529 +size 505 diff --git a/tests/model_identification/stripped_models/01e489b0-ba16-4db8-9f36-df86ce2887ac/minimax_h3_ref2va_pruned_int8_convrot.safetensors b/tests/model_identification/stripped_models/01e489b0-ba16-4db8-9f36-df86ce2887ac/minimax_h3_ref2va_pruned_int8_convrot.safetensors new file mode 100644 index 00000000000..e7171cb1ed2 --- /dev/null +++ b/tests/model_identification/stripped_models/01e489b0-ba16-4db8-9f36-df86ce2887ac/minimax_h3_ref2va_pruned_int8_convrot.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6de94503fdd8d61d99abdd27a8468e43106859cd7f562f5c516a37d5d69131a9 +size 123726 diff --git a/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/__test_metadata__.json b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/__test_metadata__.json new file mode 100644 index 00000000000..b13e28729d4 --- /dev/null +++ b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/__test_metadata__.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1b1e889cc839a01f682e592fda97e6b975a085742198ff358fa12669113bbc3 +size 617 diff --git a/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/audio_vae/config.json b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/audio_vae/config.json new file mode 100644 index 00000000000..83b0d085b96 --- /dev/null +++ b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/audio_vae/config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a3c645ff892b376c6f5f4c8685964cd75474731af594ff058492a0000caabb6 +size 2271 diff --git a/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/audio_vae/model.safetensors b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/audio_vae/model.safetensors new file mode 100644 index 00000000000..237b961654c --- /dev/null +++ b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/audio_vae/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb2e2a2f7686fd2e45fa37dd632d66cdf9f4274d888b842fb6fa7ee01776a819 +size 95 diff --git a/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/modular_model_index.json b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/modular_model_index.json new file mode 100644 index 00000000000..20c48eddc34 --- /dev/null +++ b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/modular_model_index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2b6a210e482ffb78e613b553f570c44e101afce6741bd4ed91429d0559af031 +size 2935 diff --git a/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/processor/preprocessor_config.json b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/processor/preprocessor_config.json new file mode 100644 index 00000000000..e7a1091ec59 --- /dev/null +++ b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/processor/preprocessor_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516 +size 390 diff --git a/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/processor/video_preprocessor_config.json b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/processor/video_preprocessor_config.json new file mode 100644 index 00000000000..32579be08bc --- /dev/null +++ b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/processor/video_preprocessor_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13 +size 385 diff --git a/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/tokenizer/tokenizer_config.json b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/tokenizer/tokenizer_config.json new file mode 100644 index 00000000000..98cd9c27d57 --- /dev/null +++ b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/tokenizer/tokenizer_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a07e942ac874baa13758de8d1fbdb186683cc03416b5589e1b6671c6b3057c68 +size 11003 diff --git a/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/transformer_ref/config.json b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/transformer_ref/config.json new file mode 100644 index 00000000000..d2de6f23b60 --- /dev/null +++ b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/transformer_ref/config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74c11bff524336576096993cbfcdcdc2ef4fa2fa4409df693bdcbc6c666282ae +size 546 diff --git a/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/vae/config.json b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/vae/config.json new file mode 100644 index 00000000000..aa311976f36 --- /dev/null +++ b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/vae/config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78f67deec3d63aae807f2bfe7154bc1e26f6372cb20b63265fcbae1b62bb5745 +size 2011 diff --git a/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/vae/diffusion_pytorch_model-00001-of-00003.safetensors b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/vae/diffusion_pytorch_model-00001-of-00003.safetensors new file mode 100644 index 00000000000..0df1855547d --- /dev/null +++ b/tests/model_identification/stripped_models/23e100f6-636c-40ce-b86e-54eb956260e6/vae/diffusion_pytorch_model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc8edc91c5380231b7b3944c9cfb644a4d50bad2610e6c4518d7aefcbe6be05d +size 102 diff --git a/tests/model_identification/stripped_models/e4798fc6-ca48-404e-9624-c11ab3418ae7/__test_metadata__.json b/tests/model_identification/stripped_models/e4798fc6-ca48-404e-9624-c11ab3418ae7/__test_metadata__.json new file mode 100644 index 00000000000..6a9dac14422 --- /dev/null +++ b/tests/model_identification/stripped_models/e4798fc6-ca48-404e-9624-c11ab3418ae7/__test_metadata__.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c65b835fd273f9d5c3280f8e7559945d55b90e8faea261671648f83ac26e47a +size 559 diff --git a/tests/model_identification/stripped_models/e4798fc6-ca48-404e-9624-c11ab3418ae7/minimax_h3_transformer_pruned.safetensors b/tests/model_identification/stripped_models/e4798fc6-ca48-404e-9624-c11ab3418ae7/minimax_h3_transformer_pruned.safetensors new file mode 100644 index 00000000000..e7171cb1ed2 --- /dev/null +++ b/tests/model_identification/stripped_models/e4798fc6-ca48-404e-9624-c11ab3418ae7/minimax_h3_transformer_pruned.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6de94503fdd8d61d99abdd27a8468e43106859cd7f562f5c516a37d5d69131a9 +size 123726