NVIDIA · pamelap-nvidia · Aug 7, 2026 · May 12, 2026 · May 15, 2026 · May 21, 2026
@@ -1,5 +1,5 @@
 #
-# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION &
+# SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION &
 # AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
@@ -67,7 +67,7 @@ list(FILTER SRC_CPP EXCLUDE REGEX "mhcKernels/.*")
 list(FILTER SRC_CU EXCLUDE REGEX "mhcKernels/.*")
 list(FILTER SRC_CPP EXCLUDE REGEX "compressorKernels/.*")
 list(FILTER SRC_CU EXCLUDE REGEX "compressorKernels/.*")
-# Marlin is built as its own Hopper-only OBJECT library below.
+# Marlin is built as its own architecture-scoped OBJECT library below.
 list(FILTER SRC_CPP EXCLUDE REGEX "marlin/.*")
 list(FILTER SRC_CU EXCLUDE REGEX "marlin/.*")
 
@@ -88,8 +88,8 @@ if(FAST_BUILD)
     STATUS "FAST_BUILD enabled for kernels: using -O1 for CUDA compilation")
 endif()
 
-# Marlin NVFP4: Ada (SM89, e.g. L40S) and Hopper (SM90) OBJECT library. Pinned
-# to sm_89/sm_90 so the global CMAKE_CUDA_ARCHITECTURES doesn't propagate.
+# Marlin NVFP4: build Ada/Hopper kernels and the dense W4A16/repack kernels used
+# on SM12x. Unsupported device passes emit empty kernel stubs.
 file(GLOB_RECURSE MARLIN_SRC "marlin/*.cu" "marlin/*.cpp")
 if(MARLIN_SRC)
   add_library(marlin_src OBJECT ${MARLIN_SRC})
@@ -101,7 +101,7 @@ if(MARLIN_SRC)
       $<TARGET_PROPERTY:${INTERNAL_CUTLASS_KERNELS_TARGET},INTERFACE_INCLUDE_DIRECTORIES>
   )
   target_link_libraries(marlin_src PRIVATE trtllm_gen_fmha_interface)
-  set_cuda_architectures(marlin_src 89 90)
+  set_cuda_architectures(marlin_src 89 90 120f)
 endif()
 
 add_library(

@@ -41,6 +41,15 @@
 #define MARLIN_NVFP4_DEVICE_SUPPORTED 0
 #endif
 
+// Dense Marlin also supports Blackwell GeForce (SM120/121). Keep this
+// separate from MARLIN_NVFP4_DEVICE_SUPPORTED because Marlin MoE does not.
+#if defined(__CUDA_ARCH__)                                                                                             \
+    && ((__CUDA_ARCH__ >= 890 && __CUDA_ARCH__ < 1000) || (__CUDA_ARCH__ >= 1200 && __CUDA_ARCH__ < 1300))
+#define MARLIN_NVFP4_DENSE_DEVICE_SUPPORTED 1
+#else
+#define MARLIN_NVFP4_DENSE_DEVICE_SUPPORTED 0
+#endif
+
 namespace MARLIN_NAMESPACE_NAME
 {
 

@@ -30,6 +30,11 @@ inline bool isMarlinNvfp4SmSupported(int sm)
     return sm >= 89 && sm < 100;
 }
 
+inline bool isMarlinNvfp4DenseSmSupported(int sm)
+{
+    return isMarlinNvfp4SmSupported(sm) || sm == 120 || sm == 121;
+}
+
 void dequantFp4Activations(
     void const* act_fp4, void const* act_sf, float const* alpha, void* act_bf16, int m, int k, cudaStream_t stream);
 

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -327,8 +327,8 @@ void marlinNvfp4Gemm(void const* act_bf16, void const* weight, void* output, voi
     bool use_fp32_reduce, cudaStream_t stream)
 {
     int const sm = tensorrt_llm::common::getSMVersion();
-    TLLM_CHECK_WITH_INFO(isMarlinNvfp4SmSupported(sm),
-        "Marlin NVFP4 GEMM is only supported on Ada (SM89) and Hopper (SM90-99); current SM = %d", sm);
+    TLLM_CHECK_WITH_INFO(isMarlinNvfp4DenseSmSupported(sm),
+        "Marlin NVFP4 GEMM is only supported on SM89, SM90-99, and SM120/121; current SM = %d", sm);
 
     int dev;
     cudaGetDevice(&dev);

@@ -49,8 +49,8 @@
 namespace MARLIN_NAMESPACE_NAME
 {
 
-// Empty kernel stub for unsupported device passes; see marlin.cuh.
-#if defined(__CUDA_ARCH__) && !MARLIN_NVFP4_DEVICE_SUPPORTED
+// Empty kernel stub outside the architectures supported by dense Marlin.
+#if defined(__CUDA_ARCH__) && !MARLIN_NVFP4_DENSE_DEVICE_SUPPORTED
 
 template <typename scalar_t,   // compute dtype, nv_bfloat16
     int const threads,         // number of threads in a threadblock

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -323,8 +323,8 @@ void gptq_marlin_repack_dispatch(uint32_t const* b_q_weight_ptr, uint32_t const*
     int size_k, int size_n, int num_bits, bool has_perm, bool is_a_8bit, cudaStream_t stream)
 {
     int const sm = tensorrt_llm::common::getSMVersion();
-    TLLM_CHECK_WITH_INFO(isMarlinNvfp4SmSupported(sm),
-        "Marlin NVFP4 repack is only supported on Ada (SM89) and Hopper (SM90-99); current SM = %d", sm);
+    TLLM_CHECK_WITH_INFO(isMarlinNvfp4DenseSmSupported(sm),
+        "Marlin NVFP4 repack is only supported on SM89, SM90-99, and SM120/121; current SM = %d", sm);
 
     int blocks;
     int dev;

@@ -1232,7 +1232,7 @@ def nvfp4_gemm(
         allowed_backends: Comma-separated list of backends to consider for auto-selection.
             Default: "cutlass,cublaslt,cuda_core" (excludes cutedsl for faster build)
             Add 'cutedsl' for extreme performance at the cost of longer build time.
-            Valid backends: 'cutlass', 'cublaslt', 'cutedsl', 'cuda_core'.
+            Valid backends: 'cutlass', 'cublaslt', 'cutedsl', 'cuda_core', 'marlin'.
 
     Returns:
         Output tensor [m, n] with dtype=output_dtype

@@ -352,6 +352,17 @@ def resolve_moe_backend(moe_backend: str,
             if 100 <= sm_version < 120:
                 return "TRTLLM"
 
+        is_w4a16_nvfp4 = (quant_config is not None and quant_config.quant_algo
+                          in (QuantAlgo.W4A16_NVFP4, "W4A16_NVFP4"))
+        if is_w4a16_nvfp4:
+            sm_version = get_sm_version()
+            # CuteDslB12xFusedMoE on SM120/121, MarlinFusedMoE on Hopper. Any
+            # other SM falls through to CUTLASS, which dequantizes on the fly.
+            if sm_version in (120, 121):
+                return "CUTEDSL"
+            if 90 <= sm_version < 100:
+                return "MARLIN"
+
         if architecture == "GptOssForCausalLM":
             sm_version = get_sm_version()
             # Select the best performing backend based on SM version

@@ -162,7 +162,6 @@ def forward(
                                **kwargs)
 
 
-# Ref code: https://huggingface.co/nvidia/Nemotron-Nano-3-30B-A3.5B-dev-1024/blob/main/modeling_nemotron_h.py#L818
 class NemotronHMOE(nn.Module):
 
     def __init__(
@@ -252,12 +251,12 @@ def _moe(name):
         # UnquantizedFusedMoEMethod and allocate BF16 weight buffers, causing a shape mismatch
         # when loading NVFP4/W4A8_NVFP4_FP8 quantized expert weights.
         # Look up the per-expert quant config from quant_config_dict and use it for create_moe.
-        moe_model_config = model_config
+        override_quant_config = None
         if model_config.quant_config_dict is not None:
             experts_prefix = f"model.layers.{layer_idx}.mixer.experts."
             for key, cfg in model_config.quant_config_dict.items():
                 if key.startswith(experts_prefix):
-                    moe_model_config = replace(model_config, quant_config=cfg)
+                    override_quant_config = cfg
                     break
 
         # Setup MoE experts.
@@ -269,7 +268,8 @@ def _moe(name):
             aux_stream_dict=aux_stream_dict,
             dtype=config.torch_dtype,
             reduce_results=self.reduce_results,
-            model_config=moe_model_config,
+            model_config=model_config,
+            override_quant_config=override_quant_config,
             layer_idx=self.layer_idx,
             weight_loading_mode=MoEWeightLoadingMode.VANILLA,
             bias=self.mlp_bias,

@@ -23,8 +23,9 @@
 from ..model_config import ModelConfig
 from ..peft.lora.layer import LoraLayer, LoraModuleType
 from ..utils import (Fp4QuantizedTensor, get_model_extra_attrs,
-                     is_nvfp4_marlin_enabled, is_torch_compiling)
-from .linear import Linear, TensorParallelMode, WeightMode, WeightsLoadingConfig
+                     is_torch_compiling)
+from .linear import (Linear, TensorParallelMode, WeightMode,
+                     WeightsLoadingConfig, is_static_nvfp4_input_eligible)
 from .multi_stream_utils import maybe_execute_in_parallel
 from .rotary_embedding import MRotaryEmbedding, RotaryEmbedding
 
@@ -606,7 +607,7 @@ def __init__(
         attn_cls = get_attention_backend(self.attn_backend,
                                          sparse_params=sparse_params)
 
-        self.is_marlin_enabled: bool = is_nvfp4_marlin_enabled()
+        self.is_marlin_enabled = False
 
         # These two modules are mutually exclusive - either splitted_qkv_lora or fused_qkv_lora will be used,
         # but never both at the same time. splitted_qkv_lora handles Q,K,V separately while fused_qkv_lora
@@ -697,7 +698,9 @@ def create_weights(self):
         self.attn.update_quant_config(self.quant_config)
 
         self.o_proj.create_weights()
-        self.has_quant_scale = (self.o_proj.has_fp8_qdq or self.o_proj.has_nvfp4
+        self.is_marlin_enabled = self.o_proj.uses_marlin_nvfp4
+        self.has_quant_scale = (self.o_proj.has_fp8_qdq
+                                or self.o_proj.has_nvfp4_activation_quantization
                                 or self.o_proj.has_fp8_block_scales
                                 or self.o_proj.has_fp8_rowwise
                                 or self.o_proj.has_w4a8_nvfp4_fp8)
@@ -762,6 +765,12 @@ def _use_quantize_output(self):
         if self.o_proj.force_dynamic_quantization:
             return False
 
+        # Producing FP4 output requires a calibrated activation scale on the
+        # consumer. Weight-only W4A16 has NVFP4 weights but consumes BF16/FP16.
+        if (self.o_proj.has_nvfp4
+                and not is_static_nvfp4_input_eligible(self.o_proj)):
+            return False
+
         # If no quant is applied, no need to quantize the output
         if self.quant_config is not None and not self.quant_config.layer_quant_mode.has_any_quant(
                 exclude_kv_cache=True):

@@ -78,17 +78,28 @@ def get_moe_cls(
     elif moe_backend.upper() == "VANILLA":
         return VanillaMoE
     elif moe_backend.upper() == "CUTEDSL":
+        has_w4a16_nvfp4 = (quant_config is not None
+                           and quant_config.quant_algo == QuantAlgo.W4A16_NVFP4)
         if quant_config is not None and (
                 quant_config.quant_mode.has_fp8_block_scales()
-                or quant_config.quant_mode.has_nvfp4()):
-            # On SM120 / SM121 + NVFP4 the cuteDSL family member is the
+                or quant_config.quant_mode.has_nvfp4() or has_w4a16_nvfp4):
+            # On SM120 / SM121 + NVFP4/W4A16_NVFP4 the cuteDSL family member is the
             # hybrid CUTLASS-prefill / FlashInfer NVFP4 MoE decode backend
             # (CuteDslB12xFusedMoE). Prefer it when flashinfer is importable;
             # otherwise fall through to CuteDslFusedMoE for SM100 / SM103.
-            if quant_config.quant_mode.has_nvfp4():
+            has_nvfp4 = (quant_config.quant_mode.has_nvfp4()
+                         and not has_w4a16_nvfp4)
+            if has_nvfp4 or has_w4a16_nvfp4:
                 from tensorrt_llm._utils import get_sm_version
                 sm_version = get_sm_version()
                 if sm_version in CuteDslB12xFusedMoE._SUPPORTED_SM_VERSIONS:
+                    mapping = model_config.mapping
+                    if mapping.moe_ep_size > 1 or mapping.dp_size > 1:
+                        logger.warning(
+                            "CuteDslB12xFusedMoE does not support expert "
+                            "parallelism or attention-DP/all-to-all; selecting "
+                            "CutlassFusedMoE.")
+                        return CutlassFusedMoE
                     try:
                         import flashinfer  # noqa: F401
                         logger.info(
@@ -101,13 +112,24 @@ def get_moe_cls(
                     except ImportError:
                         logger.warning(
                             "CuteDslB12xFusedMoE eligible (SM%d + NVFP4) "
-                            "but flashinfer is not importable; using CuteDslFusedMoE.",
+                            "but flashinfer is not importable; using %s.",
                             sm_version,
+                            "CutlassFusedMoE"
+                            if has_w4a16_nvfp4 else "CuteDslFusedMoE",
                         )
+                        if has_w4a16_nvfp4:
+                            return CutlassFusedMoE
+                elif has_w4a16_nvfp4:
+                    logger.warning(
+                        "CuteDslB12xFusedMoE requires SM120/121 for W4A16_NVFP4 "
+                        "(got SM%d). Using CutlassFusedMoE.",
+                        sm_version,
+                    )
+                    return CutlassFusedMoE
             return CuteDslFusedMoE
         else:
             logger.warning(
-                f"{layer_prefix}CuteDslFusedMoE only supports fp8_block_scales and nvfp4. "
+                f"{layer_prefix}CuteDslFusedMoE only supports fp8_block_scales, nvfp4, and w4a16_nvfp4. "
                 f"Check out details in quant_config: {quant_config}. Using CutlassFusedMoE instead."
             )
             return CutlassFusedMoE

@@ -44,24 +44,25 @@
 
 
 class CuteDslB12xFusedMoE(CuteDslFusedMoE):
-    """Hybrid CUTLASS-prefill / b12x-decode NVFP4 fused-MoE backend for SM120 / SM121.
+    """B12x NVFP4 fused-MoE backend for SM120 / SM121.
 
     Member of the cuteDSL backend family: the decode kernel
     (``flashinfer.B12xMoEWrapper.run``) is JIT-compiled CuTe DSL, so the
     backend slots in next to :class:`CuteDslFusedMoE` (which targets SM100 /
-    SM103). The hybrid prefill path still routes through the C++ CUTLASS
-    NVFP4 GroupGEMM via explicit :class:`CutlassFusedMoE` method calls; the
-    parent class on the MRO does not change which kernels execute, only
-    where the b12x backend sits in the family.
+    SM103). Plain NVFP4 prefill can route through the C++ CUTLASS NVFP4
+    GroupGEMM via explicit :class:`CutlassFusedMoE` method calls; the parent
+    class on the MRO does not change which kernels execute, only where the
+    b12x backend sits in the family.
 
     Composition (see ``MOE_DEVELOPER_GUIDE.md`` for the full explainer):
 
-    - **Prefill (``m >= _PREFILL_VIA_CUTLASS_THRESHOLD``)** explicitly
+    - **NVFP4 prefill (``m >= _PREFILL_VIA_CUTLASS_THRESHOLD``)** explicitly
       invokes :class:`CutlassFusedMoE` NVFP4 GroupGEMM. The b12x kernel's
       12-CTA-per-token MMA pattern is suboptimal at large ``m``.
     - **Decode (``m <  _PREFILL_VIA_CUTLASS_THRESHOLD``)** dispatches to
       FlashInfer's ``B12xMoEWrapper.run`` — a kernel purpose-built for
       ``m=1`` / small routed-row counts.
+    - **W4A16_NVFP4** stays on the b12x path for both prefill and decode.
 
     NVFP4 weights are loaded via :class:`NVFP4CuteDslB12xFusedMoEMethod`
     (an :class:`NVFP4CutlassFusedMoEMethod` subclass returned by
@@ -79,8 +80,9 @@ class CuteDslB12xFusedMoE(CuteDslFusedMoE):
     The backend hard-rejects EP (b12x has no dispatch / combine kernel),
     MoE alltoall, ``Fp4QuantizedTensor`` input, ``swiglu_gptoss_style``
     biased SwiGLU, and activations outside ``{Relu2, Swiglu}``. It is
-    selected on the ``CUTEDSL`` MoE path when SM120 / SM121 + NVFP4 +
-    flashinfer-importable gates pass (see ``create_moe.get_moe_cls``).
+    selected on the ``CUTEDSL`` MoE path when SM120 / SM121 + NVFP4 or
+    W4A16_NVFP4 + flashinfer-importable gates pass (see
+    ``create_moe.get_moe_cls``).
     """
 
     # SM versions on which the FlashInfer b12x NVFP4 MoE kernel is available.
@@ -105,9 +107,9 @@ def can_implement(
         if sm_version not in cls._SUPPORTED_SM_VERSIONS:
             sm_list = "/".join(f"SM{v}" for v in sorted(cls._SUPPORTED_SM_VERSIONS))
             return _warn_and_return(f"CuteDslB12xFusedMoE requires {sm_list}, got SM{sm_version}")
-        if quant_algo != QuantAlgo.NVFP4:
+        if quant_algo not in {QuantAlgo.NVFP4, QuantAlgo.W4A16_NVFP4}:
             return _warn_and_return(
-                f"CuteDslB12xFusedMoE only supports NVFP4 quantization "
+                f"CuteDslB12xFusedMoE only supports NVFP4 or W4A16_NVFP4 quantization "
                 f"(got quant_algo={quant_algo})"
             )
         if dtype_activation not in {torch.float16, torch.bfloat16}:
@@ -167,9 +169,12 @@ def _get_quant_method(self):
 
     def _route_to_cutlass(self, x) -> bool:
         """Return ``True`` iff this call should fall back to the inherited
-        CUTLASS path (prefill chunk). ``Fp4QuantizedTensor`` inputs always
-        stay on the b12x path (which rejects them) so the existing error
-        message is preserved."""
+        CUTLASS path (NVFP4 prefill chunk). ``Fp4QuantizedTensor`` inputs
+        always stay on the b12x path (which rejects them) so the existing
+        error message is preserved."""
+        quant_config = getattr(self, "quant_config", None)
+        if quant_config is not None and quant_config.quant_algo == QuantAlgo.W4A16_NVFP4:
+            return False
         return isinstance(x, torch.Tensor) and x.shape[0] >= self._PREFILL_VIA_CUTLASS_THRESHOLD
 
     # ``post_load_weights`` is inherited from ``CutlassFusedMoE`` and
@@ -190,12 +195,12 @@ def quantize_input(
     ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
         """Hybrid dispatch entrypoint for activation handling.
 
-        Prefill chunks (``x.shape[0] >= _PREFILL_VIA_CUTLASS_THRESHOLD``) take
-        the inherited :meth:`CutlassFusedMoE.quantize_input` path so the
-        downstream ``run_moe`` can call CUTLASS NVFP4 GroupGEMM. Decode
-        chunks pass through unchanged because b12x quantizes activations
-        internally (consumes a bf16 / fp16 ``x`` and produces its own scale
-        factors).
+        NVFP4 prefill chunks take the inherited
+        :meth:`CutlassFusedMoE.quantize_input` path so the downstream
+        ``run_moe`` can call CUTLASS NVFP4 GroupGEMM. Decode chunks and
+        W4A16_NVFP4 chunks pass through unchanged because b12x quantizes
+        activations internally (consumes a bf16 / fp16 ``x`` and produces its
+        own scale factors).
         """
         if self._route_to_cutlass(x):
             return CutlassFusedMoE.quantize_input(

@@ -813,6 +813,8 @@ def _get_quant_method(self):
                 return FP8QDQFusedMoEMethod()
             elif self.quant_config.layer_quant_mode.has_fp8_block_scales():
                 return DeepSeekFP8BlockScalesFusedMoEMethod()
+            elif self.quant_config.quant_algo == QuantAlgo.W4A16_NVFP4:
+                return W4A16NVFP4CutlassFusedMoEMethod()
             elif self.quant_config.layer_quant_mode.has_nvfp4():
                 return NVFP4CutlassFusedMoEMethod()
             elif self.quant_config.layer_quant_mode.is_int4_weight_only_per_group(

@@ -57,6 +57,10 @@ class MarlinFusedMoE(CutlassFusedMoE):
             "sm_constraint": ("in", set(range(89, 100))),
             "dtypes": {torch.bfloat16},
         },
+        QuantAlgo.W4A16_NVFP4: {
+            "sm_constraint": ("in", set(range(89, 100))),
+            "dtypes": {torch.bfloat16},
+        },
     }
 
     @classmethod
@@ -68,9 +72,9 @@ def can_implement(
     ) -> Tuple[bool, Optional[str]]:
         sm_version = get_sm_version()
 
-        if quant_algo != QuantAlgo.NVFP4:
+        if quant_algo not in cls._QUANT_SUPPORT_TABLE:
             return _warn_and_return(
-                f"MarlinFusedMoE only supports NVFP4 (got quant_algo={quant_algo})"
+                f"MarlinFusedMoE only supports NVFP4 or W4A16_NVFP4 (got quant_algo={quant_algo})"
             )
 
         if not is_nvfp4_marlin_supported_sm(sm_version):