From f20e8ad1d10944f14eb6b95d4835891b6ebc07dc Mon Sep 17 00:00:00 2001 From: Xiaowei Ren Date: Wed, 11 Nov 2026 06:40:58 -0800 Subject: [PATCH 1/5] [PyTorch] Run NVFP4 FC2 fprop and FC1 dgrad of the fused grouped MLP on CuTe DSL The fused grouped MLP runs the NVFP4 FC2 forward and FC1 input-gradient GEMMs on cuBLAS. Run them on the cuDNN frontend CuTe DSL grouped GEMM kernel instead, like the MXFP8 path. Add NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM to move the GEMMs without a fused activation that both backends implement back to cuBLAS: the weight gradient GEMMs and the NVFP4 FC2 forward and FC1 input-gradient GEMMs. It replaces NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP, which covered the weight-gradient GEMMs only and stays as a deprecated alias, honored when the new variable is unset. A single-group MLP on the dense single-group path (every activation but the SReLU ones) keeps its dense GEMMs. NVFP4 with single_grouped_weight=True is not supported by the fused op (the cuBLAS path already rejects its unswizzled weight scales); the new paths raise NotImplementedError for it instead of failing in a kernel. Tests: backend routing for grouped and single-group MLPs, the environment variable and its alias, CUDA graph capture with NVFP4, and the single-grouped-weight error. Co-Authored-By: Claude Signed-off-by: Xiaowei Ren --- docs/envvars.rst | 12 + tests/pytorch/test_grouped_mlp.py | 207 +++++++++++++++++- .../pytorch/cpp_extensions/gemm.py | 103 ++++++++- .../pytorch/ops/fused/grouped_mlp.py | 193 +++++++++++++++- 4 files changed, 500 insertions(+), 15 deletions(-) diff --git a/docs/envvars.rst b/docs/envvars.rst index a0a64b3f63..e61c5c5be5 100644 --- a/docs/envvars.rst +++ b/docs/envvars.rst @@ -178,6 +178,18 @@ General When set to ``1``, overrides the operation's declared split alignment to 256, enabling grouped MLP fusion when its other requirements are met. +.. envvar:: NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM + + :Type: ``int`` (0 or 1) + :Default: ``0`` + :Description: When set to ``1``, the fused grouped MLP runs its plain GEMMs, i.e. those + without a fused activation, on cuBLAS instead of the cuDNN frontend CuTe + DSL kernels where both implement them: the weight-gradient GEMMs and, for + NVFP4, the FC2 forward and FC1 input-gradient GEMMs. A single-group MLP + on the dense single-group path keeps its dense GEMMs either way. + ``NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP`` + is a deprecated alias, honored when this variable is unset. + Attention Backend Selection ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/tests/pytorch/test_grouped_mlp.py b/tests/pytorch/test_grouped_mlp.py index 8c3bf5b2c0..a32847470b 100644 --- a/tests/pytorch/test_grouped_mlp.py +++ b/tests/pytorch/test_grouped_mlp.py @@ -1896,6 +1896,136 @@ def test_grouped_mlp_situglu_nvfp4_real_cudnn_hadamard_fusion( }, ] + @pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) + @pytest.mark.parametrize("activation", ("scaled_swiglu", "scaled_srelu", "scaled_tanh_srelu")) + @pytest.mark.parametrize("disable_cutedsl", (None, "1"), ids=("cutedsl", "cublas")) + @pytest.mark.parametrize("group_size", (4, 1), ids=("grouped", "single_group")) + def test_grouped_mlp_nvfp4_gemm_backend( + self, + monkeypatch, + activation: str, + disable_cutedsl: Optional[str], + group_size: int, + ) -> None: + """FC2 fprop, FC1 dgrad and wgrad run on CuTe DSL unless it is disabled for plain GEMMs. + + A single group on the dense single-group path (every activation but the SReLU ones) + keeps its dense GEMMs and calls no grouped GEMM at all, whatever the setting. + """ + fused_cls = ( + te.ops.fused.GroupedMLP_CuTeGEMMGLU + if activation == "scaled_swiglu" + else te.ops.fused.GroupedMLP_CuTeGEMMUnary + ) + if not fused_cls.is_supported(): + pytest.skip("Fused grouped MLP is not supported on this system") + import cudnn + + gemm_calls = [] + + def _traced(fn, label): + @functools.wraps(fn) + def traced(*args, **kwargs): + gemm_calls.append(kwargs["layout"] if label is None else label) + return fn(*args, **kwargs) + + return traced + + monkeypatch.setattr( + cudnn, + "grouped_gemm_quant_wrapper_sm100", + _traced(cudnn.grouped_gemm_quant_wrapper_sm100, "cutedsl"), + ) + monkeypatch.setattr( + cudnn, + "grouped_gemm_wgrad_wrapper_sm100", + _traced(cudnn.grouped_gemm_wgrad_wrapper_sm100, "cutedsl_wgrad"), + ) + # cuBLAS calls are recorded by their GEMM layout + monkeypatch.setattr( + grouped_mlp_module, + "general_grouped_gemm_for_grouped_tensor", + _traced(grouped_mlp_module.general_grouped_gemm_for_grouped_tensor, None), + ) + if disable_cutedsl is None: + monkeypatch.delenv("NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM", raising=False) + else: + monkeypatch.setenv("NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM", disable_cutedsl) + + def _clear_caches(): + fused_cls.grouped_gemm_quant_kernel.cache_clear() + fused_cls.grouped_gemm_wgrad_kernel.cache_clear() + grouped_mlp_module._use_cutedsl_plain_gemm.cache_clear() + + _clear_caches() + try: + self.test_grouped_mlp( + enable_fusion=True, + group_size=group_size, + bias=False, + hidden_size=256, + quantization="nvfp4_rht", + single_grouped_weight=False, + activation=activation, + strict_fusion=True, + ) + torch.cuda.synchronize() + finally: + _clear_caches() + + # FC2 fprop, FC2 wgrad, FC1 dgrad, FC1 wgrad + expected_calls = { + None: ["cutedsl", "cutedsl_wgrad", "cutedsl", "cutedsl_wgrad"], + "1": ["TN", "NT", "NN", "NT"], + } + dense_single_group = group_size == 1 and activation == "scaled_swiglu" + assert gemm_calls == ([] if dense_single_group else expected_calls[disable_cutedsl]) + + @pytest.mark.parametrize( + "env,expected,warns", + ( + ({}, True, False), + ({"NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM": "1"}, False, False), + ({"NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM": "0"}, True, False), + # Deprecated alias: honored only while the new variable is unset + ({"NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP": "1"}, False, True), + ({"NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP": "0"}, True, True), + ( + { + "NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM": "0", + "NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP": "1", + }, + True, + False, + ), + ), + ids=("unset", "disabled", "enabled", "alias_disabled", "alias_enabled", "new_wins"), + ) + def test_cutedsl_plain_gemm_env_var( + self, + monkeypatch, + recwarn, + env: dict[str, str], + expected: bool, + warns: bool, + ) -> None: + """The plain-GEMM backend switch, including its deprecated wgrad-only alias.""" + for name in ( + "NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM", + "NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP", + ): + monkeypatch.delenv(name, raising=False) + for name, value in env.items(): + monkeypatch.setenv(name, value) + + grouped_mlp_module._use_cutedsl_plain_gemm.cache_clear() + try: + assert grouped_mlp_module._use_cutedsl_plain_gemm() is expected + finally: + grouped_mlp_module._use_cutedsl_plain_gemm.cache_clear() + deprecations = [w for w in recwarn if issubclass(w.category, DeprecationWarning)] + assert bool(deprecations) is warns + @pytest.mark.parametrize("weight_requires_grad", (False, True)) def test_grouped_mlp_prequantized_mxfp8_input( self, @@ -3080,13 +3210,14 @@ def test_grouped_mlp_cuda_graph_safe_mxfp8( split_alignment: int = 256, glu_interleave_size: int = 32, token_padding: int = 2048, + quantization: str = "mxfp8", ) -> None: """Grouped MLP forward+backward should be CUDA graph capturable (MXFP8).""" if not te.ops.fused.GroupedMLP_CuTeGEMMGLU.is_supported(): - pytest.skip("MXFP8 fused grouped MLP is not supported on this system") + pytest.skip("Fused grouped MLP is not supported on this system") if dtype not in (torch.bfloat16, torch.float16): - pytest.skip("MXFP8 fused grouped MLP is only supported with BF16/FP16") + pytest.skip("Fused grouped MLP is only supported with BF16/FP16") split_sizes = [split_alignment * (i + 1) for i in range(group_size)] random.shuffle(split_sizes) @@ -3094,7 +3225,7 @@ def test_grouped_mlp_cuda_graph_safe_mxfp8( # Pad the input tokens to validate the sync-free MOE num_active_tokens = split_sizes.sum().item() in_shape = (num_active_tokens + token_padding, hidden_size) - recipe = make_recipe("mxfp8") + recipe = make_recipe(quantization) with te.quantized_model_init(enabled=True, recipe=recipe): fc1 = te.ops.GroupedLinear( group_size, @@ -3343,6 +3474,76 @@ def train_step( for graph_grad, param in zip(graph_param_grads, reference_module.parameters()): assert_close(graph_grad, param.grad, **tols) + @pytest.mark.parametrize("accumulate_into_main_grad", (False, True)) + @pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) + def test_grouped_mlp_cuda_graph_safe_nvfp4( + self, + monkeypatch, + accumulate_into_main_grad: bool, + ) -> None: + """Grouped MLP forward+backward should be CUDA graph capturable (NVFP4). + + With NVFP4 every GEMM of the fused op runs on a CuTe DSL kernel by default, + including the FC2 forward and FC1 input-gradient GEMMs, which re-swizzle or + re-pack the weight scales on each call. + """ + monkeypatch.delenv("NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM", raising=False) + monkeypatch.delenv("NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP", raising=False) + grouped_mlp_module._use_cutedsl_plain_gemm.cache_clear() + try: + # NVFP4 supports discrete weights only. + self.test_grouped_mlp_cuda_graph_safe_mxfp8( + single_grouped_weight=False, + accumulate_into_main_grad=accumulate_into_main_grad, + activation="scaled_swiglu", + quantization="nvfp4_rht", + ) + finally: + grouped_mlp_module._use_cutedsl_plain_gemm.cache_clear() + + @pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) + def test_grouped_mlp_nvfp4_rejects_single_grouped_weight( + self, + monkeypatch, + *, + dtype: torch.dtype = torch.bfloat16, + device: torch.device = "cuda", + group_size: int = 4, + hidden_size: int = 256, + split_alignment: int = 256, + ) -> None: + """NVFP4 with a single grouped weight fails with a clear error, not inside a kernel.""" + if not te.ops.fused.GroupedMLP_CuTeGEMMGLU.is_supported(): + pytest.skip("Fused grouped MLP is not supported on this system") + monkeypatch.delenv("NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM", raising=False) + monkeypatch.delenv("NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP", raising=False) + grouped_mlp_module._use_cutedsl_plain_gemm.cache_clear() + + split_sizes = torch.full((group_size,), split_alignment, dtype=torch.int64, device=device) + in_shape = (group_size * split_alignment, hidden_size) + recipe = make_recipe("nvfp4_rht") + with te.quantized_model_init(enabled=True, recipe=recipe): + fc_kwargs = dict( + bias=False, + device=device, + dtype=dtype, + single_grouped_weight=True, + split_alignment=split_alignment, + ) + module = te.ops.Sequential( + te.ops.GroupedLinear(group_size, hidden_size, 2 * hidden_size, **fc_kwargs), + te.ops.ScaledSwiGLU(glu_interleave_size=32), + te.ops.GroupedLinear(group_size, hidden_size, hidden_size, **fc_kwargs), + ) + x = torch.randn(in_shape, device=device, dtype=dtype) + probs = torch.randn((in_shape[0],), device=device, dtype=dtype) + try: + with pytest.raises(NotImplementedError, match="single_grouped_weight=True"): + with te.autocast(enabled=True, recipe=recipe): + module(x, split_sizes, probs, split_sizes) + finally: + grouped_mlp_module._use_cutedsl_plain_gemm.cache_clear() + # (group_size, tokens_per_group): token counts chosen so the quantized inputs' byte sizes # never coincide with a weight's, which the saved-tensor counting below relies on. @pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) diff --git a/transformer_engine/pytorch/cpp_extensions/gemm.py b/transformer_engine/pytorch/cpp_extensions/gemm.py index 96bbd490a0..d9bdd1ae29 100644 --- a/transformer_engine/pytorch/cpp_extensions/gemm.py +++ b/transformer_engine/pytorch/cpp_extensions/gemm.py @@ -12,8 +12,9 @@ import torch import transformer_engine_torch as tex from transformer_engine.pytorch.device import te_device_type -from ..constants import TE_DType, DType +from ..constants import MXFP8_BLOCK_SCALING_SIZE, NVFP4_BLOCK_SCALING_SIZE, TE_DType, DType from ..utils import ( + ceil_div, get_device_compute_capability, get_sm_count, _empty_tensor, @@ -223,6 +224,106 @@ def validate_or_alloc_output( return buffer +def _convert_to_cudnn_grouped_gemm_tensor_format( + data: torch.Tensor, + scale_inv: torch.Tensor, + *, + data_dtype: torch.dtype, + scale_dtype: torch.dtype, + valid_M_or_N: int, + k_logical: int, + L: int = 1, + sf_swizzled: bool = False, + use_N_major_for_B: bool = False, +) -> tuple[torch.Tensor, torch.Tensor]: + """Reshape a plain buffer into the layout cuDNN's grouped GEMM expects. + + cuDNN requirements: + A: (valid_m, K, 1), K-major + B: (N, K, L), K-major (FP8 also supports N-major) + + SFA: (32, 4, ceil(valid_m/128), 4, ceil(ceil(K/sf_vec_size)/4), 1) + SFB: (32, 4, ceil(N/128), 4, ceil(ceil(K/sf_vec_size)/4), L) + + whereas TE stores flat buffers which can be intepreted as contiguous tensors + with the following layouts: + + Note: K_packed is K/2 for FP4 (two values per byte) and K for FP8 + + A (K-major): (1, valid_m, K_packed) + B (K-major): (L, N, K_packed) -- used for FP4 only now + B (N-major): (L, K, N) -- used for FP8 only now + + SFA (unswizzled): (1, ceil(valid_m/128), 4, 32, ceil(ceil(K/sf_vec_size)/4), 4) + SFB (unswizzled): (L, ceil(N/128), 4, 32, ceil(ceil(K/sf_vec_size)/4), 4) + SFA (swizzled): (1, ceil(valid_m/128), ceil(ceil(K/sf_vec_size)/4), 32, 4, 4) + SFB (swizzled): (L, ceil(N/128), ceil(ceil(K/sf_vec_size)/4), 32, 4, 4) + + """ + + if use_N_major_for_B: + assert data_dtype in ( + torch.float8_e4m3fn, + torch.float8_e5m2, + ), f"Using N-major layout for B is only supported for FP8, but got {data_dtype}." + + available_scalings = { + # NVFP4 recipe (UE5M3 scales ride as E4M3 since torch has no ue5m3 dtype) + (torch.float4_e2m1fn_x2, torch.float8_e4m3fn): NVFP4_BLOCK_SCALING_SIZE, + # MXFP8 recipe + (torch.float8_e4m3fn, torch.float8_e8m0fnu): MXFP8_BLOCK_SCALING_SIZE, + } + assert (data_dtype, scale_dtype) in available_scalings, ( + "Unsupported (data_dtype, scale_dtype) pair for a cuDNN block-scaled operand: " + f"({data_dtype}, {scale_dtype}). Expected NVFP4 (float4_e2m1fn_x2, " + "float8_e4m3fn) or MXFP8 (float8_e4m3fn, float8_e8m0fnu)." + ) + sf_vec_size = available_scalings[(data_dtype, scale_dtype)] + + k_sf_tiles = ceil_div(k_logical, 4 * sf_vec_size) + + if data_dtype == torch.float4_e2m1fn_x2: + k_packed = k_logical // 2 # fp4 packs two values per byte + else: + k_packed = k_logical # fp8 packs one value per byte + + data = data.view(dtype=data_dtype) + if use_N_major_for_B: + # B is stored untransposed, i.e. (L, K, N); permuting to (N, K, L) leaves + # stride 1 on N. Only FP8 accepts this, asserted above. + data = data.view(L, k_packed, valid_M_or_N) + data = data.permute(2, 1, 0) + else: + # (L, N, K) -> (N, K, L), stride 1 on K. + data = data.view(L, valid_M_or_N, k_packed) + data = data.permute(1, 2, 0) + + if sf_swizzled: + scale_inv = scale_inv.view(dtype=scale_dtype) + scale_inv = scale_inv.view( + L, + ceil_div(valid_M_or_N, 128), + k_sf_tiles, + 32, + 4, + 4, + ) + scale_inv = scale_inv.permute(3, 4, 1, 5, 2, 0) + return data, scale_inv + + scale_inv = scale_inv.view(dtype=scale_dtype) + scale_inv = scale_inv.view( + L, + ceil_div(valid_M_or_N, 128), + 4, + 32, + k_sf_tiles, + 4, + ) + scale_inv = scale_inv.permute(3, 2, 1, 5, 4, 0) + return data, scale_inv + + def general_gemm( A: torch.Tensor, B: torch.Tensor, diff --git a/transformer_engine/pytorch/ops/fused/grouped_mlp.py b/transformer_engine/pytorch/ops/fused/grouped_mlp.py index 6092bd89b5..67f88fac7a 100644 --- a/transformer_engine/pytorch/ops/fused/grouped_mlp.py +++ b/transformer_engine/pytorch/ops/fused/grouped_mlp.py @@ -22,6 +22,7 @@ from ...constants import DType, MXFP8_BLOCK_SCALING_SIZE, NVFP4_BLOCK_SCALING_SIZE, TE_DType from ...cpu_offload import is_cpu_offload_enabled, mark_activation_offload, start_offload from ...cpp_extensions import general_gemm, general_grouped_gemm_for_grouped_tensor +from ...cpp_extensions.gemm import _convert_to_cudnn_grouped_gemm_tensor_format from ...distributed_weight import ( is_distributed_weight, materialize_weight_for_forward, @@ -532,6 +533,31 @@ def _use_tmem_post_rht_amax() -> bool: return os.environ.get("NVTE_CUTEDSL_FUSED_GROUPED_MLP_FC1_GLU_RHT_AMAX_TMEM", "0") == "1" +@functools.lru_cache(maxsize=1) +def _use_cutedsl_plain_gemm() -> bool: + """Whether the plain GEMMs run on the CuTe DSL kernels instead of cuBLAS. + + Applies to the GEMMs without a fused activation that both backends implement: + wgrad, and the NVFP4 FC2 fprop and FC1 dgrad. They use cuBLAS when the environment + variable ``NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM`` is set to ``1``. + + ``NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP``, which used to cover wgrad only, is + still honored as a deprecated alias when the new variable is unset. + """ + value = os.environ.get("NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM") + if value is None: + value = os.environ.get("NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP") + if value is not None: + warnings.warn( + "NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP is deprecated, use " + "NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM instead. Note that it also moves " + "the NVFP4 FC2 forward and FC1 input-gradient GEMMs to cuBLAS.", + DeprecationWarning, + stacklevel=2, + ) + return int(value or "0") < 1 + + def _single_group_wgrad_gemm( grouped_x: GroupedTensor, grouped_dy: GroupedTensor, @@ -1139,14 +1165,8 @@ def grouped_gemm_quant_kernel(cls) -> Callable: @classmethod @functools.lru_cache(maxsize=None) - def grouped_gemm_wgrad_kernel(cls) -> Optional[Callable]: - """CuTe DSL kernel for grouped GEMM wgrad on SM100+. - - Returns ``None`` when the environment variable - ``NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP`` is set to ``1``. - """ - if int(os.environ.get("NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP", "0")) >= 1: - return None + def grouped_gemm_wgrad_kernel(cls) -> Callable: + """CuTe DSL kernel for grouped GEMM wgrad on SM100+.""" from cudnn import grouped_gemm_wgrad_wrapper_sm100 # pylint: disable=no-name-in-module return grouped_gemm_wgrad_wrapper_sm100 @@ -1960,7 +1980,9 @@ def fuser_forward( # FC2 GEMM fc2_out_shape = in_shape[:-1] + [fc2_weight_shape[0]] fc2_scales = basic_op_extra_inputs[2][1] if fc2_op._scale_bias else None - if use_nvfp4: + # A single dense group keeps its dense GEMM. + use_cutedsl_fc2_gemm = _use_cutedsl_plain_gemm() and not use_dense_single_group + if use_nvfp4 and not use_cutedsl_fc2_gemm: fc2_bias_for_gemm = None fc2_bias_scale = None if fc2_bias_packed is not None: @@ -2007,6 +2029,76 @@ def fuser_forward( bias_scale=fc2_bias_scale, ) fc2_out = fc2_out_buf + elif use_nvfp4: + fc2_x_data, fc2_x_scales = _convert_to_cudnn_grouped_gemm_tensor_format( + grouped_fc2_x.rowwise_data, + grouped_fc2_x.scale_inv, + data_dtype=data_dtype, + scale_dtype=scale_view_dtype, + valid_M_or_N=in_shape[0], + k_logical=fc2_weight_shape[1], + sf_swizzled=grouped_fc2_x._with_gemm_swizzled_scales, + ) + + fc2_fwd_alpha_tensor = _nvfp4_global_scale_pair( + grouped_fc2_x, + fc2_input_quantizer, + False, + grouped_fc2_weight, + fc2_weight_quantizer, + False, + num_groups=num_groups, + device=device, + combine="product", + ) + + fc2_scales_tensor = ( + fc2_scales.detach().to(dtype=torch.float32).reshape(-1, 1, 1) + if fc2_scales is not None + else torch.ones((in_shape[0], 1, 1), dtype=torch.float32, device=device) + ) + fc2_quant_kwargs = { + "a_tensor": fc2_x_data, + "sfa_tensor": fc2_x_scales, + "padded_offsets": split_points, + "alpha_tensor": fc2_fwd_alpha_tensor, + "bias_tensor": fc2_bias_packed, + "norm_const_tensor": None, + "prob_tensor": fc2_scales_tensor, + "acc_dtype": torch.float32, + "d_dtype": dtype, + "cd_major": "n", + "sf_vec_size": sf_vec_size, + "sf_fp8_dtype_override": _nvfp4_sf_dtype_override(fc2_input_quantizer), + "current_stream": current_stream, + "use_dynamic_sched": True, + } + + if fc2_op.single_grouped_weight: + raise NotImplementedError( + "The fused grouped MLP does not support NVFP4 with single_grouped_weight=True." + ) + fc2_b_ptrs, fc2_sfb_ptrs, _fc2_sfb_buffer = ( + tex.grouped_mlp_experimental.swizzle_scales_and_pack_ptrs_for_discrete_weights( + [w._rowwise_data for w in grouped_fc2_weight], + [w._rowwise_scale_inv for w in grouped_fc2_weight], + "nvfp4", + device, + ) + ) + fc2_quant_kwargs["b_ptrs"] = fc2_b_ptrs + fc2_quant_kwargs["sfb_ptrs"] = fc2_sfb_ptrs + fc2_quant_kwargs["n"] = fc2_weight_shape[0] + fc2_quant_kwargs["b_dtype"] = data_dtype + fc2_quant_kwargs["b_major"] = "k" + + output_buffer = validate_or_alloc_output(output_buffer, fc2_out_shape, dtype, device) + fc2_quant_kwargs["d_tensor"] = output_buffer.as_strided( + (in_shape[0], fc2_weight_shape[0], 1), + (fc2_weight_shape[0], 1, in_shape[0] * fc2_weight_shape[0]), + ) + self.grouped_gemm_quant_kernel()(**fc2_quant_kwargs) + fc2_out = output_buffer else: use_single_group_dense_fc2 = use_dense_single_group fc2_out_buf = validate_or_alloc_output(output_buffer, fc2_out_shape, dtype, device) @@ -2770,7 +2862,7 @@ def fuser_backward( ) # FC2 wgrad GEMM - wgrad_kernel_fn = self.grouped_gemm_wgrad_kernel() + wgrad_kernel_fn = self.grouped_gemm_wgrad_kernel() if _use_cutedsl_plain_gemm() else None fc2_grad_params = _compute_grad_params( fc_op=fc2_op, ctx=fc2_ctx, @@ -2832,7 +2924,7 @@ def fuser_backward( single_grouped_weight=fc1_op.single_grouped_weight, dtype=dtype, ) - elif use_nvfp4: + elif use_nvfp4 and not _use_cutedsl_plain_gemm(): grad_input = validate_or_alloc_output(grad_input_buffer, in_shape, dtype, device) grouped_grad_input = GroupedTensor( shape=(out_shape[0], fc1_weight_shape[1]), @@ -2849,6 +2941,85 @@ def fuser_backward( grouped_grad_input, layout="NN", ) + elif use_nvfp4: + # This assertion should never fail because we set fc1_grad_output_quantizer.optimize_for_gemm = True + assert ( + grouped_fc1_dy._with_gemm_swizzled_scales + ), "cuDNN NVFP4 dgrad requires GEMM-swizzled grad-output scale factors." + + grad_input_buffer = validate_or_alloc_output( + grad_input_buffer, in_shape, dtype, device + ) + + dgrad_k = fc1_weight_shape[0] # contraction dim + dgrad_valid_m = out_shape[0] # batch dim + + # Create A and its sf tensor for cuDNN that satisfies its layout requirements + fc1_dgrad_a_data, fc1_dgrad_a_scales = _convert_to_cudnn_grouped_gemm_tensor_format( + grouped_fc1_dy.rowwise_data, + grouped_fc1_dy.scale_inv, + data_dtype=data_dtype, + scale_dtype=scale_view_dtype, + valid_M_or_N=dgrad_valid_m, + k_logical=dgrad_k, + sf_swizzled=grouped_fc1_dy._with_gemm_swizzled_scales, + ) + + fc1_dgrad_alpha = _nvfp4_global_scale_pair( + grouped_fc1_dy, + fc1_grad_output_quantizer, + False, + grouped_fc1_weight, + fc1_ctx.weight_quantizers[0], + True, + num_groups=num_groups, + device=device, + combine="product", + ) + + fc1_dgrad_kwargs = { + "a_tensor": fc1_dgrad_a_data, + "sfa_tensor": fc1_dgrad_a_scales, + "padded_offsets": split_points, + "alpha_tensor": fc1_dgrad_alpha, + "norm_const_tensor": None, # must be None for FP4 inputs + "acc_dtype": torch.float32, + "d_dtype": dtype, # high precision -> no output quantization + "cd_major": "n", + "sf_vec_size": sf_vec_size, + "sf_fp8_dtype_override": _nvfp4_sf_dtype_override(fc1_grad_output_quantizer), + "current_stream": current_stream, + "discrete_col_sfd": False, + "use_dynamic_sched": True, + } + + if fc1_op.single_grouped_weight: + raise NotImplementedError( + "The fused grouped MLP does not support NVFP4 with" + " single_grouped_weight=True." + ) + fc1_b_ptrs, fc1_sfb_ptrs, _fc1_sfb_buffer = ( + tex.grouped_mlp_experimental.swizzle_scales_and_pack_ptrs_for_discrete_weights( + [w._columnwise_data for w in grouped_fc1_weight], + [w._columnwise_scale_inv for w in grouped_fc1_weight], + "nvfp4", + device, + ) + ) + fc1_dgrad_kwargs["b_ptrs"] = fc1_b_ptrs + fc1_dgrad_kwargs["sfb_ptrs"] = fc1_sfb_ptrs + fc1_dgrad_kwargs["n"] = fc1_weight_shape[1] + fc1_dgrad_kwargs["b_dtype"] = torch.float4_e2m1fn_x2 + # FP4 has no N-major operand support, and the column-wise buffer is + # already transposed, so it is K-major. + fc1_dgrad_kwargs["b_major"] = "k" + + fc1_dgrad_kwargs["d_tensor"] = grad_input_buffer.as_strided( + (out_shape[0], fc1_weight_shape[1], 1), + (fc1_weight_shape[1], 1, out_shape[0] * fc1_weight_shape[1]), + ) + self.grouped_gemm_quant_kernel()(**fc1_dgrad_kwargs) + grad_input = grad_input_buffer else: fc1_dgrad_a_data = fc2_dgrad_kernel_out["d_row_tensor"] fc1_dgrad_a_scales = fc2_dgrad_kernel_out["sfd_row_tensor"] From 506da5c54f36993bf5f9860eb685a79973b3ede7 Mon Sep 17 00:00:00 2001 From: Xiaowei Ren Date: Fri, 20 Nov 2026 00:24:11 -0800 Subject: [PATCH 2/5] [PyTorch] Pass canonical layouts to cuDNN for NVFP4 FC2 fprop and FC1 dgrad Pass the operands of the NVFP4 FC2 fprop and FC1 dgrad CuTe DSL GEMMs in cuDNN's canonical layouts instead of reshaping them into the kernel-facing strided views, and drop the reshape helper from cpp_extensions/gemm.py. These two GEMMs fall back to cuBLAS when the cuDNN frontend predates canonical layouts, whatever the plain-GEMM backend setting. Reject NVFP4 with single_grouped_weight=True once, up front, so the error does not depend on the GEMM backend. Co-Authored-By: Claude Signed-off-by: Xiaowei Ren --- docs/envvars.rst | 4 +- tests/pytorch/test_grouped_mlp.py | 12 +- .../pytorch/cpp_extensions/gemm.py | 103 +----------------- .../pytorch/ops/fused/grouped_mlp.py | 96 +++++++--------- 4 files changed, 55 insertions(+), 160 deletions(-) diff --git a/docs/envvars.rst b/docs/envvars.rst index e61c5c5be5..7f394dde2f 100644 --- a/docs/envvars.rst +++ b/docs/envvars.rst @@ -186,7 +186,9 @@ General without a fused activation, on cuBLAS instead of the cuDNN frontend CuTe DSL kernels where both implement them: the weight-gradient GEMMs and, for NVFP4, the FC2 forward and FC1 input-gradient GEMMs. A single-group MLP - on the dense single-group path keeps its dense GEMMs either way. + on the dense single-group path keeps its dense GEMMs either way, and the + two NVFP4 GEMMs always run on cuBLAS when the cuDNN frontend predates + canonical data layouts (1.32.1). ``NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP`` is a deprecated alias, honored when this variable is unset. diff --git a/tests/pytorch/test_grouped_mlp.py b/tests/pytorch/test_grouped_mlp.py index a32847470b..7e746bbf3d 100644 --- a/tests/pytorch/test_grouped_mlp.py +++ b/tests/pytorch/test_grouped_mlp.py @@ -1909,6 +1909,9 @@ def test_grouped_mlp_nvfp4_gemm_backend( ) -> None: """FC2 fprop, FC1 dgrad and wgrad run on CuTe DSL unless it is disabled for plain GEMMs. + FC2 fprop and FC1 dgrad also need a cuDNN frontend with canonical layouts, and + fall back to cuBLAS without one. + A single group on the dense single-group path (every activation but the SReLU ones) keeps its dense GEMMs and calls no grouped GEMM at all, whatever the setting. """ @@ -1974,8 +1977,15 @@ def _clear_caches(): _clear_caches() # FC2 fprop, FC2 wgrad, FC1 dgrad, FC1 wgrad + canonical = getattr( + cudnn.grouped_gemm_quant_wrapper_sm100, "supports_canonical_layouts", False + ) expected_calls = { - None: ["cutedsl", "cutedsl_wgrad", "cutedsl", "cutedsl_wgrad"], + None: ( + ["cutedsl", "cutedsl_wgrad", "cutedsl", "cutedsl_wgrad"] + if canonical + else ["TN", "cutedsl_wgrad", "NN", "cutedsl_wgrad"] + ), "1": ["TN", "NT", "NN", "NT"], } dense_single_group = group_size == 1 and activation == "scaled_swiglu" diff --git a/transformer_engine/pytorch/cpp_extensions/gemm.py b/transformer_engine/pytorch/cpp_extensions/gemm.py index d9bdd1ae29..96bbd490a0 100644 --- a/transformer_engine/pytorch/cpp_extensions/gemm.py +++ b/transformer_engine/pytorch/cpp_extensions/gemm.py @@ -12,9 +12,8 @@ import torch import transformer_engine_torch as tex from transformer_engine.pytorch.device import te_device_type -from ..constants import MXFP8_BLOCK_SCALING_SIZE, NVFP4_BLOCK_SCALING_SIZE, TE_DType, DType +from ..constants import TE_DType, DType from ..utils import ( - ceil_div, get_device_compute_capability, get_sm_count, _empty_tensor, @@ -224,106 +223,6 @@ def validate_or_alloc_output( return buffer -def _convert_to_cudnn_grouped_gemm_tensor_format( - data: torch.Tensor, - scale_inv: torch.Tensor, - *, - data_dtype: torch.dtype, - scale_dtype: torch.dtype, - valid_M_or_N: int, - k_logical: int, - L: int = 1, - sf_swizzled: bool = False, - use_N_major_for_B: bool = False, -) -> tuple[torch.Tensor, torch.Tensor]: - """Reshape a plain buffer into the layout cuDNN's grouped GEMM expects. - - cuDNN requirements: - A: (valid_m, K, 1), K-major - B: (N, K, L), K-major (FP8 also supports N-major) - - SFA: (32, 4, ceil(valid_m/128), 4, ceil(ceil(K/sf_vec_size)/4), 1) - SFB: (32, 4, ceil(N/128), 4, ceil(ceil(K/sf_vec_size)/4), L) - - whereas TE stores flat buffers which can be intepreted as contiguous tensors - with the following layouts: - - Note: K_packed is K/2 for FP4 (two values per byte) and K for FP8 - - A (K-major): (1, valid_m, K_packed) - B (K-major): (L, N, K_packed) -- used for FP4 only now - B (N-major): (L, K, N) -- used for FP8 only now - - SFA (unswizzled): (1, ceil(valid_m/128), 4, 32, ceil(ceil(K/sf_vec_size)/4), 4) - SFB (unswizzled): (L, ceil(N/128), 4, 32, ceil(ceil(K/sf_vec_size)/4), 4) - SFA (swizzled): (1, ceil(valid_m/128), ceil(ceil(K/sf_vec_size)/4), 32, 4, 4) - SFB (swizzled): (L, ceil(N/128), ceil(ceil(K/sf_vec_size)/4), 32, 4, 4) - - """ - - if use_N_major_for_B: - assert data_dtype in ( - torch.float8_e4m3fn, - torch.float8_e5m2, - ), f"Using N-major layout for B is only supported for FP8, but got {data_dtype}." - - available_scalings = { - # NVFP4 recipe (UE5M3 scales ride as E4M3 since torch has no ue5m3 dtype) - (torch.float4_e2m1fn_x2, torch.float8_e4m3fn): NVFP4_BLOCK_SCALING_SIZE, - # MXFP8 recipe - (torch.float8_e4m3fn, torch.float8_e8m0fnu): MXFP8_BLOCK_SCALING_SIZE, - } - assert (data_dtype, scale_dtype) in available_scalings, ( - "Unsupported (data_dtype, scale_dtype) pair for a cuDNN block-scaled operand: " - f"({data_dtype}, {scale_dtype}). Expected NVFP4 (float4_e2m1fn_x2, " - "float8_e4m3fn) or MXFP8 (float8_e4m3fn, float8_e8m0fnu)." - ) - sf_vec_size = available_scalings[(data_dtype, scale_dtype)] - - k_sf_tiles = ceil_div(k_logical, 4 * sf_vec_size) - - if data_dtype == torch.float4_e2m1fn_x2: - k_packed = k_logical // 2 # fp4 packs two values per byte - else: - k_packed = k_logical # fp8 packs one value per byte - - data = data.view(dtype=data_dtype) - if use_N_major_for_B: - # B is stored untransposed, i.e. (L, K, N); permuting to (N, K, L) leaves - # stride 1 on N. Only FP8 accepts this, asserted above. - data = data.view(L, k_packed, valid_M_or_N) - data = data.permute(2, 1, 0) - else: - # (L, N, K) -> (N, K, L), stride 1 on K. - data = data.view(L, valid_M_or_N, k_packed) - data = data.permute(1, 2, 0) - - if sf_swizzled: - scale_inv = scale_inv.view(dtype=scale_dtype) - scale_inv = scale_inv.view( - L, - ceil_div(valid_M_or_N, 128), - k_sf_tiles, - 32, - 4, - 4, - ) - scale_inv = scale_inv.permute(3, 4, 1, 5, 2, 0) - return data, scale_inv - - scale_inv = scale_inv.view(dtype=scale_dtype) - scale_inv = scale_inv.view( - L, - ceil_div(valid_M_or_N, 128), - 4, - 32, - k_sf_tiles, - 4, - ) - scale_inv = scale_inv.permute(3, 2, 1, 5, 4, 0) - return data, scale_inv - - def general_gemm( A: torch.Tensor, B: torch.Tensor, diff --git a/transformer_engine/pytorch/ops/fused/grouped_mlp.py b/transformer_engine/pytorch/ops/fused/grouped_mlp.py index 67f88fac7a..371be9d0eb 100644 --- a/transformer_engine/pytorch/ops/fused/grouped_mlp.py +++ b/transformer_engine/pytorch/ops/fused/grouped_mlp.py @@ -22,7 +22,6 @@ from ...constants import DType, MXFP8_BLOCK_SCALING_SIZE, NVFP4_BLOCK_SCALING_SIZE, TE_DType from ...cpu_offload import is_cpu_offload_enabled, mark_activation_offload, start_offload from ...cpp_extensions import general_gemm, general_grouped_gemm_for_grouped_tensor -from ...cpp_extensions.gemm import _convert_to_cudnn_grouped_gemm_tensor_format from ...distributed_weight import ( is_distributed_weight, materialize_weight_for_forward, @@ -558,6 +557,20 @@ def _use_cutedsl_plain_gemm() -> bool: return int(value or "0") < 1 +def _use_cutedsl_nvfp4_plain_gemm(kernel: Callable, grouped_input: GroupedTensorStorage) -> bool: + """Whether an NVFP4 FC2 fprop or FC1 dgrad GEMM runs on its CuTe DSL kernel. + + The operands are passed in cuDNN's canonical layouts, so the kernel has to accept + them and the input scales must be swizzled for GEMM. Otherwise the GEMM runs on + cuBLAS, whatever the plain-GEMM backend setting. + """ + return ( + _use_cutedsl_plain_gemm() + and getattr(kernel, "supports_canonical_layouts", False) + and grouped_input._with_gemm_swizzled_scales + ) + + def _single_group_wgrad_gemm( grouped_x: GroupedTensor, grouped_dy: GroupedTensor, @@ -1530,6 +1543,10 @@ def fuser_forward( use_nvfp4 = isinstance(fc1_input_quantizer, NVFP4Quantizer) or isinstance( fc1_weight_param, NVFP4Tensor ) + if use_nvfp4 and (fc1_op.single_grouped_weight or fc2_op.single_grouped_weight): + raise NotImplementedError( + "The fused grouped MLP does not support NVFP4 with single_grouped_weight=True." + ) data_dtype = torch.float4_e2m1fn_x2 if use_nvfp4 else torch.float8_e4m3fn scale_view_dtype = torch.float8_e4m3fn if use_nvfp4 else torch.float8_e8m0fnu sf_vec_size = NVFP4_BLOCK_SCALING_SIZE if use_nvfp4 else MXFP8_BLOCK_SCALING_SIZE @@ -1981,7 +1998,11 @@ def fuser_forward( fc2_out_shape = in_shape[:-1] + [fc2_weight_shape[0]] fc2_scales = basic_op_extra_inputs[2][1] if fc2_op._scale_bias else None # A single dense group keeps its dense GEMM. - use_cutedsl_fc2_gemm = _use_cutedsl_plain_gemm() and not use_dense_single_group + use_cutedsl_fc2_gemm = ( + use_nvfp4 + and not use_dense_single_group + and _use_cutedsl_nvfp4_plain_gemm(self.grouped_gemm_quant_kernel(), grouped_fc2_x) + ) if use_nvfp4 and not use_cutedsl_fc2_gemm: fc2_bias_for_gemm = None fc2_bias_scale = None @@ -2030,15 +2051,10 @@ def fuser_forward( ) fc2_out = fc2_out_buf elif use_nvfp4: - fc2_x_data, fc2_x_scales = _convert_to_cudnn_grouped_gemm_tensor_format( - grouped_fc2_x.rowwise_data, - grouped_fc2_x.scale_inv, - data_dtype=data_dtype, - scale_dtype=scale_view_dtype, - valid_M_or_N=in_shape[0], - k_logical=fc2_weight_shape[1], - sf_swizzled=grouped_fc2_x._with_gemm_swizzled_scales, - ) + # Canonical layouts: A is (sum(m), k) and its scales are the packed buffer as-is. + fc2_x_data = grouped_fc2_x.rowwise_data.view(dtype=data_dtype) + fc2_x_data = fc2_x_data.view(in_shape[0], fc2_weight_shape[1] // 2) + fc2_x_scales = grouped_fc2_x.scale_inv.view(dtype=scale_view_dtype) fc2_fwd_alpha_tensor = _nvfp4_global_scale_pair( grouped_fc2_x, @@ -2053,9 +2069,9 @@ def fuser_forward( ) fc2_scales_tensor = ( - fc2_scales.detach().to(dtype=torch.float32).reshape(-1, 1, 1) + fc2_scales.detach().to(dtype=torch.float32).reshape(-1) if fc2_scales is not None - else torch.ones((in_shape[0], 1, 1), dtype=torch.float32, device=device) + else torch.ones(in_shape[0], dtype=torch.float32, device=device) ) fc2_quant_kwargs = { "a_tensor": fc2_x_data, @@ -2074,10 +2090,6 @@ def fuser_forward( "use_dynamic_sched": True, } - if fc2_op.single_grouped_weight: - raise NotImplementedError( - "The fused grouped MLP does not support NVFP4 with single_grouped_weight=True." - ) fc2_b_ptrs, fc2_sfb_ptrs, _fc2_sfb_buffer = ( tex.grouped_mlp_experimental.swizzle_scales_and_pack_ptrs_for_discrete_weights( [w._rowwise_data for w in grouped_fc2_weight], @@ -2092,13 +2104,9 @@ def fuser_forward( fc2_quant_kwargs["b_dtype"] = data_dtype fc2_quant_kwargs["b_major"] = "k" - output_buffer = validate_or_alloc_output(output_buffer, fc2_out_shape, dtype, device) - fc2_quant_kwargs["d_tensor"] = output_buffer.as_strided( - (in_shape[0], fc2_weight_shape[0], 1), - (fc2_weight_shape[0], 1, in_shape[0] * fc2_weight_shape[0]), - ) + fc2_out = validate_or_alloc_output(output_buffer, fc2_out_shape, dtype, device) + fc2_quant_kwargs["d_tensor"] = fc2_out self.grouped_gemm_quant_kernel()(**fc2_quant_kwargs) - fc2_out = output_buffer else: use_single_group_dense_fc2 = use_dense_single_group fc2_out_buf = validate_or_alloc_output(output_buffer, fc2_out_shape, dtype, device) @@ -2924,7 +2932,9 @@ def fuser_backward( single_grouped_weight=fc1_op.single_grouped_weight, dtype=dtype, ) - elif use_nvfp4 and not _use_cutedsl_plain_gemm(): + elif use_nvfp4 and not _use_cutedsl_nvfp4_plain_gemm( + self.grouped_gemm_quant_kernel(), grouped_fc1_dy + ): grad_input = validate_or_alloc_output(grad_input_buffer, in_shape, dtype, device) grouped_grad_input = GroupedTensor( shape=(out_shape[0], fc1_weight_shape[1]), @@ -2942,28 +2952,10 @@ def fuser_backward( layout="NN", ) elif use_nvfp4: - # This assertion should never fail because we set fc1_grad_output_quantizer.optimize_for_gemm = True - assert ( - grouped_fc1_dy._with_gemm_swizzled_scales - ), "cuDNN NVFP4 dgrad requires GEMM-swizzled grad-output scale factors." - - grad_input_buffer = validate_or_alloc_output( - grad_input_buffer, in_shape, dtype, device - ) - - dgrad_k = fc1_weight_shape[0] # contraction dim - dgrad_valid_m = out_shape[0] # batch dim - - # Create A and its sf tensor for cuDNN that satisfies its layout requirements - fc1_dgrad_a_data, fc1_dgrad_a_scales = _convert_to_cudnn_grouped_gemm_tensor_format( - grouped_fc1_dy.rowwise_data, - grouped_fc1_dy.scale_inv, - data_dtype=data_dtype, - scale_dtype=scale_view_dtype, - valid_M_or_N=dgrad_valid_m, - k_logical=dgrad_k, - sf_swizzled=grouped_fc1_dy._with_gemm_swizzled_scales, - ) + # Canonical layouts: A is (sum(m), k) and its scales are the packed buffer as-is. + fc1_dgrad_a_data = grouped_fc1_dy.rowwise_data.view(dtype=data_dtype) + fc1_dgrad_a_data = fc1_dgrad_a_data.view(out_shape[0], fc1_weight_shape[0] // 2) + fc1_dgrad_a_scales = grouped_fc1_dy.scale_inv.view(dtype=scale_view_dtype) fc1_dgrad_alpha = _nvfp4_global_scale_pair( grouped_fc1_dy, @@ -2993,11 +2985,6 @@ def fuser_backward( "use_dynamic_sched": True, } - if fc1_op.single_grouped_weight: - raise NotImplementedError( - "The fused grouped MLP does not support NVFP4 with" - " single_grouped_weight=True." - ) fc1_b_ptrs, fc1_sfb_ptrs, _fc1_sfb_buffer = ( tex.grouped_mlp_experimental.swizzle_scales_and_pack_ptrs_for_discrete_weights( [w._columnwise_data for w in grouped_fc1_weight], @@ -3014,12 +3001,9 @@ def fuser_backward( # already transposed, so it is K-major. fc1_dgrad_kwargs["b_major"] = "k" - fc1_dgrad_kwargs["d_tensor"] = grad_input_buffer.as_strided( - (out_shape[0], fc1_weight_shape[1], 1), - (fc1_weight_shape[1], 1, out_shape[0] * fc1_weight_shape[1]), - ) + grad_input = validate_or_alloc_output(grad_input_buffer, in_shape, dtype, device) + fc1_dgrad_kwargs["d_tensor"] = grad_input self.grouped_gemm_quant_kernel()(**fc1_dgrad_kwargs) - grad_input = grad_input_buffer else: fc1_dgrad_a_data = fc2_dgrad_kernel_out["d_row_tensor"] fc1_dgrad_a_scales = fc2_dgrad_kernel_out["sfd_row_tensor"] From 9db1cc4d44b2e2414ffbfe8cf021d97d68ecf8ab Mon Sep 17 00:00:00 2001 From: Xiaowei Ren Date: Fri, 20 Nov 2026 00:35:31 -0800 Subject: [PATCH 3/5] [PyTorch] Rename the grouped MLP unfused GEMM backend variable Rename NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM to NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP, since "plain GEMM" does not say which GEMMs are meant, and rename the helpers and tests to match. Co-Authored-By: Claude Signed-off-by: Xiaowei Ren --- docs/envvars.rst | 4 +- tests/pytorch/test_grouped_mlp.py | 54 +++++++++---------- .../pytorch/ops/fused/grouped_mlp.py | 30 ++++++----- 3 files changed, 45 insertions(+), 43 deletions(-) diff --git a/docs/envvars.rst b/docs/envvars.rst index 7f394dde2f..71bc749adb 100644 --- a/docs/envvars.rst +++ b/docs/envvars.rst @@ -178,11 +178,11 @@ General When set to ``1``, overrides the operation's declared split alignment to 256, enabling grouped MLP fusion when its other requirements are met. -.. envvar:: NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM +.. envvar:: NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP :Type: ``int`` (0 or 1) :Default: ``0`` - :Description: When set to ``1``, the fused grouped MLP runs its plain GEMMs, i.e. those + :Description: When set to ``1``, the fused grouped MLP runs its unfused GEMMs, i.e. those without a fused activation, on cuBLAS instead of the cuDNN frontend CuTe DSL kernels where both implement them: the weight-gradient GEMMs and, for NVFP4, the FC2 forward and FC1 input-gradient GEMMs. A single-group MLP diff --git a/tests/pytorch/test_grouped_mlp.py b/tests/pytorch/test_grouped_mlp.py index 7e746bbf3d..9d523459a4 100644 --- a/tests/pytorch/test_grouped_mlp.py +++ b/tests/pytorch/test_grouped_mlp.py @@ -1898,16 +1898,16 @@ def test_grouped_mlp_situglu_nvfp4_real_cudnn_hadamard_fusion( @pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) @pytest.mark.parametrize("activation", ("scaled_swiglu", "scaled_srelu", "scaled_tanh_srelu")) - @pytest.mark.parametrize("disable_cutedsl", (None, "1"), ids=("cutedsl", "cublas")) + @pytest.mark.parametrize("disable_cudnn", (None, "1"), ids=("cutedsl", "cublas")) @pytest.mark.parametrize("group_size", (4, 1), ids=("grouped", "single_group")) def test_grouped_mlp_nvfp4_gemm_backend( self, monkeypatch, activation: str, - disable_cutedsl: Optional[str], + disable_cudnn: Optional[str], group_size: int, ) -> None: - """FC2 fprop, FC1 dgrad and wgrad run on CuTe DSL unless it is disabled for plain GEMMs. + """FC2 fprop, FC1 dgrad and wgrad run on CuTe DSL unless the cuDNN unfused GEMMs are disabled. FC2 fprop and FC1 dgrad also need a cuDNN frontend with canonical layouts, and fall back to cuBLAS without one. @@ -1950,15 +1950,15 @@ def traced(*args, **kwargs): "general_grouped_gemm_for_grouped_tensor", _traced(grouped_mlp_module.general_grouped_gemm_for_grouped_tensor, None), ) - if disable_cutedsl is None: - monkeypatch.delenv("NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM", raising=False) + if disable_cudnn is None: + monkeypatch.delenv("NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP", raising=False) else: - monkeypatch.setenv("NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM", disable_cutedsl) + monkeypatch.setenv("NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP", disable_cudnn) def _clear_caches(): fused_cls.grouped_gemm_quant_kernel.cache_clear() fused_cls.grouped_gemm_wgrad_kernel.cache_clear() - grouped_mlp_module._use_cutedsl_plain_gemm.cache_clear() + grouped_mlp_module._cudnn_unfused_gemm_disabled.cache_clear() _clear_caches() try: @@ -1989,29 +1989,29 @@ def _clear_caches(): "1": ["TN", "NT", "NN", "NT"], } dense_single_group = group_size == 1 and activation == "scaled_swiglu" - assert gemm_calls == ([] if dense_single_group else expected_calls[disable_cutedsl]) + assert gemm_calls == ([] if dense_single_group else expected_calls[disable_cudnn]) @pytest.mark.parametrize( "env,expected,warns", ( - ({}, True, False), - ({"NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM": "1"}, False, False), - ({"NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM": "0"}, True, False), + ({}, False, False), + ({"NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP": "1"}, True, False), + ({"NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP": "0"}, False, False), # Deprecated alias: honored only while the new variable is unset - ({"NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP": "1"}, False, True), - ({"NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP": "0"}, True, True), + ({"NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP": "1"}, True, True), + ({"NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP": "0"}, False, True), ( { - "NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM": "0", + "NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP": "0", "NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP": "1", }, - True, + False, False, ), ), ids=("unset", "disabled", "enabled", "alias_disabled", "alias_enabled", "new_wins"), ) - def test_cutedsl_plain_gemm_env_var( + def test_cudnn_unfused_gemm_disabled_env_var( self, monkeypatch, recwarn, @@ -2019,20 +2019,20 @@ def test_cutedsl_plain_gemm_env_var( expected: bool, warns: bool, ) -> None: - """The plain-GEMM backend switch, including its deprecated wgrad-only alias.""" + """The cuDNN unfused GEMM switch, including its deprecated wgrad-only alias.""" for name in ( - "NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM", + "NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP", "NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP", ): monkeypatch.delenv(name, raising=False) for name, value in env.items(): monkeypatch.setenv(name, value) - grouped_mlp_module._use_cutedsl_plain_gemm.cache_clear() + grouped_mlp_module._cudnn_unfused_gemm_disabled.cache_clear() try: - assert grouped_mlp_module._use_cutedsl_plain_gemm() is expected + assert grouped_mlp_module._cudnn_unfused_gemm_disabled() is expected finally: - grouped_mlp_module._use_cutedsl_plain_gemm.cache_clear() + grouped_mlp_module._cudnn_unfused_gemm_disabled.cache_clear() deprecations = [w for w in recwarn if issubclass(w.category, DeprecationWarning)] assert bool(deprecations) is warns @@ -3497,9 +3497,9 @@ def test_grouped_mlp_cuda_graph_safe_nvfp4( including the FC2 forward and FC1 input-gradient GEMMs, which re-swizzle or re-pack the weight scales on each call. """ - monkeypatch.delenv("NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM", raising=False) + monkeypatch.delenv("NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP", raising=False) monkeypatch.delenv("NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP", raising=False) - grouped_mlp_module._use_cutedsl_plain_gemm.cache_clear() + grouped_mlp_module._cudnn_unfused_gemm_disabled.cache_clear() try: # NVFP4 supports discrete weights only. self.test_grouped_mlp_cuda_graph_safe_mxfp8( @@ -3509,7 +3509,7 @@ def test_grouped_mlp_cuda_graph_safe_nvfp4( quantization="nvfp4_rht", ) finally: - grouped_mlp_module._use_cutedsl_plain_gemm.cache_clear() + grouped_mlp_module._cudnn_unfused_gemm_disabled.cache_clear() @pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) def test_grouped_mlp_nvfp4_rejects_single_grouped_weight( @@ -3525,9 +3525,9 @@ def test_grouped_mlp_nvfp4_rejects_single_grouped_weight( """NVFP4 with a single grouped weight fails with a clear error, not inside a kernel.""" if not te.ops.fused.GroupedMLP_CuTeGEMMGLU.is_supported(): pytest.skip("Fused grouped MLP is not supported on this system") - monkeypatch.delenv("NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM", raising=False) + monkeypatch.delenv("NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP", raising=False) monkeypatch.delenv("NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP", raising=False) - grouped_mlp_module._use_cutedsl_plain_gemm.cache_clear() + grouped_mlp_module._cudnn_unfused_gemm_disabled.cache_clear() split_sizes = torch.full((group_size,), split_alignment, dtype=torch.int64, device=device) in_shape = (group_size * split_alignment, hidden_size) @@ -3552,7 +3552,7 @@ def test_grouped_mlp_nvfp4_rejects_single_grouped_weight( with te.autocast(enabled=True, recipe=recipe): module(x, split_sizes, probs, split_sizes) finally: - grouped_mlp_module._use_cutedsl_plain_gemm.cache_clear() + grouped_mlp_module._cudnn_unfused_gemm_disabled.cache_clear() # (group_size, tokens_per_group): token counts chosen so the quantized inputs' byte sizes # never coincide with a weight's, which the saved-tensor counting below relies on. diff --git a/transformer_engine/pytorch/ops/fused/grouped_mlp.py b/transformer_engine/pytorch/ops/fused/grouped_mlp.py index 371be9d0eb..162fd6e047 100644 --- a/transformer_engine/pytorch/ops/fused/grouped_mlp.py +++ b/transformer_engine/pytorch/ops/fused/grouped_mlp.py @@ -533,39 +533,39 @@ def _use_tmem_post_rht_amax() -> bool: @functools.lru_cache(maxsize=1) -def _use_cutedsl_plain_gemm() -> bool: - """Whether the plain GEMMs run on the CuTe DSL kernels instead of cuBLAS. +def _cudnn_unfused_gemm_disabled() -> bool: + """Whether the unfused GEMMs run on cuBLAS instead of the cuDNN CuTe DSL kernels. - Applies to the GEMMs without a fused activation that both backends implement: - wgrad, and the NVFP4 FC2 fprop and FC1 dgrad. They use cuBLAS when the environment - variable ``NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM`` is set to ``1``. + Applies to the GEMMs without a fused activation that both backends implement: wgrad, + and the NVFP4 FC2 fprop and FC1 dgrad. Set by the environment variable + ``NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP``. ``NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP``, which used to cover wgrad only, is still honored as a deprecated alias when the new variable is unset. """ - value = os.environ.get("NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM") + value = os.environ.get("NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP") if value is None: value = os.environ.get("NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP") if value is not None: warnings.warn( "NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP is deprecated, use " - "NVTE_DISABLE_CUTEDSL_PLAIN_GROUPED_MLP_GEMM instead. Note that it also moves " + "NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP instead. Note that it also moves " "the NVFP4 FC2 forward and FC1 input-gradient GEMMs to cuBLAS.", DeprecationWarning, stacklevel=2, ) - return int(value or "0") < 1 + return int(value or "0") >= 1 -def _use_cutedsl_nvfp4_plain_gemm(kernel: Callable, grouped_input: GroupedTensorStorage) -> bool: +def _use_cutedsl_nvfp4_gemm(kernel: Callable, grouped_input: GroupedTensorStorage) -> bool: """Whether an NVFP4 FC2 fprop or FC1 dgrad GEMM runs on its CuTe DSL kernel. The operands are passed in cuDNN's canonical layouts, so the kernel has to accept them and the input scales must be swizzled for GEMM. Otherwise the GEMM runs on - cuBLAS, whatever the plain-GEMM backend setting. + cuBLAS, even when the cuDNN unfused GEMMs are not disabled. """ return ( - _use_cutedsl_plain_gemm() + not _cudnn_unfused_gemm_disabled() and getattr(kernel, "supports_canonical_layouts", False) and grouped_input._with_gemm_swizzled_scales ) @@ -2001,7 +2001,7 @@ def fuser_forward( use_cutedsl_fc2_gemm = ( use_nvfp4 and not use_dense_single_group - and _use_cutedsl_nvfp4_plain_gemm(self.grouped_gemm_quant_kernel(), grouped_fc2_x) + and _use_cutedsl_nvfp4_gemm(self.grouped_gemm_quant_kernel(), grouped_fc2_x) ) if use_nvfp4 and not use_cutedsl_fc2_gemm: fc2_bias_for_gemm = None @@ -2870,7 +2870,9 @@ def fuser_backward( ) # FC2 wgrad GEMM - wgrad_kernel_fn = self.grouped_gemm_wgrad_kernel() if _use_cutedsl_plain_gemm() else None + wgrad_kernel_fn = ( + None if _cudnn_unfused_gemm_disabled() else self.grouped_gemm_wgrad_kernel() + ) fc2_grad_params = _compute_grad_params( fc_op=fc2_op, ctx=fc2_ctx, @@ -2932,7 +2934,7 @@ def fuser_backward( single_grouped_weight=fc1_op.single_grouped_weight, dtype=dtype, ) - elif use_nvfp4 and not _use_cutedsl_nvfp4_plain_gemm( + elif use_nvfp4 and not _use_cutedsl_nvfp4_gemm( self.grouped_gemm_quant_kernel(), grouped_fc1_dy ): grad_input = validate_or_alloc_output(grad_input_buffer, in_shape, dtype, device) From 1ed26b2e1216813574ed37fd54b9582cabbf301d Mon Sep 17 00:00:00 2001 From: Xiaowei Ren Date: Fri, 20 Nov 2026 01:07:38 -0800 Subject: [PATCH 4/5] [PyTorch] Respect the grouped MLP unfused GEMM backend choice for MXFP8 Run the MXFP8 FC2 fprop and FC1 dgrad GEMMs of the fused grouped MLP on cuBLAS when NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP is set, as is already done for NVFP4 and for the wgrad GEMMs. A dense single group now takes the branch shared with cuBLAS for both recipes, so drop the unreachable dense case from the MXFP8 kernel branch. Co-Authored-By: Claude Signed-off-by: Xiaowei Ren --- docs/envvars.rst | 10 +- tests/pytorch/test_grouped_mlp.py | 25 ++- .../pytorch/ops/fused/grouped_mlp.py | 212 +++++++++--------- 3 files changed, 123 insertions(+), 124 deletions(-) diff --git a/docs/envvars.rst b/docs/envvars.rst index 71bc749adb..031a85cd21 100644 --- a/docs/envvars.rst +++ b/docs/envvars.rst @@ -184,11 +184,11 @@ General :Default: ``0`` :Description: When set to ``1``, the fused grouped MLP runs its unfused GEMMs, i.e. those without a fused activation, on cuBLAS instead of the cuDNN frontend CuTe - DSL kernels where both implement them: the weight-gradient GEMMs and, for - NVFP4, the FC2 forward and FC1 input-gradient GEMMs. A single-group MLP - on the dense single-group path keeps its dense GEMMs either way, and the - two NVFP4 GEMMs always run on cuBLAS when the cuDNN frontend predates - canonical data layouts (1.32.1). + DSL kernels: the FC2 forward, FC1 input-gradient and weight-gradient + GEMMs. A single-group MLP on the dense single-group path keeps its dense + GEMMs either way, and the NVFP4 FC2 forward and FC1 input-gradient GEMMs + always run on cuBLAS when the cuDNN frontend predates canonical data + layouts (1.32.1). ``NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP`` is a deprecated alias, honored when this variable is unset. diff --git a/tests/pytorch/test_grouped_mlp.py b/tests/pytorch/test_grouped_mlp.py index 9d523459a4..ad0807edfa 100644 --- a/tests/pytorch/test_grouped_mlp.py +++ b/tests/pytorch/test_grouped_mlp.py @@ -1896,21 +1896,27 @@ def test_grouped_mlp_situglu_nvfp4_real_cudnn_hadamard_fusion( }, ] - @pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) @pytest.mark.parametrize("activation", ("scaled_swiglu", "scaled_srelu", "scaled_tanh_srelu")) @pytest.mark.parametrize("disable_cudnn", (None, "1"), ids=("cutedsl", "cublas")) @pytest.mark.parametrize("group_size", (4, 1), ids=("grouped", "single_group")) - def test_grouped_mlp_nvfp4_gemm_backend( + @pytest.mark.parametrize( + "quantization,single_grouped_weight", + (("nvfp4_rht", False), ("mxfp8", False), ("mxfp8", True)), + ids=("nvfp4", "mxfp8", "mxfp8_single_grouped_weight"), + ) + def test_grouped_mlp_unfused_gemm_backend( self, monkeypatch, activation: str, disable_cudnn: Optional[str], group_size: int, + quantization: str, + single_grouped_weight: bool, ) -> None: """FC2 fprop, FC1 dgrad and wgrad run on CuTe DSL unless the cuDNN unfused GEMMs are disabled. - FC2 fprop and FC1 dgrad also need a cuDNN frontend with canonical layouts, and - fall back to cuBLAS without one. + The NVFP4 FC2 fprop and FC1 dgrad also need a cuDNN frontend with canonical + layouts, and fall back to cuBLAS without one. A single group on the dense single-group path (every activation but the SReLU ones) keeps its dense GEMMs and calls no grouped GEMM at all, whatever the setting. @@ -1965,10 +1971,11 @@ def _clear_caches(): self.test_grouped_mlp( enable_fusion=True, group_size=group_size, - bias=False, + # The NVFP4 coverage of test_grouped_mlp is limited to no-bias. + bias=quantization == "mxfp8", hidden_size=256, - quantization="nvfp4_rht", - single_grouped_weight=False, + quantization=quantization, + single_grouped_weight=single_grouped_weight, activation=activation, strict_fusion=True, ) @@ -1977,13 +1984,13 @@ def _clear_caches(): _clear_caches() # FC2 fprop, FC2 wgrad, FC1 dgrad, FC1 wgrad - canonical = getattr( + cutedsl_fprop_dgrad = quantization == "mxfp8" or getattr( cudnn.grouped_gemm_quant_wrapper_sm100, "supports_canonical_layouts", False ) expected_calls = { None: ( ["cutedsl", "cutedsl_wgrad", "cutedsl", "cutedsl_wgrad"] - if canonical + if cutedsl_fprop_dgrad else ["TN", "cutedsl_wgrad", "NN", "cutedsl_wgrad"] ), "1": ["TN", "NT", "NN", "NT"], diff --git a/transformer_engine/pytorch/ops/fused/grouped_mlp.py b/transformer_engine/pytorch/ops/fused/grouped_mlp.py index 162fd6e047..f162bc0b7a 100644 --- a/transformer_engine/pytorch/ops/fused/grouped_mlp.py +++ b/transformer_engine/pytorch/ops/fused/grouped_mlp.py @@ -536,9 +536,8 @@ def _use_tmem_post_rht_amax() -> bool: def _cudnn_unfused_gemm_disabled() -> bool: """Whether the unfused GEMMs run on cuBLAS instead of the cuDNN CuTe DSL kernels. - Applies to the GEMMs without a fused activation that both backends implement: wgrad, - and the NVFP4 FC2 fprop and FC1 dgrad. Set by the environment variable - ``NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP``. + Applies to the GEMMs without a fused activation: FC2 fprop, FC1 dgrad and wgrad. Set by + the environment variable ``NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP``. ``NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP``, which used to cover wgrad only, is still honored as a deprecated alias when the new variable is unset. @@ -550,23 +549,31 @@ def _cudnn_unfused_gemm_disabled() -> bool: warnings.warn( "NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP is deprecated, use " "NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP instead. Note that it also moves " - "the NVFP4 FC2 forward and FC1 input-gradient GEMMs to cuBLAS.", + "the FC2 forward and FC1 input-gradient GEMMs to cuBLAS.", DeprecationWarning, stacklevel=2, ) return int(value or "0") >= 1 -def _use_cutedsl_nvfp4_gemm(kernel: Callable, grouped_input: GroupedTensorStorage) -> bool: - """Whether an NVFP4 FC2 fprop or FC1 dgrad GEMM runs on its CuTe DSL kernel. +def _use_cutedsl_fprop_dgrad_gemm( + kernel: Callable, + grouped_input: GroupedTensorStorage, + *, + use_nvfp4: bool, +) -> bool: + """Whether an FC2 fprop or FC1 dgrad GEMM runs on its CuTe DSL kernel instead of cuBLAS. - The operands are passed in cuDNN's canonical layouts, so the kernel has to accept - them and the input scales must be swizzled for GEMM. Otherwise the GEMM runs on - cuBLAS, even when the cuDNN unfused GEMMs are not disabled. + NVFP4 operands are passed in cuDNN's canonical layouts, so the kernel has to accept + them and the input scales must be swizzled for GEMM. Otherwise the NVFP4 GEMM runs + on cuBLAS, even when the cuDNN unfused GEMMs are not disabled. """ + if _cudnn_unfused_gemm_disabled(): + return False + if not use_nvfp4: + return True return ( - not _cudnn_unfused_gemm_disabled() - and getattr(kernel, "supports_canonical_layouts", False) + getattr(kernel, "supports_canonical_layouts", False) and grouped_input._with_gemm_swizzled_scales ) @@ -1998,12 +2005,10 @@ def fuser_forward( fc2_out_shape = in_shape[:-1] + [fc2_weight_shape[0]] fc2_scales = basic_op_extra_inputs[2][1] if fc2_op._scale_bias else None # A single dense group keeps its dense GEMM. - use_cutedsl_fc2_gemm = ( - use_nvfp4 - and not use_dense_single_group - and _use_cutedsl_nvfp4_gemm(self.grouped_gemm_quant_kernel(), grouped_fc2_x) + use_cutedsl_fc2_gemm = not use_dense_single_group and _use_cutedsl_fprop_dgrad_gemm( + self.grouped_gemm_quant_kernel(), grouped_fc2_x, use_nvfp4=use_nvfp4 ) - if use_nvfp4 and not use_cutedsl_fc2_gemm: + if not use_cutedsl_fc2_gemm: fc2_bias_for_gemm = None fc2_bias_scale = None if fc2_bias_packed is not None: @@ -2108,105 +2113,92 @@ def fuser_forward( fc2_quant_kwargs["d_tensor"] = fc2_out self.grouped_gemm_quant_kernel()(**fc2_quant_kwargs) else: - use_single_group_dense_fc2 = use_dense_single_group fc2_out_buf = validate_or_alloc_output(output_buffer, fc2_out_shape, dtype, device) - if use_single_group_dense_fc2: - fc2_out = _single_group_fc2_gemm( - grouped_fc2_x, - grouped_fc2_weight, - fc2_input_quantizer, - fc2_out_buf, - single_grouped_weight=fc2_op.single_grouped_weight, - bias=fc2_bias_packed, - bias_scale=fc2_scales, - dtype=dtype, + fc2_scales_tensor = None + if fc2_scales is not None: + fc2_scales_tensor = ( + fc2_scales.detach() + .to(dtype=torch.float32) + .reshape((-1,) if canonical else (-1, 1, 1)) + ) + fc2_quant_kwargs = { + "a_tensor": fc1_kernel_out["d_tensor"], + "sfa_tensor": fc1_kernel_out["sfd_row_tensor"], + "padded_offsets": split_points, + "alpha_tensor": alpha_tensor, + "bias_tensor": fc2_bias_packed, + "norm_const_tensor": None, + "prob_tensor": fc2_scales_tensor, + "acc_dtype": torch.float32, + "d_dtype": dtype, + "cd_major": "n", + "sf_vec_size": MXFP8_BLOCK_SCALING_SIZE, + "current_stream": current_stream, + "use_dynamic_sched": True, + } + fc2_quant_kernel = self.grouped_gemm_quant_kernel() + if getattr(fc2_quant_kernel, "supports_optional_amax", False): + # The FC2 output amax is not used. + fc2_quant_kwargs["generate_amax"] = False + if supports_single_group_runtime_offsets: + fc2_quant_kwargs["use_single_group_runtime_offsets"] = use_dense_single_group + + if fc2_op.single_grouped_weight: + # Clone and swizzle scales for GEMM + # (original stays unmodified for save_for_backward). + fc2_weight_for_gemm = grouped_fc2_weight.copy() + tex.grouped_swizzle_for_gemm( + fc2_weight_for_gemm, + rowwise=True, + columnwise=False, ) - else: - fc2_scales_tensor = None - if fc2_scales is not None: - fc2_scales_tensor = ( - fc2_scales.detach() - .to(dtype=torch.float32) - .reshape((-1,) if canonical else (-1, 1, 1)) - ) - fc2_quant_kwargs = { - "a_tensor": fc1_kernel_out["d_tensor"], - "sfa_tensor": fc1_kernel_out["sfd_row_tensor"], - "padded_offsets": split_points, - "alpha_tensor": alpha_tensor, - "bias_tensor": fc2_bias_packed, - "norm_const_tensor": None, - "prob_tensor": fc2_scales_tensor, - "acc_dtype": torch.float32, - "d_dtype": dtype, - "cd_major": "n", - "sf_vec_size": MXFP8_BLOCK_SCALING_SIZE, - "current_stream": current_stream, - "use_dynamic_sched": True, - } - fc2_quant_kernel = self.grouped_gemm_quant_kernel() - if getattr(fc2_quant_kernel, "supports_optional_amax", False): - # The FC2 output amax is not used. - fc2_quant_kwargs["generate_amax"] = False - if supports_single_group_runtime_offsets: - fc2_quant_kwargs["use_single_group_runtime_offsets"] = use_dense_single_group - - if fc2_op.single_grouped_weight: - # Clone and swizzle scales for GEMM - # (original stays unmodified for save_for_backward). - fc2_weight_for_gemm = grouped_fc2_weight.copy() - tex.grouped_swizzle_for_gemm( - fc2_weight_for_gemm, - rowwise=True, - columnwise=False, - ) - fc2_w_data = fc2_weight_for_gemm.rowwise_data - fc2_w_data = fc2_w_data.view(dtype=torch.float8_e4m3fn) - fc2_w_data = fc2_w_data.view( + fc2_w_data = fc2_weight_for_gemm.rowwise_data + fc2_w_data = fc2_w_data.view(dtype=torch.float8_e4m3fn) + fc2_w_data = fc2_w_data.view( + num_groups, + fc2_weight_shape[0], + fc2_weight_shape[1], + ) + fc2_w_scales = fc2_weight_for_gemm.scale_inv.view(dtype=torch.float8_e8m0fnu) + if not canonical: + fc2_w_data = fc2_w_data.permute(1, 2, 0) + fc2_w_scales = fc2_w_scales.view( num_groups, - fc2_weight_shape[0], - fc2_weight_shape[1], - ) - fc2_w_scales = fc2_weight_for_gemm.scale_inv.view(dtype=torch.float8_e8m0fnu) - if not canonical: - fc2_w_data = fc2_w_data.permute(1, 2, 0) - fc2_w_scales = fc2_w_scales.view( - num_groups, - ceil_div(fc2_weight_shape[0], 128), - ceil_div(fc2_weight_shape[1], 128), - MXFP8_BLOCK_SCALING_SIZE, - 4, - 4, - ) - fc2_w_scales = fc2_w_scales.permute(3, 4, 1, 5, 2, 0) - fc2_quant_kwargs["b_tensor"] = fc2_w_data - fc2_quant_kwargs["sfb_tensor"] = fc2_w_scales - else: - fc2_b_ptrs, fc2_sfb_ptrs, _fc2_sfb_buffer = ( - tex.grouped_mlp_experimental.swizzle_scales_and_pack_ptrs_for_discrete_weights( - [w._rowwise_data for w in grouped_fc2_weight], - [w._rowwise_scale_inv for w in grouped_fc2_weight], - "mxfp8_rowwise", - device, - ) + ceil_div(fc2_weight_shape[0], 128), + ceil_div(fc2_weight_shape[1], 128), + MXFP8_BLOCK_SCALING_SIZE, + 4, + 4, ) - fc2_quant_kwargs["b_ptrs"] = fc2_b_ptrs - fc2_quant_kwargs["sfb_ptrs"] = fc2_sfb_ptrs - fc2_quant_kwargs["n"] = fc2_weight_shape[0] - fc2_quant_kwargs["b_dtype"] = torch.float8_e4m3fn - fc2_quant_kwargs["b_major"] = "k" - - fc2_quant_kwargs["d_tensor"] = ( - fc2_out_buf - if canonical - else fc2_out_buf.as_strided( - (in_shape[0], fc2_weight_shape[0], 1), - (fc2_weight_shape[0], 1, in_shape[0] * fc2_weight_shape[0]), + fc2_w_scales = fc2_w_scales.permute(3, 4, 1, 5, 2, 0) + fc2_quant_kwargs["b_tensor"] = fc2_w_data + fc2_quant_kwargs["sfb_tensor"] = fc2_w_scales + else: + fc2_b_ptrs, fc2_sfb_ptrs, _fc2_sfb_buffer = ( + tex.grouped_mlp_experimental.swizzle_scales_and_pack_ptrs_for_discrete_weights( + [w._rowwise_data for w in grouped_fc2_weight], + [w._rowwise_scale_inv for w in grouped_fc2_weight], + "mxfp8_rowwise", + device, ) ) - fc2_quant_kernel(**fc2_quant_kwargs) - fc2_out = fc2_out_buf + fc2_quant_kwargs["b_ptrs"] = fc2_b_ptrs + fc2_quant_kwargs["sfb_ptrs"] = fc2_sfb_ptrs + fc2_quant_kwargs["n"] = fc2_weight_shape[0] + fc2_quant_kwargs["b_dtype"] = torch.float8_e4m3fn + fc2_quant_kwargs["b_major"] = "k" + + fc2_quant_kwargs["d_tensor"] = ( + fc2_out_buf + if canonical + else fc2_out_buf.as_strided( + (in_shape[0], fc2_weight_shape[0], 1), + (fc2_weight_shape[0], 1, in_shape[0] * fc2_weight_shape[0]), + ) + ) + fc2_quant_kernel(**fc2_quant_kwargs) + fc2_out = fc2_out_buf # Save state for backward pass if requires_grad: @@ -2934,8 +2926,8 @@ def fuser_backward( single_grouped_weight=fc1_op.single_grouped_weight, dtype=dtype, ) - elif use_nvfp4 and not _use_cutedsl_nvfp4_gemm( - self.grouped_gemm_quant_kernel(), grouped_fc1_dy + elif not _use_cutedsl_fprop_dgrad_gemm( + self.grouped_gemm_quant_kernel(), grouped_fc1_dy, use_nvfp4=use_nvfp4 ): grad_input = validate_or_alloc_output(grad_input_buffer, in_shape, dtype, device) grouped_grad_input = GroupedTensor( From 54032766242a091a0a44a0573d00a97bd0199dbe Mon Sep 17 00:00:00 2001 From: Xiaowei Ren Date: Fri, 20 Nov 2026 01:13:31 -0800 Subject: [PATCH 5/5] [PyTorch] Parametrize the grouped MLP CUDA graph test over quantization Turn test_grouped_mlp_cuda_graph_safe_mxfp8 into test_grouped_mlp_cuda_graph_safe, parametrized over MXFP8 and NVFP4, and drop the NVFP4 wrapper that called the MXFP8 test. Co-Authored-By: Claude Signed-off-by: Xiaowei Ren --- tests/pytorch/test_grouped_mlp.py | 38 ++++++------------------------- 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/tests/pytorch/test_grouped_mlp.py b/tests/pytorch/test_grouped_mlp.py index ad0807edfa..59c1f60d4e 100644 --- a/tests/pytorch/test_grouped_mlp.py +++ b/tests/pytorch/test_grouped_mlp.py @@ -3213,28 +3213,31 @@ def _run_backward(module, fc1, fc2): @pytest.mark.parametrize("single_grouped_weight", (False, True)) @pytest.mark.parametrize("accumulate_into_main_grad", (False, True)) @pytest.mark.parametrize("activation", ("scaled_swiglu", "scaled_clamped_qgeglu")) - @pytest.mark.skipif(not mxfp8_available, reason=reason_for_no_mxfp8) - def test_grouped_mlp_cuda_graph_safe_mxfp8( + @pytest.mark.parametrize("quantization", ("mxfp8", "nvfp4_rht")) + def test_grouped_mlp_cuda_graph_safe( self, *, dtype: torch.dtype = torch.bfloat16, single_grouped_weight: bool, accumulate_into_main_grad: bool, activation: str, + quantization: str, device: torch.device = "cuda", group_size: int = 4, hidden_size: int = 256, split_alignment: int = 256, glu_interleave_size: int = 32, token_padding: int = 2048, - quantization: str = "mxfp8", ) -> None: - """Grouped MLP forward+backward should be CUDA graph capturable (MXFP8).""" + """Grouped MLP forward+backward should be CUDA graph capturable.""" + maybe_skip_quantization(quantization, device=device, dtype=dtype) if not te.ops.fused.GroupedMLP_CuTeGEMMGLU.is_supported(): pytest.skip("Fused grouped MLP is not supported on this system") if dtype not in (torch.bfloat16, torch.float16): pytest.skip("Fused grouped MLP is only supported with BF16/FP16") + if single_grouped_weight and quantization != "mxfp8": + pytest.skip("single_grouped_weight is only supported for MXFP8") split_sizes = [split_alignment * (i + 1) for i in range(group_size)] random.shuffle(split_sizes) @@ -3491,33 +3494,6 @@ def train_step( for graph_grad, param in zip(graph_param_grads, reference_module.parameters()): assert_close(graph_grad, param.grad, **tols) - @pytest.mark.parametrize("accumulate_into_main_grad", (False, True)) - @pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) - def test_grouped_mlp_cuda_graph_safe_nvfp4( - self, - monkeypatch, - accumulate_into_main_grad: bool, - ) -> None: - """Grouped MLP forward+backward should be CUDA graph capturable (NVFP4). - - With NVFP4 every GEMM of the fused op runs on a CuTe DSL kernel by default, - including the FC2 forward and FC1 input-gradient GEMMs, which re-swizzle or - re-pack the weight scales on each call. - """ - monkeypatch.delenv("NVTE_DISABLE_CUDNN_UNFUSED_GEMM_IN_GROUPED_MLP", raising=False) - monkeypatch.delenv("NVTE_DISABLE_CUTEDSL_WGRAD_FUSED_GROUPED_MLP", raising=False) - grouped_mlp_module._cudnn_unfused_gemm_disabled.cache_clear() - try: - # NVFP4 supports discrete weights only. - self.test_grouped_mlp_cuda_graph_safe_mxfp8( - single_grouped_weight=False, - accumulate_into_main_grad=accumulate_into_main_grad, - activation="scaled_swiglu", - quantization="nvfp4_rht", - ) - finally: - grouped_mlp_module._cudnn_unfused_gemm_disabled.cache_clear() - @pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) def test_grouped_mlp_nvfp4_rejects_single_grouped_weight( self,