diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index 4b5b543815..f2a1992148 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -6,6 +6,7 @@ import importlib import inspect +from types import SimpleNamespace import jax import jax.numpy as jnp @@ -13,6 +14,7 @@ import pytest from jax.sharding import Mesh +from transformer_engine.jax.cpp_extensions.activation import ActivationParams from transformer_engine.jax.flax import _MoEBlock from transformer_engine.jax.moe import ( RouterComputationInfo, @@ -22,6 +24,65 @@ from transformer_engine.jax.sharding import MeshResource, global_mesh_resource, global_shard_guard +@pytest.mark.parametrize("soft_caps", [None, {"gate_beta": 2.0, "up_beta": 8.0}]) +def test_situ_activation_params_reach_grouped_gemm_adapters(monkeypatch, soft_caps): + module = importlib.import_module("transformer_engine.jax.moe") + adapter = importlib.import_module("transformer_engine.jax.cpp_extensions.grouped_gemm_glu") + params = module._resolve_moe_activation_params("situglu", soft_caps) + assert isinstance(params, ActivationParams) + calls = [] + scales = jnp.ones((512,), dtype=jnp.uint8) + + def fused(**kwargs): + calls.append(kwargs) + return dict( + c_tensor=jnp.ones((1, 64)), + d_tensor=jnp.ones((1, 32)), + d_col_tensor=jnp.ones((1, 32)), + d_row_tensor=jnp.ones((1, 64)), + dprob_tensor=jnp.ones((1,)), + sfd_row_tensor=scales, + sfd_col_tensor=scales, + ) + + cudnn = SimpleNamespace( + grouped_gemm_glu_wrapper_sm100=fused, grouped_gemm_dglu_wrapper_sm100=fused + ) + original_import = importlib.import_module + monkeypatch.setattr( + adapter.importlib, + "import_module", + lambda name: cudnn if name == "cudnn" else original_import(name), + ) + adapter.grouped_gemm_glu( + jnp.ones((1, 32, 1)), + jnp.ones((1, 64, 32)), + scales, + scales, + jnp.asarray([1]), + jnp.ones((1,)), + compute_dtype=jnp.bfloat16, + output_dtype=jnp.float8_e4m3fn, + **module._fused_glu_activation_kwargs("situglu", params), + ) + adapter.grouped_gemm_dglu( + jnp.ones((1, 32)), + jnp.ones((1, 32, 32)), + jnp.ones((1, 64)), + scales, + scales, + jnp.asarray([1]), + jnp.ones((1,)), + output_dtype=jnp.float8_e4m3fn, + **module._fused_glu_activation_kwargs("situglu", params, backward=True), + ) + expected = soft_caps or {"gate_beta": 4.0, "up_beta": 25.0} + assert [call["act_func"] for call in calls] == ["situglu", "dsituglu"] + for call in calls: + assert call["situ_beta1"] == expected["gate_beta"] + assert call["situ_beta2"] == expected["up_beta"] + + @pytest.fixture(autouse=True) def no_global_resource(): with global_shard_guard(None): diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index 71535d2f93..888b0c257d 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -282,6 +282,7 @@ def mesh(use_cudnn_fusion): "num_experts_per_tok", "aux_loss_coeff", "score_function", + "activation_type", ), ) def _pure_jax_moe_reference( @@ -298,6 +299,7 @@ def _pure_jax_moe_reference( aux_loss_coeff: float = 0.0, score_function: str = "softmax", scaling_factor: float = 1.0, + activation_type="silu", ): B, S, H = x.shape T = B * S @@ -348,7 +350,13 @@ def _pure_jax_moe_reference( # Activation runs in x.dtype (typically bf16) to mirror the impl -- # the impl keeps silu+multiply in the wi GEMM output dtype because # storing higher precision than the consumer (wo) GEMM buys nothing. - intermediate = jax.nn.silu(layer_w0) * layer_w1 + if activation_type == "situglu": + gate, up = layer_w0.astype(jnp.float32), layer_w1.astype(jnp.float32) + capped_gate = 4.0 * jnp.tanh(gate / 4.0) + capped_up = 25.0 * jnp.tanh(up / 25.0) + intermediate = (capped_gate * jax.nn.sigmoid(gate) * capped_up).astype(x.dtype) + else: + intermediate = jax.nn.silu(layer_w0) * layer_w1 expert_out = jnp.einsum("tem,emh->teh", intermediate, wo) # [T, E, H] output_2d = jnp.einsum("te,teh->th", routing_weights_full.astype(x.dtype), expert_out) output = output_2d.reshape(B, S, H).astype(x.dtype) @@ -403,8 +411,12 @@ def _make_block( mesh_resource=None, dispatch_quantization_recipe=None, use_cudnn_fusion, + activation_type="silu", + activation_params=None, ): kwargs = dict( + activation_type=activation_type, + activation_params=activation_params, num_experts=NUM_EXPERTS, num_experts_per_tok=TOPK, intermediate_size=INTER, @@ -1263,7 +1275,9 @@ def unexpected_fused_kernel(*args, **kwargs): # independently forcing the baseline to execute the unfused FFN. moe_module = importlib.import_module("transformer_engine.jax.moe") with monkeypatch.context() as baseline_patch: - baseline_patch.setattr(moe_module, "_select_cudnn_jax_fusion", lambda _: False) + baseline_patch.setattr( + moe_module, "_select_cudnn_jax_fusion", lambda *args, **kwargs: False + ) variables, baseline_output, _, _ = _init_apply( baseline, mesh, x, jax.random.PRNGKey(52) ) @@ -1298,8 +1312,16 @@ def native_moe(*args, **kwargs): _to_global_numpy(_unwrap(baseline_grads["params"][name]), mesh), ) - @pytest.mark.parametrize("native_layout", [False, True]) - @pytest.mark.parametrize("apply_topk_weights_early", [False, True]) + @pytest.mark.parametrize( + "activation_type,native_layout,apply_topk_weights_early", + [ + pytest.param("silu", False, False, id="swiglu-standard-late"), + pytest.param("silu", False, True, id="swiglu-standard-early"), + pytest.param("silu", True, False, id="swiglu-native-late"), + pytest.param("silu", True, True, id="swiglu-native-early"), + pytest.param("situglu", False, False, id="situ-default"), + ], + ) def test_mxfp8_forward_and_backward( self, mesh, @@ -1308,12 +1330,15 @@ def test_mxfp8_forward_and_backward( make_block, use_cudnn_fusion, native_layout, + activation_type, ): if not use_cudnn_fusion: pytest.skip("requires the cuDNN fusion fixture parameter") from transformer_engine.jax import cpp_extensions as tex - available, reason = tex.grouped_gemm_glu_dependencies_available() + available, reason = tex.grouped_gemm_glu_dependencies_available( + act_func="situglu" if activation_type == "situglu" else "swiglu" + ) if not available: pytest.skip(reason) @@ -1336,10 +1361,18 @@ def native_moe(*args, **kwargs): original_dglu = tex.grouped_gemm_dglu def checked_glu(*args, **kwargs): + if activation_type == "situglu": + assert kwargs["act_func"] == "situglu" + caps = kwargs["act_params"].situglu + assert (caps.gate_beta, caps.up_beta) == (4.0, 25.0) shared_calls.append("glu") return original_glu(*args, **kwargs) def checked_dglu(*args, **kwargs): + if activation_type == "situglu": + assert kwargs["act_func"] == "dsituglu" + caps = kwargs["act_params"].situglu + assert (caps.gate_beta, caps.up_beta) == (4.0, 25.0) shared_calls.append("dglu") return original_dglu(*args, **kwargs) @@ -1348,6 +1381,10 @@ def checked_dglu(*args, **kwargs): block = make_block( apply_topk_weights_early=apply_topk_weights_early, quantization_recipe=MXFP8BlockScaling(), + activation_type=activation_type, + activation_params=( + {"gate_beta": 4.0, "up_beta": 25.0} if activation_type == "situglu" else None + ), ) x = _make_inputs(jax.random.PRNGKey(30)) variables, output, aux, _ = _init_apply(block, mesh, x, jax.random.PRNGKey(31)) @@ -1383,6 +1420,7 @@ def loss_fn(params, inputs): num_experts_per_tok=TOPK, score_function="softmax", expert_bias=None, + activation_type=activation_type, ) return jnp.mean(out.astype(jnp.float32) ** 2) @@ -1391,16 +1429,19 @@ def loss_fn(params, inputs): jnp.asarray(x_np), ) for name in ("gate_kernel", "wi", "wo"): + grad_np = _to_global_numpy(_unwrap(grads["params"][name]), mesh).astype(np.float32) + grad_ref_np = np.asarray(jax.device_get(grads_ref[name])).astype(np.float32) np.testing.assert_allclose( - _to_global_numpy(_unwrap(grads["params"][name]), mesh).astype(np.float32), - np.asarray(jax.device_get(grads_ref[name])).astype(np.float32), - **( - GRAD_GATE_TOLERANCE["mxfp8"] - if name == "gate_kernel" - else GRAD_FFN_TOLERANCE["mxfp8"] - ), + grad_np, + grad_ref_np, + atol=3e-6, + rtol=1.3e-6, err_msg=f"{name} fused MXFP8 gradient parity breach", ) + relative_error = np.linalg.norm(grad_np - grad_ref_np) / np.linalg.norm(grad_ref_np) + assert ( + relative_error < 0.1 + ), f"{name} MXFP8 relative L2 error {relative_error:.3%} >= 10%" grad_x_ref_np = np.asarray(jax.device_get(grad_x_ref)).astype(np.float32) np.testing.assert_allclose( grad_x_np, diff --git a/transformer_engine/jax/cpp_extensions/activation.py b/transformer_engine/jax/cpp_extensions/activation.py index 5058192c3f..a1d6aed607 100644 --- a/transformer_engine/jax/cpp_extensions/activation.py +++ b/transformer_engine/jax/cpp_extensions/activation.py @@ -3,6 +3,7 @@ # See LICENSE for license information. """JAX/TE custom ops for activation""" from typing import Sequence, Union, Callable, Optional, Tuple +import math import operator from functools import reduce, partial from dataclasses import dataclass @@ -88,13 +89,27 @@ def to_ffi_lowering_dict(self): } +@dataclass(frozen=True) +class SiTUParams: + """Soft caps for the gate and up branches of SiTU-GLU.""" + + gate_beta: float = 4.0 + up_beta: float = 25.0 + + def __post_init__(self): + for name in ("gate_beta", "up_beta"): + value = float(getattr(self, name)) + if not math.isfinite(value) or value <= 0: + raise ValueError(f"{name} must be finite and positive, got {value}") + object.__setattr__(self, name, value) + + @dataclass(frozen=True) class ActivationParams: - """Parameters for various activation functions. - Currently only Clamped SwiGLU activation has parameters. - """ + """Parameters for activation functions.""" clamped_swiglu: ClampedSwigluParams = ClampedSwigluParams() + situglu: SiTUParams = SiTUParams() @staticmethod def create(activation_type, **kwargs): @@ -106,11 +121,13 @@ def create(activation_type, **kwargs): } if activation_type in CLAMPED_ACTIVATION_TYPES: return ActivationParams(ClampedSwigluParams(**kwargs)) + if activation_type == "situglu": + return ActivationParams(situglu=SiTUParams(**kwargs)) return ActivationParams() # Default params for activations without parameters def __hash__(self): """Custom hash function to ensure dataclass is hashable for jax jit to work""" - return hash((self.clamped_swiglu,)) + return hash((self.clamped_swiglu, self.situglu)) def to_ffi_lowering_dict(self): """Convert the activation parameters to a dictionary format for FFI lowering. diff --git a/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py b/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py index 86c497218c..72de2e607b 100644 --- a/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py +++ b/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py @@ -12,6 +12,8 @@ import jax.numpy as jnp from packaging.version import Version +from .activation import ActivationParams + __all__ = [ "grouped_gemm_dglu", "grouped_gemm_glu", @@ -108,15 +110,21 @@ def _compact_sf(scale: jax.Array, shape: tuple[int, ...], name: str) -> jax.Arra ) -def grouped_gemm_glu_dependencies_available() -> tuple[bool, str]: - """Require Frontend >= 1.32.0 and shared JAX APIs supporting dense MXFP8.""" +def grouped_gemm_glu_dependencies_available(*, act_func: str = "swiglu") -> tuple[bool, str]: + """Require Frontend >= 1.33.0 for SiTU or >= 1.32.0 for other GLU activations.""" + if act_func not in ("swiglu", "geglu", "situglu"): + return False, f"Unsupported grouped GLU activation: {act_func}" try: cutlass_jax = importlib.import_module("cutlass.jax") if not cutlass_jax.is_available(): return False, "CuTeDSL JAX support is unavailable" cudnn = importlib.import_module("cudnn") - if Version(cudnn.__version__) < Version("1.32.0"): - return False, f"cuDNN Frontend >= 1.32.0 is required, got {cudnn.__version__}" + minimum_version = "1.33.0" if act_func == "situglu" else "1.32.0" + if Version(cudnn.__version__) < Version(minimum_version): + return ( + False, + f"cuDNN Frontend >= {minimum_version} is required, got {cudnn.__version__}", + ) for module, name, arguments in ( (cudnn, "grouped_gemm_glu_wrapper_sm100", _FORWARD_ARGS), (cudnn, "grouped_gemm_dglu_wrapper_sm100", _BACKWARD_ARGS), @@ -126,6 +134,8 @@ def grouped_gemm_glu_dependencies_available() -> tuple[bool, str]: api = getattr(module, name, None) if not callable(api): return False, f"{module.__name__}.{name} is unavailable" + if act_func == "situglu": + arguments += ("situ_beta1", "situ_beta2") signature = inspect.signature(api) missing = set(arguments) - signature.parameters.keys() if missing: @@ -151,8 +161,10 @@ def grouped_gemm_glu( *, compute_dtype, output_dtype, + act_func: str = "swiglu", + act_params: ActivationParams = ActivationParams(), ) -> tuple[jax.Array, jax.Array, jax.Array, jax.Array, jax.Array]: - """Run cuDNN's shared grouped MXFP8 GLU API in SwiGLU mode.""" + """Run cuDNN's shared grouped MXFP8 GLU API with the selected activation.""" cudnn_grouped_gemm_glu = getattr( importlib.import_module("cudnn"), "grouped_gemm_glu_wrapper_sm100" ) @@ -167,6 +179,8 @@ def grouped_gemm_glu( prob, compute_dtype=compute_dtype, output_dtype=output_dtype, + act_func=act_func, + act_params=act_params, ) @@ -181,6 +195,8 @@ def _grouped_gemm_forward( *, compute_dtype, output_dtype, + act_func: str = "swiglu", + act_params: ActivationParams = ActivationParams(), ): rows, hidden, _ = a.shape experts, combined, b_hidden = b.shape @@ -188,6 +204,12 @@ def _grouped_gemm_forward( raise ValueError(f"A K={hidden} does not match B K={b_hidden}") alpha = jnp.ones((experts,), dtype=jnp.float32) norm_const = jnp.ones((1,), dtype=jnp.float32) + situ_kwargs = {} + if act_func == "situglu": + situ_kwargs = { + "situ_beta1": act_params.situglu.gate_beta, + "situ_beta2": act_params.situglu.up_beta, + } result = cudnn_forward( a_tensor=a.reshape(rows, hidden), b_tensor=b, @@ -201,8 +223,9 @@ def _grouped_gemm_forward( d_dtype=jnp.dtype(output_dtype), discrete_col_sfd=True, sf_vec_size=32, - act_func="swiglu", + act_func=act_func, generate_c=True, + **situ_kwargs, ) return ( result["c_tensor"].reshape(rows, combined), @@ -223,8 +246,10 @@ def grouped_gemm_dglu( prob: jax.Array, *, output_dtype, + act_func: str = "dswiglu", + act_params: ActivationParams = ActivationParams(), ) -> tuple[jax.Array, jax.Array, jax.Array, jax.Array, jax.Array]: - """Run cuDNN's shared grouped MXFP8 dGLU API in dSwiGLU mode.""" + """Run cuDNN's shared grouped MXFP8 dGLU API with the selected activation.""" if a.ndim != 2: raise ValueError(f"Expected A[M,K], got {a.shape}") if b.ndim != 3: @@ -246,6 +271,12 @@ def grouped_gemm_dglu( alpha = jnp.ones((experts,), dtype=jnp.float32) beta = jnp.ones((experts,), dtype=jnp.float32) norm_const = jnp.ones((1,), dtype=jnp.float32) + situ_kwargs = {} + if act_func == "dsituglu": + situ_kwargs = { + "situ_beta1": act_params.situglu.gate_beta, + "situ_beta2": act_params.situglu.up_beta, + } result = cudnn_grouped_gemm_dglu( a_tensor=a, b_tensor=b, @@ -261,7 +292,8 @@ def grouped_gemm_dglu( discrete_col_sfd=True, dprob_tensor=None, sf_vec_size=32, - act_func="dswiglu", + act_func=act_func, + **situ_kwargs, ) return ( result["d_row_tensor"], diff --git a/transformer_engine/jax/flax/moe.py b/transformer_engine/jax/flax/moe.py index add9769700..20cf2e0bd6 100644 --- a/transformer_engine/jax/flax/moe.py +++ b/transformer_engine/jax/flax/moe.py @@ -34,7 +34,12 @@ from flax import linen as nn from transformer_engine.common.recipe import Recipe -from ..moe import RouterComputationInfo, _moe_mesh_axes, _resolve_moe_mesh_resource, moe +from ..moe import ( + RouterComputationInfo, + _moe_mesh_axes, + _resolve_moe_mesh_resource, + moe, +) from ..quantize import QuantizerSet from ..router import ScoreFunction from ..sharding import MeshResource, _get_mesh, global_shard_guard @@ -69,7 +74,13 @@ class _MoEBlock(TransformerEngineBase): Hidden dim of the per-expert FFN (the inner ``mlp`` axis). activation_type : str Activation between ``layer_w0 @ wi_0`` and the elementwise - product with ``layer_w0 @ wi_1``. Default ``"silu"``. + product with ``layer_w0 @ wi_1``. Default ``"silu"``. Pass + ``"situglu"`` for fused MXFP8 + SiTU-GLU on SM100/SM103. SiTU requires cuDNN Frontend >= 1.33.0 with compatible JAX + APIs and ``use_cudnn_fusion=True``; there is no unfused SiTU path. + activation_params : Optional[dict] + Static SiTU soft caps: ``gate_beta`` (default 4.0) and ``up_beta`` + (default 25.0). Values must be finite and positive. score_function : Union[str, ScoreFunction] ``"softmax"`` (default), ``"sigmoid"``, or ``"sqrtsoftplus"`` for the routing scores. @@ -181,6 +192,7 @@ class _MoEBlock(TransformerEngineBase): expert_bias_init: Initializer = nn.initializers.zeros use_ffn_bias: bool = False quantization_recipe: Optional[Recipe] = None + activation_params: Optional[dict] = None def __post_init__(self): if self.kernel_init is None: @@ -341,6 +353,7 @@ def make_grouped_quantizer_set(postfix): wo_bias, num_experts=self.num_experts, activation_type=self.activation_type, + activation_params=self.activation_params, use_cudnn_fusion=self.use_cudnn_fusion, cudnn_native_weight_layout=False, apply_topk_weights_early=self.apply_topk_weights_early, diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index a9c5366e0b..83a6d5c5fd 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -43,6 +43,7 @@ import warnings from dataclasses import replace from functools import partial +from collections.abc import Mapping from typing import Any, Optional, Tuple, Union import flax.struct @@ -53,6 +54,7 @@ from jax.sharding import NamedSharding, PartitionSpec as P from . import cpp_extensions as tex +from .cpp_extensions.activation import ActivationParams from .cpp_extensions.misc import jax_dtype_to_te_dtype from .quantize import ( GroupedScaledTensor1x, @@ -82,6 +84,26 @@ ] +def _resolve_moe_activation_params(activation_type, activation_params): + if activation_params is None: + activation_params = {} + if not isinstance(activation_params, Mapping): + raise TypeError("activation_params must be a mapping or None") + if activation_type != "situglu": + if activation_params: + raise ValueError("activation_params are only supported for activation_type='situglu'") + return ActivationParams() + unknown = activation_params.keys() - {"gate_beta", "up_beta"} + if unknown: + raise ValueError(f"Unsupported SiTU activation_params: {sorted(unknown, key=str)}") + return ActivationParams.create("situglu", **activation_params) + + +def _fused_glu_activation_kwargs(activation_type, act_params, *, backward=False): + act_func = "situglu" if activation_type == "situglu" else "swiglu" + return {"act_func": ("d" if backward else "") + act_func, "act_params": act_params} + + @flax.struct.dataclass class RouterComputationInfo: """Router parameters and settings for :func:`moe` to compute routing itself. @@ -213,7 +235,7 @@ def _resolve_moe_mesh_resource( _CUDNN_JAX_ALIGN_SIZE = 256 -def _select_cudnn_jax_fusion(rejection_reasons: list[str]) -> bool: +def _select_cudnn_jax_fusion(rejection_reasons: list[str], *, activation_type="silu") -> bool: """Select shared cuDNN GLU/dGLU fusion or the unfused TE path.""" from transformer_engine_jax import get_device_compute_capability @@ -225,11 +247,18 @@ def _select_cudnn_jax_fusion(rejection_reasons: list[str]) -> bool: else: if capability < 100: reasons.append(f"fused kernels require SM100+, got SM{capability}") + is_situ = activation_type == "situglu" + if is_situ and capability not in (100, 103): + reasons.append(f"SiTU fusion requires SM100/SM103, got SM{capability}") if not reasons: - available, error = tex.grouped_gemm_glu_dependencies_available() + available, error = tex.grouped_gemm_glu_dependencies_available( + act_func="situglu" if is_situ else "swiglu" + ) if available: return True reasons.append(f"shared fused API is incompatible: {error}") + if activation_type == "situglu": + raise ValueError("SiTU requires cuDNN MXFP8 fusion: " + "; ".join(reasons)) warnings.warn( "use_cudnn_fusion=True: falling back to the unfused TE grouped-GEMM path: " + "; ".join(reasons) @@ -254,8 +283,8 @@ def _cudnn_jax_fusion_rejection_reasons( ) -> list[str]: """Return reasons this call cannot use cuDNN's shared grouped GLU/dGLU JAX APIs.""" errors = [] - if str(activation_type).lower() != "silu": - errors.append("requires activation_type='silu'") + if activation_type != "situglu" and str(activation_type).lower() != "silu": + errors.append("requires activation_type='silu' or 'situglu'") if wi_0_bias is not None or wi_1_bias is not None: errors.append("does not support FC1 gate/up bias") hidden = x.shape[-1] @@ -927,6 +956,7 @@ def _ffn_fwd_per_shard( fsdp_size: int, wi_fsdp_axis: Optional[int], wo_fsdp_axis: Optional[int], + act_params: ActivationParams = ActivationParams(), ): """Run the grouped FFN on one shard's EP receive buffer. @@ -1007,6 +1037,7 @@ def _ffn_fwd_per_shard( prob, compute_dtype=compute_dtype, output_dtype=fc2_quantizer_set.x.q_dtype, + **_fused_glu_activation_kwargs(activation_type, act_params), ) combined_out = combined_out_3d.reshape(sorted_x.shape[0], combined) if wi_0_checkpoint_name is not None: @@ -1080,8 +1111,8 @@ def _ffn_fwd_per_shard( # dtype the wo GEMM / wo's quantized input consumes. For bf16 compute # that's all bf16; for FP8/FP4 the downstream grouped_quantize is what # transitions to the target precision. - act_fn = _convert_to_activation_function(activation_type) if not use_cudnn_jax_fusion: + act_fn = _convert_to_activation_function(activation_type) intermediate = act_fn(gate_proj_out) * up_proj_out if apply_topk_weights_early and not use_cudnn_jax_fusion: # Fold the per-token combine weights into the FFN intermediate; @@ -1146,6 +1177,7 @@ def _ffn_bwd_per_shard( has_bias: bool, use_cudnn_jax_fusion: bool, cudnn_native_weight_layout: bool, + act_params: ActivationParams = ActivationParams(), ): """Backward mirror of :func:`_ffn_fwd_per_shard`.""" group_sizes = local_group_sizes.reshape(-1).astype(jnp.int32) @@ -1193,6 +1225,7 @@ def _ffn_bwd_per_shard( padded_offsets, prob, output_dtype=fc1_quantizer_set.dgrad.q_dtype, + **_fused_glu_activation_kwargs(activation_type, act_params, backward=True), ) d_recv_w_from_intermediate = ( dprob.astype(recv_w_flat.dtype) @@ -1356,6 +1389,7 @@ def _moe_fwd_rule( quant_before_fsdp_ag, use_cudnn_fusion: bool = True, cudnn_native_weight_layout: Optional[bool] = None, + act_params: ActivationParams = ActivationParams(), ): """Forward: [gate -> route ->] ep_dispatch -> FFN -> ep_combine. @@ -1680,6 +1714,7 @@ def _ffn_fwd_body(*args): compute_dtype=x.dtype, num_local_experts=num_local_experts, activation_type=activation_type, + act_params=act_params, apply_topk_weights_early=apply_topk_weights_early, use_cudnn_jax_fusion=use_cudnn_jax_fusion, wi_0_checkpoint_name=wi_0_checkpoint_name, @@ -1798,6 +1833,7 @@ def _moe_bwd_rule( quant_before_fsdp_ag, use_cudnn_fusion, cudnn_native_weight_layout, + act_params, residuals, cotangents, ): @@ -1890,6 +1926,7 @@ def _ffn_bwd_body(*args): *args, ctx.quantizer_sets, activation_type=activation_type, + act_params=act_params, apply_topk_weights_early=apply_topk_weights_early, has_bias=has_bias, use_cudnn_jax_fusion=use_cudnn_jax_fusion, @@ -2088,7 +2125,7 @@ def _moe_aux_loss_bwd(ctx, d_aux_loss, K, num_experts, mesh, ep2_spec): # ============================================================================= -@partial(jax.custom_vjp, nondiff_argnums=tuple(range(9, 27))) +@partial(jax.custom_vjp, nondiff_argnums=tuple(range(9, 28))) def _moe( x, router_info, @@ -2117,6 +2154,7 @@ def _moe( quant_before_fsdp_ag, use_cudnn_fusion: bool = True, cudnn_native_weight_layout: Optional[bool] = None, + act_params: ActivationParams = ActivationParams(), ): primal, _ = _moe_fwd_rule( x, @@ -2146,6 +2184,7 @@ def _moe( quant_before_fsdp_ag, use_cudnn_fusion, cudnn_native_weight_layout, + act_params, ) return primal @@ -2184,6 +2223,7 @@ def moe( use_cudnn_fusion: bool = True, cudnn_native_weight_layout: Optional[bool] = None, dispatch_quantizer: Optional[Quantizer] = None, + activation_params: Optional[dict] = None, ) -> Tuple[jnp.ndarray, Optional[jnp.ndarray], jnp.ndarray, jnp.ndarray]: """Run a full MoE block under a single fused custom_vjp on the TE EP path. @@ -2197,6 +2237,13 @@ def moe( Parameters ---------- + activation_type : str + Gate activation (default ``"silu"``), or ``"situglu"`` for fused MXFP8 + SiTU-GLU on SM100/SM103. Unsupported SiTU configurations raise ``ValueError``. + activation_params : Optional[dict] + Static SiTU soft caps: ``gate_beta`` (default 4.0) and ``up_beta`` + (default 25.0). Values must be finite and positive. Other activations + do not accept parameters. router_info : Union[RouterComputationInfo, RoutingMapInfo] :class:`RouterComputationInfo` makes TE compute the gate and top-k from the given router parameters. :class:`RoutingMapInfo` supplies @@ -2251,10 +2298,14 @@ def moe( Defaults to ``True``: try cuDNN's shared JAX grouped MXFP8 GLU/dGLU APIs on SM100+ GPUs; cuDNN selects the architecture-specific kernels. ``False`` uses unfused TE grouped GEMM. Requires cuDNN Frontend >= 1.32.0 with compatible JAX APIs. + SiTU additionally requires SM100/SM103 and JAX GLU/dGLU APIs exposing + ``situ_beta1`` and ``situ_beta2`` with Frontend >= 1.33.0. + Ineligible SiTU calls raise + ``ValueError``; there is no unfused SiTU path. Frontends whose JAX GLU/dGLU entry points only support BF16 discrete weights fall back to unfused TE grouped GEMM. - Ineligible calls warn and fall back to TE's regular grouped-GEMM - implementation. API signatures and GPU capability determine support. + Except for SiTU, ineligible calls warn and fall back to TE's regular + grouped-GEMM implementation. API signatures and GPU capability determine support. cudnn_native_weight_layout : Optional[bool] ``False`` selects standard contiguous gate/up [E,K,2N] storage; ``True`` selects native [E,2N,K] storage with alternating 32-column @@ -2288,6 +2339,9 @@ def moe( raise TypeError("use_cudnn_fusion must be a bool") if dispatch_quantizer is not None and not isinstance(dispatch_quantizer, Quantizer): raise TypeError("dispatch_quantizer must be a Quantizer or None") + act_params = _resolve_moe_activation_params(activation_type, activation_params) + if activation_type == "situglu" and not use_cudnn_fusion: + raise ValueError("SiTU requires use_cudnn_fusion=True") mesh_resource, quant_before_fsdp_ag = _resolve_moe_mesh_resource( mesh_resource, quant_before_fsdp_ag ) @@ -2345,7 +2399,9 @@ def moe( activation_type=activation_type, ep_axis=ep_axis, ) - use_cudnn_jax_fusion = _select_cudnn_jax_fusion(rejection_reasons) + use_cudnn_jax_fusion = _select_cudnn_jax_fusion( + rejection_reasons, activation_type=activation_type + ) output, aux_loss, total_recv_tokens, expert_counts = _moe( x, @@ -2375,6 +2431,7 @@ def moe( quant_before_fsdp_ag, use_cudnn_fusion, cudnn_native_weight_layout, + act_params, ) if not has_aux_loss: aux_loss = None