From 1e5ae43c5f9c84d00c224882e3d13b20359744f7 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 16 Nov 2026 11:45:44 +0100 Subject: [PATCH 01/11] [JAX] Enable cuDNN SiTU-GLU fusion in MoEBlock Signed-off-by: Pawel Gadzinski --- docs/api/jax.rst | 2 + tests/jax/test_custom_call_compute.py | 160 +++++++++++++++++- tests/jax/test_te_ep_moe.py | 36 +++- .../jax/cpp_extensions/grouped_gemm_glu.py | 32 +++- transformer_engine/jax/flax/moe.py | 17 +- transformer_engine/jax/moe.py | 75 +++++++- 6 files changed, 301 insertions(+), 21 deletions(-) diff --git a/docs/api/jax.rst b/docs/api/jax.rst index 99282be422c..970303cfdcf 100644 --- a/docs/api/jax.rst +++ b/docs/api/jax.rst @@ -74,6 +74,8 @@ Router, routing kernels, and grouped dense for MoE layers. See .. autoapifunction:: transformer_engine.jax.dense.grouped_dense +.. autoapiclass:: transformer_engine.jax.moe.SiTUActivation + .. autoapifunction:: transformer_engine.jax.moe.moe .. autoapifunction:: transformer_engine.jax.ep.ep_bootstrap diff --git a/tests/jax/test_custom_call_compute.py b/tests/jax/test_custom_call_compute.py index 61759d74746..eb6cf924af5 100644 --- a/tests/jax/test_custom_call_compute.py +++ b/tests/jax/test_custom_call_compute.py @@ -2156,6 +2156,11 @@ def function(**kwargs): if change == "positional" and name == "a_tensor" else inspect.Parameter.POSITIONAL_OR_KEYWORD ), + default=( + 4.0 + if name == "situ_beta1" + else 25.0 if name == "situ_beta2" else inspect.Parameter.empty + ), ) for name in arguments if not (change == "renamed" and name == "discrete_col_sfd") @@ -2282,6 +2287,151 @@ def test_api_signature(self, frontend, name, args, change): assert available == (change == "optional") assert bool(reason) == (change != "optional") + @pytest.mark.parametrize("missing", [None, "situ_beta1", "situ_beta2"]) + @pytest.mark.parametrize( + "name,arguments", + [ + ("grouped_gemm_glu_wrapper_sm100", _glu_adapter._FORWARD_ARGS), + ("grouped_gemm_dglu_wrapper_sm100", _glu_adapter._BACKWARD_ARGS), + ("grouped_gemm_glu_jax_sm100", _glu_adapter._JAX_FORWARD_ARGS), + ("grouped_gemm_dglu_jax_sm100", _glu_adapter._JAX_BACKWARD_ARGS), + ], + ) + def test_situ_requires_scalar_parameters_in_both_framework_apis( + self, frontend, name, arguments, missing + ): + apis = ( + ("grouped_gemm_glu_wrapper_sm100", _glu_adapter._FORWARD_ARGS), + ("grouped_gemm_dglu_wrapper_sm100", _glu_adapter._BACKWARD_ARGS), + ("grouped_gemm_glu_jax_sm100", _glu_adapter._JAX_FORWARD_ARGS), + ("grouped_gemm_dglu_jax_sm100", _glu_adapter._JAX_BACKWARD_ARGS), + ) + for api_name, args in apis: + setattr(frontend, api_name, self._api(args + ("situ_beta1", "situ_beta2"))) + if missing: + setattr( + frontend, + name, + self._api( + arguments + tuple(arg for arg in ("situ_beta1", "situ_beta2") if arg != missing) + ), + ) + assert _glu_adapter.grouped_gemm_glu_dependencies_available()[0] + available, reason = _glu_adapter.grouped_gemm_glu_dependencies_available(require_situ=True) + assert available is (missing is None) + if missing: + assert name in reason and missing in reason + + @pytest.mark.parametrize("capability", [90, 100, 103, 107, 120]) + def test_situ_fusion_device_support(self, frontend, monkeypatch, capability): + for name, args in ( + ("grouped_gemm_glu_wrapper_sm100", _glu_adapter._FORWARD_ARGS), + ("grouped_gemm_dglu_wrapper_sm100", _glu_adapter._BACKWARD_ARGS), + ("grouped_gemm_glu_jax_sm100", _glu_adapter._JAX_FORWARD_ARGS), + ("grouped_gemm_dglu_jax_sm100", _glu_adapter._JAX_BACKWARD_ARGS), + ): + setattr(frontend, name, self._api(args + ("situ_beta1", "situ_beta2"))) + monkeypatch.setattr( + transformer_engine_jax, "get_device_compute_capability", lambda _: capability + ) + if capability in (100, 103): + assert _moe_module._select_cudnn_jax_fusion( + [], activation_type=_moe_module.SiTUActivation() + ) + else: + with pytest.raises(ValueError, match="SiTU.*SM100/SM103"): + _moe_module._select_cudnn_jax_fusion( + [], activation_type=_moe_module.SiTUActivation() + ) + + @pytest.mark.parametrize("name", ["gate_beta", "up_beta"]) + @pytest.mark.parametrize("value", [0, -1, float("nan"), float("inf")]) + def test_situ_invalid_cap(self, name, value): + with pytest.raises(ValueError, match=name): + _moe_module.SiTUActivation(**{name: value}) + + def test_situ_cannot_disable_fusion(self): + with pytest.raises(ValueError, match="use_cudnn_fusion=True"): + _moe_module.moe( + None, + None, + None, + None, + num_experts=2, + activation_type=_moe_module.SiTUActivation(), + use_cudnn_fusion=False, + ) + + def test_situ_missing_frontend_support(self, frontend, monkeypatch): + monkeypatch.setattr(transformer_engine_jax, "get_device_compute_capability", lambda _: 100) + with pytest.raises(ValueError, match="SiTU.*situ_beta"): + _moe_module._select_cudnn_jax_fusion([], activation_type=_moe_module.SiTUActivation()) + with pytest.raises(ValueError, match="SiTU.*MXFP8 quantizers"): + _moe_module._select_cudnn_jax_fusion( + ["requires MXFP8 quantizers"], activation_type=_moe_module.SiTUActivation() + ) + + @pytest.mark.parametrize("backward", [False, True], ids=["forward", "backward"]) + @pytest.mark.parametrize( + "caps", [None, (4.0, 25.0), (2.0, 8.0)], ids=["swiglu", "situ-default", "situ-custom"] + ) + def test_activation_reaches_shared_api(self, frontend, monkeypatch, backward, caps): + rows, hidden, width = 256, 128, 128 + calls = [] + + def shared(**kwargs): + expected = "dswiglu" if backward else "swiglu" + if caps is None: + assert "situ_beta1" not in kwargs and "situ_beta2" not in kwargs + else: + expected = "dsituglu" if backward else "situglu" + assert (kwargs["situ_beta1"], kwargs["situ_beta2"]) == caps + assert kwargs["act_func"] == expected + calls.append(expected) + output = jnp.zeros((rows, width * (2 if backward else 1)), jnp.float8_e4m3fn) + return dict( + c_tensor=jnp.zeros((rows, 2 * width), jnp.bfloat16), + d_tensor=output, + d_row_tensor=output, + d_col_tensor=output, + dprob_tensor=jnp.zeros((rows,), jnp.float32), + sfd_row_tensor=jnp.zeros((1,), jnp.uint8), + sfd_col_tensor=jnp.zeros((1,), jnp.uint8), + ) + + name = "grouped_gemm_dglu_wrapper_sm100" if backward else "grouped_gemm_glu_wrapper_sm100" + monkeypatch.setattr(frontend, name, shared) + sfa = jnp.zeros(_glu_adapter._sf_atom_shape(1, rows, hidden), jnp.uint8) + combined = width if backward else 2 * width + sfb = jnp.zeros(_glu_adapter._sf_atom_shape(1, combined, hidden), jnp.uint8) + b = jnp.zeros((1, combined, hidden), jnp.float8_e4m3fn) + offsets = jnp.asarray([rows], jnp.int32) + prob = jnp.ones((rows,), jnp.float32) + options = {"output_dtype": jnp.float8_e4m3fn} + if caps is not None: + options.update( + act_func="dsituglu" if backward else "situglu", + situ_beta1=caps[0], + situ_beta2=caps[1], + ) + if backward: + a = jnp.zeros((rows, hidden), jnp.float8_e4m3fn) + c = jnp.zeros((rows, 2 * width), jnp.bfloat16) + result = jax.jit( + lambda a: _glu_adapter.grouped_gemm_dglu( + a, b, c, sfa, sfb, offsets, prob, **options + ) + )(a) + else: + a = jnp.zeros((rows, hidden, 1), jnp.float8_e4m3fn) + result = jax.jit( + lambda a: _glu_adapter.grouped_gemm_glu( + a, b, sfa, sfb, offsets, prob, compute_dtype=jnp.bfloat16, **options + ) + )(a) + jax.block_until_ready(result) + assert len(calls) == 1 + @pytest.mark.parametrize("name", ["grouped_gemm_glu_jax_sm100", "grouped_gemm_dglu_jax_sm100"]) def test_bf16_only_jax_api_falls_back(self, frontend, monkeypatch, name): # Frontend 1.29 exposes JAX entry points for BF16 discrete weights, but @@ -2424,8 +2574,11 @@ def execute(*args): @pytest.mark.parametrize("native_layout", [False, True]) @pytest.mark.parametrize("gather_gated_dimension", [False, True]) + @pytest.mark.parametrize( + "activation_type", ["silu", _moe_module.SiTUActivation(), _moe_module.SiTUActivation(2, 8)] + ) def test_fused_forward_uses_shared_glu( - self, monkeypatch, native_layout, gather_gated_dimension + self, monkeypatch, native_layout, gather_gated_dimension, activation_type ): class SelectedKernel(Exception): @@ -2433,6 +2586,10 @@ class SelectedKernel(Exception): def selected(*args, **kwargs): assert args[1].shape == (1, 256, 128) + if isinstance(activation_type, _moe_module.SiTUActivation): + assert kwargs["act_func"] == "situglu" + assert kwargs["situ_beta1"] == activation_type.gate_beta + assert kwargs["situ_beta2"] == activation_type.up_beta raise SelectedKernel monkeypatch.setattr(_moe_module.tex, "grouped_gemm_glu", selected) @@ -2470,6 +2627,7 @@ def reorder(tensor, *, interleave): wo=jnp.ones((1, 128, 128), jnp.bfloat16), quantizer_sets=(quantizers, quantizers), num_local_experts=1, + activation_type=activation_type, use_cudnn_jax_fusion=True, cudnn_native_weight_layout=native_layout, quant_before_fsdp_ag=gather_gated_dimension, diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index ad5c897b8d5..754b2aea27f 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -122,6 +122,7 @@ def _read_mp_options(): from transformer_engine.jax.flax import _MoEBlock as MoEBlock from transformer_engine.jax.moe import ( + SiTUActivation, RouterComputationInfo, RoutingMapInfo, get_moe_recv_capacity_per_rank, @@ -298,6 +299,7 @@ def _pure_jax_moe_reference( aux_loss_coeff: float = 0.0, score_function: str = "softmax", scaling_factor: float = 1.0, + activation_type="silu", ): B, S, H = x.shape T = B * S @@ -345,7 +347,13 @@ def _pure_jax_moe_reference( # Activation runs in x.dtype (typically bf16) to mirror the impl -- # the impl keeps silu+multiply in the wi GEMM output dtype because # storing higher precision than the consumer (wo) GEMM buys nothing. - intermediate = jax.nn.silu(layer_w0) * layer_w1 + if isinstance(activation_type, SiTUActivation): + gate, up = layer_w0.astype(jnp.float32), layer_w1.astype(jnp.float32) + capped_gate = activation_type.gate_beta * jnp.tanh(gate / activation_type.gate_beta) + capped_up = activation_type.up_beta * jnp.tanh(up / activation_type.up_beta) + intermediate = (capped_gate * jax.nn.sigmoid(gate) * capped_up).astype(x.dtype) + else: + intermediate = jax.nn.silu(layer_w0) * layer_w1 expert_out = jnp.einsum("tem,emh->teh", intermediate, wo) # [T, E, H] output_2d = jnp.einsum("te,teh->th", routing_weights_full.astype(x.dtype), expert_out) output = output_2d.reshape(B, S, H).astype(x.dtype) @@ -396,8 +404,10 @@ def _make_block( quant_before_fsdp_ag=False, mesh_resource=None, use_cudnn_fusion, + activation_type="silu", ): kwargs = dict( + activation_type=activation_type, num_experts=NUM_EXPERTS, num_experts_per_tok=TOPK, intermediate_size=INTER, @@ -1219,7 +1229,9 @@ def unexpected_fused_kernel(*args, **kwargs): # independently forcing the baseline to execute the unfused FFN. moe_module = importlib.import_module("transformer_engine.jax.moe") with monkeypatch.context() as baseline_patch: - baseline_patch.setattr(moe_module, "_select_cudnn_jax_fusion", lambda _: False) + baseline_patch.setattr( + moe_module, "_select_cudnn_jax_fusion", lambda *args, **kwargs: False + ) variables, baseline_output, _, _ = _init_apply( baseline, mesh, x, jax.random.PRNGKey(52) ) @@ -1254,6 +1266,11 @@ def native_moe(*args, **kwargs): _to_global_numpy(_unwrap(baseline_grads["params"][name]), mesh), ) + @pytest.mark.parametrize( + "activation_type", + ["silu", SiTUActivation(), SiTUActivation(2.0, 8.0)], + ids=["swiglu", "situ-k3", "situ-custom"], + ) @pytest.mark.parametrize("native_layout", [False, True]) @pytest.mark.parametrize("apply_topk_weights_early", [False, True]) def test_mxfp8_forward_and_backward( @@ -1264,12 +1281,15 @@ def test_mxfp8_forward_and_backward( make_block, use_cudnn_fusion, native_layout, + activation_type, ): if not use_cudnn_fusion: pytest.skip("requires the cuDNN fusion fixture parameter") from transformer_engine.jax import cpp_extensions as tex - available, reason = tex.grouped_gemm_glu_dependencies_available() + available, reason = tex.grouped_gemm_glu_dependencies_available( + require_situ=isinstance(activation_type, SiTUActivation) + ) if not available: pytest.skip(reason) @@ -1292,10 +1312,18 @@ def native_moe(*args, **kwargs): original_dglu = tex.grouped_gemm_dglu def checked_glu(*args, **kwargs): + if isinstance(activation_type, SiTUActivation): + assert kwargs["act_func"] == "situglu" + assert kwargs["situ_beta1"] == activation_type.gate_beta + assert kwargs["situ_beta2"] == activation_type.up_beta shared_calls.append("glu") return original_glu(*args, **kwargs) def checked_dglu(*args, **kwargs): + if isinstance(activation_type, SiTUActivation): + assert kwargs["act_func"] == "dsituglu" + assert kwargs["situ_beta1"] == activation_type.gate_beta + assert kwargs["situ_beta2"] == activation_type.up_beta shared_calls.append("dglu") return original_dglu(*args, **kwargs) @@ -1304,6 +1332,7 @@ def checked_dglu(*args, **kwargs): block = make_block( apply_topk_weights_early=apply_topk_weights_early, quantization_recipe=MXFP8BlockScaling(), + activation_type=activation_type, ) x = _make_inputs(jax.random.PRNGKey(30)) variables, output, aux, _ = _init_apply(block, mesh, x, jax.random.PRNGKey(31)) @@ -1339,6 +1368,7 @@ def loss_fn(params, inputs): num_experts_per_tok=TOPK, score_function="softmax", expert_bias=None, + activation_type=activation_type, ) return jnp.mean(out.astype(jnp.float32) ** 2) diff --git a/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py b/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py index 86c497218ce..9238975f023 100644 --- a/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py +++ b/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py @@ -108,7 +108,7 @@ def _compact_sf(scale: jax.Array, shape: tuple[int, ...], name: str) -> jax.Arra ) -def grouped_gemm_glu_dependencies_available() -> tuple[bool, str]: +def grouped_gemm_glu_dependencies_available(*, require_situ: bool = False) -> tuple[bool, str]: """Require Frontend >= 1.32.0 and shared JAX APIs supporting dense MXFP8.""" try: cutlass_jax = importlib.import_module("cutlass.jax") @@ -126,6 +126,8 @@ def grouped_gemm_glu_dependencies_available() -> tuple[bool, str]: api = getattr(module, name, None) if not callable(api): return False, f"{module.__name__}.{name} is unavailable" + if require_situ: + arguments += ("situ_beta1", "situ_beta2") signature = inspect.signature(api) missing = set(arguments) - signature.parameters.keys() if missing: @@ -151,8 +153,11 @@ def grouped_gemm_glu( *, compute_dtype, output_dtype, + act_func: str = "swiglu", + situ_beta1: float = 4.0, + situ_beta2: float = 25.0, ) -> tuple[jax.Array, jax.Array, jax.Array, jax.Array, jax.Array]: - """Run cuDNN's shared grouped MXFP8 GLU API in SwiGLU mode.""" + """Run cuDNN's shared grouped MXFP8 GLU API with the selected activation.""" cudnn_grouped_gemm_glu = getattr( importlib.import_module("cudnn"), "grouped_gemm_glu_wrapper_sm100" ) @@ -167,6 +172,9 @@ def grouped_gemm_glu( prob, compute_dtype=compute_dtype, output_dtype=output_dtype, + act_func=act_func, + situ_beta1=situ_beta1, + situ_beta2=situ_beta2, ) @@ -181,6 +189,9 @@ def _grouped_gemm_forward( *, compute_dtype, output_dtype, + act_func: str = "swiglu", + situ_beta1: float = 4.0, + situ_beta2: float = 25.0, ): rows, hidden, _ = a.shape experts, combined, b_hidden = b.shape @@ -188,6 +199,9 @@ def _grouped_gemm_forward( raise ValueError(f"A K={hidden} does not match B K={b_hidden}") alpha = jnp.ones((experts,), dtype=jnp.float32) norm_const = jnp.ones((1,), dtype=jnp.float32) + situ_kwargs = {} + if act_func == "situglu": + situ_kwargs = {"situ_beta1": situ_beta1, "situ_beta2": situ_beta2} result = cudnn_forward( a_tensor=a.reshape(rows, hidden), b_tensor=b, @@ -201,8 +215,9 @@ def _grouped_gemm_forward( d_dtype=jnp.dtype(output_dtype), discrete_col_sfd=True, sf_vec_size=32, - act_func="swiglu", + act_func=act_func, generate_c=True, + **situ_kwargs, ) return ( result["c_tensor"].reshape(rows, combined), @@ -223,8 +238,11 @@ def grouped_gemm_dglu( prob: jax.Array, *, output_dtype, + act_func: str = "dswiglu", + situ_beta1: float = 4.0, + situ_beta2: float = 25.0, ) -> tuple[jax.Array, jax.Array, jax.Array, jax.Array, jax.Array]: - """Run cuDNN's shared grouped MXFP8 dGLU API in dSwiGLU mode.""" + """Run cuDNN's shared grouped MXFP8 dGLU API with the selected activation.""" if a.ndim != 2: raise ValueError(f"Expected A[M,K], got {a.shape}") if b.ndim != 3: @@ -246,6 +264,9 @@ def grouped_gemm_dglu( alpha = jnp.ones((experts,), dtype=jnp.float32) beta = jnp.ones((experts,), dtype=jnp.float32) norm_const = jnp.ones((1,), dtype=jnp.float32) + situ_kwargs = {} + if act_func == "dsituglu": + situ_kwargs = {"situ_beta1": situ_beta1, "situ_beta2": situ_beta2} result = cudnn_grouped_gemm_dglu( a_tensor=a, b_tensor=b, @@ -261,7 +282,8 @@ def grouped_gemm_dglu( discrete_col_sfd=True, dprob_tensor=None, sf_vec_size=32, - act_func="dswiglu", + act_func=act_func, + **situ_kwargs, ) return ( result["d_row_tensor"], diff --git a/transformer_engine/jax/flax/moe.py b/transformer_engine/jax/flax/moe.py index 19b1a41778f..140a9042609 100644 --- a/transformer_engine/jax/flax/moe.py +++ b/transformer_engine/jax/flax/moe.py @@ -34,7 +34,13 @@ from flax import linen as nn from transformer_engine.common.recipe import Recipe -from ..moe import RouterComputationInfo, _moe_mesh_axes, _resolve_moe_mesh_resource, moe +from ..moe import ( + RouterComputationInfo, + SiTUActivation, + _moe_mesh_axes, + _resolve_moe_mesh_resource, + moe, +) from ..quantize import QuantizerSet from ..router import ScoreFunction from ..sharding import MeshResource, _get_mesh, global_shard_guard @@ -67,9 +73,12 @@ class _MoEBlock(TransformerEngineBase): Top-k value for routing. intermediate_size : int Hidden dim of the per-expert FFN (the inner ``mlp`` axis). - activation_type : str + activation_type : Union[str, SiTUActivation] Activation between ``layer_w0 @ wi_0`` and the elementwise - product with ``layer_w0 @ wi_1``. Default ``"silu"``. + product with ``layer_w0 @ wi_1``. Default ``"silu"``. Pass + :class:`~transformer_engine.jax.moe.SiTUActivation` for fused MXFP8 + SiTU-GLU on SM100/SM103. SiTU requires compatible cuDNN Frontend JAX + APIs and ``use_cudnn_fusion=True``; there is no unfused SiTU path. score_function : Union[str, ScoreFunction] ``"softmax"`` (default), ``"sigmoid"``, or ``"sqrtsoftplus"`` for the routing scores. @@ -140,7 +149,7 @@ class _MoEBlock(TransformerEngineBase): num_experts: int = 8 num_experts_per_tok: int = 2 intermediate_size: int = 2048 - activation_type: str = "silu" + activation_type: Union[str, SiTUActivation] = "silu" # Routing score_function: Union[str, ScoreFunction] = "softmax" diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index 4e9bfe4aa8d..92622f7a1f6 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -41,7 +41,7 @@ import math import warnings -from dataclasses import replace +from dataclasses import dataclass, replace from functools import partial from typing import Any, Optional, Tuple, Union @@ -72,6 +72,7 @@ from .sharding import MeshResource, _get_mesh, global_mesh_resource, global_shard_guard __all__ = [ + "SiTUActivation", "RouterComputationInfo", "RoutingMapInfo", "get_moe_recv_capacity_per_rank", @@ -79,6 +80,27 @@ ] +@dataclass(frozen=True) +class SiTUActivation: + """SiTU-GLU configuration for cuDNN-fused MXFP8 MoE on SM100/SM103. + + Computes ``gate_beta * tanh(gate / gate_beta) * sigmoid(gate)`` times + ``up_beta * tanh(up / up_beta)``. Both caps must be finite and positive. + Requires cuDNN Frontend JAX APIs exposing both SiTU parameters. + Unfused SiTU execution is unsupported. + """ + + gate_beta: float = 4.0 + up_beta: float = 25.0 + + def __post_init__(self): + for name in ("gate_beta", "up_beta"): + value = float(getattr(self, name)) + if not math.isfinite(value) or value <= 0: + raise ValueError(f"{name} must be finite and positive, got {value}") + object.__setattr__(self, name, value) + + @flax.struct.dataclass class RouterComputationInfo: """Router parameters and settings for :func:`moe` to compute routing itself. @@ -210,7 +232,7 @@ def _resolve_moe_mesh_resource( _CUDNN_JAX_ALIGN_SIZE = 256 -def _select_cudnn_jax_fusion(rejection_reasons: list[str]) -> bool: +def _select_cudnn_jax_fusion(rejection_reasons: list[str], *, activation_type="silu") -> bool: """Select shared cuDNN GLU/dGLU fusion or the unfused TE path.""" from transformer_engine_jax import get_device_compute_capability @@ -222,11 +244,18 @@ def _select_cudnn_jax_fusion(rejection_reasons: list[str]) -> bool: else: if capability < 100: reasons.append(f"fused kernels require SM100+, got SM{capability}") + require_situ = isinstance(activation_type, SiTUActivation) + if require_situ and capability not in (100, 103): + reasons.append(f"SiTU fusion requires SM100/SM103, got SM{capability}") if not reasons: - available, error = tex.grouped_gemm_glu_dependencies_available() + available, error = tex.grouped_gemm_glu_dependencies_available( + require_situ=require_situ + ) if available: return True reasons.append(f"shared fused API is incompatible: {error}") + if isinstance(activation_type, SiTUActivation): + raise ValueError("SiTU requires cuDNN MXFP8 fusion: " + "; ".join(reasons)) warnings.warn( "use_cudnn_fusion=True: falling back to the unfused TE grouped-GEMM path: " + "; ".join(reasons) @@ -251,8 +280,8 @@ def _cudnn_jax_fusion_rejection_reasons( ) -> list[str]: """Return reasons this call cannot use cuDNN's shared grouped GLU/dGLU JAX APIs.""" errors = [] - if str(activation_type).lower() != "silu": - errors.append("requires activation_type='silu'") + if not isinstance(activation_type, SiTUActivation) and str(activation_type).lower() != "silu": + errors.append("requires activation_type='silu' or SiTUActivation") if wi_0_bias is not None or wi_1_bias is not None: errors.append("does not support FC1 gate/up bias") hidden = x.shape[-1] @@ -968,6 +997,15 @@ def _ffn_fwd_per_shard( prob, compute_dtype=sorted_x.dtype, output_dtype=fc2_quantizer_set.x.q_dtype, + **( + { + "act_func": "situglu", + "situ_beta1": activation_type.gate_beta, + "situ_beta2": activation_type.up_beta, + } + if isinstance(activation_type, SiTUActivation) + else {} + ), ) combined_out = combined_out_3d.reshape(sorted_x.shape[0], combined) if wi_0_checkpoint_name is not None: @@ -1041,8 +1079,8 @@ def _ffn_fwd_per_shard( # dtype the wo GEMM / wo's quantized input consumes. For bf16 compute # that's all bf16; for FP8/FP4 the downstream grouped_quantize is what # transitions to the target precision. - act_fn = _convert_to_activation_function(activation_type) if not use_cudnn_jax_fusion: + act_fn = _convert_to_activation_function(activation_type) intermediate = act_fn(gate_proj_out) * up_proj_out if apply_topk_weights_early and not use_cudnn_jax_fusion: # Fold the per-token combine weights into the FFN intermediate; @@ -1154,6 +1192,15 @@ def _ffn_bwd_per_shard( padded_offsets, prob, output_dtype=fc1_quantizer_set.dgrad.q_dtype, + **( + { + "act_func": "dsituglu", + "situ_beta1": activation_type.gate_beta, + "situ_beta2": activation_type.up_beta, + } + if isinstance(activation_type, SiTUActivation) + else {} + ), ) d_recv_w_from_intermediate = ( dprob.astype(recv_w_flat.dtype) @@ -2094,7 +2141,7 @@ def moe( wo_bias: Optional[jnp.ndarray] = None, *, num_experts: int, - activation_type: str = "silu", + activation_type: Union[str, SiTUActivation] = "silu", apply_topk_weights_early: bool = False, quantizer_sets: Tuple[QuantizerSet, QuantizerSet] = ( noop_quantizer_set, @@ -2127,6 +2174,10 @@ def moe( Parameters ---------- + activation_type : Union[str, SiTUActivation] + Gate activation (default ``"silu"``), or :class:`SiTUActivation` for + fused MXFP8 SiTU-GLU. SiTU requires cuDNN fusion on SM100/SM103; + unsupported configurations raise ``ValueError``. router_info : Union[RouterComputationInfo, RoutingMapInfo] :class:`RouterComputationInfo` makes TE compute the gate and top-k from the given router parameters. :class:`RoutingMapInfo` supplies @@ -2181,6 +2232,10 @@ def moe( Defaults to ``True``: try cuDNN's shared JAX grouped MXFP8 GLU/dGLU APIs on SM100+ GPUs; cuDNN selects the architecture-specific kernels. ``False`` uses unfused TE grouped GEMM. Requires cuDNN Frontend >= 1.32.0 with compatible JAX APIs. + SiTU additionally requires SM100/SM103 and JAX GLU/dGLU APIs exposing + ``situ_beta1`` and ``situ_beta2`` (Frontend 1.33.0 or a develop build + containing NVIDIA/cudnn-frontend#2540). Ineligible SiTU calls raise + ``ValueError``; there is no unfused SiTU path. Frontends whose JAX GLU/dGLU entry points only support BF16 discrete weights fall back to unfused TE grouped GEMM. Ineligible calls warn and fall back to TE's regular grouped-GEMM @@ -2210,6 +2265,8 @@ def moe( """ if not isinstance(use_cudnn_fusion, bool): raise TypeError("use_cudnn_fusion must be a bool") + if isinstance(activation_type, SiTUActivation) and not use_cudnn_fusion: + raise ValueError("SiTU requires use_cudnn_fusion=True and compatible MXFP8 quantizers") mesh_resource, quant_before_fsdp_ag = _resolve_moe_mesh_resource( mesh_resource, quant_before_fsdp_ag ) @@ -2267,7 +2324,9 @@ def moe( activation_type=activation_type, ep_axis=ep_axis, ) - use_cudnn_jax_fusion = _select_cudnn_jax_fusion(rejection_reasons) + use_cudnn_jax_fusion = _select_cudnn_jax_fusion( + rejection_reasons, activation_type=activation_type + ) output, aux_loss, total_recv_tokens, expert_counts = _moe( x, From 544535cf513f8a9dfa15483e9317dd46589a6afd Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 16 Nov 2026 12:16:20 +0100 Subject: [PATCH 02/11] [JAX] Check fused GLU availability by activation Signed-off-by: Pawel Gadzinski --- tests/jax/test_custom_call_compute.py | 130 ++++-------------- tests/jax/test_te_ep_moe.py | 16 ++- .../jax/cpp_extensions/grouped_gemm_glu.py | 8 +- transformer_engine/jax/moe.py | 6 +- 4 files changed, 47 insertions(+), 113 deletions(-) diff --git a/tests/jax/test_custom_call_compute.py b/tests/jax/test_custom_call_compute.py index eb6cf924af5..cde8badbd11 100644 --- a/tests/jax/test_custom_call_compute.py +++ b/tests/jax/test_custom_call_compute.py @@ -2287,42 +2287,40 @@ def test_api_signature(self, frontend, name, args, change): assert available == (change == "optional") assert bool(reason) == (change != "optional") - @pytest.mark.parametrize("missing", [None, "situ_beta1", "situ_beta2"]) @pytest.mark.parametrize( - "name,arguments", + "name,missing", [ - ("grouped_gemm_glu_wrapper_sm100", _glu_adapter._FORWARD_ARGS), - ("grouped_gemm_dglu_wrapper_sm100", _glu_adapter._BACKWARD_ARGS), - ("grouped_gemm_glu_jax_sm100", _glu_adapter._JAX_FORWARD_ARGS), - ("grouped_gemm_dglu_jax_sm100", _glu_adapter._JAX_BACKWARD_ARGS), + ("grouped_gemm_glu_wrapper_sm100", "situ_beta1"), + ("grouped_gemm_dglu_wrapper_sm100", "situ_beta2"), + ("grouped_gemm_glu_jax_sm100", "situ_beta2"), + ("grouped_gemm_dglu_jax_sm100", "situ_beta1"), ], ) - def test_situ_requires_scalar_parameters_in_both_framework_apis( - self, frontend, name, arguments, missing - ): - apis = ( - ("grouped_gemm_glu_wrapper_sm100", _glu_adapter._FORWARD_ARGS), - ("grouped_gemm_dglu_wrapper_sm100", _glu_adapter._BACKWARD_ARGS), - ("grouped_gemm_glu_jax_sm100", _glu_adapter._JAX_FORWARD_ARGS), - ("grouped_gemm_dglu_jax_sm100", _glu_adapter._JAX_BACKWARD_ARGS), - ) - for api_name, args in apis: + def test_situ_requires_scalar_parameters_in_both_framework_apis(self, frontend, name, missing): + apis = { + "grouped_gemm_glu_wrapper_sm100": _glu_adapter._FORWARD_ARGS, + "grouped_gemm_dglu_wrapper_sm100": _glu_adapter._BACKWARD_ARGS, + "grouped_gemm_glu_jax_sm100": _glu_adapter._JAX_FORWARD_ARGS, + "grouped_gemm_dglu_jax_sm100": _glu_adapter._JAX_BACKWARD_ARGS, + } + for api_name, args in apis.items(): setattr(frontend, api_name, self._api(args + ("situ_beta1", "situ_beta2"))) - if missing: - setattr( - frontend, - name, - self._api( - arguments + tuple(arg for arg in ("situ_beta1", "situ_beta2") if arg != missing) - ), - ) + assert _glu_adapter.grouped_gemm_glu_dependencies_available(act_func="situglu")[0] + arguments = apis[name] + tuple( + arg for arg in ("situ_beta1", "situ_beta2") if arg != missing + ) + setattr(frontend, name, self._api(arguments)) assert _glu_adapter.grouped_gemm_glu_dependencies_available()[0] - available, reason = _glu_adapter.grouped_gemm_glu_dependencies_available(require_situ=True) - assert available is (missing is None) - if missing: - assert name in reason and missing in reason + available, reason = _glu_adapter.grouped_gemm_glu_dependencies_available(act_func="situglu") + assert not available and name in reason and missing in reason - @pytest.mark.parametrize("capability", [90, 100, 103, 107, 120]) + @pytest.mark.parametrize("act_func,expected", [("geglu", True), ("unknown", False)]) + def test_fused_activation_support(self, frontend, act_func, expected): + available, reason = _glu_adapter.grouped_gemm_glu_dependencies_available(act_func=act_func) + assert available is expected + assert bool(reason) is not expected + + @pytest.mark.parametrize("capability", [100, 103, 107]) def test_situ_fusion_device_support(self, frontend, monkeypatch, capability): for name, args in ( ("grouped_gemm_glu_wrapper_sm100", _glu_adapter._FORWARD_ARGS), @@ -2344,8 +2342,7 @@ def test_situ_fusion_device_support(self, frontend, monkeypatch, capability): [], activation_type=_moe_module.SiTUActivation() ) - @pytest.mark.parametrize("name", ["gate_beta", "up_beta"]) - @pytest.mark.parametrize("value", [0, -1, float("nan"), float("inf")]) + @pytest.mark.parametrize("name,value", [("gate_beta", 0), ("up_beta", float("nan"))]) def test_situ_invalid_cap(self, name, value): with pytest.raises(ValueError, match=name): _moe_module.SiTUActivation(**{name: value}) @@ -2371,67 +2368,6 @@ def test_situ_missing_frontend_support(self, frontend, monkeypatch): ["requires MXFP8 quantizers"], activation_type=_moe_module.SiTUActivation() ) - @pytest.mark.parametrize("backward", [False, True], ids=["forward", "backward"]) - @pytest.mark.parametrize( - "caps", [None, (4.0, 25.0), (2.0, 8.0)], ids=["swiglu", "situ-default", "situ-custom"] - ) - def test_activation_reaches_shared_api(self, frontend, monkeypatch, backward, caps): - rows, hidden, width = 256, 128, 128 - calls = [] - - def shared(**kwargs): - expected = "dswiglu" if backward else "swiglu" - if caps is None: - assert "situ_beta1" not in kwargs and "situ_beta2" not in kwargs - else: - expected = "dsituglu" if backward else "situglu" - assert (kwargs["situ_beta1"], kwargs["situ_beta2"]) == caps - assert kwargs["act_func"] == expected - calls.append(expected) - output = jnp.zeros((rows, width * (2 if backward else 1)), jnp.float8_e4m3fn) - return dict( - c_tensor=jnp.zeros((rows, 2 * width), jnp.bfloat16), - d_tensor=output, - d_row_tensor=output, - d_col_tensor=output, - dprob_tensor=jnp.zeros((rows,), jnp.float32), - sfd_row_tensor=jnp.zeros((1,), jnp.uint8), - sfd_col_tensor=jnp.zeros((1,), jnp.uint8), - ) - - name = "grouped_gemm_dglu_wrapper_sm100" if backward else "grouped_gemm_glu_wrapper_sm100" - monkeypatch.setattr(frontend, name, shared) - sfa = jnp.zeros(_glu_adapter._sf_atom_shape(1, rows, hidden), jnp.uint8) - combined = width if backward else 2 * width - sfb = jnp.zeros(_glu_adapter._sf_atom_shape(1, combined, hidden), jnp.uint8) - b = jnp.zeros((1, combined, hidden), jnp.float8_e4m3fn) - offsets = jnp.asarray([rows], jnp.int32) - prob = jnp.ones((rows,), jnp.float32) - options = {"output_dtype": jnp.float8_e4m3fn} - if caps is not None: - options.update( - act_func="dsituglu" if backward else "situglu", - situ_beta1=caps[0], - situ_beta2=caps[1], - ) - if backward: - a = jnp.zeros((rows, hidden), jnp.float8_e4m3fn) - c = jnp.zeros((rows, 2 * width), jnp.bfloat16) - result = jax.jit( - lambda a: _glu_adapter.grouped_gemm_dglu( - a, b, c, sfa, sfb, offsets, prob, **options - ) - )(a) - else: - a = jnp.zeros((rows, hidden, 1), jnp.float8_e4m3fn) - result = jax.jit( - lambda a: _glu_adapter.grouped_gemm_glu( - a, b, sfa, sfb, offsets, prob, compute_dtype=jnp.bfloat16, **options - ) - )(a) - jax.block_until_ready(result) - assert len(calls) == 1 - @pytest.mark.parametrize("name", ["grouped_gemm_glu_jax_sm100", "grouped_gemm_dglu_jax_sm100"]) def test_bf16_only_jax_api_falls_back(self, frontend, monkeypatch, name): # Frontend 1.29 exposes JAX entry points for BF16 discrete weights, but @@ -2574,11 +2510,8 @@ def execute(*args): @pytest.mark.parametrize("native_layout", [False, True]) @pytest.mark.parametrize("gather_gated_dimension", [False, True]) - @pytest.mark.parametrize( - "activation_type", ["silu", _moe_module.SiTUActivation(), _moe_module.SiTUActivation(2, 8)] - ) def test_fused_forward_uses_shared_glu( - self, monkeypatch, native_layout, gather_gated_dimension, activation_type + self, monkeypatch, native_layout, gather_gated_dimension ): class SelectedKernel(Exception): @@ -2586,10 +2519,6 @@ class SelectedKernel(Exception): def selected(*args, **kwargs): assert args[1].shape == (1, 256, 128) - if isinstance(activation_type, _moe_module.SiTUActivation): - assert kwargs["act_func"] == "situglu" - assert kwargs["situ_beta1"] == activation_type.gate_beta - assert kwargs["situ_beta2"] == activation_type.up_beta raise SelectedKernel monkeypatch.setattr(_moe_module.tex, "grouped_gemm_glu", selected) @@ -2627,7 +2556,6 @@ def reorder(tensor, *, interleave): wo=jnp.ones((1, 128, 128), jnp.bfloat16), quantizer_sets=(quantizers, quantizers), num_local_experts=1, - activation_type=activation_type, use_cudnn_jax_fusion=True, cudnn_native_weight_layout=native_layout, quant_before_fsdp_ag=gather_gated_dimension, diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index 754b2aea27f..b6b56f9d83b 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -1267,12 +1267,16 @@ def native_moe(*args, **kwargs): ) @pytest.mark.parametrize( - "activation_type", - ["silu", SiTUActivation(), SiTUActivation(2.0, 8.0)], - ids=["swiglu", "situ-k3", "situ-custom"], + "activation_type,native_layout,apply_topk_weights_early", + [ + pytest.param("silu", False, False, id="swiglu-standard-late"), + pytest.param("silu", False, True, id="swiglu-standard-early"), + pytest.param("silu", True, False, id="swiglu-native-late"), + pytest.param("silu", True, True, id="swiglu-native-early"), + pytest.param(SiTUActivation(), False, False, id="situ-standard-late"), + pytest.param(SiTUActivation(2, 8), True, True, id="situ-custom-native-early"), + ], ) - @pytest.mark.parametrize("native_layout", [False, True]) - @pytest.mark.parametrize("apply_topk_weights_early", [False, True]) def test_mxfp8_forward_and_backward( self, mesh, @@ -1288,7 +1292,7 @@ def test_mxfp8_forward_and_backward( from transformer_engine.jax import cpp_extensions as tex available, reason = tex.grouped_gemm_glu_dependencies_available( - require_situ=isinstance(activation_type, SiTUActivation) + act_func="situglu" if isinstance(activation_type, SiTUActivation) else "swiglu" ) if not available: pytest.skip(reason) diff --git a/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py b/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py index 9238975f023..f28797e32a7 100644 --- a/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py +++ b/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py @@ -108,8 +108,10 @@ def _compact_sf(scale: jax.Array, shape: tuple[int, ...], name: str) -> jax.Arra ) -def grouped_gemm_glu_dependencies_available(*, require_situ: bool = False) -> tuple[bool, str]: - """Require Frontend >= 1.32.0 and shared JAX APIs supporting dense MXFP8.""" +def grouped_gemm_glu_dependencies_available(*, act_func: str = "swiglu") -> tuple[bool, str]: + """Check dense MXFP8 forward/backward APIs for the selected GLU activation.""" + if act_func not in ("swiglu", "geglu", "situglu"): + return False, f"Unsupported grouped GLU activation: {act_func}" try: cutlass_jax = importlib.import_module("cutlass.jax") if not cutlass_jax.is_available(): @@ -126,7 +128,7 @@ def grouped_gemm_glu_dependencies_available(*, require_situ: bool = False) -> tu api = getattr(module, name, None) if not callable(api): return False, f"{module.__name__}.{name} is unavailable" - if require_situ: + if act_func == "situglu": arguments += ("situ_beta1", "situ_beta2") signature = inspect.signature(api) missing = set(arguments) - signature.parameters.keys() diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index 92622f7a1f6..8dcce3703a4 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -244,12 +244,12 @@ def _select_cudnn_jax_fusion(rejection_reasons: list[str], *, activation_type="s else: if capability < 100: reasons.append(f"fused kernels require SM100+, got SM{capability}") - require_situ = isinstance(activation_type, SiTUActivation) - if require_situ and capability not in (100, 103): + is_situ = isinstance(activation_type, SiTUActivation) + if is_situ and capability not in (100, 103): reasons.append(f"SiTU fusion requires SM100/SM103, got SM{capability}") if not reasons: available, error = tex.grouped_gemm_glu_dependencies_available( - require_situ=require_situ + act_func="situglu" if is_situ else "swiglu" ) if available: return True From 4193a8dd80b175b03548bee774a05e99ad09894e Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 16 Nov 2026 12:21:29 +0100 Subject: [PATCH 03/11] [JAX] Keep SiTU compatibility tests focused Signed-off-by: Pawel Gadzinski --- tests/jax/test_custom_call_compute.py | 48 --------------------------- tests/jax/test_te_ep_moe.py | 40 +++------------------- 2 files changed, 4 insertions(+), 84 deletions(-) diff --git a/tests/jax/test_custom_call_compute.py b/tests/jax/test_custom_call_compute.py index cde8badbd11..e63787731ad 100644 --- a/tests/jax/test_custom_call_compute.py +++ b/tests/jax/test_custom_call_compute.py @@ -2320,54 +2320,6 @@ def test_fused_activation_support(self, frontend, act_func, expected): assert available is expected assert bool(reason) is not expected - @pytest.mark.parametrize("capability", [100, 103, 107]) - def test_situ_fusion_device_support(self, frontend, monkeypatch, capability): - for name, args in ( - ("grouped_gemm_glu_wrapper_sm100", _glu_adapter._FORWARD_ARGS), - ("grouped_gemm_dglu_wrapper_sm100", _glu_adapter._BACKWARD_ARGS), - ("grouped_gemm_glu_jax_sm100", _glu_adapter._JAX_FORWARD_ARGS), - ("grouped_gemm_dglu_jax_sm100", _glu_adapter._JAX_BACKWARD_ARGS), - ): - setattr(frontend, name, self._api(args + ("situ_beta1", "situ_beta2"))) - monkeypatch.setattr( - transformer_engine_jax, "get_device_compute_capability", lambda _: capability - ) - if capability in (100, 103): - assert _moe_module._select_cudnn_jax_fusion( - [], activation_type=_moe_module.SiTUActivation() - ) - else: - with pytest.raises(ValueError, match="SiTU.*SM100/SM103"): - _moe_module._select_cudnn_jax_fusion( - [], activation_type=_moe_module.SiTUActivation() - ) - - @pytest.mark.parametrize("name,value", [("gate_beta", 0), ("up_beta", float("nan"))]) - def test_situ_invalid_cap(self, name, value): - with pytest.raises(ValueError, match=name): - _moe_module.SiTUActivation(**{name: value}) - - def test_situ_cannot_disable_fusion(self): - with pytest.raises(ValueError, match="use_cudnn_fusion=True"): - _moe_module.moe( - None, - None, - None, - None, - num_experts=2, - activation_type=_moe_module.SiTUActivation(), - use_cudnn_fusion=False, - ) - - def test_situ_missing_frontend_support(self, frontend, monkeypatch): - monkeypatch.setattr(transformer_engine_jax, "get_device_compute_capability", lambda _: 100) - with pytest.raises(ValueError, match="SiTU.*situ_beta"): - _moe_module._select_cudnn_jax_fusion([], activation_type=_moe_module.SiTUActivation()) - with pytest.raises(ValueError, match="SiTU.*MXFP8 quantizers"): - _moe_module._select_cudnn_jax_fusion( - ["requires MXFP8 quantizers"], activation_type=_moe_module.SiTUActivation() - ) - @pytest.mark.parametrize("name", ["grouped_gemm_glu_jax_sm100", "grouped_gemm_dglu_jax_sm100"]) def test_bf16_only_jax_api_falls_back(self, frontend, monkeypatch, name): # Frontend 1.29 exposes JAX entry points for BF16 discrete weights, but diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index b6b56f9d83b..a864199568b 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -122,7 +122,6 @@ def _read_mp_options(): from transformer_engine.jax.flax import _MoEBlock as MoEBlock from transformer_engine.jax.moe import ( - SiTUActivation, RouterComputationInfo, RoutingMapInfo, get_moe_recv_capacity_per_rank, @@ -299,7 +298,6 @@ def _pure_jax_moe_reference( aux_loss_coeff: float = 0.0, score_function: str = "softmax", scaling_factor: float = 1.0, - activation_type="silu", ): B, S, H = x.shape T = B * S @@ -347,13 +345,7 @@ def _pure_jax_moe_reference( # Activation runs in x.dtype (typically bf16) to mirror the impl -- # the impl keeps silu+multiply in the wi GEMM output dtype because # storing higher precision than the consumer (wo) GEMM buys nothing. - if isinstance(activation_type, SiTUActivation): - gate, up = layer_w0.astype(jnp.float32), layer_w1.astype(jnp.float32) - capped_gate = activation_type.gate_beta * jnp.tanh(gate / activation_type.gate_beta) - capped_up = activation_type.up_beta * jnp.tanh(up / activation_type.up_beta) - intermediate = (capped_gate * jax.nn.sigmoid(gate) * capped_up).astype(x.dtype) - else: - intermediate = jax.nn.silu(layer_w0) * layer_w1 + intermediate = jax.nn.silu(layer_w0) * layer_w1 expert_out = jnp.einsum("tem,emh->teh", intermediate, wo) # [T, E, H] output_2d = jnp.einsum("te,teh->th", routing_weights_full.astype(x.dtype), expert_out) output = output_2d.reshape(B, S, H).astype(x.dtype) @@ -404,10 +396,8 @@ def _make_block( quant_before_fsdp_ag=False, mesh_resource=None, use_cudnn_fusion, - activation_type="silu", ): kwargs = dict( - activation_type=activation_type, num_experts=NUM_EXPERTS, num_experts_per_tok=TOPK, intermediate_size=INTER, @@ -1266,17 +1256,8 @@ def native_moe(*args, **kwargs): _to_global_numpy(_unwrap(baseline_grads["params"][name]), mesh), ) - @pytest.mark.parametrize( - "activation_type,native_layout,apply_topk_weights_early", - [ - pytest.param("silu", False, False, id="swiglu-standard-late"), - pytest.param("silu", False, True, id="swiglu-standard-early"), - pytest.param("silu", True, False, id="swiglu-native-late"), - pytest.param("silu", True, True, id="swiglu-native-early"), - pytest.param(SiTUActivation(), False, False, id="situ-standard-late"), - pytest.param(SiTUActivation(2, 8), True, True, id="situ-custom-native-early"), - ], - ) + @pytest.mark.parametrize("native_layout", [False, True]) + @pytest.mark.parametrize("apply_topk_weights_early", [False, True]) def test_mxfp8_forward_and_backward( self, mesh, @@ -1285,15 +1266,12 @@ def test_mxfp8_forward_and_backward( make_block, use_cudnn_fusion, native_layout, - activation_type, ): if not use_cudnn_fusion: pytest.skip("requires the cuDNN fusion fixture parameter") from transformer_engine.jax import cpp_extensions as tex - available, reason = tex.grouped_gemm_glu_dependencies_available( - act_func="situglu" if isinstance(activation_type, SiTUActivation) else "swiglu" - ) + available, reason = tex.grouped_gemm_glu_dependencies_available() if not available: pytest.skip(reason) @@ -1316,18 +1294,10 @@ def native_moe(*args, **kwargs): original_dglu = tex.grouped_gemm_dglu def checked_glu(*args, **kwargs): - if isinstance(activation_type, SiTUActivation): - assert kwargs["act_func"] == "situglu" - assert kwargs["situ_beta1"] == activation_type.gate_beta - assert kwargs["situ_beta2"] == activation_type.up_beta shared_calls.append("glu") return original_glu(*args, **kwargs) def checked_dglu(*args, **kwargs): - if isinstance(activation_type, SiTUActivation): - assert kwargs["act_func"] == "dsituglu" - assert kwargs["situ_beta1"] == activation_type.gate_beta - assert kwargs["situ_beta2"] == activation_type.up_beta shared_calls.append("dglu") return original_dglu(*args, **kwargs) @@ -1336,7 +1306,6 @@ def checked_dglu(*args, **kwargs): block = make_block( apply_topk_weights_early=apply_topk_weights_early, quantization_recipe=MXFP8BlockScaling(), - activation_type=activation_type, ) x = _make_inputs(jax.random.PRNGKey(30)) variables, output, aux, _ = _init_apply(block, mesh, x, jax.random.PRNGKey(31)) @@ -1372,7 +1341,6 @@ def loss_fn(params, inputs): num_experts_per_tok=TOPK, score_function="softmax", expert_bias=None, - activation_type=activation_type, ) return jnp.mean(out.astype(jnp.float32) ** 2) From 152264ebff90c282bf1341b98fbdc758c1e88380 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 16 Nov 2026 12:39:40 +0100 Subject: [PATCH 04/11] [JAX] Restore existing custom-call compute tests Signed-off-by: Pawel Gadzinski --- tests/jax/test_custom_call_compute.py | 38 --------------------------- 1 file changed, 38 deletions(-) diff --git a/tests/jax/test_custom_call_compute.py b/tests/jax/test_custom_call_compute.py index e63787731ad..61759d74746 100644 --- a/tests/jax/test_custom_call_compute.py +++ b/tests/jax/test_custom_call_compute.py @@ -2156,11 +2156,6 @@ def function(**kwargs): if change == "positional" and name == "a_tensor" else inspect.Parameter.POSITIONAL_OR_KEYWORD ), - default=( - 4.0 - if name == "situ_beta1" - else 25.0 if name == "situ_beta2" else inspect.Parameter.empty - ), ) for name in arguments if not (change == "renamed" and name == "discrete_col_sfd") @@ -2287,39 +2282,6 @@ def test_api_signature(self, frontend, name, args, change): assert available == (change == "optional") assert bool(reason) == (change != "optional") - @pytest.mark.parametrize( - "name,missing", - [ - ("grouped_gemm_glu_wrapper_sm100", "situ_beta1"), - ("grouped_gemm_dglu_wrapper_sm100", "situ_beta2"), - ("grouped_gemm_glu_jax_sm100", "situ_beta2"), - ("grouped_gemm_dglu_jax_sm100", "situ_beta1"), - ], - ) - def test_situ_requires_scalar_parameters_in_both_framework_apis(self, frontend, name, missing): - apis = { - "grouped_gemm_glu_wrapper_sm100": _glu_adapter._FORWARD_ARGS, - "grouped_gemm_dglu_wrapper_sm100": _glu_adapter._BACKWARD_ARGS, - "grouped_gemm_glu_jax_sm100": _glu_adapter._JAX_FORWARD_ARGS, - "grouped_gemm_dglu_jax_sm100": _glu_adapter._JAX_BACKWARD_ARGS, - } - for api_name, args in apis.items(): - setattr(frontend, api_name, self._api(args + ("situ_beta1", "situ_beta2"))) - assert _glu_adapter.grouped_gemm_glu_dependencies_available(act_func="situglu")[0] - arguments = apis[name] + tuple( - arg for arg in ("situ_beta1", "situ_beta2") if arg != missing - ) - setattr(frontend, name, self._api(arguments)) - assert _glu_adapter.grouped_gemm_glu_dependencies_available()[0] - available, reason = _glu_adapter.grouped_gemm_glu_dependencies_available(act_func="situglu") - assert not available and name in reason and missing in reason - - @pytest.mark.parametrize("act_func,expected", [("geglu", True), ("unknown", False)]) - def test_fused_activation_support(self, frontend, act_func, expected): - available, reason = _glu_adapter.grouped_gemm_glu_dependencies_available(act_func=act_func) - assert available is expected - assert bool(reason) is not expected - @pytest.mark.parametrize("name", ["grouped_gemm_glu_jax_sm100", "grouped_gemm_dglu_jax_sm100"]) def test_bf16_only_jax_api_falls_back(self, frontend, monkeypatch, name): # Frontend 1.29 exposes JAX entry points for BF16 discrete weights, but From 8c83556e1f423f719088c0ba01f88e0e2e71272b Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 16 Nov 2026 13:02:10 +0100 Subject: [PATCH 05/11] [JAX] Check default SiTU MoE numerics against JAX Signed-off-by: Pawel Gadzinski --- tests/jax/test_te_ep_moe.py | 51 ++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index a864199568b..4caae569d6a 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -122,6 +122,7 @@ def _read_mp_options(): from transformer_engine.jax.flax import _MoEBlock as MoEBlock from transformer_engine.jax.moe import ( + SiTUActivation, RouterComputationInfo, RoutingMapInfo, get_moe_recv_capacity_per_rank, @@ -282,6 +283,7 @@ def mesh(use_cudnn_fusion): "num_experts_per_tok", "aux_loss_coeff", "score_function", + "activation_type", ), ) def _pure_jax_moe_reference( @@ -298,6 +300,7 @@ def _pure_jax_moe_reference( aux_loss_coeff: float = 0.0, score_function: str = "softmax", scaling_factor: float = 1.0, + activation_type="silu", ): B, S, H = x.shape T = B * S @@ -345,7 +348,13 @@ def _pure_jax_moe_reference( # Activation runs in x.dtype (typically bf16) to mirror the impl -- # the impl keeps silu+multiply in the wi GEMM output dtype because # storing higher precision than the consumer (wo) GEMM buys nothing. - intermediate = jax.nn.silu(layer_w0) * layer_w1 + if isinstance(activation_type, SiTUActivation): + gate, up = layer_w0.astype(jnp.float32), layer_w1.astype(jnp.float32) + capped_gate = activation_type.gate_beta * jnp.tanh(gate / activation_type.gate_beta) + capped_up = activation_type.up_beta * jnp.tanh(up / activation_type.up_beta) + intermediate = (capped_gate * jax.nn.sigmoid(gate) * capped_up).astype(x.dtype) + else: + intermediate = jax.nn.silu(layer_w0) * layer_w1 expert_out = jnp.einsum("tem,emh->teh", intermediate, wo) # [T, E, H] output_2d = jnp.einsum("te,teh->th", routing_weights_full.astype(x.dtype), expert_out) output = output_2d.reshape(B, S, H).astype(x.dtype) @@ -396,8 +405,10 @@ def _make_block( quant_before_fsdp_ag=False, mesh_resource=None, use_cudnn_fusion, + activation_type="silu", ): kwargs = dict( + activation_type=activation_type, num_experts=NUM_EXPERTS, num_experts_per_tok=TOPK, intermediate_size=INTER, @@ -1256,8 +1267,16 @@ def native_moe(*args, **kwargs): _to_global_numpy(_unwrap(baseline_grads["params"][name]), mesh), ) - @pytest.mark.parametrize("native_layout", [False, True]) - @pytest.mark.parametrize("apply_topk_weights_early", [False, True]) + @pytest.mark.parametrize( + "activation_type,native_layout,apply_topk_weights_early", + [ + pytest.param("silu", False, False, id="swiglu-standard-late"), + pytest.param("silu", False, True, id="swiglu-standard-early"), + pytest.param("silu", True, False, id="swiglu-native-late"), + pytest.param("silu", True, True, id="swiglu-native-early"), + pytest.param(SiTUActivation(), False, False, id="situ-default"), + ], + ) def test_mxfp8_forward_and_backward( self, mesh, @@ -1266,12 +1285,15 @@ def test_mxfp8_forward_and_backward( make_block, use_cudnn_fusion, native_layout, + activation_type, ): if not use_cudnn_fusion: pytest.skip("requires the cuDNN fusion fixture parameter") from transformer_engine.jax import cpp_extensions as tex - available, reason = tex.grouped_gemm_glu_dependencies_available() + available, reason = tex.grouped_gemm_glu_dependencies_available( + act_func="situglu" if isinstance(activation_type, SiTUActivation) else "swiglu" + ) if not available: pytest.skip(reason) @@ -1294,10 +1316,16 @@ def native_moe(*args, **kwargs): original_dglu = tex.grouped_gemm_dglu def checked_glu(*args, **kwargs): + if isinstance(activation_type, SiTUActivation): + assert kwargs["act_func"] == "situglu" + assert (kwargs["situ_beta1"], kwargs["situ_beta2"]) == (4.0, 25.0) shared_calls.append("glu") return original_glu(*args, **kwargs) def checked_dglu(*args, **kwargs): + if isinstance(activation_type, SiTUActivation): + assert kwargs["act_func"] == "dsituglu" + assert (kwargs["situ_beta1"], kwargs["situ_beta2"]) == (4.0, 25.0) shared_calls.append("dglu") return original_dglu(*args, **kwargs) @@ -1306,6 +1334,7 @@ def checked_dglu(*args, **kwargs): block = make_block( apply_topk_weights_early=apply_topk_weights_early, quantization_recipe=MXFP8BlockScaling(), + activation_type=activation_type, ) x = _make_inputs(jax.random.PRNGKey(30)) variables, output, aux, _ = _init_apply(block, mesh, x, jax.random.PRNGKey(31)) @@ -1341,13 +1370,23 @@ def loss_fn(params, inputs): num_experts_per_tok=TOPK, score_function="softmax", expert_bias=None, + activation_type=activation_type, ) - return jnp.mean(out.astype(jnp.float32) ** 2) + return jnp.mean(out.astype(jnp.float32) ** 2), out - grads_ref, grad_x_ref = jax.jit(jax.grad(loss_fn, argnums=(0, 1)))( + (grads_ref, grad_x_ref), output_ref = jax.jit( + jax.grad(loss_fn, argnums=(0, 1), has_aux=True) + )( {k: jnp.asarray(v) for k, v in params_np.items() if k != "expert_bias"}, jnp.asarray(x_np), ) + if isinstance(activation_type, SiTUActivation): + np.testing.assert_allclose( + output_np, + np.asarray(jax.device_get(output_ref)).astype(np.float32), + **FWD_TOLERANCE["mxfp8"], + err_msg="SiTU fused MXFP8 forward parity breach", + ) for name in ("gate_kernel", "wi", "wo"): np.testing.assert_allclose( _to_global_numpy(_unwrap(grads["params"][name]), mesh).astype(np.float32), From 834ce5eb280d8da83b29f97ba51b01c427114f64 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 16 Nov 2026 15:00:35 +0100 Subject: [PATCH 06/11] [JAX] Configure MoE SiTU through activation parameters Signed-off-by: Pawel Gadzinski --- docs/api/jax.rst | 2 - tests/jax/test_te_ep_moe.py | 22 +++--- transformer_engine/jax/flax/moe.py | 12 ++-- transformer_engine/jax/moe.py | 112 +++++++++++++++-------------- 4 files changed, 80 insertions(+), 68 deletions(-) diff --git a/docs/api/jax.rst b/docs/api/jax.rst index 970303cfdcf..99282be422c 100644 --- a/docs/api/jax.rst +++ b/docs/api/jax.rst @@ -74,8 +74,6 @@ Router, routing kernels, and grouped dense for MoE layers. See .. autoapifunction:: transformer_engine.jax.dense.grouped_dense -.. autoapiclass:: transformer_engine.jax.moe.SiTUActivation - .. autoapifunction:: transformer_engine.jax.moe.moe .. autoapifunction:: transformer_engine.jax.ep.ep_bootstrap diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index 4caae569d6a..10c738c1b75 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -122,7 +122,6 @@ def _read_mp_options(): from transformer_engine.jax.flax import _MoEBlock as MoEBlock from transformer_engine.jax.moe import ( - SiTUActivation, RouterComputationInfo, RoutingMapInfo, get_moe_recv_capacity_per_rank, @@ -348,10 +347,10 @@ def _pure_jax_moe_reference( # Activation runs in x.dtype (typically bf16) to mirror the impl -- # the impl keeps silu+multiply in the wi GEMM output dtype because # storing higher precision than the consumer (wo) GEMM buys nothing. - if isinstance(activation_type, SiTUActivation): + if activation_type == "situglu": gate, up = layer_w0.astype(jnp.float32), layer_w1.astype(jnp.float32) - capped_gate = activation_type.gate_beta * jnp.tanh(gate / activation_type.gate_beta) - capped_up = activation_type.up_beta * jnp.tanh(up / activation_type.up_beta) + capped_gate = 4.0 * jnp.tanh(gate / 4.0) + capped_up = 25.0 * jnp.tanh(up / 25.0) intermediate = (capped_gate * jax.nn.sigmoid(gate) * capped_up).astype(x.dtype) else: intermediate = jax.nn.silu(layer_w0) * layer_w1 @@ -406,9 +405,11 @@ def _make_block( mesh_resource=None, use_cudnn_fusion, activation_type="silu", + activation_params=None, ): kwargs = dict( activation_type=activation_type, + activation_params=activation_params, num_experts=NUM_EXPERTS, num_experts_per_tok=TOPK, intermediate_size=INTER, @@ -1274,7 +1275,7 @@ def native_moe(*args, **kwargs): pytest.param("silu", False, True, id="swiglu-standard-early"), pytest.param("silu", True, False, id="swiglu-native-late"), pytest.param("silu", True, True, id="swiglu-native-early"), - pytest.param(SiTUActivation(), False, False, id="situ-default"), + pytest.param("situglu", False, False, id="situ-default"), ], ) def test_mxfp8_forward_and_backward( @@ -1292,7 +1293,7 @@ def test_mxfp8_forward_and_backward( from transformer_engine.jax import cpp_extensions as tex available, reason = tex.grouped_gemm_glu_dependencies_available( - act_func="situglu" if isinstance(activation_type, SiTUActivation) else "swiglu" + act_func="situglu" if activation_type == "situglu" else "swiglu" ) if not available: pytest.skip(reason) @@ -1316,14 +1317,14 @@ def native_moe(*args, **kwargs): original_dglu = tex.grouped_gemm_dglu def checked_glu(*args, **kwargs): - if isinstance(activation_type, SiTUActivation): + if activation_type == "situglu": assert kwargs["act_func"] == "situglu" assert (kwargs["situ_beta1"], kwargs["situ_beta2"]) == (4.0, 25.0) shared_calls.append("glu") return original_glu(*args, **kwargs) def checked_dglu(*args, **kwargs): - if isinstance(activation_type, SiTUActivation): + if activation_type == "situglu": assert kwargs["act_func"] == "dsituglu" assert (kwargs["situ_beta1"], kwargs["situ_beta2"]) == (4.0, 25.0) shared_calls.append("dglu") @@ -1335,6 +1336,9 @@ def checked_dglu(*args, **kwargs): apply_topk_weights_early=apply_topk_weights_early, quantization_recipe=MXFP8BlockScaling(), activation_type=activation_type, + activation_params=( + {"gate_beta": 4.0, "up_beta": 25.0} if activation_type == "situglu" else None + ), ) x = _make_inputs(jax.random.PRNGKey(30)) variables, output, aux, _ = _init_apply(block, mesh, x, jax.random.PRNGKey(31)) @@ -1380,7 +1384,7 @@ def loss_fn(params, inputs): {k: jnp.asarray(v) for k, v in params_np.items() if k != "expert_bias"}, jnp.asarray(x_np), ) - if isinstance(activation_type, SiTUActivation): + if activation_type == "situglu": np.testing.assert_allclose( output_np, np.asarray(jax.device_get(output_ref)).astype(np.float32), diff --git a/transformer_engine/jax/flax/moe.py b/transformer_engine/jax/flax/moe.py index 140a9042609..5d5a4526399 100644 --- a/transformer_engine/jax/flax/moe.py +++ b/transformer_engine/jax/flax/moe.py @@ -36,7 +36,6 @@ from transformer_engine.common.recipe import Recipe from ..moe import ( RouterComputationInfo, - SiTUActivation, _moe_mesh_axes, _resolve_moe_mesh_resource, moe, @@ -73,12 +72,15 @@ class _MoEBlock(TransformerEngineBase): Top-k value for routing. intermediate_size : int Hidden dim of the per-expert FFN (the inner ``mlp`` axis). - activation_type : Union[str, SiTUActivation] + activation_type : str Activation between ``layer_w0 @ wi_0`` and the elementwise product with ``layer_w0 @ wi_1``. Default ``"silu"``. Pass - :class:`~transformer_engine.jax.moe.SiTUActivation` for fused MXFP8 + ``"situglu"`` for fused MXFP8 SiTU-GLU on SM100/SM103. SiTU requires compatible cuDNN Frontend JAX APIs and ``use_cudnn_fusion=True``; there is no unfused SiTU path. + activation_params : Optional[dict] + Static SiTU soft caps: ``gate_beta`` (default 4.0) and ``up_beta`` + (default 25.0). Values must be finite and positive. score_function : Union[str, ScoreFunction] ``"softmax"`` (default), ``"sigmoid"``, or ``"sqrtsoftplus"`` for the routing scores. @@ -149,7 +151,7 @@ class _MoEBlock(TransformerEngineBase): num_experts: int = 8 num_experts_per_tok: int = 2 intermediate_size: int = 2048 - activation_type: Union[str, SiTUActivation] = "silu" + activation_type: str = "silu" # Routing score_function: Union[str, ScoreFunction] = "softmax" @@ -184,6 +186,7 @@ class _MoEBlock(TransformerEngineBase): expert_bias_init: Initializer = nn.initializers.zeros use_ffn_bias: bool = False quantization_recipe: Optional[Recipe] = None + activation_params: Optional[dict] = None def __post_init__(self): if self.kernel_init is None: @@ -338,6 +341,7 @@ def make_grouped_quantizer_set(postfix): wo_bias, num_experts=self.num_experts, activation_type=self.activation_type, + activation_params=self.activation_params, use_cudnn_fusion=self.use_cudnn_fusion, cudnn_native_weight_layout=False, apply_topk_weights_early=self.apply_topk_weights_early, diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index 8dcce3703a4..67995c69d1b 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -41,8 +41,9 @@ import math import warnings -from dataclasses import dataclass, replace +from dataclasses import replace from functools import partial +from collections.abc import Mapping from typing import Any, Optional, Tuple, Union import flax.struct @@ -72,7 +73,6 @@ from .sharding import MeshResource, _get_mesh, global_mesh_resource, global_shard_guard __all__ = [ - "SiTUActivation", "RouterComputationInfo", "RoutingMapInfo", "get_moe_recv_capacity_per_rank", @@ -80,25 +80,33 @@ ] -@dataclass(frozen=True) -class SiTUActivation: - """SiTU-GLU configuration for cuDNN-fused MXFP8 MoE on SM100/SM103. - - Computes ``gate_beta * tanh(gate / gate_beta) * sigmoid(gate)`` times - ``up_beta * tanh(up / up_beta)``. Both caps must be finite and positive. - Requires cuDNN Frontend JAX APIs exposing both SiTU parameters. - Unfused SiTU execution is unsupported. - """ - - gate_beta: float = 4.0 - up_beta: float = 25.0 - - def __post_init__(self): - for name in ("gate_beta", "up_beta"): - value = float(getattr(self, name)) - if not math.isfinite(value) or value <= 0: - raise ValueError(f"{name} must be finite and positive, got {value}") - object.__setattr__(self, name, value) +def _resolve_moe_activation_params(activation_type, activation_params): + if activation_params is None: + activation_params = {} + if not isinstance(activation_params, Mapping): + raise TypeError("activation_params must be a mapping or None") + if activation_type != "situglu": + if activation_params: + raise ValueError("activation_params are only supported for activation_type='situglu'") + return (4.0, 25.0) + unknown = activation_params.keys() - {"gate_beta", "up_beta"} + if unknown: + raise ValueError(f"Unsupported SiTU activation_params: {sorted(unknown, key=str)}") + values = [] + for name, default in (("gate_beta", 4.0), ("up_beta", 25.0)): + value = float(activation_params.get(name, default)) + if not math.isfinite(value) or value <= 0: + raise ValueError(f"{name} must be finite and positive, got {value}") + values.append(value) + return tuple(values) + + +def _fused_glu_activation_kwargs(activation_type, situ_params, *, backward=False): + act_func = "situglu" if activation_type == "situglu" else "swiglu" + kwargs = {"act_func": ("d" if backward else "") + act_func} + if activation_type == "situglu": + kwargs.update(situ_beta1=situ_params[0], situ_beta2=situ_params[1]) + return kwargs @flax.struct.dataclass @@ -244,7 +252,7 @@ def _select_cudnn_jax_fusion(rejection_reasons: list[str], *, activation_type="s else: if capability < 100: reasons.append(f"fused kernels require SM100+, got SM{capability}") - is_situ = isinstance(activation_type, SiTUActivation) + is_situ = activation_type == "situglu" if is_situ and capability not in (100, 103): reasons.append(f"SiTU fusion requires SM100/SM103, got SM{capability}") if not reasons: @@ -254,7 +262,7 @@ def _select_cudnn_jax_fusion(rejection_reasons: list[str], *, activation_type="s if available: return True reasons.append(f"shared fused API is incompatible: {error}") - if isinstance(activation_type, SiTUActivation): + if activation_type == "situglu": raise ValueError("SiTU requires cuDNN MXFP8 fusion: " + "; ".join(reasons)) warnings.warn( "use_cudnn_fusion=True: falling back to the unfused TE grouped-GEMM path: " @@ -280,8 +288,8 @@ def _cudnn_jax_fusion_rejection_reasons( ) -> list[str]: """Return reasons this call cannot use cuDNN's shared grouped GLU/dGLU JAX APIs.""" errors = [] - if not isinstance(activation_type, SiTUActivation) and str(activation_type).lower() != "silu": - errors.append("requires activation_type='silu' or SiTUActivation") + if activation_type != "situglu" and str(activation_type).lower() != "silu": + errors.append("requires activation_type='silu' or 'situglu'") if wi_0_bias is not None or wi_1_bias is not None: errors.append("does not support FC1 gate/up bias") hidden = x.shape[-1] @@ -931,6 +939,7 @@ def _ffn_fwd_per_shard( fsdp_size: int, wi_fsdp_axis: Optional[int], wo_fsdp_axis: Optional[int], + situ_params: Tuple[float, float] = (4.0, 25.0), ): """Run the grouped FFN on one shard's EP receive buffer.""" hidden = recv_tokens_local.shape[-1] @@ -997,15 +1006,7 @@ def _ffn_fwd_per_shard( prob, compute_dtype=sorted_x.dtype, output_dtype=fc2_quantizer_set.x.q_dtype, - **( - { - "act_func": "situglu", - "situ_beta1": activation_type.gate_beta, - "situ_beta2": activation_type.up_beta, - } - if isinstance(activation_type, SiTUActivation) - else {} - ), + **_fused_glu_activation_kwargs(activation_type, situ_params), ) combined_out = combined_out_3d.reshape(sorted_x.shape[0], combined) if wi_0_checkpoint_name is not None: @@ -1145,6 +1146,7 @@ def _ffn_bwd_per_shard( has_bias: bool, use_cudnn_jax_fusion: bool, cudnn_native_weight_layout: bool, + situ_params: Tuple[float, float] = (4.0, 25.0), ): """Backward mirror of :func:`_ffn_fwd_per_shard`.""" group_sizes = local_group_sizes.reshape(-1).astype(jnp.int32) @@ -1192,15 +1194,7 @@ def _ffn_bwd_per_shard( padded_offsets, prob, output_dtype=fc1_quantizer_set.dgrad.q_dtype, - **( - { - "act_func": "dsituglu", - "situ_beta1": activation_type.gate_beta, - "situ_beta2": activation_type.up_beta, - } - if isinstance(activation_type, SiTUActivation) - else {} - ), + **_fused_glu_activation_kwargs(activation_type, situ_params, backward=True), ) d_recv_w_from_intermediate = ( dprob.astype(recv_w_flat.dtype) @@ -1358,6 +1352,7 @@ def _moe_fwd_rule( quant_before_fsdp_ag, use_cudnn_fusion: bool = True, cudnn_native_weight_layout: Optional[bool] = None, + situ_params: Tuple[float, float] = (4.0, 25.0), ): """Forward: [gate -> route ->] ep_dispatch -> FFN -> ep_combine. @@ -1658,6 +1653,7 @@ def _ffn_fwd_body(*args): quantizer_sets, num_local_experts=num_local_experts, activation_type=activation_type, + situ_params=situ_params, apply_topk_weights_early=apply_topk_weights_early, use_cudnn_jax_fusion=use_cudnn_jax_fusion, wi_0_checkpoint_name=wi_0_checkpoint_name, @@ -1777,6 +1773,7 @@ def _moe_bwd_rule( quant_before_fsdp_ag, use_cudnn_fusion, cudnn_native_weight_layout, + situ_params, residuals, cotangents, ): @@ -1869,6 +1866,7 @@ def _ffn_bwd_body(*args): *args, ctx.quantizer_sets, activation_type=activation_type, + situ_params=situ_params, apply_topk_weights_early=apply_topk_weights_early, has_bias=has_bias, use_cudnn_jax_fusion=use_cudnn_jax_fusion, @@ -2068,7 +2066,7 @@ def _moe_aux_loss_bwd(ctx, d_aux_loss, K): # ============================================================================= -@partial(jax.custom_vjp, nondiff_argnums=tuple(range(8, 26))) +@partial(jax.custom_vjp, nondiff_argnums=tuple(range(8, 27))) def _moe( x, router_info, @@ -2096,6 +2094,7 @@ def _moe( quant_before_fsdp_ag, use_cudnn_fusion: bool = True, cudnn_native_weight_layout: Optional[bool] = None, + situ_params: Tuple[float, float] = (4.0, 25.0), ): primal, _ = _moe_fwd_rule( x, @@ -2124,6 +2123,7 @@ def _moe( quant_before_fsdp_ag, use_cudnn_fusion, cudnn_native_weight_layout, + situ_params, ) return primal @@ -2141,7 +2141,7 @@ def moe( wo_bias: Optional[jnp.ndarray] = None, *, num_experts: int, - activation_type: Union[str, SiTUActivation] = "silu", + activation_type: str = "silu", apply_topk_weights_early: bool = False, quantizer_sets: Tuple[QuantizerSet, QuantizerSet] = ( noop_quantizer_set, @@ -2161,6 +2161,7 @@ def moe( dispatch_checkpoint_name: Optional[str] = None, use_cudnn_fusion: bool = True, cudnn_native_weight_layout: Optional[bool] = None, + activation_params: Optional[dict] = None, ) -> Tuple[jnp.ndarray, Optional[jnp.ndarray], jnp.ndarray, jnp.ndarray]: """Run a full MoE block under a single fused custom_vjp on the TE EP path. @@ -2174,10 +2175,13 @@ def moe( Parameters ---------- - activation_type : Union[str, SiTUActivation] - Gate activation (default ``"silu"``), or :class:`SiTUActivation` for - fused MXFP8 SiTU-GLU. SiTU requires cuDNN fusion on SM100/SM103; - unsupported configurations raise ``ValueError``. + activation_type : str + Gate activation (default ``"silu"``), or ``"situglu"`` for fused MXFP8 + SiTU-GLU on SM100/SM103. Unsupported SiTU configurations raise ``ValueError``. + activation_params : Optional[dict] + Static SiTU soft caps: ``gate_beta`` (default 4.0) and ``up_beta`` + (default 25.0). Values must be finite and positive. Other activations + do not accept parameters. router_info : Union[RouterComputationInfo, RoutingMapInfo] :class:`RouterComputationInfo` makes TE compute the gate and top-k from the given router parameters. :class:`RoutingMapInfo` supplies @@ -2238,8 +2242,8 @@ def moe( ``ValueError``; there is no unfused SiTU path. Frontends whose JAX GLU/dGLU entry points only support BF16 discrete weights fall back to unfused TE grouped GEMM. - Ineligible calls warn and fall back to TE's regular grouped-GEMM - implementation. API signatures and GPU capability determine support. + Except for SiTU, ineligible calls warn and fall back to TE's regular + grouped-GEMM implementation. API signatures and GPU capability determine support. cudnn_native_weight_layout : Optional[bool] ``False`` selects standard contiguous gate/up [E,K,2N] storage; ``True`` selects native [E,2N,K] storage with alternating 32-column @@ -2265,8 +2269,9 @@ def moe( """ if not isinstance(use_cudnn_fusion, bool): raise TypeError("use_cudnn_fusion must be a bool") - if isinstance(activation_type, SiTUActivation) and not use_cudnn_fusion: - raise ValueError("SiTU requires use_cudnn_fusion=True and compatible MXFP8 quantizers") + situ_params = _resolve_moe_activation_params(activation_type, activation_params) + if activation_type == "situglu" and not use_cudnn_fusion: + raise ValueError("SiTU requires use_cudnn_fusion=True") mesh_resource, quant_before_fsdp_ag = _resolve_moe_mesh_resource( mesh_resource, quant_before_fsdp_ag ) @@ -2355,6 +2360,7 @@ def moe( quant_before_fsdp_ag, use_cudnn_fusion, cudnn_native_weight_layout, + situ_params, ) if not has_aux_loss: aux_loss = None From 88c3cc86c9802b2ae27a7d670fce7d993b02b304 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 16 Nov 2026 17:33:25 +0100 Subject: [PATCH 07/11] [JAX] Require cuDNN Frontend 1.33 for SiTU fusion Signed-off-by: Pawel Gadzinski --- .../jax/cpp_extensions/grouped_gemm_glu.py | 10 +++++++--- transformer_engine/jax/flax/moe.py | 2 +- transformer_engine/jax/moe.py | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py b/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py index f28797e32a7..2eeb5e22d7a 100644 --- a/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py +++ b/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py @@ -109,7 +109,7 @@ def _compact_sf(scale: jax.Array, shape: tuple[int, ...], name: str) -> jax.Arra def grouped_gemm_glu_dependencies_available(*, act_func: str = "swiglu") -> tuple[bool, str]: - """Check dense MXFP8 forward/backward APIs for the selected GLU activation.""" + """Require Frontend >= 1.33.0 for SiTU or >= 1.32.0 for other GLU activations.""" if act_func not in ("swiglu", "geglu", "situglu"): return False, f"Unsupported grouped GLU activation: {act_func}" try: @@ -117,8 +117,12 @@ def grouped_gemm_glu_dependencies_available(*, act_func: str = "swiglu") -> tupl if not cutlass_jax.is_available(): return False, "CuTeDSL JAX support is unavailable" cudnn = importlib.import_module("cudnn") - if Version(cudnn.__version__) < Version("1.32.0"): - return False, f"cuDNN Frontend >= 1.32.0 is required, got {cudnn.__version__}" + minimum_version = "1.33.0" if act_func == "situglu" else "1.32.0" + if Version(cudnn.__version__) < Version(minimum_version): + return ( + False, + f"cuDNN Frontend >= {minimum_version} is required, got {cudnn.__version__}", + ) for module, name, arguments in ( (cudnn, "grouped_gemm_glu_wrapper_sm100", _FORWARD_ARGS), (cudnn, "grouped_gemm_dglu_wrapper_sm100", _BACKWARD_ARGS), diff --git a/transformer_engine/jax/flax/moe.py b/transformer_engine/jax/flax/moe.py index 5d5a4526399..0402c84f3a7 100644 --- a/transformer_engine/jax/flax/moe.py +++ b/transformer_engine/jax/flax/moe.py @@ -76,7 +76,7 @@ class _MoEBlock(TransformerEngineBase): Activation between ``layer_w0 @ wi_0`` and the elementwise product with ``layer_w0 @ wi_1``. Default ``"silu"``. Pass ``"situglu"`` for fused MXFP8 - SiTU-GLU on SM100/SM103. SiTU requires compatible cuDNN Frontend JAX + SiTU-GLU on SM100/SM103. SiTU requires cuDNN Frontend >= 1.33.0 with compatible JAX APIs and ``use_cudnn_fusion=True``; there is no unfused SiTU path. activation_params : Optional[dict] Static SiTU soft caps: ``gate_beta`` (default 4.0) and ``up_beta`` diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index 67995c69d1b..d36c8b6d6df 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -2237,8 +2237,8 @@ def moe( on SM100+ GPUs; cuDNN selects the architecture-specific kernels. ``False`` uses unfused TE grouped GEMM. Requires cuDNN Frontend >= 1.32.0 with compatible JAX APIs. SiTU additionally requires SM100/SM103 and JAX GLU/dGLU APIs exposing - ``situ_beta1`` and ``situ_beta2`` (Frontend 1.33.0 or a develop build - containing NVIDIA/cudnn-frontend#2540). Ineligible SiTU calls raise + ``situ_beta1`` and ``situ_beta2`` with Frontend >= 1.33.0. + Ineligible SiTU calls raise ``ValueError``; there is no unfused SiTU path. Frontends whose JAX GLU/dGLU entry points only support BF16 discrete weights fall back to unfused TE grouped GEMM. From b1a3f0d39d5365090436f5fcf643576890eb957f Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Tue, 17 Nov 2026 11:25:59 +0100 Subject: [PATCH 08/11] [JAX] Reuse ActivationParams for fused SiTU-GLU Signed-off-by: Pawel Gadzinski --- tests/jax/moe/test_moe_resource_api.py | 61 +++++++++++++++++++ .../jax/cpp_extensions/activation.py | 25 ++++++-- .../jax/cpp_extensions/grouped_gemm_glu.py | 24 +++++--- transformer_engine/jax/moe.py | 42 ++++++------- 4 files changed, 113 insertions(+), 39 deletions(-) diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index ab6c02932d6..2764d9ca54b 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -6,6 +6,7 @@ import importlib import inspect +from types import SimpleNamespace import jax import jax.numpy as jnp @@ -13,6 +14,7 @@ import pytest from jax.sharding import Mesh +from transformer_engine.jax.cpp_extensions.activation import ActivationParams from transformer_engine.jax.flax import _MoEBlock from transformer_engine.jax.moe import ( RouterComputationInfo, @@ -22,6 +24,65 @@ from transformer_engine.jax.sharding import MeshResource, global_mesh_resource, global_shard_guard +@pytest.mark.parametrize("soft_caps", [None, {"gate_beta": 2.0, "up_beta": 8.0}]) +def test_situ_activation_params_reach_grouped_gemm_adapters(monkeypatch, soft_caps): + module = importlib.import_module("transformer_engine.jax.moe") + adapter = importlib.import_module("transformer_engine.jax.cpp_extensions.grouped_gemm_glu") + params = module._resolve_moe_activation_params("situglu", soft_caps) + assert isinstance(params, ActivationParams) + calls = [] + scales = jnp.ones((512,), dtype=jnp.uint8) + + def fused(**kwargs): + calls.append(kwargs) + return dict( + c_tensor=jnp.ones((1, 64)), + d_tensor=jnp.ones((1, 32)), + d_col_tensor=jnp.ones((1, 32)), + d_row_tensor=jnp.ones((1, 64)), + dprob_tensor=jnp.ones((1,)), + sfd_row_tensor=scales, + sfd_col_tensor=scales, + ) + + cudnn = SimpleNamespace( + grouped_gemm_glu_wrapper_sm100=fused, grouped_gemm_dglu_wrapper_sm100=fused + ) + original_import = importlib.import_module + monkeypatch.setattr( + adapter.importlib, + "import_module", + lambda name: cudnn if name == "cudnn" else original_import(name), + ) + adapter.grouped_gemm_glu( + jnp.ones((1, 32, 1)), + jnp.ones((1, 64, 32)), + scales, + scales, + jnp.asarray([1]), + jnp.ones((1,)), + compute_dtype=jnp.bfloat16, + output_dtype=jnp.float8_e4m3fn, + **module._fused_glu_activation_kwargs("situglu", params), + ) + adapter.grouped_gemm_dglu( + jnp.ones((1, 32)), + jnp.ones((1, 32, 32)), + jnp.ones((1, 64)), + scales, + scales, + jnp.asarray([1]), + jnp.ones((1,)), + output_dtype=jnp.float8_e4m3fn, + **module._fused_glu_activation_kwargs("situglu", params, backward=True), + ) + expected = soft_caps or {"gate_beta": 4.0, "up_beta": 25.0} + assert [call["act_func"] for call in calls] == ["situglu", "dsituglu"] + for call in calls: + assert call["situ_beta1"] == expected["gate_beta"] + assert call["situ_beta2"] == expected["up_beta"] + + @pytest.fixture(autouse=True) def no_global_resource(): with global_shard_guard(None): diff --git a/transformer_engine/jax/cpp_extensions/activation.py b/transformer_engine/jax/cpp_extensions/activation.py index 5058192c3f3..a1d6aed6073 100644 --- a/transformer_engine/jax/cpp_extensions/activation.py +++ b/transformer_engine/jax/cpp_extensions/activation.py @@ -3,6 +3,7 @@ # See LICENSE for license information. """JAX/TE custom ops for activation""" from typing import Sequence, Union, Callable, Optional, Tuple +import math import operator from functools import reduce, partial from dataclasses import dataclass @@ -88,13 +89,27 @@ def to_ffi_lowering_dict(self): } +@dataclass(frozen=True) +class SiTUParams: + """Soft caps for the gate and up branches of SiTU-GLU.""" + + gate_beta: float = 4.0 + up_beta: float = 25.0 + + def __post_init__(self): + for name in ("gate_beta", "up_beta"): + value = float(getattr(self, name)) + if not math.isfinite(value) or value <= 0: + raise ValueError(f"{name} must be finite and positive, got {value}") + object.__setattr__(self, name, value) + + @dataclass(frozen=True) class ActivationParams: - """Parameters for various activation functions. - Currently only Clamped SwiGLU activation has parameters. - """ + """Parameters for activation functions.""" clamped_swiglu: ClampedSwigluParams = ClampedSwigluParams() + situglu: SiTUParams = SiTUParams() @staticmethod def create(activation_type, **kwargs): @@ -106,11 +121,13 @@ def create(activation_type, **kwargs): } if activation_type in CLAMPED_ACTIVATION_TYPES: return ActivationParams(ClampedSwigluParams(**kwargs)) + if activation_type == "situglu": + return ActivationParams(situglu=SiTUParams(**kwargs)) return ActivationParams() # Default params for activations without parameters def __hash__(self): """Custom hash function to ensure dataclass is hashable for jax jit to work""" - return hash((self.clamped_swiglu,)) + return hash((self.clamped_swiglu, self.situglu)) def to_ffi_lowering_dict(self): """Convert the activation parameters to a dictionary format for FFI lowering. diff --git a/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py b/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py index 2eeb5e22d7a..72de2e607b7 100644 --- a/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py +++ b/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py @@ -12,6 +12,8 @@ import jax.numpy as jnp from packaging.version import Version +from .activation import ActivationParams + __all__ = [ "grouped_gemm_dglu", "grouped_gemm_glu", @@ -160,8 +162,7 @@ def grouped_gemm_glu( compute_dtype, output_dtype, act_func: str = "swiglu", - situ_beta1: float = 4.0, - situ_beta2: float = 25.0, + act_params: ActivationParams = ActivationParams(), ) -> tuple[jax.Array, jax.Array, jax.Array, jax.Array, jax.Array]: """Run cuDNN's shared grouped MXFP8 GLU API with the selected activation.""" cudnn_grouped_gemm_glu = getattr( @@ -179,8 +180,7 @@ def grouped_gemm_glu( compute_dtype=compute_dtype, output_dtype=output_dtype, act_func=act_func, - situ_beta1=situ_beta1, - situ_beta2=situ_beta2, + act_params=act_params, ) @@ -196,8 +196,7 @@ def _grouped_gemm_forward( compute_dtype, output_dtype, act_func: str = "swiglu", - situ_beta1: float = 4.0, - situ_beta2: float = 25.0, + act_params: ActivationParams = ActivationParams(), ): rows, hidden, _ = a.shape experts, combined, b_hidden = b.shape @@ -207,7 +206,10 @@ def _grouped_gemm_forward( norm_const = jnp.ones((1,), dtype=jnp.float32) situ_kwargs = {} if act_func == "situglu": - situ_kwargs = {"situ_beta1": situ_beta1, "situ_beta2": situ_beta2} + situ_kwargs = { + "situ_beta1": act_params.situglu.gate_beta, + "situ_beta2": act_params.situglu.up_beta, + } result = cudnn_forward( a_tensor=a.reshape(rows, hidden), b_tensor=b, @@ -245,8 +247,7 @@ def grouped_gemm_dglu( *, output_dtype, act_func: str = "dswiglu", - situ_beta1: float = 4.0, - situ_beta2: float = 25.0, + act_params: ActivationParams = ActivationParams(), ) -> tuple[jax.Array, jax.Array, jax.Array, jax.Array, jax.Array]: """Run cuDNN's shared grouped MXFP8 dGLU API with the selected activation.""" if a.ndim != 2: @@ -272,7 +273,10 @@ def grouped_gemm_dglu( norm_const = jnp.ones((1,), dtype=jnp.float32) situ_kwargs = {} if act_func == "dsituglu": - situ_kwargs = {"situ_beta1": situ_beta1, "situ_beta2": situ_beta2} + situ_kwargs = { + "situ_beta1": act_params.situglu.gate_beta, + "situ_beta2": act_params.situglu.up_beta, + } result = cudnn_grouped_gemm_dglu( a_tensor=a, b_tensor=b, diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index d36c8b6d6df..e299724ea8c 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -54,6 +54,7 @@ from jax.sharding import NamedSharding, PartitionSpec as P from . import cpp_extensions as tex +from .cpp_extensions.activation import ActivationParams from .cpp_extensions.misc import jax_dtype_to_te_dtype from .quantize import ( GroupedScaledTensor1x, @@ -88,25 +89,16 @@ def _resolve_moe_activation_params(activation_type, activation_params): if activation_type != "situglu": if activation_params: raise ValueError("activation_params are only supported for activation_type='situglu'") - return (4.0, 25.0) + return ActivationParams() unknown = activation_params.keys() - {"gate_beta", "up_beta"} if unknown: raise ValueError(f"Unsupported SiTU activation_params: {sorted(unknown, key=str)}") - values = [] - for name, default in (("gate_beta", 4.0), ("up_beta", 25.0)): - value = float(activation_params.get(name, default)) - if not math.isfinite(value) or value <= 0: - raise ValueError(f"{name} must be finite and positive, got {value}") - values.append(value) - return tuple(values) + return ActivationParams.create("situglu", **activation_params) -def _fused_glu_activation_kwargs(activation_type, situ_params, *, backward=False): +def _fused_glu_activation_kwargs(activation_type, act_params, *, backward=False): act_func = "situglu" if activation_type == "situglu" else "swiglu" - kwargs = {"act_func": ("d" if backward else "") + act_func} - if activation_type == "situglu": - kwargs.update(situ_beta1=situ_params[0], situ_beta2=situ_params[1]) - return kwargs + return {"act_func": ("d" if backward else "") + act_func, "act_params": act_params} @flax.struct.dataclass @@ -939,7 +931,7 @@ def _ffn_fwd_per_shard( fsdp_size: int, wi_fsdp_axis: Optional[int], wo_fsdp_axis: Optional[int], - situ_params: Tuple[float, float] = (4.0, 25.0), + act_params: ActivationParams = ActivationParams(), ): """Run the grouped FFN on one shard's EP receive buffer.""" hidden = recv_tokens_local.shape[-1] @@ -1006,7 +998,7 @@ def _ffn_fwd_per_shard( prob, compute_dtype=sorted_x.dtype, output_dtype=fc2_quantizer_set.x.q_dtype, - **_fused_glu_activation_kwargs(activation_type, situ_params), + **_fused_glu_activation_kwargs(activation_type, act_params), ) combined_out = combined_out_3d.reshape(sorted_x.shape[0], combined) if wi_0_checkpoint_name is not None: @@ -1146,7 +1138,7 @@ def _ffn_bwd_per_shard( has_bias: bool, use_cudnn_jax_fusion: bool, cudnn_native_weight_layout: bool, - situ_params: Tuple[float, float] = (4.0, 25.0), + act_params: ActivationParams = ActivationParams(), ): """Backward mirror of :func:`_ffn_fwd_per_shard`.""" group_sizes = local_group_sizes.reshape(-1).astype(jnp.int32) @@ -1194,7 +1186,7 @@ def _ffn_bwd_per_shard( padded_offsets, prob, output_dtype=fc1_quantizer_set.dgrad.q_dtype, - **_fused_glu_activation_kwargs(activation_type, situ_params, backward=True), + **_fused_glu_activation_kwargs(activation_type, act_params, backward=True), ) d_recv_w_from_intermediate = ( dprob.astype(recv_w_flat.dtype) @@ -1352,7 +1344,7 @@ def _moe_fwd_rule( quant_before_fsdp_ag, use_cudnn_fusion: bool = True, cudnn_native_weight_layout: Optional[bool] = None, - situ_params: Tuple[float, float] = (4.0, 25.0), + act_params: ActivationParams = ActivationParams(), ): """Forward: [gate -> route ->] ep_dispatch -> FFN -> ep_combine. @@ -1653,7 +1645,7 @@ def _ffn_fwd_body(*args): quantizer_sets, num_local_experts=num_local_experts, activation_type=activation_type, - situ_params=situ_params, + act_params=act_params, apply_topk_weights_early=apply_topk_weights_early, use_cudnn_jax_fusion=use_cudnn_jax_fusion, wi_0_checkpoint_name=wi_0_checkpoint_name, @@ -1773,7 +1765,7 @@ def _moe_bwd_rule( quant_before_fsdp_ag, use_cudnn_fusion, cudnn_native_weight_layout, - situ_params, + act_params, residuals, cotangents, ): @@ -1866,7 +1858,7 @@ def _ffn_bwd_body(*args): *args, ctx.quantizer_sets, activation_type=activation_type, - situ_params=situ_params, + act_params=act_params, apply_topk_weights_early=apply_topk_weights_early, has_bias=has_bias, use_cudnn_jax_fusion=use_cudnn_jax_fusion, @@ -2094,7 +2086,7 @@ def _moe( quant_before_fsdp_ag, use_cudnn_fusion: bool = True, cudnn_native_weight_layout: Optional[bool] = None, - situ_params: Tuple[float, float] = (4.0, 25.0), + act_params: ActivationParams = ActivationParams(), ): primal, _ = _moe_fwd_rule( x, @@ -2123,7 +2115,7 @@ def _moe( quant_before_fsdp_ag, use_cudnn_fusion, cudnn_native_weight_layout, - situ_params, + act_params, ) return primal @@ -2269,7 +2261,7 @@ def moe( """ if not isinstance(use_cudnn_fusion, bool): raise TypeError("use_cudnn_fusion must be a bool") - situ_params = _resolve_moe_activation_params(activation_type, activation_params) + act_params = _resolve_moe_activation_params(activation_type, activation_params) if activation_type == "situglu" and not use_cudnn_fusion: raise ValueError("SiTU requires use_cudnn_fusion=True") mesh_resource, quant_before_fsdp_ag = _resolve_moe_mesh_resource( @@ -2360,7 +2352,7 @@ def moe( quant_before_fsdp_ag, use_cudnn_fusion, cudnn_native_weight_layout, - situ_params, + act_params, ) if not has_aux_loss: aux_loss = None From a9dd38d8f88c65ef1ce87ea86955010b75fef8bf Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Tue, 17 Nov 2026 11:31:33 +0100 Subject: [PATCH 09/11] [JAX] Preserve gradient-only fused MoE parity testing Signed-off-by: Pawel Gadzinski --- tests/jax/test_te_ep_moe.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index 10c738c1b75..bc2eb804e4f 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -1376,21 +1376,12 @@ def loss_fn(params, inputs): expert_bias=None, activation_type=activation_type, ) - return jnp.mean(out.astype(jnp.float32) ** 2), out + return jnp.mean(out.astype(jnp.float32) ** 2) - (grads_ref, grad_x_ref), output_ref = jax.jit( - jax.grad(loss_fn, argnums=(0, 1), has_aux=True) - )( + grads_ref, grad_x_ref = jax.jit(jax.grad(loss_fn, argnums=(0, 1)))( {k: jnp.asarray(v) for k, v in params_np.items() if k != "expert_bias"}, jnp.asarray(x_np), ) - if activation_type == "situglu": - np.testing.assert_allclose( - output_np, - np.asarray(jax.device_get(output_ref)).astype(np.float32), - **FWD_TOLERANCE["mxfp8"], - err_msg="SiTU fused MXFP8 forward parity breach", - ) for name in ("gate_kernel", "wi", "wo"): np.testing.assert_allclose( _to_global_numpy(_unwrap(grads["params"][name]), mesh).astype(np.float32), From 11af8e07d9e17c7e5b8a0ef4e3f67e82cfe0c913 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Tue, 17 Nov 2026 12:27:23 +0100 Subject: [PATCH 10/11] [JAX] Check SiTU caps through ActivationParams in fused MoE test Co-Authored-By: Claude Fable 5.1 Signed-off-by: Pawel Gadzinski --- tests/jax/test_te_ep_moe.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index bc2eb804e4f..6ec9060776c 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -1319,14 +1319,16 @@ def native_moe(*args, **kwargs): def checked_glu(*args, **kwargs): if activation_type == "situglu": assert kwargs["act_func"] == "situglu" - assert (kwargs["situ_beta1"], kwargs["situ_beta2"]) == (4.0, 25.0) + caps = kwargs["act_params"].situglu + assert (caps.gate_beta, caps.up_beta) == (4.0, 25.0) shared_calls.append("glu") return original_glu(*args, **kwargs) def checked_dglu(*args, **kwargs): if activation_type == "situglu": assert kwargs["act_func"] == "dsituglu" - assert (kwargs["situ_beta1"], kwargs["situ_beta2"]) == (4.0, 25.0) + caps = kwargs["act_params"].situglu + assert (caps.gate_beta, caps.up_beta) == (4.0, 25.0) shared_calls.append("dglu") return original_dglu(*args, **kwargs) From 7dc00c67801992b1b69c5a47558b3264fa74c6f6 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Fri, 20 Nov 2026 12:28:43 +0100 Subject: [PATCH 11/11] [JAX] Bound fused MoE parameter gradients by relative error Signed-off-by: Pawel Gadzinski --- tests/jax/test_te_ep_moe.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index 521f37d8d64..888b0c257d2 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -1429,16 +1429,19 @@ def loss_fn(params, inputs): jnp.asarray(x_np), ) for name in ("gate_kernel", "wi", "wo"): + grad_np = _to_global_numpy(_unwrap(grads["params"][name]), mesh).astype(np.float32) + grad_ref_np = np.asarray(jax.device_get(grads_ref[name])).astype(np.float32) np.testing.assert_allclose( - _to_global_numpy(_unwrap(grads["params"][name]), mesh).astype(np.float32), - np.asarray(jax.device_get(grads_ref[name])).astype(np.float32), - **( - GRAD_GATE_TOLERANCE["mxfp8"] - if name == "gate_kernel" - else GRAD_FFN_TOLERANCE["mxfp8"] - ), + grad_np, + grad_ref_np, + atol=3e-6, + rtol=1.3e-6, err_msg=f"{name} fused MXFP8 gradient parity breach", ) + relative_error = np.linalg.norm(grad_np - grad_ref_np) / np.linalg.norm(grad_ref_np) + assert ( + relative_error < 0.1 + ), f"{name} MXFP8 relative L2 error {relative_error:.3%} >= 10%" grad_x_ref_np = np.asarray(jax.device_get(grad_x_ref)).astype(np.float32) np.testing.assert_allclose( grad_x_np,