From ab2ae5e447be4d5919ef8a123a9dc4ddaf5be7d1 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Tue, 10 Nov 2026 17:46:07 +0100 Subject: [PATCH] [JAX] Enable fused SiTU-GLU in MoEBlock Select the shared cuDNN grouped GEMM SiTU forward/backward APIs when their JAX bridge exposes both soft-cap parameters. Preserve fallback for incompatible frontends and devices, and pass device expert offsets through the existing fused path. [skip ci] Signed-off-by: Pawel Gadzinski --- tests/jax/test_custom_call_compute.py | 62 ++++++++++++++++++- tests/jax/test_te_ep_moe.py | 24 ++++++- .../jax/cpp_extensions/grouped_gemm_glu.py | 32 ++++++++-- transformer_engine/jax/flax/moe.py | 4 +- transformer_engine/jax/moe.py | 51 +++++++++++---- 5 files changed, 153 insertions(+), 20 deletions(-) diff --git a/tests/jax/test_custom_call_compute.py b/tests/jax/test_custom_call_compute.py index 61759d74746..e0d04406d59 100644 --- a/tests/jax/test_custom_call_compute.py +++ b/tests/jax/test_custom_call_compute.py @@ -2156,6 +2156,11 @@ def function(**kwargs): if change == "positional" and name == "a_tensor" else inspect.Parameter.POSITIONAL_OR_KEYWORD ), + default=( + 4.0 + if name == "situ_beta1" + else 25.0 if name == "situ_beta2" else inspect.Parameter.empty + ), ) for name in arguments if not (change == "renamed" and name == "discrete_col_sfd") @@ -2282,6 +2287,61 @@ def test_api_signature(self, frontend, name, args, change): assert available == (change == "optional") assert bool(reason) == (change != "optional") + @pytest.mark.parametrize("missing", [None, "situ_beta1", "situ_beta2"]) + @pytest.mark.parametrize( + "name,arguments", + [ + ("grouped_gemm_glu_wrapper_sm100", _glu_adapter._FORWARD_ARGS), + ("grouped_gemm_dglu_wrapper_sm100", _glu_adapter._BACKWARD_ARGS), + ("grouped_gemm_glu_jax_sm100", _glu_adapter._JAX_FORWARD_ARGS), + ("grouped_gemm_dglu_jax_sm100", _glu_adapter._JAX_BACKWARD_ARGS), + ], + ) + def test_situ_requires_scalar_parameters_in_both_framework_apis( + self, frontend, name, arguments, missing + ): + apis = ( + ("grouped_gemm_glu_wrapper_sm100", _glu_adapter._FORWARD_ARGS), + ("grouped_gemm_dglu_wrapper_sm100", _glu_adapter._BACKWARD_ARGS), + ("grouped_gemm_glu_jax_sm100", _glu_adapter._JAX_FORWARD_ARGS), + ("grouped_gemm_dglu_jax_sm100", _glu_adapter._JAX_BACKWARD_ARGS), + ) + for api_name, args in apis: + setattr(frontend, api_name, self._api(args + ("situ_beta1", "situ_beta2"))) + if missing: + setattr( + frontend, + name, + self._api( + arguments + tuple(arg for arg in ("situ_beta1", "situ_beta2") if arg != missing) + ), + ) + assert _glu_adapter.grouped_gemm_glu_dependencies_available()[0] + available, reason = _glu_adapter.grouped_gemm_glu_dependencies_available(require_situ=True) + assert available is (missing is None) + if missing: + assert name in reason and missing in reason + + @pytest.mark.parametrize("capability", [90, 100, 103, 107, 120]) + def test_situ_fusion_device_support(self, frontend, monkeypatch, capability): + for name, args in ( + ("grouped_gemm_glu_wrapper_sm100", _glu_adapter._FORWARD_ARGS), + ("grouped_gemm_dglu_wrapper_sm100", _glu_adapter._BACKWARD_ARGS), + ("grouped_gemm_glu_jax_sm100", _glu_adapter._JAX_FORWARD_ARGS), + ("grouped_gemm_dglu_jax_sm100", _glu_adapter._JAX_BACKWARD_ARGS), + ): + setattr(frontend, name, self._api(args + ("situ_beta1", "situ_beta2"))) + monkeypatch.setattr( + transformer_engine_jax, "get_device_compute_capability", lambda _: capability + ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + selected = _moe_module._select_cudnn_jax_fusion( + [], activation_type=_moe_module.SiTUActivation() + ) + assert selected is (capability in (100, 103)) + assert bool(caught) is not selected + @pytest.mark.parametrize("name", ["grouped_gemm_glu_jax_sm100", "grouped_gemm_dglu_jax_sm100"]) def test_bf16_only_jax_api_falls_back(self, frontend, monkeypatch, name): # Frontend 1.29 exposes JAX entry points for BF16 discrete weights, but @@ -2396,7 +2456,7 @@ def execute(*args): assert bound["use_cudnn_fusion"] is (request_fusion is not False) assert bound["cudnn_native_weight_layout"] is native_layout received.append(bound["use_cudnn_jax_fusion"]) - return args[0], None, jnp.asarray(0), jnp.zeros((2,), jnp.int32) + return args[0], None, jnp.asarray(0), jnp.zeros((2,), jnp.int32), None monkeypatch.setattr(_moe_module, "_moe", execute) x = jnp.ones((1, 1, 128), jnp.bfloat16) diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index ddfed8e6fa0..8636d60d46f 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -1438,7 +1438,9 @@ def unexpected_fused_kernel(*args, **kwargs): # independently forcing the baseline to execute the unfused FFN. moe_module = importlib.import_module("transformer_engine.jax.moe") with monkeypatch.context() as baseline_patch: - baseline_patch.setattr(moe_module, "_select_cudnn_jax_fusion", lambda _: False) + baseline_patch.setattr( + moe_module, "_select_cudnn_jax_fusion", lambda *args, **kwargs: False + ) variables, baseline_output, _, _ = _init_apply( baseline, mesh, x, jax.random.PRNGKey(52) ) @@ -1473,6 +1475,11 @@ def native_moe(*args, **kwargs): _to_global_numpy(_unwrap(baseline_grads["params"][name]), mesh), ) + @pytest.mark.parametrize( + "activation_type", + ["silu", SiTUActivation(), SiTUActivation(2.0, 8.0)], + ids=["swiglu", "situ-k3", "situ-custom"], + ) @pytest.mark.parametrize("native_layout", [False, True]) @pytest.mark.parametrize("apply_topk_weights_early", [False, True]) def test_mxfp8_forward_and_backward( @@ -1483,12 +1490,15 @@ def test_mxfp8_forward_and_backward( make_block, use_cudnn_fusion, native_layout, + activation_type, ): if not use_cudnn_fusion: pytest.skip("requires the cuDNN fusion fixture parameter") from transformer_engine.jax import cpp_extensions as tex - available, reason = tex.grouped_gemm_glu_dependencies_available() + available, reason = tex.grouped_gemm_glu_dependencies_available( + require_situ=isinstance(activation_type, SiTUActivation) + ) if not available: pytest.skip(reason) @@ -1511,10 +1521,18 @@ def native_moe(*args, **kwargs): original_dglu = tex.grouped_gemm_dglu def checked_glu(*args, **kwargs): + if isinstance(activation_type, SiTUActivation): + assert kwargs["act_func"] == "situglu" + assert kwargs["situ_beta1"] == activation_type.gate_beta + assert kwargs["situ_beta2"] == activation_type.up_beta shared_calls.append("glu") return original_glu(*args, **kwargs) def checked_dglu(*args, **kwargs): + if isinstance(activation_type, SiTUActivation): + assert kwargs["act_func"] == "dsituglu" + assert kwargs["situ_beta1"] == activation_type.gate_beta + assert kwargs["situ_beta2"] == activation_type.up_beta shared_calls.append("dglu") return original_dglu(*args, **kwargs) @@ -1523,6 +1541,7 @@ def checked_dglu(*args, **kwargs): block = make_block( apply_topk_weights_early=apply_topk_weights_early, quantization_recipe=MXFP8BlockScaling(), + activation_type=activation_type, ) x = _make_inputs(jax.random.PRNGKey(30)) variables, output, aux, _ = _init_apply(block, mesh, x, jax.random.PRNGKey(31)) @@ -1558,6 +1577,7 @@ def loss_fn(params, inputs): num_experts_per_tok=TOPK, score_function="softmax", expert_bias=None, + activation_type=activation_type, ) return jnp.mean(out.astype(jnp.float32) ** 2) diff --git a/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py b/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py index 86c497218ce..9238975f023 100644 --- a/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py +++ b/transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py @@ -108,7 +108,7 @@ def _compact_sf(scale: jax.Array, shape: tuple[int, ...], name: str) -> jax.Arra ) -def grouped_gemm_glu_dependencies_available() -> tuple[bool, str]: +def grouped_gemm_glu_dependencies_available(*, require_situ: bool = False) -> tuple[bool, str]: """Require Frontend >= 1.32.0 and shared JAX APIs supporting dense MXFP8.""" try: cutlass_jax = importlib.import_module("cutlass.jax") @@ -126,6 +126,8 @@ def grouped_gemm_glu_dependencies_available() -> tuple[bool, str]: api = getattr(module, name, None) if not callable(api): return False, f"{module.__name__}.{name} is unavailable" + if require_situ: + arguments += ("situ_beta1", "situ_beta2") signature = inspect.signature(api) missing = set(arguments) - signature.parameters.keys() if missing: @@ -151,8 +153,11 @@ def grouped_gemm_glu( *, compute_dtype, output_dtype, + act_func: str = "swiglu", + situ_beta1: float = 4.0, + situ_beta2: float = 25.0, ) -> tuple[jax.Array, jax.Array, jax.Array, jax.Array, jax.Array]: - """Run cuDNN's shared grouped MXFP8 GLU API in SwiGLU mode.""" + """Run cuDNN's shared grouped MXFP8 GLU API with the selected activation.""" cudnn_grouped_gemm_glu = getattr( importlib.import_module("cudnn"), "grouped_gemm_glu_wrapper_sm100" ) @@ -167,6 +172,9 @@ def grouped_gemm_glu( prob, compute_dtype=compute_dtype, output_dtype=output_dtype, + act_func=act_func, + situ_beta1=situ_beta1, + situ_beta2=situ_beta2, ) @@ -181,6 +189,9 @@ def _grouped_gemm_forward( *, compute_dtype, output_dtype, + act_func: str = "swiglu", + situ_beta1: float = 4.0, + situ_beta2: float = 25.0, ): rows, hidden, _ = a.shape experts, combined, b_hidden = b.shape @@ -188,6 +199,9 @@ def _grouped_gemm_forward( raise ValueError(f"A K={hidden} does not match B K={b_hidden}") alpha = jnp.ones((experts,), dtype=jnp.float32) norm_const = jnp.ones((1,), dtype=jnp.float32) + situ_kwargs = {} + if act_func == "situglu": + situ_kwargs = {"situ_beta1": situ_beta1, "situ_beta2": situ_beta2} result = cudnn_forward( a_tensor=a.reshape(rows, hidden), b_tensor=b, @@ -201,8 +215,9 @@ def _grouped_gemm_forward( d_dtype=jnp.dtype(output_dtype), discrete_col_sfd=True, sf_vec_size=32, - act_func="swiglu", + act_func=act_func, generate_c=True, + **situ_kwargs, ) return ( result["c_tensor"].reshape(rows, combined), @@ -223,8 +238,11 @@ def grouped_gemm_dglu( prob: jax.Array, *, output_dtype, + act_func: str = "dswiglu", + situ_beta1: float = 4.0, + situ_beta2: float = 25.0, ) -> tuple[jax.Array, jax.Array, jax.Array, jax.Array, jax.Array]: - """Run cuDNN's shared grouped MXFP8 dGLU API in dSwiGLU mode.""" + """Run cuDNN's shared grouped MXFP8 dGLU API with the selected activation.""" if a.ndim != 2: raise ValueError(f"Expected A[M,K], got {a.shape}") if b.ndim != 3: @@ -246,6 +264,9 @@ def grouped_gemm_dglu( alpha = jnp.ones((experts,), dtype=jnp.float32) beta = jnp.ones((experts,), dtype=jnp.float32) norm_const = jnp.ones((1,), dtype=jnp.float32) + situ_kwargs = {} + if act_func == "dsituglu": + situ_kwargs = {"situ_beta1": situ_beta1, "situ_beta2": situ_beta2} result = cudnn_grouped_gemm_dglu( a_tensor=a, b_tensor=b, @@ -261,7 +282,8 @@ def grouped_gemm_dglu( discrete_col_sfd=True, dprob_tensor=None, sf_vec_size=32, - act_func="dswiglu", + act_func=act_func, + **situ_kwargs, ) return ( result["d_row_tensor"], diff --git a/transformer_engine/jax/flax/moe.py b/transformer_engine/jax/flax/moe.py index 283902756ef..ab720674978 100644 --- a/transformer_engine/jax/flax/moe.py +++ b/transformer_engine/jax/flax/moe.py @@ -77,7 +77,9 @@ class _MoEBlock(TransformerEngineBase): Activation between ``layer_w0 @ wi_0`` and the elementwise product with ``layer_w0 @ wi_1``. Default ``"silu"``. Pass :class:`~transformer_engine.jax.moe.SiTUActivation` to soft-cap both - branches in FP32 using native grouped SiTU and separate grouped GEMMs. + branches in FP32. Compatible MXFP8 calls can use cuDNN SiTU fusion + on SM100/SM103; other configurations use native grouped SiTU kernels + with separate grouped GEMMs. score_function : Union[str, ScoreFunction] ``"softmax"`` (default), ``"sigmoid"``, or ``"sqrtsoftplus"`` for the routing scores. diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index 44d37a3f3e1..eb0cb61325f 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -89,8 +89,9 @@ class SiTUActivation: The defaults match Kimi-K3. Both soft-cap parameters must be finite and positive. Pass an instance as ``moe(..., activation_type=...)``. - BF16 and MXFP8 use native grouped SiTU forward/backward with separate GEMMs. - Requesting cuDNN fusion retains its normal warning and fallback behavior. + MXFP8 can use cuDNN's fused grouped GEMM on SM100/SM103 with compatible + JAX SiTU APIs. Other configurations use native grouped SiTU forward/backward + with separate grouped GEMMs. """ gate_beta: float = 4.0 @@ -272,7 +273,7 @@ def _resolve_moe_mesh_resource( _CUDNN_JAX_ALIGN_SIZE = 256 -def _select_cudnn_jax_fusion(rejection_reasons: list[str]) -> bool: +def _select_cudnn_jax_fusion(rejection_reasons: list[str], *, activation_type="silu") -> bool: """Select shared cuDNN GLU/dGLU fusion or the unfused TE path.""" from transformer_engine_jax import get_device_compute_capability @@ -284,8 +285,13 @@ def _select_cudnn_jax_fusion(rejection_reasons: list[str]) -> bool: else: if capability < 100: reasons.append(f"fused kernels require SM100+, got SM{capability}") + require_situ = isinstance(activation_type, SiTUActivation) + if require_situ and capability not in (100, 103): + reasons.append(f"SiTU fusion requires SM100/SM103, got SM{capability}") if not reasons: - available, error = tex.grouped_gemm_glu_dependencies_available() + available, error = tex.grouped_gemm_glu_dependencies_available( + require_situ=require_situ + ) if available: return True reasons.append(f"shared fused API is incompatible: {error}") @@ -313,8 +319,8 @@ def _cudnn_jax_fusion_rejection_reasons( ) -> list[str]: """Return reasons this call cannot use cuDNN's shared grouped GLU/dGLU JAX APIs.""" errors = [] - if str(activation_type).lower() != "silu": - errors.append("requires activation_type='silu'") + if not isinstance(activation_type, SiTUActivation) and str(activation_type).lower() != "silu": + errors.append("requires activation_type='silu' or SiTUActivation") if wi_0_bias is not None or wi_1_bias is not None: errors.append("does not support FC1 gate/up bias") hidden = x.shape[-1] @@ -1030,6 +1036,15 @@ def _ffn_fwd_per_shard( prob, compute_dtype=sorted_x.dtype, output_dtype=fc2_quantizer_set.x.q_dtype, + **( + { + "act_func": "situglu", + "situ_beta1": activation_type.gate_beta, + "situ_beta2": activation_type.up_beta, + } + if isinstance(activation_type, SiTUActivation) + else {} + ), ) combined_out = combined_out_3d.reshape(sorted_x.shape[0], combined) if wi_0_checkpoint_name is not None: @@ -1214,6 +1229,15 @@ def _ffn_bwd_per_shard( padded_offsets, prob, output_dtype=fc1_quantizer_set.dgrad.q_dtype, + **( + { + "act_func": "dsituglu", + "situ_beta1": activation_type.gate_beta, + "situ_beta2": activation_type.up_beta, + } + if isinstance(activation_type, SiTUActivation) + else {} + ), ) d_recv_w_from_intermediate = ( dprob.astype(recv_w_flat.dtype) @@ -2263,9 +2287,9 @@ def moe( ---------- activation_type : Union[str, SiTUActivation] Gate activation name (default ``"silu"``), or :class:`SiTUActivation` - for FP32 soft-capping of both gate and up branches. SiTU uses native - grouped activation kernels and separate grouped GEMMs with BF16 or - MXFP8 quantizer sets. cuDNN GLU/dGLU uses the existing fallback path. + for FP32 soft-capping of both gate and up branches. Compatible MXFP8 + calls use cuDNN fusion on SM100/SM103; other configurations use native + grouped SiTU kernels with separate grouped GEMMs. router_info : Union[RouterComputationInfo, RoutingMapInfo] :class:`RouterComputationInfo` makes TE compute the gate and top-k from the given router parameters. :class:`RoutingMapInfo` supplies @@ -2319,7 +2343,10 @@ def moe( use_cudnn_fusion : bool Defaults to ``True``: try cuDNN's shared JAX grouped MXFP8 GLU/dGLU APIs on SM100+ GPUs; cuDNN selects the architecture-specific kernels. ``False`` uses unfused TE grouped GEMM. - Requires cuDNN Frontend >= 1.32.0 with compatible JAX APIs. + Requires cuDNN Frontend >= 1.32.0 with compatible JAX APIs. SiTU + additionally requires SM100/SM103 and JAX GLU/dGLU APIs exposing + ``situ_beta1`` and ``situ_beta2``. Device offsets bound active expert + rows; unused receive-buffer capacity is not part of any expert. Frontends whose JAX GLU/dGLU entry points only support BF16 discrete weights fall back to unfused TE grouped GEMM. Ineligible calls warn and fall back to TE's regular grouped-GEMM @@ -2453,7 +2480,9 @@ def moe( activation_type=activation_type, ep_axis=ep_axis, ) - use_cudnn_jax_fusion = _select_cudnn_jax_fusion(rejection_reasons) + use_cudnn_jax_fusion = _select_cudnn_jax_fusion( + rejection_reasons, activation_type=activation_type + ) output, aux_loss, total_recv_tokens, expert_counts, qb_histograms = _moe( x,