From b8be87d42493ba32fc2202b86ceb1dc03fcae12f Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Wed, 18 Nov 2026 14:50:41 +0100 Subject: [PATCH 1/3] [JAX] Add SiTU fallback on top of cuDNN support Reuse the upstream SiTU activation string and ActivationParams container. Keep eligible cuDNN fusion and use masked JAX activation operations with a VJP for unfused execution. Signed-off-by: Pawel Gadzinski --- docs/api/jax.rst | 8 ++ tests/jax/moe/test_moe_resource_api.py | 132 +++++++++++++++++++++++++ tests/jax/test_te_ep_moe.py | 8 ++ transformer_engine/jax/flax/moe.py | 6 +- transformer_engine/jax/moe.py | 60 +++++++---- 5 files changed, 192 insertions(+), 22 deletions(-) diff --git a/docs/api/jax.rst b/docs/api/jax.rst index 99282be422c..01bf6a0680e 100644 --- a/docs/api/jax.rst +++ b/docs/api/jax.rst @@ -74,6 +74,14 @@ Router, routing kernels, and grouped dense for MoE layers. See .. autoapifunction:: transformer_engine.jax.dense.grouped_dense +MoE callers select SiTU with ``activation_type="situglu"`` and optional +``activation_params={"gate_beta": 4.0, "up_beta": 25.0}``. Eligible MXFP8 +configurations use cuDNN fusion. With ``use_cudnn_fusion=False`` or unsupported +fusion settings, the activation uses FP32 JAX operations and ``jax.vjp`` between +the grouped GEMMs. Device expert counts mask inactive receive capacity in forward +and backward, including non-finite inactive values. Outputs keep the projection +dtype; MXFP8 quantization remains separate in the fallback. + .. autoapifunction:: transformer_engine.jax.moe.moe .. autoapifunction:: transformer_engine.jax.ep.ep_bootstrap diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index 2764d9ca54b..e728550b31b 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -6,6 +6,7 @@ import importlib import inspect +from functools import partial from types import SimpleNamespace import jax @@ -18,6 +19,7 @@ from transformer_engine.jax.flax import _MoEBlock from transformer_engine.jax.moe import ( RouterComputationInfo, + _apply_ffn_activation, _moe_mesh_axes, _resolve_moe_mesh_resource, ) @@ -83,6 +85,136 @@ def fused(**kwargs): assert call["situ_beta2"] == expected["up_beta"] +@pytest.mark.parametrize("betas", [(4.0, 25.0), (2.0, 7.0)]) +def test_situ_value_and_both_branch_derivatives(betas): + (gate_beta, up_beta) = betas + gate = jnp.asarray([-10000.0, -12.0, -0.5, 0.0, 0.5, 12.0, 10000.0]) + up = jnp.asarray([10000.0, 40.0, -3.0, 2.0, 3.0, -40.0, -10000.0]) + params = ActivationParams.create("situglu", gate_beta=gate_beta, up_beta=up_beta) + counts = jnp.asarray([1], jnp.int32) + + def activate(g, u): + return _apply_ffn_activation("situglu", g[None, :], u[None, :], counts, params)[0] + + (output, pullback) = jax.vjp(activate, gate, up) + (dg, du) = pullback(jnp.ones_like(output)) + (g, u) = (np.asarray(gate, np.float64), np.asarray(up, np.float64)) + sigmoid = 0.5 + 0.5 * np.tanh(g / 2) + (tg, tu) = (np.tanh(g / gate_beta), np.tanh(u / up_beta)) + (capped_gate, capped_up) = (gate_beta * tg * sigmoid, up_beta * tu) + derivative_gate = (1 - tg**2) * sigmoid + gate_beta * tg * sigmoid * (1 - sigmoid) + for actual, expected in ( + (output, capped_gate * capped_up), + (dg, derivative_gate * capped_up), + (du, capped_gate * (1 - tu**2)), + ): + np.testing.assert_allclose(actual, expected, atol=2e-05, rtol=2e-05) + + +def test_situ_bf16_uses_fp32_before_final_cast(): + gate = jnp.asarray([0.25, 1.5, -3.0, 12.0, 64.0], dtype=jnp.bfloat16) + up = jnp.asarray([0.75, -2.0, 32.0, 64.0, -128.0], dtype=jnp.bfloat16) + actual = _apply_ffn_activation( + "situglu", gate[None, :], up[None, :], jnp.asarray([1], jnp.int32) + )[0] + (g, u) = (np.asarray(gate, np.float64), np.asarray(up, np.float64)) + expected = jnp.asarray(100 * np.tanh(g / 4) / (1 + np.exp(-g)) * np.tanh(u / 25), jnp.bfloat16) + assert actual.dtype == jnp.bfloat16 + np.testing.assert_array_equal(actual, expected) + + +@pytest.mark.parametrize("dtype", [jnp.float32, jnp.float16, jnp.bfloat16]) +@pytest.mark.parametrize("betas", [(4.0, 25.0), (2.0, 8.0)]) +@pytest.mark.parametrize("counts", [[3, 0, 5], [0, 0, 0], [0] * 257 + [13], [0]]) +def test_situ_masks_inactive_values_and_gradients(dtype, betas, counts): + capacity = 0 if counts == [0] else 13 + active = sum(counts) + rng = np.random.default_rng(72) + (gate, up, dy) = [rng.uniform(-5, 5, (capacity, 6)).astype(np.float32) for _ in range(3)] + for value in (gate, up, dy): + value[active:] = np.nan + (gate, up, dy) = [jnp.asarray(value, dtype) for value in (gate, up, dy)] + sizes = jnp.asarray(counts, jnp.int32) + activate = lambda g, u: _apply_ffn_activation( + "situglu", + g, + u, + sizes, + ActivationParams.create("situglu", gate_beta=betas[0], up_beta=betas[1]), + ) + output = jax.jit(activate)(gate, up) + (_, pullback) = jax.vjp(activate, gate, up) + (dg, du) = jax.jit(pullback)(dy) + (g, u, d) = [np.asarray(value, np.float64)[:active] for value in (gate, up, dy)] + (gate_beta, up_beta) = betas + (tg, tu, sigmoid) = (np.tanh(g / gate_beta), np.tanh(u / up_beta), 1 / (1 + np.exp(-g))) + ref = ( + gate_beta * tg * sigmoid * up_beta * tu, + d * ((1 - tg**2) * sigmoid + gate_beta * tg * sigmoid * (1 - sigmoid)) * up_beta * tu, + d * gate_beta * tg * sigmoid * (1 - tu**2), + ) + tolerance = {jnp.float32: 2e-05, jnp.float16: 0.003, jnp.bfloat16: 0.02}[dtype] + for actual, expected in zip((output, dg, du), ref): + actual = np.asarray(actual, np.float32) + assert np.isfinite(actual).all() + np.testing.assert_array_equal(actual[active:], 0) + np.testing.assert_allclose(actual[:active], expected, atol=tolerance, rtol=tolerance) + + +def test_situ_dynamic_counts_remat_and_vmap(): + gate = jnp.arange(13 * 6, dtype=jnp.float32).reshape(13, 6) / 20 - 2 + up = -gate + activation = "situglu" + activate = partial(_apply_ffn_activation, activation) + compiled = jax.jit(activate) + for counts in (jnp.array([3, 0, 5]), jnp.array([1, 2, 2])): + expected = activate(gate, up, counts) + np.testing.assert_allclose(compiled(gate, up, counts), expected, atol=2e-05, rtol=2e-05) + loss = lambda g, u: activate(g, u, counts).sum() + gradients = jax.jit(jax.grad(loss, (0, 1)))(gate, up) + remat = jax.jit(jax.grad(jax.checkpoint(loss), (0, 1)))(gate, up) + for actual, reference in zip(remat, gradients): + np.testing.assert_allclose(actual, reference, atol=2e-05, rtol=2e-05) + counts = jnp.array([[3, 0, 5], [1, 2, 2]], dtype=jnp.int32) + (gates, ups) = (jnp.stack([gate, gate]), jnp.stack([up, up])) + mapped = jax.jit(jax.vmap(activate))(gates, ups, counts) + for actual, sizes in zip(mapped, counts): + np.testing.assert_allclose(actual, activate(gate, up, sizes), atol=2e-05, rtol=2e-05) + hlo = compiled.lower(gate, up, counts[0]).compile().as_text() + assert "te_grouped_situglu_ffi" not in hlo + + +@pytest.mark.parametrize( + "capability,available,expected", + [(89, True, False), (100, False, False), (103, True, True), (107, True, False)], +) +def test_situ_fusion_or_jax_fallback(monkeypatch, capability, available, expected): + import transformer_engine_jax + + module = importlib.import_module("transformer_engine.jax.moe") + monkeypatch.setattr( + transformer_engine_jax, "get_device_compute_capability", lambda _: capability + ) + monkeypatch.setattr( + module.tex, + "grouped_gemm_glu_dependencies_available", + lambda **_: (available, "unavailable frontend"), + ) + if expected: + assert module._select_cudnn_jax_fusion([], activation_type="situglu") + else: + with pytest.warns(UserWarning, match="falling back"): + assert not module._select_cudnn_jax_fusion([], activation_type="situglu") + + +def test_situ_can_disable_fusion(): + module = importlib.import_module("transformer_engine.jax.moe") + with pytest.raises(ValueError, match="active global_shard_guard"): + module.moe( + None, None, None, None, num_experts=2, activation_type="situglu", use_cudnn_fusion=False + ) + + @pytest.fixture(autouse=True) def no_global_resource(): with global_shard_guard(None): diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index 6ec9060776c..4c0c259b8a8 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -584,6 +584,13 @@ def _quantization_recipe(quantization): # ----------------------------------------------------------------------------- _CONFIGS = [ + *[ + pytest.param( + dict(activation_type="situglu", apply_topk_weights_early=early), + id=f"situ-{'early' if early else 'late'}-weighting", + ) + for early in (False, True) + ], pytest.param( dict(score_function="softmax"), id="softmax", @@ -739,6 +746,7 @@ def loss_fn(variables, inputs): def _reference_kwargs_from_config(config, params_np): """Pick out the reference-relevant pieces of a parametrize config.""" return dict( + activation_type=config.get("activation_type", "silu"), score_function=config.get("score_function", "softmax"), expert_bias=( jnp.asarray(params_np["expert_bias"]) diff --git a/transformer_engine/jax/flax/moe.py b/transformer_engine/jax/flax/moe.py index 0402c84f3a7..4dca3bb8693 100644 --- a/transformer_engine/jax/flax/moe.py +++ b/transformer_engine/jax/flax/moe.py @@ -75,9 +75,9 @@ class _MoEBlock(TransformerEngineBase): activation_type : str Activation between ``layer_w0 @ wi_0`` and the elementwise product with ``layer_w0 @ wi_1``. Default ``"silu"``. Pass - ``"situglu"`` for fused MXFP8 - SiTU-GLU on SM100/SM103. SiTU requires cuDNN Frontend >= 1.33.0 with compatible JAX - APIs and ``use_cudnn_fusion=True``; there is no unfused SiTU path. + ``"situglu"`` for SiTU-GLU. Eligible MXFP8 calls use cuDNN fusion on + SM100/SM103 with cuDNN Frontend >= 1.33.0 and compatible JAX APIs. + Other configurations use JAX activation operations and a VJP between grouped GEMMs. activation_params : Optional[dict] Static SiTU soft caps: ``gate_beta`` (default 4.0) and ``up_beta`` (default 25.0). Values must be finite and positive. diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index e299724ea8c..11514a1b143 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -101,6 +101,18 @@ def _fused_glu_activation_kwargs(activation_type, act_params, *, backward=False) return {"act_func": ("d" if backward else "") + act_func, "act_params": act_params} +def _apply_ffn_activation(activation_type, gate, up, group_sizes, act_params=ActivationParams()): + if activation_type == "situglu": + active = (jnp.arange(gate.shape[0]) < group_sizes.sum())[:, None] + gate_fp32 = jnp.where(active, gate, 0).astype(jnp.float32) + up_fp32 = jnp.where(active, up, 0).astype(jnp.float32) + caps = act_params.situglu + gated = caps.gate_beta * jnp.tanh(gate_fp32 / caps.gate_beta) * jax.nn.sigmoid(gate_fp32) + linear = caps.up_beta * jnp.tanh(up_fp32 / caps.up_beta) + return jnp.where(active, (gated * linear).astype(gate.dtype), 0) + return _convert_to_activation_function(activation_type)(gate) * up + + @flax.struct.dataclass class RouterComputationInfo: """Router parameters and settings for :func:`moe` to compute routing itself. @@ -254,8 +266,6 @@ def _select_cudnn_jax_fusion(rejection_reasons: list[str], *, activation_type="s if available: return True reasons.append(f"shared fused API is incompatible: {error}") - if activation_type == "situglu": - raise ValueError("SiTU requires cuDNN MXFP8 fusion: " + "; ".join(reasons)) warnings.warn( "use_cudnn_fusion=True: falling back to the unfused TE grouped-GEMM path: " + "; ".join(reasons) @@ -1073,8 +1083,9 @@ def _ffn_fwd_per_shard( # that's all bf16; for FP8/FP4 the downstream grouped_quantize is what # transitions to the target precision. if not use_cudnn_jax_fusion: - act_fn = _convert_to_activation_function(activation_type) - intermediate = act_fn(gate_proj_out) * up_proj_out + intermediate = _apply_ffn_activation( + activation_type, gate_proj_out, up_proj_out, group_sizes, act_params + ) if apply_topk_weights_early and not use_cudnn_jax_fusion: # Fold the per-token combine weights into the FFN intermediate; # the downstream wo GEMM is linear so this is equivalent to the @@ -1239,12 +1250,13 @@ def _ffn_bwd_per_shard( casted_wo_rhs_trans, contracting_dims=((1,), (2,)), ) - act_fn = _convert_to_activation_function(activation_type) if apply_topk_weights_early: # intermediate' = intermediate * w. # Masking is not required as grouped GEMMs consume only group-size rows. w_b = recv_w_flat[:, None].astype(d_intermediate.dtype) - intermediate_unweighted = act_fn(gate_proj_out) * up_proj_out + intermediate_unweighted = _apply_ffn_activation( + activation_type, gate_proj_out, up_proj_out, group_sizes, act_params + ) d_recv_w_from_intermediate = jnp.sum( d_intermediate * intermediate_unweighted, axis=-1, @@ -1253,10 +1265,23 @@ def _ffn_bwd_per_shard( else: d_recv_w_from_intermediate = jnp.zeros_like(recv_w_flat) - # Activation bwd stays in the GEMM dtype, matching the forward path. - act_gp, dact_pullback = jax.vjp(act_fn, gate_proj_out) - d_up_proj_out = d_intermediate * act_gp - (d_gate_proj_out,) = dact_pullback(d_intermediate * up_proj_out) + if activation_type == "situglu": + _, dact_pullback = jax.vjp( + partial( + _apply_ffn_activation, + activation_type, + group_sizes=group_sizes, + act_params=act_params, + ), + gate_proj_out, + up_proj_out, + ) + d_gate_proj_out, d_up_proj_out = dact_pullback(d_intermediate) + else: + act_fn = _convert_to_activation_function(activation_type) + act_gp, dact_pullback = jax.vjp(act_fn, gate_proj_out) + d_up_proj_out = d_intermediate * act_gp + (d_gate_proj_out,) = dact_pullback(d_intermediate * up_proj_out) d_combined_for_bias = jnp.concatenate([d_gate_proj_out, d_up_proj_out], axis=-1) casted_d_combined = tex.grouped_quantize( d_combined_for_bias, @@ -2168,8 +2193,8 @@ def moe( Parameters ---------- activation_type : str - Gate activation (default ``"silu"``), or ``"situglu"`` for fused MXFP8 - SiTU-GLU on SM100/SM103. Unsupported SiTU configurations raise ``ValueError``. + Gate activation (default ``"silu"``), or ``"situglu"`` for SiTU-GLU. + SiTU uses cuDNN MXFP8 fusion when supported and a JAX/VJP fallback otherwise. activation_params : Optional[dict] Static SiTU soft caps: ``gate_beta`` (default 4.0) and ``up_beta`` (default 25.0). Values must be finite and positive. Other activations @@ -2228,14 +2253,13 @@ def moe( Defaults to ``True``: try cuDNN's shared JAX grouped MXFP8 GLU/dGLU APIs on SM100+ GPUs; cuDNN selects the architecture-specific kernels. ``False`` uses unfused TE grouped GEMM. Requires cuDNN Frontend >= 1.32.0 with compatible JAX APIs. - SiTU additionally requires SM100/SM103 and JAX GLU/dGLU APIs exposing + SiTU fusion additionally requires SM100/SM103 and JAX GLU/dGLU APIs exposing ``situ_beta1`` and ``situ_beta2`` with Frontend >= 1.33.0. - Ineligible SiTU calls raise - ``ValueError``; there is no unfused SiTU path. Frontends whose JAX GLU/dGLU entry points only support BF16 discrete weights fall back to unfused TE grouped GEMM. - Except for SiTU, ineligible calls warn and fall back to TE's regular - grouped-GEMM implementation. API signatures and GPU capability determine support. + Ineligible calls warn and fall back to TE's regular grouped-GEMM implementation, + with JAX activation operations and a VJP for SiTU. API signatures and GPU + capability determine fusion support. cudnn_native_weight_layout : Optional[bool] ``False`` selects standard contiguous gate/up [E,K,2N] storage; ``True`` selects native [E,2N,K] storage with alternating 32-column @@ -2262,8 +2286,6 @@ def moe( if not isinstance(use_cudnn_fusion, bool): raise TypeError("use_cudnn_fusion must be a bool") act_params = _resolve_moe_activation_params(activation_type, activation_params) - if activation_type == "situglu" and not use_cudnn_fusion: - raise ValueError("SiTU requires use_cudnn_fusion=True") mesh_resource, quant_before_fsdp_ag = _resolve_moe_mesh_resource( mesh_resource, quant_before_fsdp_ag ) From bc85a863b0153d721f924a676cd8535f9062c82d Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Wed, 18 Nov 2026 15:02:55 +0100 Subject: [PATCH 2/3] [JAX] Share activation conversion and VJP in MoE fallback Define SiTU gate and up operations in the existing parameter-aware activation converter. Use one masked gated-activation helper and VJP for SiTU and SwiGLU, and move fallback documentation from the API index to the moe parameters. Signed-off-by: Pawel Gadzinski --- docs/api/jax.rst | 8 --- .../jax/cpp_extensions/activation.py | 11 ++++ transformer_engine/jax/moe.py | 57 +++++++++---------- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/docs/api/jax.rst b/docs/api/jax.rst index 01bf6a0680e..99282be422c 100644 --- a/docs/api/jax.rst +++ b/docs/api/jax.rst @@ -74,14 +74,6 @@ Router, routing kernels, and grouped dense for MoE layers. See .. autoapifunction:: transformer_engine.jax.dense.grouped_dense -MoE callers select SiTU with ``activation_type="situglu"`` and optional -``activation_params={"gate_beta": 4.0, "up_beta": 25.0}``. Eligible MXFP8 -configurations use cuDNN fusion. With ``use_cudnn_fusion=False`` or unsupported -fusion settings, the activation uses FP32 JAX operations and ``jax.vjp`` between -the grouped GEMMs. Device expert counts mask inactive receive capacity in forward -and backward, including non-finite inactive values. Outputs keep the projection -dtype; MXFP8 quantization remains separate in the fallback. - .. autoapifunction:: transformer_engine.jax.moe.moe .. autoapifunction:: transformer_engine.jax.ep.ep_bootstrap diff --git a/transformer_engine/jax/cpp_extensions/activation.py b/transformer_engine/jax/cpp_extensions/activation.py index a1d6aed6073..ec5e9d758b1 100644 --- a/transformer_engine/jax/cpp_extensions/activation.py +++ b/transformer_engine/jax/cpp_extensions/activation.py @@ -142,6 +142,17 @@ def _convert_to_activation_function(fn_or_string, act_params: ActivationParams): """Convert a string to an activation function.""" if fn_or_string == "linear": return lambda x: x + if fn_or_string == "situ": + beta = act_params.situglu.gate_beta + + def situ(x): + x = x.astype(jnp.float32) + return beta * jnp.tanh(x / beta) * jax.nn.sigmoid(x) + + return situ + if fn_or_string == "situ_linear": + beta = act_params.situglu.up_beta + return lambda x: beta * jnp.tanh(x.astype(jnp.float32) / beta) if fn_or_string == "clamped_linear": limit = act_params.clamped_swiglu.limit offset = act_params.clamped_swiglu.glu_linear_offset diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index 11514a1b143..d7a594c97d8 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -54,7 +54,7 @@ from jax.sharding import NamedSharding, PartitionSpec as P from . import cpp_extensions as tex -from .cpp_extensions.activation import ActivationParams +from .cpp_extensions.activation import ActivationParams, _convert_to_activation_function from .cpp_extensions.misc import jax_dtype_to_te_dtype from .quantize import ( GroupedScaledTensor1x, @@ -69,7 +69,6 @@ ) from .quantize.dequantizer import _unswizzle_mxfp8_grouped_scale from .cpp_extensions.gemm import swizzled_scale -from .flax.module import _convert_to_activation_function from .router import ScoreFunction, _validate_score_function from .sharding import MeshResource, _get_mesh, global_mesh_resource, global_shard_guard @@ -102,15 +101,14 @@ def _fused_glu_activation_kwargs(activation_type, act_params, *, backward=False) def _apply_ffn_activation(activation_type, gate, up, group_sizes, act_params=ActivationParams()): - if activation_type == "situglu": - active = (jnp.arange(gate.shape[0]) < group_sizes.sum())[:, None] - gate_fp32 = jnp.where(active, gate, 0).astype(jnp.float32) - up_fp32 = jnp.where(active, up, 0).astype(jnp.float32) - caps = act_params.situglu - gated = caps.gate_beta * jnp.tanh(gate_fp32 / caps.gate_beta) * jax.nn.sigmoid(gate_fp32) - linear = caps.up_beta * jnp.tanh(up_fp32 / caps.up_beta) - return jnp.where(active, (gated * linear).astype(gate.dtype), 0) - return _convert_to_activation_function(activation_type)(gate) * up + gate_activation, up_activation = ( + ("situ", "situ_linear") if activation_type == "situglu" else (activation_type, "linear") + ) + active = (jnp.arange(gate.shape[0]) < group_sizes.sum())[:, None] + gate_fn = _convert_to_activation_function(gate_activation, act_params) + up_fn = _convert_to_activation_function(up_activation, act_params) + result = gate_fn(jnp.where(active, gate, 0)) * up_fn(jnp.where(active, up, 0)) + return jnp.where(active, result.astype(gate.dtype), 0) @flax.struct.dataclass @@ -1265,23 +1263,17 @@ def _ffn_bwd_per_shard( else: d_recv_w_from_intermediate = jnp.zeros_like(recv_w_flat) - if activation_type == "situglu": - _, dact_pullback = jax.vjp( - partial( - _apply_ffn_activation, - activation_type, - group_sizes=group_sizes, - act_params=act_params, - ), - gate_proj_out, - up_proj_out, - ) - d_gate_proj_out, d_up_proj_out = dact_pullback(d_intermediate) - else: - act_fn = _convert_to_activation_function(activation_type) - act_gp, dact_pullback = jax.vjp(act_fn, gate_proj_out) - d_up_proj_out = d_intermediate * act_gp - (d_gate_proj_out,) = dact_pullback(d_intermediate * up_proj_out) + _, dact_pullback = jax.vjp( + partial( + _apply_ffn_activation, + activation_type, + group_sizes=group_sizes, + act_params=act_params, + ), + gate_proj_out, + up_proj_out, + ) + d_gate_proj_out, d_up_proj_out = dact_pullback(d_intermediate) d_combined_for_bias = jnp.concatenate([d_gate_proj_out, d_up_proj_out], axis=-1) casted_d_combined = tex.grouped_quantize( d_combined_for_bias, @@ -2194,11 +2186,16 @@ def moe( ---------- activation_type : str Gate activation (default ``"silu"``), or ``"situglu"`` for SiTU-GLU. - SiTU uses cuDNN MXFP8 fusion when supported and a JAX/VJP fallback otherwise. + Eligible MXFP8 SiTU configurations use cuDNN fusion. Otherwise, SiTU + uses FP32 JAX operations and ``jax.vjp`` between the grouped GEMMs, + keeping the projection dtype at the output. Device expert counts mask + inactive receive capacity in forward and backward, including non-finite + inactive values. MXFP8 quantization remains separate in the fallback. activation_params : Optional[dict] Static SiTU soft caps: ``gate_beta`` (default 4.0) and ``up_beta`` (default 25.0). Values must be finite and positive. Other activations - do not accept parameters. + do not accept parameters. For example, + ``activation_params={"gate_beta": 4.0, "up_beta": 25.0}``. router_info : Union[RouterComputationInfo, RoutingMapInfo] :class:`RouterComputationInfo` makes TE compute the gate and top-k from the given router parameters. :class:`RoutingMapInfo` supplies From 5510fbfdbdf9ce6e69ea380b3de50deb9e25afe7 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Wed, 18 Nov 2026 16:25:41 +0100 Subject: [PATCH 3/3] [JAX] Reuse existing activation fallbacks in MoE Remove _apply_ffn_activation and call the existing JAX activation and VJP routines. Keep receive-capacity masking at the MoE call sites and exercise the actual forward/backward composition. Signed-off-by: Pawel Gadzinski --- tests/jax/moe/test_moe_resource_api.py | 48 ++++++++++++-------- transformer_engine/jax/moe.py | 61 ++++++++++++++------------ 2 files changed, 63 insertions(+), 46 deletions(-) diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index e728550b31b..41f1b078164 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -15,11 +15,14 @@ import pytest from jax.sharding import Mesh -from transformer_engine.jax.cpp_extensions.activation import ActivationParams +from transformer_engine.jax.cpp_extensions.activation import ( + ActivationParams, + _jax_act_lu, + _jax_quantize_dact_dbias, +) from transformer_engine.jax.flax import _MoEBlock from transformer_engine.jax.moe import ( RouterComputationInfo, - _apply_ffn_activation, _moe_mesh_axes, _resolve_moe_mesh_resource, ) @@ -91,10 +94,11 @@ def test_situ_value_and_both_branch_derivatives(betas): gate = jnp.asarray([-10000.0, -12.0, -0.5, 0.0, 0.5, 12.0, 10000.0]) up = jnp.asarray([10000.0, 40.0, -3.0, 2.0, 3.0, -40.0, -10000.0]) params = ActivationParams.create("situglu", gate_beta=gate_beta, up_beta=up_beta) - counts = jnp.asarray([1], jnp.int32) def activate(g, u): - return _apply_ffn_activation("situglu", g[None, :], u[None, :], counts, params)[0] + return _jax_act_lu( + jnp.stack([g, u], axis=-2), ("situ", "situ_linear"), act_params=params + ).data (output, pullback) = jax.vjp(activate, gate, up) (dg, du) = pullback(jnp.ones_like(output)) @@ -114,9 +118,9 @@ def activate(g, u): def test_situ_bf16_uses_fp32_before_final_cast(): gate = jnp.asarray([0.25, 1.5, -3.0, 12.0, 64.0], dtype=jnp.bfloat16) up = jnp.asarray([0.75, -2.0, 32.0, 64.0, -128.0], dtype=jnp.bfloat16) - actual = _apply_ffn_activation( - "situglu", gate[None, :], up[None, :], jnp.asarray([1], jnp.int32) - )[0] + actual = _jax_act_lu(jnp.stack([gate, up], axis=-2), ("situ", "situ_linear")).data.astype( + gate.dtype + ) (g, u) = (np.asarray(gate, np.float64), np.asarray(up, np.float64)) expected = jnp.asarray(100 * np.tanh(g / 4) / (1 + np.exp(-g)) * np.tanh(u / 25), jnp.bfloat16) assert actual.dtype == jnp.bfloat16 @@ -135,16 +139,19 @@ def test_situ_masks_inactive_values_and_gradients(dtype, betas, counts): value[active:] = np.nan (gate, up, dy) = [jnp.asarray(value, dtype) for value in (gate, up, dy)] sizes = jnp.asarray(counts, jnp.int32) - activate = lambda g, u: _apply_ffn_activation( - "situglu", - g, - u, - sizes, - ActivationParams.create("situglu", gate_beta=betas[0], up_beta=betas[1]), + mask = (jnp.arange(capacity) < sizes.sum())[:, None] + inputs = jnp.where(mask[:, None, :], jnp.stack([gate, up], axis=-2), 0) + params = ActivationParams.create("situglu", gate_beta=betas[0], up_beta=betas[1]) + activate = partial(_jax_act_lu, activation_type=("situ", "situ_linear"), act_params=params) + output = jnp.where(mask, jax.jit(activate)(inputs).data.astype(dtype), 0) + backward = partial( + _jax_quantize_dact_dbias, + activation_type=("situ", "situ_linear"), + is_dbias=False, + act_params=params, ) - output = jax.jit(activate)(gate, up) - (_, pullback) = jax.vjp(activate, gate, up) - (dg, du) = jax.jit(pullback)(dy) + gradients, _ = jax.jit(backward)(jnp.where(mask, dy, 0), inputs) + dg, du = gradients.data[:, 0, :], gradients.data[:, 1, :] (g, u, d) = [np.asarray(value, np.float64)[:active] for value in (gate, up, dy)] (gate_beta, up_beta) = betas (tg, tu, sigmoid) = (np.tanh(g / gate_beta), np.tanh(u / up_beta), 1 / (1 + np.exp(-g))) @@ -164,8 +171,13 @@ def test_situ_masks_inactive_values_and_gradients(dtype, betas, counts): def test_situ_dynamic_counts_remat_and_vmap(): gate = jnp.arange(13 * 6, dtype=jnp.float32).reshape(13, 6) / 20 - 2 up = -gate - activation = "situglu" - activate = partial(_apply_ffn_activation, activation) + + def activate(g, u, sizes): + active = (jnp.arange(g.shape[0]) < sizes.sum())[:, None] + inputs = jnp.where(active[:, None, :], jnp.stack([g, u], axis=-2), 0) + output = _jax_act_lu(inputs, ("situ", "situ_linear")).data + return jnp.where(active, output.astype(g.dtype), 0) + compiled = jax.jit(activate) for counts in (jnp.array([3, 0, 5]), jnp.array([1, 2, 2])): expected = activate(gate, up, counts) diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index d7a594c97d8..828e58cca5d 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -54,7 +54,11 @@ from jax.sharding import NamedSharding, PartitionSpec as P from . import cpp_extensions as tex -from .cpp_extensions.activation import ActivationParams, _convert_to_activation_function +from .cpp_extensions.activation import ( + ActivationParams, + _jax_act_lu, + _jax_quantize_dact_dbias, +) from .cpp_extensions.misc import jax_dtype_to_te_dtype from .quantize import ( GroupedScaledTensor1x, @@ -100,17 +104,6 @@ def _fused_glu_activation_kwargs(activation_type, act_params, *, backward=False) return {"act_func": ("d" if backward else "") + act_func, "act_params": act_params} -def _apply_ffn_activation(activation_type, gate, up, group_sizes, act_params=ActivationParams()): - gate_activation, up_activation = ( - ("situ", "situ_linear") if activation_type == "situglu" else (activation_type, "linear") - ) - active = (jnp.arange(gate.shape[0]) < group_sizes.sum())[:, None] - gate_fn = _convert_to_activation_function(gate_activation, act_params) - up_fn = _convert_to_activation_function(up_activation, act_params) - result = gate_fn(jnp.where(active, gate, 0)) * up_fn(jnp.where(active, up, 0)) - return jnp.where(active, result.astype(gate.dtype), 0) - - @flax.struct.dataclass class RouterComputationInfo: """Router parameters and settings for :func:`moe` to compute routing itself. @@ -1081,9 +1074,15 @@ def _ffn_fwd_per_shard( # that's all bf16; for FP8/FP4 the downstream grouped_quantize is what # transitions to the target precision. if not use_cudnn_jax_fusion: - intermediate = _apply_ffn_activation( - activation_type, gate_proj_out, up_proj_out, group_sizes, act_params + activations = ( + ("situ", "situ_linear") if activation_type == "situglu" else (activation_type, "linear") ) + active = jnp.arange(gate_proj_out.shape[0]) < group_sizes.sum() + activation_inputs = jnp.where( + active[:, None, None], jnp.stack([gate_proj_out, up_proj_out], axis=-2), 0 + ) + intermediate = _jax_act_lu(activation_inputs, activations, act_params=act_params).data + intermediate = jnp.where(active[:, None], intermediate.astype(gate_proj_out.dtype), 0) if apply_topk_weights_early and not use_cudnn_jax_fusion: # Fold the per-token combine weights into the FFN intermediate; # the downstream wo GEMM is linear so this is equivalent to the @@ -1248,13 +1247,21 @@ def _ffn_bwd_per_shard( casted_wo_rhs_trans, contracting_dims=((1,), (2,)), ) + activations = ( + ("situ", "situ_linear") if activation_type == "situglu" else (activation_type, "linear") + ) + active = jnp.arange(gate_proj_out.shape[0]) < group_sizes.sum() + activation_inputs = jnp.where( + active[:, None, None], jnp.stack([gate_proj_out, up_proj_out], axis=-2), 0 + ) + d_intermediate = jnp.where(active[:, None], d_intermediate, 0) if apply_topk_weights_early: # intermediate' = intermediate * w. # Masking is not required as grouped GEMMs consume only group-size rows. w_b = recv_w_flat[:, None].astype(d_intermediate.dtype) - intermediate_unweighted = _apply_ffn_activation( - activation_type, gate_proj_out, up_proj_out, group_sizes, act_params - ) + intermediate_unweighted = _jax_act_lu( + activation_inputs, activations, act_params=act_params + ).data.astype(gate_proj_out.dtype) d_recv_w_from_intermediate = jnp.sum( d_intermediate * intermediate_unweighted, axis=-1, @@ -1263,18 +1270,16 @@ def _ffn_bwd_per_shard( else: d_recv_w_from_intermediate = jnp.zeros_like(recv_w_flat) - _, dact_pullback = jax.vjp( - partial( - _apply_ffn_activation, - activation_type, - group_sizes=group_sizes, - act_params=act_params, - ), - gate_proj_out, - up_proj_out, + d_activation, _ = _jax_quantize_dact_dbias( + jnp.where(active[:, None], d_intermediate, 0), + activation_inputs, + activations, + is_dbias=False, + act_params=act_params, + ) + d_combined_for_bias = d_activation.data.reshape( + gate_proj_out.shape[0], 2 * gate_proj_out.shape[1] ) - d_gate_proj_out, d_up_proj_out = dact_pullback(d_intermediate) - d_combined_for_bias = jnp.concatenate([d_gate_proj_out, d_up_proj_out], axis=-1) casted_d_combined = tex.grouped_quantize( d_combined_for_bias, fc1_quantizer_set.dgrad,