diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index 2764d9ca54..41f1b07816 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -6,6 +6,7 @@ import importlib import inspect +from functools import partial from types import SimpleNamespace import jax @@ -14,7 +15,11 @@ import pytest from jax.sharding import Mesh -from transformer_engine.jax.cpp_extensions.activation import ActivationParams +from transformer_engine.jax.cpp_extensions.activation import ( + ActivationParams, + _jax_act_lu, + _jax_quantize_dact_dbias, +) from transformer_engine.jax.flax import _MoEBlock from transformer_engine.jax.moe import ( RouterComputationInfo, @@ -83,6 +88,145 @@ def fused(**kwargs): assert call["situ_beta2"] == expected["up_beta"] +@pytest.mark.parametrize("betas", [(4.0, 25.0), (2.0, 7.0)]) +def test_situ_value_and_both_branch_derivatives(betas): + (gate_beta, up_beta) = betas + gate = jnp.asarray([-10000.0, -12.0, -0.5, 0.0, 0.5, 12.0, 10000.0]) + up = jnp.asarray([10000.0, 40.0, -3.0, 2.0, 3.0, -40.0, -10000.0]) + params = ActivationParams.create("situglu", gate_beta=gate_beta, up_beta=up_beta) + + def activate(g, u): + return _jax_act_lu( + jnp.stack([g, u], axis=-2), ("situ", "situ_linear"), act_params=params + ).data + + (output, pullback) = jax.vjp(activate, gate, up) + (dg, du) = pullback(jnp.ones_like(output)) + (g, u) = (np.asarray(gate, np.float64), np.asarray(up, np.float64)) + sigmoid = 0.5 + 0.5 * np.tanh(g / 2) + (tg, tu) = (np.tanh(g / gate_beta), np.tanh(u / up_beta)) + (capped_gate, capped_up) = (gate_beta * tg * sigmoid, up_beta * tu) + derivative_gate = (1 - tg**2) * sigmoid + gate_beta * tg * sigmoid * (1 - sigmoid) + for actual, expected in ( + (output, capped_gate * capped_up), + (dg, derivative_gate * capped_up), + (du, capped_gate * (1 - tu**2)), + ): + np.testing.assert_allclose(actual, expected, atol=2e-05, rtol=2e-05) + + +def test_situ_bf16_uses_fp32_before_final_cast(): + gate = jnp.asarray([0.25, 1.5, -3.0, 12.0, 64.0], dtype=jnp.bfloat16) + up = jnp.asarray([0.75, -2.0, 32.0, 64.0, -128.0], dtype=jnp.bfloat16) + actual = _jax_act_lu(jnp.stack([gate, up], axis=-2), ("situ", "situ_linear")).data.astype( + gate.dtype + ) + (g, u) = (np.asarray(gate, np.float64), np.asarray(up, np.float64)) + expected = jnp.asarray(100 * np.tanh(g / 4) / (1 + np.exp(-g)) * np.tanh(u / 25), jnp.bfloat16) + assert actual.dtype == jnp.bfloat16 + np.testing.assert_array_equal(actual, expected) + + +@pytest.mark.parametrize("dtype", [jnp.float32, jnp.float16, jnp.bfloat16]) +@pytest.mark.parametrize("betas", [(4.0, 25.0), (2.0, 8.0)]) +@pytest.mark.parametrize("counts", [[3, 0, 5], [0, 0, 0], [0] * 257 + [13], [0]]) +def test_situ_masks_inactive_values_and_gradients(dtype, betas, counts): + capacity = 0 if counts == [0] else 13 + active = sum(counts) + rng = np.random.default_rng(72) + (gate, up, dy) = [rng.uniform(-5, 5, (capacity, 6)).astype(np.float32) for _ in range(3)] + for value in (gate, up, dy): + value[active:] = np.nan + (gate, up, dy) = [jnp.asarray(value, dtype) for value in (gate, up, dy)] + sizes = jnp.asarray(counts, jnp.int32) + mask = (jnp.arange(capacity) < sizes.sum())[:, None] + inputs = jnp.where(mask[:, None, :], jnp.stack([gate, up], axis=-2), 0) + params = ActivationParams.create("situglu", gate_beta=betas[0], up_beta=betas[1]) + activate = partial(_jax_act_lu, activation_type=("situ", "situ_linear"), act_params=params) + output = jnp.where(mask, jax.jit(activate)(inputs).data.astype(dtype), 0) + backward = partial( + _jax_quantize_dact_dbias, + activation_type=("situ", "situ_linear"), + is_dbias=False, + act_params=params, + ) + gradients, _ = jax.jit(backward)(jnp.where(mask, dy, 0), inputs) + dg, du = gradients.data[:, 0, :], gradients.data[:, 1, :] + (g, u, d) = [np.asarray(value, np.float64)[:active] for value in (gate, up, dy)] + (gate_beta, up_beta) = betas + (tg, tu, sigmoid) = (np.tanh(g / gate_beta), np.tanh(u / up_beta), 1 / (1 + np.exp(-g))) + ref = ( + gate_beta * tg * sigmoid * up_beta * tu, + d * ((1 - tg**2) * sigmoid + gate_beta * tg * sigmoid * (1 - sigmoid)) * up_beta * tu, + d * gate_beta * tg * sigmoid * (1 - tu**2), + ) + tolerance = {jnp.float32: 2e-05, jnp.float16: 0.003, jnp.bfloat16: 0.02}[dtype] + for actual, expected in zip((output, dg, du), ref): + actual = np.asarray(actual, np.float32) + assert np.isfinite(actual).all() + np.testing.assert_array_equal(actual[active:], 0) + np.testing.assert_allclose(actual[:active], expected, atol=tolerance, rtol=tolerance) + + +def test_situ_dynamic_counts_remat_and_vmap(): + gate = jnp.arange(13 * 6, dtype=jnp.float32).reshape(13, 6) / 20 - 2 + up = -gate + + def activate(g, u, sizes): + active = (jnp.arange(g.shape[0]) < sizes.sum())[:, None] + inputs = jnp.where(active[:, None, :], jnp.stack([g, u], axis=-2), 0) + output = _jax_act_lu(inputs, ("situ", "situ_linear")).data + return jnp.where(active, output.astype(g.dtype), 0) + + compiled = jax.jit(activate) + for counts in (jnp.array([3, 0, 5]), jnp.array([1, 2, 2])): + expected = activate(gate, up, counts) + np.testing.assert_allclose(compiled(gate, up, counts), expected, atol=2e-05, rtol=2e-05) + loss = lambda g, u: activate(g, u, counts).sum() + gradients = jax.jit(jax.grad(loss, (0, 1)))(gate, up) + remat = jax.jit(jax.grad(jax.checkpoint(loss), (0, 1)))(gate, up) + for actual, reference in zip(remat, gradients): + np.testing.assert_allclose(actual, reference, atol=2e-05, rtol=2e-05) + counts = jnp.array([[3, 0, 5], [1, 2, 2]], dtype=jnp.int32) + (gates, ups) = (jnp.stack([gate, gate]), jnp.stack([up, up])) + mapped = jax.jit(jax.vmap(activate))(gates, ups, counts) + for actual, sizes in zip(mapped, counts): + np.testing.assert_allclose(actual, activate(gate, up, sizes), atol=2e-05, rtol=2e-05) + hlo = compiled.lower(gate, up, counts[0]).compile().as_text() + assert "te_grouped_situglu_ffi" not in hlo + + +@pytest.mark.parametrize( + "capability,available,expected", + [(89, True, False), (100, False, False), (103, True, True), (107, True, False)], +) +def test_situ_fusion_or_jax_fallback(monkeypatch, capability, available, expected): + import transformer_engine_jax + + module = importlib.import_module("transformer_engine.jax.moe") + monkeypatch.setattr( + transformer_engine_jax, "get_device_compute_capability", lambda _: capability + ) + monkeypatch.setattr( + module.tex, + "grouped_gemm_glu_dependencies_available", + lambda **_: (available, "unavailable frontend"), + ) + if expected: + assert module._select_cudnn_jax_fusion([], activation_type="situglu") + else: + with pytest.warns(UserWarning, match="falling back"): + assert not module._select_cudnn_jax_fusion([], activation_type="situglu") + + +def test_situ_can_disable_fusion(): + module = importlib.import_module("transformer_engine.jax.moe") + with pytest.raises(ValueError, match="active global_shard_guard"): + module.moe( + None, None, None, None, num_experts=2, activation_type="situglu", use_cudnn_fusion=False + ) + + @pytest.fixture(autouse=True) def no_global_resource(): with global_shard_guard(None): diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index 6ec9060776..4c0c259b8a 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -584,6 +584,13 @@ def _quantization_recipe(quantization): # ----------------------------------------------------------------------------- _CONFIGS = [ + *[ + pytest.param( + dict(activation_type="situglu", apply_topk_weights_early=early), + id=f"situ-{'early' if early else 'late'}-weighting", + ) + for early in (False, True) + ], pytest.param( dict(score_function="softmax"), id="softmax", @@ -739,6 +746,7 @@ def loss_fn(variables, inputs): def _reference_kwargs_from_config(config, params_np): """Pick out the reference-relevant pieces of a parametrize config.""" return dict( + activation_type=config.get("activation_type", "silu"), score_function=config.get("score_function", "softmax"), expert_bias=( jnp.asarray(params_np["expert_bias"]) diff --git a/transformer_engine/jax/cpp_extensions/activation.py b/transformer_engine/jax/cpp_extensions/activation.py index a1d6aed607..ec5e9d758b 100644 --- a/transformer_engine/jax/cpp_extensions/activation.py +++ b/transformer_engine/jax/cpp_extensions/activation.py @@ -142,6 +142,17 @@ def _convert_to_activation_function(fn_or_string, act_params: ActivationParams): """Convert a string to an activation function.""" if fn_or_string == "linear": return lambda x: x + if fn_or_string == "situ": + beta = act_params.situglu.gate_beta + + def situ(x): + x = x.astype(jnp.float32) + return beta * jnp.tanh(x / beta) * jax.nn.sigmoid(x) + + return situ + if fn_or_string == "situ_linear": + beta = act_params.situglu.up_beta + return lambda x: beta * jnp.tanh(x.astype(jnp.float32) / beta) if fn_or_string == "clamped_linear": limit = act_params.clamped_swiglu.limit offset = act_params.clamped_swiglu.glu_linear_offset diff --git a/transformer_engine/jax/flax/moe.py b/transformer_engine/jax/flax/moe.py index 0402c84f3a..4dca3bb869 100644 --- a/transformer_engine/jax/flax/moe.py +++ b/transformer_engine/jax/flax/moe.py @@ -75,9 +75,9 @@ class _MoEBlock(TransformerEngineBase): activation_type : str Activation between ``layer_w0 @ wi_0`` and the elementwise product with ``layer_w0 @ wi_1``. Default ``"silu"``. Pass - ``"situglu"`` for fused MXFP8 - SiTU-GLU on SM100/SM103. SiTU requires cuDNN Frontend >= 1.33.0 with compatible JAX - APIs and ``use_cudnn_fusion=True``; there is no unfused SiTU path. + ``"situglu"`` for SiTU-GLU. Eligible MXFP8 calls use cuDNN fusion on + SM100/SM103 with cuDNN Frontend >= 1.33.0 and compatible JAX APIs. + Other configurations use JAX activation operations and a VJP between grouped GEMMs. activation_params : Optional[dict] Static SiTU soft caps: ``gate_beta`` (default 4.0) and ``up_beta`` (default 25.0). Values must be finite and positive. diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index e299724ea8..828e58cca5 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -54,7 +54,11 @@ from jax.sharding import NamedSharding, PartitionSpec as P from . import cpp_extensions as tex -from .cpp_extensions.activation import ActivationParams +from .cpp_extensions.activation import ( + ActivationParams, + _jax_act_lu, + _jax_quantize_dact_dbias, +) from .cpp_extensions.misc import jax_dtype_to_te_dtype from .quantize import ( GroupedScaledTensor1x, @@ -69,7 +73,6 @@ ) from .quantize.dequantizer import _unswizzle_mxfp8_grouped_scale from .cpp_extensions.gemm import swizzled_scale -from .flax.module import _convert_to_activation_function from .router import ScoreFunction, _validate_score_function from .sharding import MeshResource, _get_mesh, global_mesh_resource, global_shard_guard @@ -254,8 +257,6 @@ def _select_cudnn_jax_fusion(rejection_reasons: list[str], *, activation_type="s if available: return True reasons.append(f"shared fused API is incompatible: {error}") - if activation_type == "situglu": - raise ValueError("SiTU requires cuDNN MXFP8 fusion: " + "; ".join(reasons)) warnings.warn( "use_cudnn_fusion=True: falling back to the unfused TE grouped-GEMM path: " + "; ".join(reasons) @@ -1073,8 +1074,15 @@ def _ffn_fwd_per_shard( # that's all bf16; for FP8/FP4 the downstream grouped_quantize is what # transitions to the target precision. if not use_cudnn_jax_fusion: - act_fn = _convert_to_activation_function(activation_type) - intermediate = act_fn(gate_proj_out) * up_proj_out + activations = ( + ("situ", "situ_linear") if activation_type == "situglu" else (activation_type, "linear") + ) + active = jnp.arange(gate_proj_out.shape[0]) < group_sizes.sum() + activation_inputs = jnp.where( + active[:, None, None], jnp.stack([gate_proj_out, up_proj_out], axis=-2), 0 + ) + intermediate = _jax_act_lu(activation_inputs, activations, act_params=act_params).data + intermediate = jnp.where(active[:, None], intermediate.astype(gate_proj_out.dtype), 0) if apply_topk_weights_early and not use_cudnn_jax_fusion: # Fold the per-token combine weights into the FFN intermediate; # the downstream wo GEMM is linear so this is equivalent to the @@ -1239,12 +1247,21 @@ def _ffn_bwd_per_shard( casted_wo_rhs_trans, contracting_dims=((1,), (2,)), ) - act_fn = _convert_to_activation_function(activation_type) + activations = ( + ("situ", "situ_linear") if activation_type == "situglu" else (activation_type, "linear") + ) + active = jnp.arange(gate_proj_out.shape[0]) < group_sizes.sum() + activation_inputs = jnp.where( + active[:, None, None], jnp.stack([gate_proj_out, up_proj_out], axis=-2), 0 + ) + d_intermediate = jnp.where(active[:, None], d_intermediate, 0) if apply_topk_weights_early: # intermediate' = intermediate * w. # Masking is not required as grouped GEMMs consume only group-size rows. w_b = recv_w_flat[:, None].astype(d_intermediate.dtype) - intermediate_unweighted = act_fn(gate_proj_out) * up_proj_out + intermediate_unweighted = _jax_act_lu( + activation_inputs, activations, act_params=act_params + ).data.astype(gate_proj_out.dtype) d_recv_w_from_intermediate = jnp.sum( d_intermediate * intermediate_unweighted, axis=-1, @@ -1253,11 +1270,16 @@ def _ffn_bwd_per_shard( else: d_recv_w_from_intermediate = jnp.zeros_like(recv_w_flat) - # Activation bwd stays in the GEMM dtype, matching the forward path. - act_gp, dact_pullback = jax.vjp(act_fn, gate_proj_out) - d_up_proj_out = d_intermediate * act_gp - (d_gate_proj_out,) = dact_pullback(d_intermediate * up_proj_out) - d_combined_for_bias = jnp.concatenate([d_gate_proj_out, d_up_proj_out], axis=-1) + d_activation, _ = _jax_quantize_dact_dbias( + jnp.where(active[:, None], d_intermediate, 0), + activation_inputs, + activations, + is_dbias=False, + act_params=act_params, + ) + d_combined_for_bias = d_activation.data.reshape( + gate_proj_out.shape[0], 2 * gate_proj_out.shape[1] + ) casted_d_combined = tex.grouped_quantize( d_combined_for_bias, fc1_quantizer_set.dgrad, @@ -2168,12 +2190,17 @@ def moe( Parameters ---------- activation_type : str - Gate activation (default ``"silu"``), or ``"situglu"`` for fused MXFP8 - SiTU-GLU on SM100/SM103. Unsupported SiTU configurations raise ``ValueError``. + Gate activation (default ``"silu"``), or ``"situglu"`` for SiTU-GLU. + Eligible MXFP8 SiTU configurations use cuDNN fusion. Otherwise, SiTU + uses FP32 JAX operations and ``jax.vjp`` between the grouped GEMMs, + keeping the projection dtype at the output. Device expert counts mask + inactive receive capacity in forward and backward, including non-finite + inactive values. MXFP8 quantization remains separate in the fallback. activation_params : Optional[dict] Static SiTU soft caps: ``gate_beta`` (default 4.0) and ``up_beta`` (default 25.0). Values must be finite and positive. Other activations - do not accept parameters. + do not accept parameters. For example, + ``activation_params={"gate_beta": 4.0, "up_beta": 25.0}``. router_info : Union[RouterComputationInfo, RoutingMapInfo] :class:`RouterComputationInfo` makes TE compute the gate and top-k from the given router parameters. :class:`RoutingMapInfo` supplies @@ -2228,14 +2255,13 @@ def moe( Defaults to ``True``: try cuDNN's shared JAX grouped MXFP8 GLU/dGLU APIs on SM100+ GPUs; cuDNN selects the architecture-specific kernels. ``False`` uses unfused TE grouped GEMM. Requires cuDNN Frontend >= 1.32.0 with compatible JAX APIs. - SiTU additionally requires SM100/SM103 and JAX GLU/dGLU APIs exposing + SiTU fusion additionally requires SM100/SM103 and JAX GLU/dGLU APIs exposing ``situ_beta1`` and ``situ_beta2`` with Frontend >= 1.33.0. - Ineligible SiTU calls raise - ``ValueError``; there is no unfused SiTU path. Frontends whose JAX GLU/dGLU entry points only support BF16 discrete weights fall back to unfused TE grouped GEMM. - Except for SiTU, ineligible calls warn and fall back to TE's regular - grouped-GEMM implementation. API signatures and GPU capability determine support. + Ineligible calls warn and fall back to TE's regular grouped-GEMM implementation, + with JAX activation operations and a VJP for SiTU. API signatures and GPU + capability determine fusion support. cudnn_native_weight_layout : Optional[bool] ``False`` selects standard contiguous gate/up [E,K,2N] storage; ``True`` selects native [E,2N,K] storage with alternating 32-column @@ -2262,8 +2288,6 @@ def moe( if not isinstance(use_cudnn_fusion, bool): raise TypeError("use_cudnn_fusion must be a bool") act_params = _resolve_moe_activation_params(activation_type, activation_params) - if activation_type == "situglu" and not use_cudnn_fusion: - raise ValueError("SiTU requires use_cudnn_fusion=True") mesh_resource, quant_before_fsdp_ag = _resolve_moe_mesh_resource( mesh_resource, quant_before_fsdp_ag )