Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 145 additions & 1 deletion tests/jax/moe/test_moe_resource_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import importlib
import inspect
from functools import partial
from types import SimpleNamespace

import jax
Expand All @@ -14,7 +15,11 @@
import pytest
from jax.sharding import Mesh

from transformer_engine.jax.cpp_extensions.activation import ActivationParams
from transformer_engine.jax.cpp_extensions.activation import (
ActivationParams,
_jax_act_lu,
_jax_quantize_dact_dbias,
)
from transformer_engine.jax.flax import _MoEBlock
from transformer_engine.jax.moe import (
RouterComputationInfo,
Expand Down Expand Up @@ -83,6 +88,145 @@ def fused(**kwargs):
assert call["situ_beta2"] == expected["up_beta"]


@pytest.mark.parametrize("betas", [(4.0, 25.0), (2.0, 7.0)])
def test_situ_value_and_both_branch_derivatives(betas):
(gate_beta, up_beta) = betas
gate = jnp.asarray([-10000.0, -12.0, -0.5, 0.0, 0.5, 12.0, 10000.0])
up = jnp.asarray([10000.0, 40.0, -3.0, 2.0, 3.0, -40.0, -10000.0])
params = ActivationParams.create("situglu", gate_beta=gate_beta, up_beta=up_beta)

def activate(g, u):
return _jax_act_lu(
jnp.stack([g, u], axis=-2), ("situ", "situ_linear"), act_params=params
).data

(output, pullback) = jax.vjp(activate, gate, up)
(dg, du) = pullback(jnp.ones_like(output))
(g, u) = (np.asarray(gate, np.float64), np.asarray(up, np.float64))
sigmoid = 0.5 + 0.5 * np.tanh(g / 2)
(tg, tu) = (np.tanh(g / gate_beta), np.tanh(u / up_beta))
(capped_gate, capped_up) = (gate_beta * tg * sigmoid, up_beta * tu)
derivative_gate = (1 - tg**2) * sigmoid + gate_beta * tg * sigmoid * (1 - sigmoid)
for actual, expected in (
(output, capped_gate * capped_up),
(dg, derivative_gate * capped_up),
(du, capped_gate * (1 - tu**2)),
):
np.testing.assert_allclose(actual, expected, atol=2e-05, rtol=2e-05)


def test_situ_bf16_uses_fp32_before_final_cast():
gate = jnp.asarray([0.25, 1.5, -3.0, 12.0, 64.0], dtype=jnp.bfloat16)
up = jnp.asarray([0.75, -2.0, 32.0, 64.0, -128.0], dtype=jnp.bfloat16)
actual = _jax_act_lu(jnp.stack([gate, up], axis=-2), ("situ", "situ_linear")).data.astype(
gate.dtype
)
(g, u) = (np.asarray(gate, np.float64), np.asarray(up, np.float64))
expected = jnp.asarray(100 * np.tanh(g / 4) / (1 + np.exp(-g)) * np.tanh(u / 25), jnp.bfloat16)
assert actual.dtype == jnp.bfloat16
np.testing.assert_array_equal(actual, expected)


@pytest.mark.parametrize("dtype", [jnp.float32, jnp.float16, jnp.bfloat16])
@pytest.mark.parametrize("betas", [(4.0, 25.0), (2.0, 8.0)])
@pytest.mark.parametrize("counts", [[3, 0, 5], [0, 0, 0], [0] * 257 + [13], [0]])
def test_situ_masks_inactive_values_and_gradients(dtype, betas, counts):
capacity = 0 if counts == [0] else 13
active = sum(counts)
rng = np.random.default_rng(72)
(gate, up, dy) = [rng.uniform(-5, 5, (capacity, 6)).astype(np.float32) for _ in range(3)]
for value in (gate, up, dy):
value[active:] = np.nan
(gate, up, dy) = [jnp.asarray(value, dtype) for value in (gate, up, dy)]
sizes = jnp.asarray(counts, jnp.int32)
mask = (jnp.arange(capacity) < sizes.sum())[:, None]
inputs = jnp.where(mask[:, None, :], jnp.stack([gate, up], axis=-2), 0)
params = ActivationParams.create("situglu", gate_beta=betas[0], up_beta=betas[1])
activate = partial(_jax_act_lu, activation_type=("situ", "situ_linear"), act_params=params)
output = jnp.where(mask, jax.jit(activate)(inputs).data.astype(dtype), 0)
backward = partial(
_jax_quantize_dact_dbias,
activation_type=("situ", "situ_linear"),
is_dbias=False,
act_params=params,
)
gradients, _ = jax.jit(backward)(jnp.where(mask, dy, 0), inputs)
dg, du = gradients.data[:, 0, :], gradients.data[:, 1, :]
(g, u, d) = [np.asarray(value, np.float64)[:active] for value in (gate, up, dy)]
(gate_beta, up_beta) = betas
(tg, tu, sigmoid) = (np.tanh(g / gate_beta), np.tanh(u / up_beta), 1 / (1 + np.exp(-g)))
ref = (
gate_beta * tg * sigmoid * up_beta * tu,
d * ((1 - tg**2) * sigmoid + gate_beta * tg * sigmoid * (1 - sigmoid)) * up_beta * tu,
d * gate_beta * tg * sigmoid * (1 - tu**2),
)
tolerance = {jnp.float32: 2e-05, jnp.float16: 0.003, jnp.bfloat16: 0.02}[dtype]
for actual, expected in zip((output, dg, du), ref):
actual = np.asarray(actual, np.float32)
assert np.isfinite(actual).all()
np.testing.assert_array_equal(actual[active:], 0)
np.testing.assert_allclose(actual[:active], expected, atol=tolerance, rtol=tolerance)


def test_situ_dynamic_counts_remat_and_vmap():
gate = jnp.arange(13 * 6, dtype=jnp.float32).reshape(13, 6) / 20 - 2
up = -gate

def activate(g, u, sizes):
active = (jnp.arange(g.shape[0]) < sizes.sum())[:, None]
inputs = jnp.where(active[:, None, :], jnp.stack([g, u], axis=-2), 0)
output = _jax_act_lu(inputs, ("situ", "situ_linear")).data
return jnp.where(active, output.astype(g.dtype), 0)

compiled = jax.jit(activate)
for counts in (jnp.array([3, 0, 5]), jnp.array([1, 2, 2])):
expected = activate(gate, up, counts)
np.testing.assert_allclose(compiled(gate, up, counts), expected, atol=2e-05, rtol=2e-05)
loss = lambda g, u: activate(g, u, counts).sum()
gradients = jax.jit(jax.grad(loss, (0, 1)))(gate, up)
remat = jax.jit(jax.grad(jax.checkpoint(loss), (0, 1)))(gate, up)
for actual, reference in zip(remat, gradients):
np.testing.assert_allclose(actual, reference, atol=2e-05, rtol=2e-05)
counts = jnp.array([[3, 0, 5], [1, 2, 2]], dtype=jnp.int32)
(gates, ups) = (jnp.stack([gate, gate]), jnp.stack([up, up]))
mapped = jax.jit(jax.vmap(activate))(gates, ups, counts)
for actual, sizes in zip(mapped, counts):
np.testing.assert_allclose(actual, activate(gate, up, sizes), atol=2e-05, rtol=2e-05)
hlo = compiled.lower(gate, up, counts[0]).compile().as_text()
assert "te_grouped_situglu_ffi" not in hlo


@pytest.mark.parametrize(
"capability,available,expected",
[(89, True, False), (100, False, False), (103, True, True), (107, True, False)],
)
def test_situ_fusion_or_jax_fallback(monkeypatch, capability, available, expected):
import transformer_engine_jax

module = importlib.import_module("transformer_engine.jax.moe")
monkeypatch.setattr(
transformer_engine_jax, "get_device_compute_capability", lambda _: capability
)
monkeypatch.setattr(
module.tex,
"grouped_gemm_glu_dependencies_available",
lambda **_: (available, "unavailable frontend"),
)
if expected:
assert module._select_cudnn_jax_fusion([], activation_type="situglu")
else:
with pytest.warns(UserWarning, match="falling back"):
assert not module._select_cudnn_jax_fusion([], activation_type="situglu")


def test_situ_can_disable_fusion():
module = importlib.import_module("transformer_engine.jax.moe")
with pytest.raises(ValueError, match="active global_shard_guard"):
module.moe(
None, None, None, None, num_experts=2, activation_type="situglu", use_cudnn_fusion=False
)


@pytest.fixture(autouse=True)
def no_global_resource():
with global_shard_guard(None):
Expand Down
8 changes: 8 additions & 0 deletions tests/jax/test_te_ep_moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -584,6 +584,13 @@ def _quantization_recipe(quantization):
# -----------------------------------------------------------------------------

_CONFIGS = [
*[
pytest.param(
dict(activation_type="situglu", apply_topk_weights_early=early),
id=f"situ-{'early' if early else 'late'}-weighting",
)
for early in (False, True)
],
pytest.param(
dict(score_function="softmax"),
id="softmax",
Expand Down Expand Up @@ -739,6 +746,7 @@ def loss_fn(variables, inputs):
def _reference_kwargs_from_config(config, params_np):
"""Pick out the reference-relevant pieces of a parametrize config."""
return dict(
activation_type=config.get("activation_type", "silu"),
score_function=config.get("score_function", "softmax"),
expert_bias=(
jnp.asarray(params_np["expert_bias"])
Expand Down
11 changes: 11 additions & 0 deletions transformer_engine/jax/cpp_extensions/activation.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,17 @@ def _convert_to_activation_function(fn_or_string, act_params: ActivationParams):
"""Convert a string to an activation function."""
if fn_or_string == "linear":
return lambda x: x
if fn_or_string == "situ":
beta = act_params.situglu.gate_beta

def situ(x):
x = x.astype(jnp.float32)
return beta * jnp.tanh(x / beta) * jax.nn.sigmoid(x)

return situ
if fn_or_string == "situ_linear":
beta = act_params.situglu.up_beta
return lambda x: beta * jnp.tanh(x.astype(jnp.float32) / beta)
if fn_or_string == "clamped_linear":
limit = act_params.clamped_swiglu.limit
offset = act_params.clamped_swiglu.glu_linear_offset
Expand Down
6 changes: 3 additions & 3 deletions transformer_engine/jax/flax/moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,9 @@ class _MoEBlock(TransformerEngineBase):
activation_type : str
Activation between ``layer_w0 @ wi_0`` and the elementwise
product with ``layer_w0 @ wi_1``. Default ``"silu"``. Pass
``"situglu"`` for fused MXFP8
SiTU-GLU on SM100/SM103. SiTU requires cuDNN Frontend >= 1.33.0 with compatible JAX
APIs and ``use_cudnn_fusion=True``; there is no unfused SiTU path.
``"situglu"`` for SiTU-GLU. Eligible MXFP8 calls use cuDNN fusion on
SM100/SM103 with cuDNN Frontend >= 1.33.0 and compatible JAX APIs.
Other configurations use JAX activation operations and a VJP between grouped GEMMs.
activation_params : Optional[dict]
Static SiTU soft caps: ``gate_beta`` (default 4.0) and ``up_beta``
(default 25.0). Values must be finite and positive.
Expand Down
70 changes: 47 additions & 23 deletions transformer_engine/jax/moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,11 @@
from jax.sharding import NamedSharding, PartitionSpec as P

from . import cpp_extensions as tex
from .cpp_extensions.activation import ActivationParams
from .cpp_extensions.activation import (
ActivationParams,
_jax_act_lu,
_jax_quantize_dact_dbias,
)
from .cpp_extensions.misc import jax_dtype_to_te_dtype
from .quantize import (
GroupedScaledTensor1x,
Expand All @@ -69,7 +73,6 @@
)
from .quantize.dequantizer import _unswizzle_mxfp8_grouped_scale
from .cpp_extensions.gemm import swizzled_scale
from .flax.module import _convert_to_activation_function
from .router import ScoreFunction, _validate_score_function
from .sharding import MeshResource, _get_mesh, global_mesh_resource, global_shard_guard

Expand Down Expand Up @@ -254,8 +257,6 @@ def _select_cudnn_jax_fusion(rejection_reasons: list[str], *, activation_type="s
if available:
return True
reasons.append(f"shared fused API is incompatible: {error}")
if activation_type == "situglu":
raise ValueError("SiTU requires cuDNN MXFP8 fusion: " + "; ".join(reasons))
warnings.warn(
"use_cudnn_fusion=True: falling back to the unfused TE grouped-GEMM path: "
+ "; ".join(reasons)
Expand Down Expand Up @@ -1073,8 +1074,15 @@ def _ffn_fwd_per_shard(
# that's all bf16; for FP8/FP4 the downstream grouped_quantize is what
# transitions to the target precision.
if not use_cudnn_jax_fusion:
act_fn = _convert_to_activation_function(activation_type)
intermediate = act_fn(gate_proj_out) * up_proj_out
activations = (
("situ", "situ_linear") if activation_type == "situglu" else (activation_type, "linear")
)
active = jnp.arange(gate_proj_out.shape[0]) < group_sizes.sum()
activation_inputs = jnp.where(
active[:, None, None], jnp.stack([gate_proj_out, up_proj_out], axis=-2), 0
)
intermediate = _jax_act_lu(activation_inputs, activations, act_params=act_params).data
intermediate = jnp.where(active[:, None], intermediate.astype(gate_proj_out.dtype), 0)
if apply_topk_weights_early and not use_cudnn_jax_fusion:
# Fold the per-token combine weights into the FFN intermediate;
# the downstream wo GEMM is linear so this is equivalent to the
Expand Down Expand Up @@ -1239,12 +1247,21 @@ def _ffn_bwd_per_shard(
casted_wo_rhs_trans,
contracting_dims=((1,), (2,)),
)
act_fn = _convert_to_activation_function(activation_type)
activations = (
("situ", "situ_linear") if activation_type == "situglu" else (activation_type, "linear")
)
active = jnp.arange(gate_proj_out.shape[0]) < group_sizes.sum()
activation_inputs = jnp.where(
active[:, None, None], jnp.stack([gate_proj_out, up_proj_out], axis=-2), 0
)
d_intermediate = jnp.where(active[:, None], d_intermediate, 0)
if apply_topk_weights_early:
# intermediate' = intermediate * w.
# Masking is not required as grouped GEMMs consume only group-size rows.
w_b = recv_w_flat[:, None].astype(d_intermediate.dtype)
intermediate_unweighted = act_fn(gate_proj_out) * up_proj_out
intermediate_unweighted = _jax_act_lu(
activation_inputs, activations, act_params=act_params
).data.astype(gate_proj_out.dtype)
d_recv_w_from_intermediate = jnp.sum(
d_intermediate * intermediate_unweighted,
axis=-1,
Expand All @@ -1253,11 +1270,16 @@ def _ffn_bwd_per_shard(
else:
d_recv_w_from_intermediate = jnp.zeros_like(recv_w_flat)

# Activation bwd stays in the GEMM dtype, matching the forward path.
act_gp, dact_pullback = jax.vjp(act_fn, gate_proj_out)
d_up_proj_out = d_intermediate * act_gp
(d_gate_proj_out,) = dact_pullback(d_intermediate * up_proj_out)
d_combined_for_bias = jnp.concatenate([d_gate_proj_out, d_up_proj_out], axis=-1)
d_activation, _ = _jax_quantize_dact_dbias(
jnp.where(active[:, None], d_intermediate, 0),
activation_inputs,
activations,
is_dbias=False,
act_params=act_params,
)
d_combined_for_bias = d_activation.data.reshape(
gate_proj_out.shape[0], 2 * gate_proj_out.shape[1]
)
casted_d_combined = tex.grouped_quantize(
d_combined_for_bias,
fc1_quantizer_set.dgrad,
Expand Down Expand Up @@ -2168,12 +2190,17 @@ def moe(
Parameters
----------
activation_type : str
Gate activation (default ``"silu"``), or ``"situglu"`` for fused MXFP8
SiTU-GLU on SM100/SM103. Unsupported SiTU configurations raise ``ValueError``.
Gate activation (default ``"silu"``), or ``"situglu"`` for SiTU-GLU.
Eligible MXFP8 SiTU configurations use cuDNN fusion. Otherwise, SiTU
uses FP32 JAX operations and ``jax.vjp`` between the grouped GEMMs,
keeping the projection dtype at the output. Device expert counts mask
inactive receive capacity in forward and backward, including non-finite
inactive values. MXFP8 quantization remains separate in the fallback.
activation_params : Optional[dict]
Static SiTU soft caps: ``gate_beta`` (default 4.0) and ``up_beta``
(default 25.0). Values must be finite and positive. Other activations
do not accept parameters.
do not accept parameters. For example,
``activation_params={"gate_beta": 4.0, "up_beta": 25.0}``.
router_info : Union[RouterComputationInfo, RoutingMapInfo]
:class:`RouterComputationInfo` makes TE compute the gate and top-k
from the given router parameters. :class:`RoutingMapInfo` supplies
Expand Down Expand Up @@ -2228,14 +2255,13 @@ def moe(
Defaults to ``True``: try cuDNN's shared JAX grouped MXFP8 GLU/dGLU APIs
on SM100+ GPUs; cuDNN selects the architecture-specific kernels. ``False`` uses unfused TE grouped GEMM.
Requires cuDNN Frontend >= 1.32.0 with compatible JAX APIs.
SiTU additionally requires SM100/SM103 and JAX GLU/dGLU APIs exposing
SiTU fusion additionally requires SM100/SM103 and JAX GLU/dGLU APIs exposing
``situ_beta1`` and ``situ_beta2`` with Frontend >= 1.33.0.
Ineligible SiTU calls raise
``ValueError``; there is no unfused SiTU path.
Frontends whose JAX GLU/dGLU entry points only support BF16 discrete
weights fall back to unfused TE grouped GEMM.
Except for SiTU, ineligible calls warn and fall back to TE's regular
grouped-GEMM implementation. API signatures and GPU capability determine support.
Ineligible calls warn and fall back to TE's regular grouped-GEMM implementation,
with JAX activation operations and a VJP for SiTU. API signatures and GPU
capability determine fusion support.
cudnn_native_weight_layout : Optional[bool]
``False`` selects standard contiguous gate/up [E,K,2N] storage;
``True`` selects native [E,2N,K] storage with alternating 32-column
Expand All @@ -2262,8 +2288,6 @@ def moe(
if not isinstance(use_cudnn_fusion, bool):
raise TypeError("use_cudnn_fusion must be a bool")
act_params = _resolve_moe_activation_params(activation_type, activation_params)
if activation_type == "situglu" and not use_cudnn_fusion:
raise ValueError("SiTU requires use_cudnn_fusion=True")
mesh_resource, quant_before_fsdp_ag = _resolve_moe_mesh_resource(
mesh_resource, quant_before_fsdp_ag
)
Expand Down