Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 61 additions & 0 deletions tests/jax/moe/test_moe_resource_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@

import importlib
import inspect
from types import SimpleNamespace

import jax
import jax.numpy as jnp
import numpy as np
import pytest
from jax.sharding import Mesh

from transformer_engine.jax.cpp_extensions.activation import ActivationParams
from transformer_engine.jax.flax import _MoEBlock
from transformer_engine.jax.moe import (
RouterComputationInfo,
Expand All @@ -22,6 +24,65 @@
from transformer_engine.jax.sharding import MeshResource, global_mesh_resource, global_shard_guard


@pytest.mark.parametrize("soft_caps", [None, {"gate_beta": 2.0, "up_beta": 8.0}])
def test_situ_activation_params_reach_grouped_gemm_adapters(monkeypatch, soft_caps):
module = importlib.import_module("transformer_engine.jax.moe")
adapter = importlib.import_module("transformer_engine.jax.cpp_extensions.grouped_gemm_glu")
params = module._resolve_moe_activation_params("situglu", soft_caps)
assert isinstance(params, ActivationParams)
calls = []
scales = jnp.ones((512,), dtype=jnp.uint8)

def fused(**kwargs):
calls.append(kwargs)
return dict(
c_tensor=jnp.ones((1, 64)),
d_tensor=jnp.ones((1, 32)),
d_col_tensor=jnp.ones((1, 32)),
d_row_tensor=jnp.ones((1, 64)),
dprob_tensor=jnp.ones((1,)),
sfd_row_tensor=scales,
sfd_col_tensor=scales,
)

cudnn = SimpleNamespace(
grouped_gemm_glu_wrapper_sm100=fused, grouped_gemm_dglu_wrapper_sm100=fused
)
original_import = importlib.import_module
monkeypatch.setattr(
adapter.importlib,
"import_module",
lambda name: cudnn if name == "cudnn" else original_import(name),
)
adapter.grouped_gemm_glu(
jnp.ones((1, 32, 1)),
jnp.ones((1, 64, 32)),
scales,
scales,
jnp.asarray([1]),
jnp.ones((1,)),
compute_dtype=jnp.bfloat16,
output_dtype=jnp.float8_e4m3fn,
**module._fused_glu_activation_kwargs("situglu", params),
)
adapter.grouped_gemm_dglu(
jnp.ones((1, 32)),
jnp.ones((1, 32, 32)),
jnp.ones((1, 64)),
scales,
scales,
jnp.asarray([1]),
jnp.ones((1,)),
output_dtype=jnp.float8_e4m3fn,
**module._fused_glu_activation_kwargs("situglu", params, backward=True),
)
expected = soft_caps or {"gate_beta": 4.0, "up_beta": 25.0}
assert [call["act_func"] for call in calls] == ["situglu", "dsituglu"]
for call in calls:
assert call["situ_beta1"] == expected["gate_beta"]
assert call["situ_beta2"] == expected["up_beta"]


@pytest.fixture(autouse=True)
def no_global_resource():
with global_shard_guard(None):
Expand Down
65 changes: 53 additions & 12 deletions tests/jax/test_te_ep_moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,7 @@ def mesh(use_cudnn_fusion):
"num_experts_per_tok",
"aux_loss_coeff",
"score_function",
"activation_type",
),
)
def _pure_jax_moe_reference(
Expand All @@ -298,6 +299,7 @@ def _pure_jax_moe_reference(
aux_loss_coeff: float = 0.0,
score_function: str = "softmax",
scaling_factor: float = 1.0,
activation_type="silu",
):
B, S, H = x.shape
T = B * S
Expand Down Expand Up @@ -348,7 +350,13 @@ def _pure_jax_moe_reference(
# Activation runs in x.dtype (typically bf16) to mirror the impl --
# the impl keeps silu+multiply in the wi GEMM output dtype because
# storing higher precision than the consumer (wo) GEMM buys nothing.
intermediate = jax.nn.silu(layer_w0) * layer_w1
if activation_type == "situglu":
gate, up = layer_w0.astype(jnp.float32), layer_w1.astype(jnp.float32)
capped_gate = 4.0 * jnp.tanh(gate / 4.0)
capped_up = 25.0 * jnp.tanh(up / 25.0)
intermediate = (capped_gate * jax.nn.sigmoid(gate) * capped_up).astype(x.dtype)
else:
intermediate = jax.nn.silu(layer_w0) * layer_w1
expert_out = jnp.einsum("tem,emh->teh", intermediate, wo) # [T, E, H]
output_2d = jnp.einsum("te,teh->th", routing_weights_full.astype(x.dtype), expert_out)
output = output_2d.reshape(B, S, H).astype(x.dtype)
Expand Down Expand Up @@ -403,8 +411,12 @@ def _make_block(
mesh_resource=None,
dispatch_quantization_recipe=None,
use_cudnn_fusion,
activation_type="silu",
activation_params=None,
):
kwargs = dict(
activation_type=activation_type,
activation_params=activation_params,
num_experts=NUM_EXPERTS,
num_experts_per_tok=TOPK,
intermediate_size=INTER,
Expand Down Expand Up @@ -1263,7 +1275,9 @@ def unexpected_fused_kernel(*args, **kwargs):
# independently forcing the baseline to execute the unfused FFN.
moe_module = importlib.import_module("transformer_engine.jax.moe")
with monkeypatch.context() as baseline_patch:
baseline_patch.setattr(moe_module, "_select_cudnn_jax_fusion", lambda _: False)
baseline_patch.setattr(
moe_module, "_select_cudnn_jax_fusion", lambda *args, **kwargs: False
)
variables, baseline_output, _, _ = _init_apply(
baseline, mesh, x, jax.random.PRNGKey(52)
)
Expand Down Expand Up @@ -1298,8 +1312,16 @@ def native_moe(*args, **kwargs):
_to_global_numpy(_unwrap(baseline_grads["params"][name]), mesh),
)

@pytest.mark.parametrize("native_layout", [False, True])
@pytest.mark.parametrize("apply_topk_weights_early", [False, True])
@pytest.mark.parametrize(
"activation_type,native_layout,apply_topk_weights_early",
[
pytest.param("silu", False, False, id="swiglu-standard-late"),
pytest.param("silu", False, True, id="swiglu-standard-early"),
pytest.param("silu", True, False, id="swiglu-native-late"),
pytest.param("silu", True, True, id="swiglu-native-early"),
pytest.param("situglu", False, False, id="situ-default"),
],
)
def test_mxfp8_forward_and_backward(
self,
mesh,
Expand All @@ -1308,12 +1330,15 @@ def test_mxfp8_forward_and_backward(
make_block,
use_cudnn_fusion,
native_layout,
activation_type,
):
if not use_cudnn_fusion:
pytest.skip("requires the cuDNN fusion fixture parameter")
from transformer_engine.jax import cpp_extensions as tex

available, reason = tex.grouped_gemm_glu_dependencies_available()
available, reason = tex.grouped_gemm_glu_dependencies_available(
act_func="situglu" if activation_type == "situglu" else "swiglu"
)
if not available:
pytest.skip(reason)

Expand All @@ -1336,10 +1361,18 @@ def native_moe(*args, **kwargs):
original_dglu = tex.grouped_gemm_dglu

def checked_glu(*args, **kwargs):
if activation_type == "situglu":
assert kwargs["act_func"] == "situglu"
caps = kwargs["act_params"].situglu
assert (caps.gate_beta, caps.up_beta) == (4.0, 25.0)
shared_calls.append("glu")
return original_glu(*args, **kwargs)

def checked_dglu(*args, **kwargs):
if activation_type == "situglu":
assert kwargs["act_func"] == "dsituglu"
caps = kwargs["act_params"].situglu
assert (caps.gate_beta, caps.up_beta) == (4.0, 25.0)
shared_calls.append("dglu")
return original_dglu(*args, **kwargs)

Expand All @@ -1348,6 +1381,10 @@ def checked_dglu(*args, **kwargs):
block = make_block(
apply_topk_weights_early=apply_topk_weights_early,
quantization_recipe=MXFP8BlockScaling(),
activation_type=activation_type,
activation_params=(
{"gate_beta": 4.0, "up_beta": 25.0} if activation_type == "situglu" else None
),
)
x = _make_inputs(jax.random.PRNGKey(30))
variables, output, aux, _ = _init_apply(block, mesh, x, jax.random.PRNGKey(31))
Expand Down Expand Up @@ -1383,6 +1420,7 @@ def loss_fn(params, inputs):
num_experts_per_tok=TOPK,
score_function="softmax",
expert_bias=None,
activation_type=activation_type,
)
return jnp.mean(out.astype(jnp.float32) ** 2)

Expand All @@ -1391,16 +1429,19 @@ def loss_fn(params, inputs):
jnp.asarray(x_np),
)
for name in ("gate_kernel", "wi", "wo"):
grad_np = _to_global_numpy(_unwrap(grads["params"][name]), mesh).astype(np.float32)
grad_ref_np = np.asarray(jax.device_get(grads_ref[name])).astype(np.float32)
np.testing.assert_allclose(
_to_global_numpy(_unwrap(grads["params"][name]), mesh).astype(np.float32),
np.asarray(jax.device_get(grads_ref[name])).astype(np.float32),
**(
GRAD_GATE_TOLERANCE["mxfp8"]
if name == "gate_kernel"
else GRAD_FFN_TOLERANCE["mxfp8"]
),
grad_np,
grad_ref_np,
atol=3e-6,
rtol=1.3e-6,
err_msg=f"{name} fused MXFP8 gradient parity breach",
)
relative_error = np.linalg.norm(grad_np - grad_ref_np) / np.linalg.norm(grad_ref_np)
assert (
relative_error < 0.1
), f"{name} MXFP8 relative L2 error {relative_error:.3%} >= 10%"
grad_x_ref_np = np.asarray(jax.device_get(grad_x_ref)).astype(np.float32)
np.testing.assert_allclose(
grad_x_np,
Expand Down
25 changes: 21 additions & 4 deletions transformer_engine/jax/cpp_extensions/activation.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
# See LICENSE for license information.
"""JAX/TE custom ops for activation"""
from typing import Sequence, Union, Callable, Optional, Tuple
import math
import operator
from functools import reduce, partial
from dataclasses import dataclass
Expand Down Expand Up @@ -88,13 +89,27 @@ def to_ffi_lowering_dict(self):
}


@dataclass(frozen=True)
class SiTUParams:
"""Soft caps for the gate and up branches of SiTU-GLU."""

gate_beta: float = 4.0
up_beta: float = 25.0

def __post_init__(self):
for name in ("gate_beta", "up_beta"):
value = float(getattr(self, name))
if not math.isfinite(value) or value <= 0:
raise ValueError(f"{name} must be finite and positive, got {value}")
object.__setattr__(self, name, value)


@dataclass(frozen=True)
class ActivationParams:
"""Parameters for various activation functions.
Currently only Clamped SwiGLU activation has parameters.
"""
"""Parameters for activation functions."""

clamped_swiglu: ClampedSwigluParams = ClampedSwigluParams()
situglu: SiTUParams = SiTUParams()

@staticmethod
def create(activation_type, **kwargs):
Expand All @@ -106,11 +121,13 @@ def create(activation_type, **kwargs):
}
if activation_type in CLAMPED_ACTIVATION_TYPES:
return ActivationParams(ClampedSwigluParams(**kwargs))
if activation_type == "situglu":
return ActivationParams(situglu=SiTUParams(**kwargs))
return ActivationParams() # Default params for activations without parameters

def __hash__(self):
"""Custom hash function to ensure dataclass is hashable for jax jit to work"""
return hash((self.clamped_swiglu,))
return hash((self.clamped_swiglu, self.situglu))

def to_ffi_lowering_dict(self):
"""Convert the activation parameters to a dictionary format for FFI lowering.
Expand Down
Loading
Loading