Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 61 additions & 1 deletion tests/jax/test_custom_call_compute.py
Original file line number Diff line number Diff line change
Expand Up @@ -2156,6 +2156,11 @@ def function(**kwargs):
if change == "positional" and name == "a_tensor"
else inspect.Parameter.POSITIONAL_OR_KEYWORD
),
default=(
4.0
if name == "situ_beta1"
else 25.0 if name == "situ_beta2" else inspect.Parameter.empty
),
)
for name in arguments
if not (change == "renamed" and name == "discrete_col_sfd")
Expand Down Expand Up @@ -2282,6 +2287,61 @@ def test_api_signature(self, frontend, name, args, change):
assert available == (change == "optional")
assert bool(reason) == (change != "optional")

@pytest.mark.parametrize("missing", [None, "situ_beta1", "situ_beta2"])
@pytest.mark.parametrize(
"name,arguments",
[
("grouped_gemm_glu_wrapper_sm100", _glu_adapter._FORWARD_ARGS),
("grouped_gemm_dglu_wrapper_sm100", _glu_adapter._BACKWARD_ARGS),
("grouped_gemm_glu_jax_sm100", _glu_adapter._JAX_FORWARD_ARGS),
("grouped_gemm_dglu_jax_sm100", _glu_adapter._JAX_BACKWARD_ARGS),
],
)
def test_situ_requires_scalar_parameters_in_both_framework_apis(
self, frontend, name, arguments, missing
):
apis = (
("grouped_gemm_glu_wrapper_sm100", _glu_adapter._FORWARD_ARGS),
("grouped_gemm_dglu_wrapper_sm100", _glu_adapter._BACKWARD_ARGS),
("grouped_gemm_glu_jax_sm100", _glu_adapter._JAX_FORWARD_ARGS),
("grouped_gemm_dglu_jax_sm100", _glu_adapter._JAX_BACKWARD_ARGS),
)
for api_name, args in apis:
setattr(frontend, api_name, self._api(args + ("situ_beta1", "situ_beta2")))
if missing:
setattr(
frontend,
name,
self._api(
arguments + tuple(arg for arg in ("situ_beta1", "situ_beta2") if arg != missing)
),
)
assert _glu_adapter.grouped_gemm_glu_dependencies_available()[0]
available, reason = _glu_adapter.grouped_gemm_glu_dependencies_available(require_situ=True)
assert available is (missing is None)
if missing:
assert name in reason and missing in reason

@pytest.mark.parametrize("capability", [90, 100, 103, 107, 120])
def test_situ_fusion_device_support(self, frontend, monkeypatch, capability):
for name, args in (
("grouped_gemm_glu_wrapper_sm100", _glu_adapter._FORWARD_ARGS),
("grouped_gemm_dglu_wrapper_sm100", _glu_adapter._BACKWARD_ARGS),
("grouped_gemm_glu_jax_sm100", _glu_adapter._JAX_FORWARD_ARGS),
("grouped_gemm_dglu_jax_sm100", _glu_adapter._JAX_BACKWARD_ARGS),
):
setattr(frontend, name, self._api(args + ("situ_beta1", "situ_beta2")))
monkeypatch.setattr(
transformer_engine_jax, "get_device_compute_capability", lambda _: capability
)
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
selected = _moe_module._select_cudnn_jax_fusion(
[], activation_type=_moe_module.SiTUActivation()
)
assert selected is (capability in (100, 103))
assert bool(caught) is not selected

@pytest.mark.parametrize("name", ["grouped_gemm_glu_jax_sm100", "grouped_gemm_dglu_jax_sm100"])
def test_bf16_only_jax_api_falls_back(self, frontend, monkeypatch, name):
# Frontend 1.29 exposes JAX entry points for BF16 discrete weights, but
Expand Down Expand Up @@ -2396,7 +2456,7 @@ def execute(*args):
assert bound["use_cudnn_fusion"] is (request_fusion is not False)
assert bound["cudnn_native_weight_layout"] is native_layout
received.append(bound["use_cudnn_jax_fusion"])
return args[0], None, jnp.asarray(0), jnp.zeros((2,), jnp.int32)
return args[0], None, jnp.asarray(0), jnp.zeros((2,), jnp.int32), None

monkeypatch.setattr(_moe_module, "_moe", execute)
x = jnp.ones((1, 1, 128), jnp.bfloat16)
Expand Down
24 changes: 22 additions & 2 deletions tests/jax/test_te_ep_moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -1438,7 +1438,9 @@ def unexpected_fused_kernel(*args, **kwargs):
# independently forcing the baseline to execute the unfused FFN.
moe_module = importlib.import_module("transformer_engine.jax.moe")
with monkeypatch.context() as baseline_patch:
baseline_patch.setattr(moe_module, "_select_cudnn_jax_fusion", lambda _: False)
baseline_patch.setattr(
moe_module, "_select_cudnn_jax_fusion", lambda *args, **kwargs: False
)
variables, baseline_output, _, _ = _init_apply(
baseline, mesh, x, jax.random.PRNGKey(52)
)
Expand Down Expand Up @@ -1473,6 +1475,11 @@ def native_moe(*args, **kwargs):
_to_global_numpy(_unwrap(baseline_grads["params"][name]), mesh),
)

@pytest.mark.parametrize(
"activation_type",
["silu", SiTUActivation(), SiTUActivation(2.0, 8.0)],
ids=["swiglu", "situ-k3", "situ-custom"],
)
@pytest.mark.parametrize("native_layout", [False, True])
@pytest.mark.parametrize("apply_topk_weights_early", [False, True])
def test_mxfp8_forward_and_backward(
Expand All @@ -1483,12 +1490,15 @@ def test_mxfp8_forward_and_backward(
make_block,
use_cudnn_fusion,
native_layout,
activation_type,
):
if not use_cudnn_fusion:
pytest.skip("requires the cuDNN fusion fixture parameter")
from transformer_engine.jax import cpp_extensions as tex

available, reason = tex.grouped_gemm_glu_dependencies_available()
available, reason = tex.grouped_gemm_glu_dependencies_available(
require_situ=isinstance(activation_type, SiTUActivation)
)
if not available:
pytest.skip(reason)

Expand All @@ -1511,10 +1521,18 @@ def native_moe(*args, **kwargs):
original_dglu = tex.grouped_gemm_dglu

def checked_glu(*args, **kwargs):
if isinstance(activation_type, SiTUActivation):
assert kwargs["act_func"] == "situglu"
assert kwargs["situ_beta1"] == activation_type.gate_beta
assert kwargs["situ_beta2"] == activation_type.up_beta
shared_calls.append("glu")
return original_glu(*args, **kwargs)

def checked_dglu(*args, **kwargs):
if isinstance(activation_type, SiTUActivation):
assert kwargs["act_func"] == "dsituglu"
assert kwargs["situ_beta1"] == activation_type.gate_beta
assert kwargs["situ_beta2"] == activation_type.up_beta
shared_calls.append("dglu")
return original_dglu(*args, **kwargs)

Expand All @@ -1523,6 +1541,7 @@ def checked_dglu(*args, **kwargs):
block = make_block(
apply_topk_weights_early=apply_topk_weights_early,
quantization_recipe=MXFP8BlockScaling(),
activation_type=activation_type,
)
x = _make_inputs(jax.random.PRNGKey(30))
variables, output, aux, _ = _init_apply(block, mesh, x, jax.random.PRNGKey(31))
Expand Down Expand Up @@ -1558,6 +1577,7 @@ def loss_fn(params, inputs):
num_experts_per_tok=TOPK,
score_function="softmax",
expert_bias=None,
activation_type=activation_type,
)
return jnp.mean(out.astype(jnp.float32) ** 2)

Expand Down
32 changes: 27 additions & 5 deletions transformer_engine/jax/cpp_extensions/grouped_gemm_glu.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ def _compact_sf(scale: jax.Array, shape: tuple[int, ...], name: str) -> jax.Arra
)


def grouped_gemm_glu_dependencies_available() -> tuple[bool, str]:
def grouped_gemm_glu_dependencies_available(*, require_situ: bool = False) -> tuple[bool, str]:
"""Require Frontend >= 1.32.0 and shared JAX APIs supporting dense MXFP8."""
try:
cutlass_jax = importlib.import_module("cutlass.jax")
Expand All @@ -126,6 +126,8 @@ def grouped_gemm_glu_dependencies_available() -> tuple[bool, str]:
api = getattr(module, name, None)
if not callable(api):
return False, f"{module.__name__}.{name} is unavailable"
if require_situ:
arguments += ("situ_beta1", "situ_beta2")
signature = inspect.signature(api)
missing = set(arguments) - signature.parameters.keys()
if missing:
Expand All @@ -151,8 +153,11 @@ def grouped_gemm_glu(
*,
compute_dtype,
output_dtype,
act_func: str = "swiglu",
situ_beta1: float = 4.0,
situ_beta2: float = 25.0,
) -> tuple[jax.Array, jax.Array, jax.Array, jax.Array, jax.Array]:
"""Run cuDNN's shared grouped MXFP8 GLU API in SwiGLU mode."""
"""Run cuDNN's shared grouped MXFP8 GLU API with the selected activation."""
cudnn_grouped_gemm_glu = getattr(
importlib.import_module("cudnn"), "grouped_gemm_glu_wrapper_sm100"
)
Expand All @@ -167,6 +172,9 @@ def grouped_gemm_glu(
prob,
compute_dtype=compute_dtype,
output_dtype=output_dtype,
act_func=act_func,
situ_beta1=situ_beta1,
situ_beta2=situ_beta2,
)


Expand All @@ -181,13 +189,19 @@ def _grouped_gemm_forward(
*,
compute_dtype,
output_dtype,
act_func: str = "swiglu",
situ_beta1: float = 4.0,
situ_beta2: float = 25.0,
):
rows, hidden, _ = a.shape
experts, combined, b_hidden = b.shape
if hidden != b_hidden:
raise ValueError(f"A K={hidden} does not match B K={b_hidden}")
alpha = jnp.ones((experts,), dtype=jnp.float32)
norm_const = jnp.ones((1,), dtype=jnp.float32)
situ_kwargs = {}
if act_func == "situglu":
situ_kwargs = {"situ_beta1": situ_beta1, "situ_beta2": situ_beta2}
result = cudnn_forward(
a_tensor=a.reshape(rows, hidden),
b_tensor=b,
Expand All @@ -201,8 +215,9 @@ def _grouped_gemm_forward(
d_dtype=jnp.dtype(output_dtype),
discrete_col_sfd=True,
sf_vec_size=32,
act_func="swiglu",
act_func=act_func,
generate_c=True,
**situ_kwargs,
)
return (
result["c_tensor"].reshape(rows, combined),
Expand All @@ -223,8 +238,11 @@ def grouped_gemm_dglu(
prob: jax.Array,
*,
output_dtype,
act_func: str = "dswiglu",
situ_beta1: float = 4.0,
situ_beta2: float = 25.0,
) -> tuple[jax.Array, jax.Array, jax.Array, jax.Array, jax.Array]:
"""Run cuDNN's shared grouped MXFP8 dGLU API in dSwiGLU mode."""
"""Run cuDNN's shared grouped MXFP8 dGLU API with the selected activation."""
if a.ndim != 2:
raise ValueError(f"Expected A[M,K], got {a.shape}")
if b.ndim != 3:
Expand All @@ -246,6 +264,9 @@ def grouped_gemm_dglu(
alpha = jnp.ones((experts,), dtype=jnp.float32)
beta = jnp.ones((experts,), dtype=jnp.float32)
norm_const = jnp.ones((1,), dtype=jnp.float32)
situ_kwargs = {}
if act_func == "dsituglu":
situ_kwargs = {"situ_beta1": situ_beta1, "situ_beta2": situ_beta2}
result = cudnn_grouped_gemm_dglu(
a_tensor=a,
b_tensor=b,
Expand All @@ -261,7 +282,8 @@ def grouped_gemm_dglu(
discrete_col_sfd=True,
dprob_tensor=None,
sf_vec_size=32,
act_func="dswiglu",
act_func=act_func,
**situ_kwargs,
)
return (
result["d_row_tensor"],
Expand Down
4 changes: 3 additions & 1 deletion transformer_engine/jax/flax/moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,9 @@ class _MoEBlock(TransformerEngineBase):
Activation between ``layer_w0 @ wi_0`` and the elementwise
product with ``layer_w0 @ wi_1``. Default ``"silu"``. Pass
:class:`~transformer_engine.jax.moe.SiTUActivation` to soft-cap both
branches in FP32 using native grouped SiTU and separate grouped GEMMs.
branches in FP32. Compatible MXFP8 calls can use cuDNN SiTU fusion
on SM100/SM103; other configurations use native grouped SiTU kernels
with separate grouped GEMMs.

score_function : Union[str, ScoreFunction]
``"softmax"`` (default), ``"sigmoid"``, or ``"sqrtsoftplus"`` for the routing scores.
Expand Down
51 changes: 40 additions & 11 deletions transformer_engine/jax/moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,9 @@ class SiTUActivation:
The defaults match Kimi-K3. Both soft-cap parameters must be finite and
positive. Pass an instance as ``moe(..., activation_type=...)``.

BF16 and MXFP8 use native grouped SiTU forward/backward with separate GEMMs.
Requesting cuDNN fusion retains its normal warning and fallback behavior.
MXFP8 can use cuDNN's fused grouped GEMM on SM100/SM103 with compatible
JAX SiTU APIs. Other configurations use native grouped SiTU forward/backward
with separate grouped GEMMs.
"""

gate_beta: float = 4.0
Expand Down Expand Up @@ -272,7 +273,7 @@ def _resolve_moe_mesh_resource(
_CUDNN_JAX_ALIGN_SIZE = 256


def _select_cudnn_jax_fusion(rejection_reasons: list[str]) -> bool:
def _select_cudnn_jax_fusion(rejection_reasons: list[str], *, activation_type="silu") -> bool:
"""Select shared cuDNN GLU/dGLU fusion or the unfused TE path."""
from transformer_engine_jax import get_device_compute_capability

Expand All @@ -284,8 +285,13 @@ def _select_cudnn_jax_fusion(rejection_reasons: list[str]) -> bool:
else:
if capability < 100:
reasons.append(f"fused kernels require SM100+, got SM{capability}")
require_situ = isinstance(activation_type, SiTUActivation)
if require_situ and capability not in (100, 103):
reasons.append(f"SiTU fusion requires SM100/SM103, got SM{capability}")
if not reasons:
available, error = tex.grouped_gemm_glu_dependencies_available()
available, error = tex.grouped_gemm_glu_dependencies_available(
require_situ=require_situ
)
if available:
return True
reasons.append(f"shared fused API is incompatible: {error}")
Expand Down Expand Up @@ -313,8 +319,8 @@ def _cudnn_jax_fusion_rejection_reasons(
) -> list[str]:
"""Return reasons this call cannot use cuDNN's shared grouped GLU/dGLU JAX APIs."""
errors = []
if str(activation_type).lower() != "silu":
errors.append("requires activation_type='silu'")
if not isinstance(activation_type, SiTUActivation) and str(activation_type).lower() != "silu":
errors.append("requires activation_type='silu' or SiTUActivation")
if wi_0_bias is not None or wi_1_bias is not None:
errors.append("does not support FC1 gate/up bias")
hidden = x.shape[-1]
Expand Down Expand Up @@ -1030,6 +1036,15 @@ def _ffn_fwd_per_shard(
prob,
compute_dtype=sorted_x.dtype,
output_dtype=fc2_quantizer_set.x.q_dtype,
**(
{
"act_func": "situglu",
"situ_beta1": activation_type.gate_beta,
"situ_beta2": activation_type.up_beta,
}
if isinstance(activation_type, SiTUActivation)
else {}
),
)
combined_out = combined_out_3d.reshape(sorted_x.shape[0], combined)
if wi_0_checkpoint_name is not None:
Expand Down Expand Up @@ -1214,6 +1229,15 @@ def _ffn_bwd_per_shard(
padded_offsets,
prob,
output_dtype=fc1_quantizer_set.dgrad.q_dtype,
**(
{
"act_func": "dsituglu",
"situ_beta1": activation_type.gate_beta,
"situ_beta2": activation_type.up_beta,
}
if isinstance(activation_type, SiTUActivation)
else {}
),
)
d_recv_w_from_intermediate = (
dprob.astype(recv_w_flat.dtype)
Expand Down Expand Up @@ -2263,9 +2287,9 @@ def moe(
----------
activation_type : Union[str, SiTUActivation]
Gate activation name (default ``"silu"``), or :class:`SiTUActivation`
for FP32 soft-capping of both gate and up branches. SiTU uses native
grouped activation kernels and separate grouped GEMMs with BF16 or
MXFP8 quantizer sets. cuDNN GLU/dGLU uses the existing fallback path.
for FP32 soft-capping of both gate and up branches. Compatible MXFP8
calls use cuDNN fusion on SM100/SM103; other configurations use native
grouped SiTU kernels with separate grouped GEMMs.
router_info : Union[RouterComputationInfo, RoutingMapInfo]
:class:`RouterComputationInfo` makes TE compute the gate and top-k
from the given router parameters. :class:`RoutingMapInfo` supplies
Expand Down Expand Up @@ -2319,7 +2343,10 @@ def moe(
use_cudnn_fusion : bool
Defaults to ``True``: try cuDNN's shared JAX grouped MXFP8 GLU/dGLU APIs
on SM100+ GPUs; cuDNN selects the architecture-specific kernels. ``False`` uses unfused TE grouped GEMM.
Requires cuDNN Frontend >= 1.32.0 with compatible JAX APIs.
Requires cuDNN Frontend >= 1.32.0 with compatible JAX APIs. SiTU
additionally requires SM100/SM103 and JAX GLU/dGLU APIs exposing
``situ_beta1`` and ``situ_beta2``. Device offsets bound active expert
rows; unused receive-buffer capacity is not part of any expert.
Frontends whose JAX GLU/dGLU entry points only support BF16 discrete
weights fall back to unfused TE grouped GEMM.
Ineligible calls warn and fall back to TE's regular grouped-GEMM
Expand Down Expand Up @@ -2453,7 +2480,9 @@ def moe(
activation_type=activation_type,
ep_axis=ep_axis,
)
use_cudnn_jax_fusion = _select_cudnn_jax_fusion(rejection_reasons)
use_cudnn_jax_fusion = _select_cudnn_jax_fusion(
rejection_reasons, activation_type=activation_type
)

output, aux_loss, total_recv_tokens, expert_counts, qb_histograms = _moe(
x,
Expand Down