From da75f29830dbe9ed8d6958d541fea77496a60d6a Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 5 Nov 2026 12:20:46 +0100 Subject: [PATCH 1/7] [JAX] Support separate router inputs in MoEBlock Keep integrated TE routing on full-width hidden states while dispatching narrower expert activations. Return independent input cotangents through RouterComputationInfo and expose the input in the Flax wrapper. Runtime validation is deferred at the author request. [skip ci] Signed-off-by: Pawel Gadzinski --- docs/api/jax.rst | 11 +++ tests/jax/moe/test_moe_resource_api.py | 33 +++++++++ tests/jax/test_te_ep_moe.py | 92 +++++++++++++++++++++++++- transformer_engine/jax/flax/moe.py | 20 +++++- transformer_engine/jax/moe.py | 63 ++++++++++++++++-- 5 files changed, 207 insertions(+), 12 deletions(-) diff --git a/docs/api/jax.rst b/docs/api/jax.rst index 99282be422c..e6d8b90e99b 100644 --- a/docs/api/jax.rst +++ b/docs/api/jax.rst @@ -76,6 +76,17 @@ Router, routing kernels, and grouped dense for MoE layers. See .. autoapifunction:: transformer_engine.jax.moe.moe +.. autoapiclass:: transformer_engine.jax.moe.RouterComputationInfo + +Latent MoE models can pass full-width hidden states through +``RouterComputationInfo(router_input=hidden_states, ...)`` while calling +``moe(latent_states, router_info, ...)`` with narrower expert activations. +Batch and sequence dimensions must agree. Router parameters use the router +width; expert parameters and EP bootstrap buffers use the latent width. +The custom VJP returns gradients to both inputs, including when the latent +is a differentiable projection of the router input. The default +``router_input=None`` preserves routing on the expert input. + .. autoapifunction:: transformer_engine.jax.ep.ep_bootstrap .. autoapifunction:: transformer_engine.jax.ep.ep_finalize diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index ab6c02932d6..661f424a013 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -36,6 +36,39 @@ def test_resource_required_for_function_and_block(): _MoEBlock().init(jax.random.PRNGKey(0), jnp.ones((1, 1, 4))) +@pytest.mark.parametrize("shape", [(1, 4), (2, 1, 8), (1, 2, 8)]) +def test_router_input_shape_is_validated(shape): + module = importlib.import_module("transformer_engine.jax.moe") + router = RouterComputationInfo( + gate_kernel=jnp.ones((8, 2)), num_experts_per_tok=1, router_input=jnp.ones(shape) + ) + with pytest.raises(ValueError, match="router_input must have shape"): + module.moe( + jnp.ones((1, 1, 4)), + router, + None, + None, + num_experts=2, + mesh_resource=MeshResource(ep_resource="ep"), + ) + + +def test_gate_kernel_uses_router_width(): + module = importlib.import_module("transformer_engine.jax.moe") + router = RouterComputationInfo( + gate_kernel=jnp.ones((4, 2)), num_experts_per_tok=1, router_input=jnp.ones((1, 1, 8)) + ) + with pytest.raises(ValueError, match="gate_kernel must have shape"): + module.moe( + jnp.ones((1, 1, 4)), + router, + None, + None, + num_experts=2, + mesh_resource=MeshResource(ep_resource="ep"), + ) + + def test_explicit_resource_overrides_global_and_is_snapshotted(): explicit = MeshResource(dp_resource="dp", fsdp_resource="fsdp", ep_resource="ep") enclosing = MeshResource(ep_resource="other") diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index ad5c897b8d5..7c8f19ccdc5 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -292,6 +292,7 @@ def _pure_jax_moe_reference( expert_bias=None, hash_input_ids=None, hash_tid2eid=None, + router_input=None, *, num_experts, num_experts_per_tok, @@ -304,8 +305,9 @@ def _pure_jax_moe_reference( K = num_experts_per_tok x_2d = x.reshape(T, H) - gate_kernel_cast = gate_kernel.astype(x.dtype) - logits = (x_2d @ gate_kernel_cast).astype(jnp.float32) # [T, E] + gate_input = x if router_input is None else router_input + gate_kernel_cast = gate_kernel.astype(gate_input.dtype) + logits = (gate_input.reshape(T, -1) @ gate_kernel_cast).astype(jnp.float32) # [T, E] if hash_input_ids is not None: if score_function != "sqrtsoftplus" or hash_tid2eid is None: @@ -724,6 +726,92 @@ def loss_fn(variables, inputs): ) +@pytest.mark.parametrize("quantization", _QUANTIZATION_CASES) +@pytest.mark.parametrize("early_weighting", [False, True]) +@pytest.mark.parametrize("router_dtype", [DTYPE, jnp.float32]) +def test_separate_router_input_latent_projection( + mesh, quantization, early_weighting, router_dtype, make_block +): + block = make_block( + score_function="sigmoid", + use_expert_routing_bias=True, + expert_bias_init=_strong_expert_bias_init, + aux_loss_coeff=1e-2, + apply_topk_weights_early=early_weighting, + quantization_recipe=_quantization_recipe(quantization), + ) + hidden = jax.random.normal(jax.random.PRNGKey(71), (BATCH, SEQ, 2 * HIDDEN), router_dtype) + projection = jax.random.normal(jax.random.PRNGKey(72), (2 * HIDDEN, HIDDEN), DTYPE) + projection = projection / jnp.sqrt(jnp.asarray(2 * HIDDEN, DTYPE)) + with _ctx(mesh): + hidden_sh = _shard_inputs(hidden, mesh) + variables = jax.jit(block.init)( + jax.random.PRNGKey(73), (hidden_sh @ projection).astype(DTYPE), router_input=hidden_sh + ) + + def loss_fn(v, h, p): + output, aux, _, _ = block.apply(v, (h @ p).astype(DTYPE), router_input=h) + return jnp.mean(output.astype(jnp.float32) ** 2) + aux + + grads, dh, dp = jax.jit(jax.grad(jax.checkpoint(loss_fn), argnums=(0, 1, 2)))( + variables, hidden_sh, projection + ) + output, aux, _, _ = jax.jit(block.apply)( + variables, (hidden_sh @ projection).astype(DTYPE), router_input=hidden_sh + ) + params_np = _params_global_numpy(variables, mesh) + assert params_np["gate_kernel"].shape == (2 * HIDDEN, NUM_EXPERTS) + assert params_np["wi"].shape == (NUM_EXPERTS, HIDDEN, 2 * INTER) + + def reference(params, h, p): + return _pure_jax_moe_reference( + (h @ p).astype(DTYPE), + params["gate_kernel"], + params["wi"], + params["wo"], + params["expert_bias"], + router_input=h, + num_experts=NUM_EXPERTS, + num_experts_per_tok=TOPK, + score_function="sigmoid", + aux_loss_coeff=1e-2, + ) + + def ref_loss(params, h, p): + out, aux_loss = reference(params, h, p) + return jnp.mean(out.astype(jnp.float32) ** 2) + aux_loss + + ref_params = {k: jnp.asarray(v) for k, v in params_np.items()} + out_ref, aux_ref = reference(ref_params, hidden, projection) + grads_ref, dh_ref, dp_ref = jax.jit(jax.grad(ref_loss, argnums=(0, 1, 2)))( + ref_params, hidden, projection + ) + np.testing.assert_allclose( + _to_global_numpy(output, mesh).astype(np.float32), + np.asarray(out_ref).astype(np.float32), + **FWD_TOLERANCE[quantization], + ) + np.testing.assert_allclose( + _to_global_numpy(aux, mesh).astype(np.float32), + np.asarray(aux_ref).astype(np.float32), + **AUX_TOLERANCE, + ) + for name in ("gate_kernel", "wi", "wo"): + tolerance = GRAD_GATE_TOLERANCE if name == "gate_kernel" else GRAD_FFN_TOLERANCE + np.testing.assert_allclose( + _to_global_numpy(_unwrap(grads["params"][name]), mesh).astype(np.float32), + np.asarray(grads_ref[name]).astype(np.float32), + **tolerance[quantization], + ) + for actual, expected, primal in ((dh, dh_ref, hidden), (dp, dp_ref, projection)): + assert actual.shape == primal.shape and actual.dtype == primal.dtype + np.testing.assert_allclose( + _to_global_numpy(actual, mesh).astype(np.float32), + np.asarray(expected).astype(np.float32), + **GRAD_FFN_TOLERANCE[quantization], + ) + + def _reference_kwargs_from_config(config, params_np): """Pick out the reference-relevant pieces of a parametrize config.""" return dict( diff --git a/transformer_engine/jax/flax/moe.py b/transformer_engine/jax/flax/moe.py index 19b1a41778f..4cd83c2bbfa 100644 --- a/transformer_engine/jax/flax/moe.py +++ b/transformer_engine/jax/flax/moe.py @@ -88,7 +88,7 @@ class _MoEBlock(TransformerEngineBase): If ``> 0``, return the MoE auxiliary load-balancing loss scalar in addition to the main output. - gate_kernel_axes, wi_kernel_axes, wo_kernel_axes, input_axes : + gate_kernel_axes, wi_kernel_axes, wo_kernel_axes, input_axes, router_input_axes : Logical sharding axis tuples (consumed by Flax's :func:`with_logical_partitioning` and our internal :func:`with_sharding_constraint_by_logical_axes`). @@ -156,6 +156,7 @@ class _MoEBlock(TransformerEngineBase): wi_kernel_axes: Tuple[Optional[str], ...] = ("exp", "embed", "mlp") wo_kernel_axes: Tuple[Optional[str], ...] = ("exp", "mlp", "embed") input_axes: Tuple[Optional[str], ...] = () + router_input_axes: Tuple[Optional[str], ...] = () # Parallelism mesh_resource: Optional[MeshResource] = None @@ -188,13 +189,19 @@ def __post_init__(self): super().__post_init__() @nn.compact - def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array]: + def __call__( + self, inputs: Array, *, router_input: Optional[Array] = None + ) -> Tuple[Array, Optional[Array], Array, Array]: """Run the MoE forward pass. Parameters ---------- inputs : jnp.ndarray ``[batch, sequence, hidden]``. + router_input : Optional[jnp.ndarray] + Separate gate input ``[batch, sequence, router_hidden]``. ``None`` + uses ``inputs``. Only the gate kernel uses ``router_hidden``; + expert kernels and EP buffers retain ``hidden``. Returns ------- @@ -218,6 +225,11 @@ def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array] inputs.ndim == 3 ), f"_MoEBlock expects [batch, sequence, hidden] input, got shape {inputs.shape}" _, _, hidden_size = inputs.shape + if router_input is not None and ( + router_input.ndim != 3 or router_input.shape[:2] != inputs.shape[:2] + ): + raise ValueError("router_input must match the batch and sequence dimensions of inputs.") + router_hidden_size = hidden_size if router_input is None else router_input.shape[-1] # Param registrations -- must run OUTSIDE any JAX transform that # alters the variable scope (e.g. shard_map). The functional @@ -226,7 +238,7 @@ def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array] gate_kernel = self.param( "gate_kernel", nn.with_logical_partitioning(self.kernel_init, self.gate_kernel_axes), - (hidden_size, self.num_experts), + (router_hidden_size, self.num_experts), self.dtype, ) # FC1 is stored as one gated-SwiGLU kernel. Keeping its two @@ -318,6 +330,8 @@ def make_grouped_quantizer_set(postfix): scaling_factor=self.scaling_factor, aux_loss_coeff=self.aux_loss_coeff, gate_kernel_axes=self.gate_kernel_axes, + router_input=router_input, + router_input_axes=self.router_input_axes, ) return moe( inputs, diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index 4e9bfe4aa8d..6de45f1a85e 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -88,7 +88,8 @@ class RouterComputationInfo: Parameters ---------- gate_kernel : jnp.ndarray - ``[hidden, num_experts]`` router projection. + ``[router_hidden, num_experts]`` router projection; ``router_hidden`` + is the width of ``router_input``, or ``x`` when no separate input is given. num_experts_per_tok : int Top-k value for routing. expert_bias : Optional[jnp.ndarray] @@ -113,6 +114,15 @@ class RouterComputationInfo: view; this lives off the dispatch critical path. gate_kernel_axes : Tuple[Optional[str], ...] Logical sharding axes of ``gate_kernel``. + router_input : Optional[jnp.ndarray] + Separate ``[batch, sequence, router_hidden]`` activations for the gate. + ``None`` uses the expert input ``x``. Batch and sequence dimensions must + match ``x``; hidden dimensions may differ. Gradients flow to this input + independently of the expert-input gradient and retain its dtype. + The gate kernel is cast to this input's dtype for the projection. + router_input_axes : Tuple[Optional[str], ...] + Logical sharding axes of ``router_input``. Its physical token layout + follows the same DP/FSDP/EP placement as ``x``. """ gate_kernel: jnp.ndarray @@ -127,6 +137,8 @@ class RouterComputationInfo: scaling_factor: float = flax.struct.field(pytree_node=False, default=1.0) aux_loss_coeff: float = flax.struct.field(pytree_node=False, default=0.0) gate_kernel_axes: Tuple[Optional[str], ...] = flax.struct.field(pytree_node=False, default=()) + router_input: Optional[jnp.ndarray] = None + router_input_axes: Tuple[Optional[str], ...] = flax.struct.field(pytree_node=False, default=()) @flax.struct.dataclass @@ -1325,6 +1337,12 @@ def _moe_fwd_rule( compute_aux_loss = router is not None and router.aux_loss_coeff > 0.0 x = with_sharding_constraint_by_logical_axes(x, input_axes) + if router is not None and router.router_input is not None: + router = router.replace( + router_input=with_sharding_constraint_by_logical_axes( + router.router_input, router.router_input_axes + ) + ) mesh = _get_mesh() if mesh is None or mesh.empty: @@ -1451,8 +1469,10 @@ def _moe_fwd_rule( # accumulate at low precision and silently produce NaNs on tokens # whose normalised weights underflow. Cast to fp32 here to stay in # the validated regime. - gate_kernel_cast = router.gate_kernel.astype(x.dtype) - gate_logits = jnp.einsum("bsh,he->bse", x, gate_kernel_cast) + gate_input = x if router.router_input is None else router.router_input + gate_input = jax.lax.with_sharding_constraint(gate_input, NamedSharding(mesh, ep3_spec)) + gate_kernel_cast = router.gate_kernel.astype(gate_input.dtype) + gate_logits = jnp.einsum("bsh,he->bse", gate_input, gate_kernel_cast) logits_2d = gate_logits.reshape(-1, num_experts).astype(jnp.float32) # ---------------- Routing (global view) ---------------- @@ -1945,19 +1965,28 @@ def _ffn_bwd_body(*args): # ---------------- Gate bwd (global view) ---------------- d_gate_logits = d_logits_2d.reshape(B, S, num_experts) - gate_kernel_cast = router.gate_kernel.astype(ctx.x.dtype) - d_x_from_gate = jnp.einsum("bse,he->bsh", d_gate_logits, gate_kernel_cast) - d_gate_kernel = jnp.einsum("bsh,bse->he", ctx.x, d_gate_logits).astype( + gate_input = ctx.x if router.router_input is None else router.router_input + gate_kernel_cast = router.gate_kernel.astype(gate_input.dtype) + d_gate_input = jnp.einsum("bse,he->bsh", d_gate_logits, gate_kernel_cast) + d_gate_kernel = jnp.einsum("bsh,bse->he", gate_input, d_gate_logits).astype( router.gate_kernel.dtype ) d_gate_kernel = with_sharding_constraint_by_logical_axes( d_gate_kernel, router.gate_kernel_axes ) - d_x = d_x_from_gate + d_x_from_dispatch + if router.router_input is None: + d_x = d_gate_input + d_x_from_dispatch + d_router_input = None + else: + d_x = d_x_from_dispatch + d_router_input = with_sharding_constraint_by_logical_axes( + d_gate_input.astype(router.router_input.dtype), router.router_input_axes + ) # expert_bias has no learnable bwd path through fused_topk: the # primitive's bwd returns None for the bias slot, so its cotangent is zero. d_router_info = router.replace( gate_kernel=d_gate_kernel, + router_input=d_router_input, expert_bias=( None if router.expert_bias is None else jnp.zeros_like(router.expert_bias) ), @@ -2216,6 +2245,17 @@ def moe( ep_axis, data_parallelism_axes = _moe_mesh_axes(mesh_resource) if isinstance(router_info, RouterComputationInfo): + gate_input = x if router_info.router_input is None else router_info.router_input + if gate_input.ndim != 3 or gate_input.shape[:2] != x.shape[:2]: + raise ValueError( + "router_input must have shape [batch, sequence, router_hidden] matching x, " + f"got x={x.shape}, router_input={gate_input.shape}." + ) + if router_info.gate_kernel.shape != (gate_input.shape[-1], num_experts): + raise ValueError( + "gate_kernel must have shape [router_hidden, num_experts], got " + f"{router_info.gate_kernel.shape} for router_input={gate_input.shape}." + ) router_info = router_info.replace( score_function=_validate_score_function(router_info.score_function), aux_loss_coeff=float(router_info.aux_loss_coeff), @@ -2250,6 +2290,15 @@ def moe( # so ep MUST be innermost in the partition spec. Soft re-pin: free if # upstream already matches, single reshard otherwise. x = _prepare_moe_input(x, ep_axis, data_parallelism_axes, caller="moe") + if isinstance(router_info, RouterComputationInfo) and router_info.router_input is not None: + router_info = router_info.replace( + router_input=_prepare_moe_input( + router_info.router_input, + ep_axis, + data_parallelism_axes, + caller="moe router_input", + ) + ) cudnn_native_weight_layout = _resolve_cudnn_native_weight_layout( wi, x.shape[-1], cudnn_native_weight_layout From 0f181ca644d0916249627b3ab7e5b12e43172def Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 5 Nov 2026 12:20:47 +0100 Subject: [PATCH 2/7] [JAX] Add two-branch SiTU activation to MoEBlock Evaluate configurable gate and up soft caps in FP32 with gradients for both branches. Keep SiTU on the unfused grouped-GEMM path for BF16 and MXFP8 and preserve existing activations. Runtime validation is deferred at the author request. [skip ci] Signed-off-by: Pawel Gadzinski --- docs/api/jax.rst | 2 + tests/jax/moe/test_moe_resource_api.py | 43 +++++++++++++- tests/jax/test_te_ep_moe.py | 35 +++++++++-- transformer_engine/jax/flax/moe.py | 18 ++++-- transformer_engine/jax/moe.py | 80 ++++++++++++++++++++------ 5 files changed, 151 insertions(+), 27 deletions(-) diff --git a/docs/api/jax.rst b/docs/api/jax.rst index 99282be422c..970303cfdcf 100644 --- a/docs/api/jax.rst +++ b/docs/api/jax.rst @@ -74,6 +74,8 @@ Router, routing kernels, and grouped dense for MoE layers. See .. autoapifunction:: transformer_engine.jax.dense.grouped_dense +.. autoapiclass:: transformer_engine.jax.moe.SiTUActivation + .. autoapifunction:: transformer_engine.jax.moe.moe .. autoapifunction:: transformer_engine.jax.ep.ep_bootstrap diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index ab6c02932d6..a7f76d56487 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -2,7 +2,7 @@ # # See LICENSE for license information. -"""MoE resource resolution and MeshResource-only API without EP kernels.""" +"""MoE resource and activation API contracts without EP kernels.""" import importlib import inspect @@ -16,12 +16,53 @@ from transformer_engine.jax.flax import _MoEBlock from transformer_engine.jax.moe import ( RouterComputationInfo, + SiTUActivation, + _apply_ffn_activation, _moe_mesh_axes, _resolve_moe_mesh_resource, ) from transformer_engine.jax.sharding import MeshResource, global_mesh_resource, global_shard_guard +@pytest.mark.parametrize("betas", [(4.0, 25.0), (2.0, 7.0)]) +def test_situ_value_and_both_branch_derivatives(betas): + gate_beta, up_beta = betas + gate = jnp.asarray([-10000.0, -12.0, -0.5, 0.0, 0.5, 12.0, 10000.0]) + up = jnp.asarray([10000.0, 40.0, -3.0, 2.0, 3.0, -40.0, -10000.0]) + activation = SiTUActivation(gate_beta, up_beta) + output, pullback = jax.vjp(lambda g, u: _apply_ffn_activation(activation, g, u), gate, up) + dg, du = pullback(jnp.ones_like(output)) + + g, u = np.asarray(gate, np.float64), np.asarray(up, np.float64) + sigmoid = 0.5 + 0.5 * np.tanh(g / 2) + tg, tu = np.tanh(g / gate_beta), np.tanh(u / up_beta) + capped_gate, capped_up = gate_beta * tg * sigmoid, up_beta * tu + derivative_gate = (1 - tg**2) * sigmoid + gate_beta * tg * sigmoid * (1 - sigmoid) + for actual, expected in ( + (output, capped_gate * capped_up), + (dg, derivative_gate * capped_up), + (du, capped_gate * (1 - tu**2)), + ): + np.testing.assert_allclose(actual, expected, atol=2e-5, rtol=2e-5) + + +def test_situ_bf16_uses_fp32_before_final_cast(): + gate = jnp.asarray([0.25, 1.5, -3.0, 12.0, 64.0], dtype=jnp.bfloat16) + up = jnp.asarray([0.75, -2.0, 32.0, 64.0, -128.0], dtype=jnp.bfloat16) + actual = _apply_ffn_activation(SiTUActivation(), gate, up) + g, u = np.asarray(gate, np.float64), np.asarray(up, np.float64) + expected = jnp.asarray(100 * np.tanh(g / 4) / (1 + np.exp(-g)) * np.tanh(u / 25), jnp.bfloat16) + assert actual.dtype == jnp.bfloat16 + np.testing.assert_array_equal(actual, expected) + + +@pytest.mark.parametrize("value", [0, -1, float("inf"), float("nan")]) +@pytest.mark.parametrize("field", ["gate_beta", "up_beta"]) +def test_situ_rejects_invalid_soft_caps(field, value): + with pytest.raises(ValueError, match=field): + SiTUActivation(**{field: value}) + + @pytest.fixture(autouse=True) def no_global_resource(): with global_shard_guard(None): diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index ad5c897b8d5..80a0e239454 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -124,6 +124,7 @@ def _read_mp_options(): from transformer_engine.jax.moe import ( RouterComputationInfo, RoutingMapInfo, + SiTUActivation, get_moe_recv_capacity_per_rank, moe, record_ep_bootstrap_signature_for_moe, @@ -282,6 +283,7 @@ def mesh(use_cudnn_fusion): "num_experts_per_tok", "aux_loss_coeff", "score_function", + "activation_type", ), ) def _pure_jax_moe_reference( @@ -298,6 +300,7 @@ def _pure_jax_moe_reference( aux_loss_coeff: float = 0.0, score_function: str = "softmax", scaling_factor: float = 1.0, + activation_type="silu", ): B, S, H = x.shape T = B * S @@ -342,10 +345,18 @@ def _pure_jax_moe_reference( wi_0, wi_1 = jnp.split(wi, 2, axis=-1) layer_w0 = jnp.einsum("th,ehm->tem", x_2d, wi_0) layer_w1 = jnp.einsum("th,ehm->tem", x_2d, wi_1) - # Activation runs in x.dtype (typically bf16) to mirror the impl -- - # the impl keeps silu+multiply in the wi GEMM output dtype because - # storing higher precision than the consumer (wo) GEMM buys nothing. - intermediate = jax.nn.silu(layer_w0) * layer_w1 + # SiTU evaluates both branches in FP32 before the final cast. + if isinstance(activation_type, SiTUActivation): + gate, up = layer_w0.astype(jnp.float32), layer_w1.astype(jnp.float32) + intermediate = ( + activation_type.gate_beta + * jnp.tanh(gate / activation_type.gate_beta) + * jax.nn.sigmoid(gate) + * activation_type.up_beta + * jnp.tanh(up / activation_type.up_beta) + ).astype(layer_w0.dtype) + else: + intermediate = jax.nn.silu(layer_w0) * layer_w1 expert_out = jnp.einsum("tem,emh->teh", intermediate, wo) # [T, E, H] output_2d = jnp.einsum("te,teh->th", routing_weights_full.astype(x.dtype), expert_out) output = output_2d.reshape(B, S, H).astype(x.dtype) @@ -384,6 +395,7 @@ def _pure_jax_moe_reference( def _make_block( *, + activation_type="silu", apply_topk_weights_early=False, aux_loss_coeff=0.0, use_expert_routing_bias=False, @@ -401,6 +413,7 @@ def _make_block( num_experts=NUM_EXPERTS, num_experts_per_tok=TOPK, intermediate_size=INTER, + activation_type=activation_type, mesh_resource=mesh_resource, apply_topk_weights_early=apply_topk_weights_early, aux_loss_coeff=aux_loss_coeff, @@ -572,6 +585,19 @@ def _quantization_recipe(quantization): # ----------------------------------------------------------------------------- _CONFIGS = [ + *[ + pytest.param( + dict( + activation_type=SiTUActivation(), + score_function="sigmoid", + use_expert_routing_bias=True, + expert_bias_init=_strong_expert_bias_init, + apply_topk_weights_early=early, + ), + id=f"situ-{'early' if early else 'late'}-weighting", + ) + for early in (False, True) + ], pytest.param( dict(score_function="softmax"), id="softmax", @@ -727,6 +753,7 @@ def loss_fn(variables, inputs): def _reference_kwargs_from_config(config, params_np): """Pick out the reference-relevant pieces of a parametrize config.""" return dict( + activation_type=config.get("activation_type", "silu"), score_function=config.get("score_function", "softmax"), expert_bias=( jnp.asarray(params_np["expert_bias"]) diff --git a/transformer_engine/jax/flax/moe.py b/transformer_engine/jax/flax/moe.py index 19b1a41778f..0bf735b0b43 100644 --- a/transformer_engine/jax/flax/moe.py +++ b/transformer_engine/jax/flax/moe.py @@ -34,7 +34,13 @@ from flax import linen as nn from transformer_engine.common.recipe import Recipe -from ..moe import RouterComputationInfo, _moe_mesh_axes, _resolve_moe_mesh_resource, moe +from ..moe import ( + RouterComputationInfo, + SiTUActivation, + _moe_mesh_axes, + _resolve_moe_mesh_resource, + moe, +) from ..quantize import QuantizerSet from ..router import ScoreFunction from ..sharding import MeshResource, _get_mesh, global_shard_guard @@ -67,9 +73,11 @@ class _MoEBlock(TransformerEngineBase): Top-k value for routing. intermediate_size : int Hidden dim of the per-expert FFN (the inner ``mlp`` axis). - activation_type : str + activation_type : Union[str, SiTUActivation] Activation between ``layer_w0 @ wi_0`` and the elementwise - product with ``layer_w0 @ wi_1``. Default ``"silu"``. + product with ``layer_w0 @ wi_1``. Default ``"silu"``. Pass + :class:`~transformer_engine.jax.moe.SiTUActivation` to soft-cap both + branches in FP32 using the unfused grouped-GEMM path. score_function : Union[str, ScoreFunction] ``"softmax"`` (default), ``"sigmoid"``, or ``"sqrtsoftplus"`` for the routing scores. @@ -140,7 +148,7 @@ class _MoEBlock(TransformerEngineBase): num_experts: int = 8 num_experts_per_tok: int = 2 intermediate_size: int = 2048 - activation_type: str = "silu" + activation_type: Union[str, SiTUActivation] = "silu" # Routing score_function: Union[str, ScoreFunction] = "softmax" @@ -229,7 +237,7 @@ def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array] (hidden_size, self.num_experts), self.dtype, ) - # FC1 is stored as one gated-SwiGLU kernel. Keeping its two + # FC1 is stored as one gated FFN kernel. Keeping its two # projections contiguous lets the functional MoE path quantize and # all-gather one FP8 data buffer (and one scale buffer), rather than # materializing a concatenate inside the custom-VJP. diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index 4e9bfe4aa8d..862248fa610 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -41,7 +41,7 @@ import math import warnings -from dataclasses import replace +from dataclasses import dataclass, replace from functools import partial from typing import Any, Optional, Tuple, Union @@ -72,6 +72,7 @@ from .sharding import MeshResource, _get_mesh, global_mesh_resource, global_shard_guard __all__ = [ + "SiTUActivation", "RouterComputationInfo", "RoutingMapInfo", "get_moe_recv_capacity_per_rank", @@ -79,6 +80,43 @@ ] +@dataclass(frozen=True) +class SiTUActivation: + """Two-branch SiTU-GLU for MoE experts. + + Computes ``gate_beta * tanh(gate / gate_beta) * sigmoid(gate)`` times + ``up_beta * tanh(up / up_beta)`` in FP32, then casts to the gate dtype. + The defaults match Kimi-K3. Both soft-cap parameters must be finite and + positive. Pass an instance as ``moe(..., activation_type=...)``. + + This activation uses the unfused grouped-GEMM path for BF16 and MXFP8; + requesting cuDNN fusion retains its normal warning and fallback behavior. + """ + + gate_beta: float = 4.0 + up_beta: float = 25.0 + + def __post_init__(self): + for name in ("gate_beta", "up_beta"): + value = float(getattr(self, name)) + if not math.isfinite(value) or value <= 0: + raise ValueError(f"{name} must be finite and positive, got {value}") + object.__setattr__(self, name, value) + + +def _apply_ffn_activation(activation_type, gate, up): + if isinstance(activation_type, SiTUActivation): + gate_fp32, up_fp32 = gate.astype(jnp.float32), up.astype(jnp.float32) + gated = ( + activation_type.gate_beta + * jnp.tanh(gate_fp32 / activation_type.gate_beta) + * jax.nn.sigmoid(gate_fp32) + ) + linear = activation_type.up_beta * jnp.tanh(up_fp32 / activation_type.up_beta) + return (gated * linear).astype(gate.dtype) + return _convert_to_activation_function(activation_type)(gate) * up + + @flax.struct.dataclass class RouterComputationInfo: """Router parameters and settings for :func:`moe` to compute routing itself. @@ -890,7 +928,7 @@ def _ffn_fwd_per_shard( quantizer_sets: Tuple[QuantizerSet, QuantizerSet], *, num_local_experts: int, - activation_type: str, + activation_type: Union[str, SiTUActivation], apply_topk_weights_early: bool, use_cudnn_jax_fusion: bool, wi_0_checkpoint_name: Optional[str], @@ -1036,14 +1074,10 @@ def _ffn_fwd_per_shard( if wi_1_checkpoint_name is not None: up_proj_out = checkpoint_name(up_proj_out, wi_1_checkpoint_name) - # Activation inputs (gate_proj_out, up_proj_out) stay in the wi GEMM - # output dtype; the activation output (`intermediate`) stays in the - # dtype the wo GEMM / wo's quantized input consumes. For bf16 compute - # that's all bf16; for FP8/FP4 the downstream grouped_quantize is what - # transitions to the target precision. - act_fn = _convert_to_activation_function(activation_type) + # Keep projection residuals and activation output in the GEMM dtype; + # SiTU evaluates its internal arithmetic in FP32 before casting back. if not use_cudnn_jax_fusion: - intermediate = act_fn(gate_proj_out) * up_proj_out + intermediate = _apply_ffn_activation(activation_type, gate_proj_out, up_proj_out) if apply_topk_weights_early and not use_cudnn_jax_fusion: # Fold the per-token combine weights into the FFN intermediate; # the downstream wo GEMM is linear so this is equivalent to the @@ -1102,7 +1136,7 @@ def _ffn_bwd_per_shard( recv_topk_weights_local: jnp.ndarray, quantizer_sets: Tuple[QuantizerSet, QuantizerSet], *, - activation_type: str, + activation_type: Union[str, SiTUActivation], apply_topk_weights_early: bool, has_bias: bool, use_cudnn_jax_fusion: bool, @@ -1206,12 +1240,13 @@ def _ffn_bwd_per_shard( casted_wo_rhs_trans, contracting_dims=((1,), (2,)), ) - act_fn = _convert_to_activation_function(activation_type) if apply_topk_weights_early: # intermediate' = intermediate * w. # Masking is not required as grouped GEMMs consume only group-size rows. w_b = recv_w_flat[:, None].astype(d_intermediate.dtype) - intermediate_unweighted = act_fn(gate_proj_out) * up_proj_out + intermediate_unweighted = _apply_ffn_activation( + activation_type, gate_proj_out, up_proj_out + ) d_recv_w_from_intermediate = jnp.sum( d_intermediate * intermediate_unweighted, axis=-1, @@ -1220,10 +1255,16 @@ def _ffn_bwd_per_shard( else: d_recv_w_from_intermediate = jnp.zeros_like(recv_w_flat) - # Activation bwd stays in the GEMM dtype, matching the forward path. - act_gp, dact_pullback = jax.vjp(act_fn, gate_proj_out) - d_up_proj_out = d_intermediate * act_gp - (d_gate_proj_out,) = dact_pullback(d_intermediate * up_proj_out) + if isinstance(activation_type, SiTUActivation): + _, dact_pullback = jax.vjp( + partial(_apply_ffn_activation, activation_type), gate_proj_out, up_proj_out + ) + d_gate_proj_out, d_up_proj_out = dact_pullback(d_intermediate) + else: + act_fn = _convert_to_activation_function(activation_type) + act_gp, dact_pullback = jax.vjp(act_fn, gate_proj_out) + d_up_proj_out = d_intermediate * act_gp + (d_gate_proj_out,) = dact_pullback(d_intermediate * up_proj_out) d_combined_for_bias = jnp.concatenate([d_gate_proj_out, d_up_proj_out], axis=-1) casted_d_combined = tex.grouped_quantize( d_combined_for_bias, @@ -2094,7 +2135,7 @@ def moe( wo_bias: Optional[jnp.ndarray] = None, *, num_experts: int, - activation_type: str = "silu", + activation_type: Union[str, SiTUActivation] = "silu", apply_topk_weights_early: bool = False, quantizer_sets: Tuple[QuantizerSet, QuantizerSet] = ( noop_quantizer_set, @@ -2127,6 +2168,11 @@ def moe( Parameters ---------- + activation_type : Union[str, SiTUActivation] + Gate activation name (default ``"silu"``), or :class:`SiTUActivation` + for FP32 soft-capping of both gate and up branches. SiTU uses unfused + grouped GEMMs with BF16 or MXFP8 quantizer sets; cuDNN GLU/dGLU does + not implement it and follows the existing fallback path. router_info : Union[RouterComputationInfo, RoutingMapInfo] :class:`RouterComputationInfo` makes TE compute the gate and top-k from the given router parameters. :class:`RoutingMapInfo` supplies From 73de05e66c67a2d89dcfa73a9b56218bd4bd3f59 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 12 Nov 2026 13:58:28 +0100 Subject: [PATCH 3/7] [JAX] Use native grouped SiTU forward and backward Expose grouped SiTU C APIs and an XLA FFI custom VJP for FP32, FP16 and BF16. Keep expert counts on device, skip inactive inputs and zero inactive outputs and gradients. Replace the JAX MoE SiTU formulas with the native calls; MXFP8 quantization remains a separate grouped operation. Validate values and FP64 analytic gradients, dynamic counts, empty experts, NaN tails, vmap and checkpoint replay, plus four-GPU BF16/MXFP8 MoE execution. Signed-off-by: Pawel Gadzinski --- docs/api/jax.rst | 12 ++ tests/jax/moe/test_moe_resource_api.py | 16 +- tests/jax/test_grouped_situglu.py | 134 +++++++++++++++++ tests/jax/test_te_ep_moe.py | 25 ++++ transformer_engine/common/CMakeLists.txt | 1 + .../common/activation/situglu_grouped.cu | 141 ++++++++++++++++++ .../include/transformer_engine/activation.h | 17 +++ transformer_engine/jax/__init__.py | 2 + transformer_engine/jax/activation.py | 28 ++++ .../jax/cpp_extensions/activation.py | 50 ++++++- transformer_engine/jax/csrc/extensions.h | 1 + .../jax/csrc/extensions/activation.cpp | 66 ++++++++ .../jax/csrc/extensions/pybind.cpp | 1 + transformer_engine/jax/flax/moe.py | 2 +- transformer_engine/jax/moe.py | 41 ++--- 15 files changed, 514 insertions(+), 23 deletions(-) create mode 100644 tests/jax/test_grouped_situglu.py create mode 100644 transformer_engine/common/activation/situglu_grouped.cu diff --git a/docs/api/jax.rst b/docs/api/jax.rst index a44068dfd47..da9a3506961 100644 --- a/docs/api/jax.rst +++ b/docs/api/jax.rst @@ -32,6 +32,7 @@ Modules .. autoapifunction:: transformer_engine.jax.fp8_autocast .. autoapifunction:: transformer_engine.jax.autocast .. autoapifunction:: transformer_engine.jax.update_collections +.. autoapifunction:: transformer_engine.jax.grouped_situglu .. autoapiclass:: transformer_engine.jax.flax.LayerNorm(epsilon=1e-6, layernorm_type='layernorm', **kwargs) @@ -76,6 +77,17 @@ Router, routing kernels, and grouped dense for MoE layers. See .. autoapiclass:: transformer_engine.jax.moe.SiTUActivation +``SiTUActivation`` uses native grouped activation kernels between the expert +GEMMs, including their backward pass. Device group counts bound active rows; +unused receive capacity is zero-filled without evaluating the activation there. +FP32, FP16 and BF16 activation outputs are supported. MXFP8 expert execution +uses grouped quantization after the activation. The standalone +``grouped_situglu(x, group_sizes, beta1=4.0, beta2=25.0)`` function has the same +math and automatic differentiation support. It expects contiguous gate/up +halves in each row of ``x[capacity, 2H]``. Counts must be nonnegative device +int32 values with a sum no greater than capacity. Distributed expert callers +should pass local arrays and counts inside ``shard_map``. + .. autoapifunction:: transformer_engine.jax.moe.moe .. autoapiclass:: transformer_engine.jax.moe.RouterComputationInfo diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index ee6f0d8b468..587fc22dd1c 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -13,6 +13,7 @@ import pytest from jax.sharding import Mesh +from transformer_engine.jax import grouped_situglu from transformer_engine.jax.flax import _MoEBlock from transformer_engine.jax.moe import ( RouterComputationInfo, @@ -30,8 +31,17 @@ def test_situ_value_and_both_branch_derivatives(betas): gate = jnp.asarray([-10000.0, -12.0, -0.5, 0.0, 0.5, 12.0, 10000.0]) up = jnp.asarray([10000.0, 40.0, -3.0, 2.0, 3.0, -40.0, -10000.0]) activation = SiTUActivation(gate_beta, up_beta) - output, pullback = jax.vjp(lambda g, u: _apply_ffn_activation(activation, g, u), gate, up) + counts = jnp.asarray([1], jnp.int32) + + def activate(g, u): + pair = jnp.concatenate([g, u])[None, :] + return grouped_situglu(pair, counts, gate_beta, up_beta)[0] + + output, pullback = jax.vjp(activate, gate, up) dg, du = pullback(jnp.ones_like(output)) + np.testing.assert_array_equal( + output, _apply_ffn_activation(activation, gate[None, :], up[None, :], counts)[0] + ) g, u = np.asarray(gate, np.float64), np.asarray(up, np.float64) sigmoid = 0.5 + 0.5 * np.tanh(g / 2) @@ -49,7 +59,9 @@ def test_situ_value_and_both_branch_derivatives(betas): def test_situ_bf16_uses_fp32_before_final_cast(): gate = jnp.asarray([0.25, 1.5, -3.0, 12.0, 64.0], dtype=jnp.bfloat16) up = jnp.asarray([0.75, -2.0, 32.0, 64.0, -128.0], dtype=jnp.bfloat16) - actual = _apply_ffn_activation(SiTUActivation(), gate, up) + actual = _apply_ffn_activation( + SiTUActivation(), gate[None, :], up[None, :], jnp.asarray([1], jnp.int32) + )[0] g, u = np.asarray(gate, np.float64), np.asarray(up, np.float64) expected = jnp.asarray(100 * np.tanh(g / 4) / (1 + np.exp(-g)) * np.tanh(u / 25), jnp.bfloat16) assert actual.dtype == jnp.bfloat16 diff --git a/tests/jax/test_grouped_situglu.py b/tests/jax/test_grouped_situglu.py new file mode 100644 index 00000000000..8bc3c5e5d78 --- /dev/null +++ b/tests/jax/test_grouped_situglu.py @@ -0,0 +1,134 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Native grouped SiTU values, derivatives and inactive-capacity isolation.""" + +from functools import partial + +import jax +import jax.numpy as jnp +import numpy as np +import pytest + +from transformer_engine.jax import grouped_situglu + + +def reference(x, counts, beta1, beta2): + active = jnp.arange(x.shape[0]) < counts.sum() + safe_x = jnp.where(active[:, None], x, 0).astype(jnp.float32) + gate, up = jnp.split(safe_x, 2, axis=-1) + result = beta1 * jnp.tanh(gate / beta1) * jax.nn.sigmoid(gate) + result *= beta2 * jnp.tanh(up / beta2) + return jnp.where(active[:, None], result.astype(x.dtype), 0) + + +def reference_backward_fp64(x, dy, active, beta1, beta2): + gate, up = np.split(np.asarray(x, np.float64)[:active], 2, axis=-1) + upstream = np.asarray(dy, np.float64)[:active] + tg, tu = np.tanh(gate / beta1), np.tanh(up / beta2) + sigmoid = 1 / (1 + np.exp(-gate)) + dx = np.zeros(x.shape, np.float64) + hidden = x.shape[1] // 2 + dx[:active, :hidden] = ( + upstream * ((1 - tg**2) * sigmoid + beta1 * tg * sigmoid * (1 - sigmoid)) * beta2 * tu + ) + dx[:active, hidden:] = upstream * beta1 * tg * sigmoid * (1 - tu**2) + return dx + + +@pytest.mark.parametrize("dtype", [jnp.float32, jnp.float16, jnp.bfloat16]) +@pytest.mark.parametrize("caps", [(4.0, 25.0), (2.0, 8.0)]) +@pytest.mark.parametrize( + "capacity,hidden,counts", + [ + (13, 6, [3, 0, 5]), + (256, 128, [128, 128]), + (513, 3584, [128, 0, 129]), + (7, 3, [0] * 257 + [3, 4]), + (9, 10, [0, 0]), + (0, 32, [0, 0]), + ], +) +def test_grouped_situ_forward_and_backward(dtype, caps, capacity, hidden, counts): + rng = np.random.default_rng(72) + host = rng.uniform(-30, 30, (capacity, 2 * hidden)).astype(np.float32) + active = sum(counts) + host[active:] = np.nan + x = jnp.asarray(host, dtype) + counts = jnp.asarray(counts, jnp.int32) + upstream = rng.normal(size=(capacity, hidden)).astype(np.float32) + upstream[active:] = np.nan + dy = jnp.asarray(upstream, dtype) + fn = partial(grouped_situglu, beta1=caps[0], beta2=caps[1]) + ref = partial(reference, beta1=caps[0], beta2=caps[1]) + y = jax.jit(fn)(x, counts) + _, pullback = jax.vjp(lambda inputs: fn(inputs, counts), x) + dx = jax.jit(pullback)(dy)[0] + expected_dx = reference_backward_fp64(x, dy, active, *caps) + tolerance = {jnp.float32: 2.0e-5, jnp.float16: 3.0e-3, jnp.bfloat16: 2.0e-2}[dtype] + np.testing.assert_allclose( + y.astype(jnp.float32), ref(x, counts).astype(jnp.float32), rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose(dx.astype(jnp.float32), expected_dx, rtol=tolerance, atol=tolerance) + assert np.isfinite(np.asarray(y, dtype=np.float32)).all() + assert np.isfinite(np.asarray(dx, dtype=np.float32)).all() + np.testing.assert_array_equal(np.asarray(y[active:], dtype=np.float32), 0) + np.testing.assert_array_equal(np.asarray(dx[active:], dtype=np.float32), 0) + + +def test_grouped_situ_dynamic_counts_replay_and_checkpoint(): + x = jnp.arange(13 * 24, dtype=jnp.float32).reshape(13, 24) / 30 - 5 + compiled = jax.jit( + grouped_situglu, compiler_options={"xla_gpu_enable_command_buffer": "FUSION,CUSTOM_CALL"} + ) + counts = jnp.array([3, 0, 5], jnp.int32) + baseline = compiled(x, counts) + poisoned = x.at[8:].set(jnp.nan) + np.testing.assert_array_equal(baseline, compiled(poisoned, counts)) + changed = jnp.array([1, 2, 2], jnp.int32) + np.testing.assert_allclose( + compiled(x, changed), reference(x, changed, 4.0, 25.0), rtol=2.0e-5, atol=2.0e-5 + ) + loss = lambda inputs: jnp.sum(grouped_situglu(inputs, counts)) + np.testing.assert_array_equal( + jax.jit(jax.grad(jax.checkpoint(loss)))(poisoned), jax.jit(jax.grad(loss))(poisoned) + ) + hlo = compiled.lower(x, counts).compile().as_text() + assert "te_grouped_situglu_ffi" in hlo + + +def test_grouped_situ_vmap(): + x = jnp.arange(2 * 9 * 16, dtype=jnp.float32).reshape(2, 9, 16) / 20 - 5 + counts = jnp.array([[2, 0, 3], [1, 1, 7]], jnp.int32) + actual = jax.jit(jax.vmap(grouped_situglu))(x, counts) + expected = jax.vmap(partial(reference, beta1=4.0, beta2=25.0))(x, counts) + np.testing.assert_allclose(actual, expected, rtol=2.0e-5, atol=2.0e-5) + actual_grad = jax.jit(jax.grad(lambda inputs: jax.vmap(grouped_situglu)(inputs, counts).sum()))( + x + ) + expected_grad = jax.grad( + lambda inputs: jax.vmap(partial(reference, beta1=4.0, beta2=25.0))(inputs, counts).sum() + )(x) + np.testing.assert_allclose(actual_grad, expected_grad, rtol=2.0e-5, atol=2.0e-5) + + +@pytest.mark.parametrize("name", ["beta1", "beta2"]) +@pytest.mark.parametrize("value", [0.0, -1.0, float("nan"), float("inf"), 1.0e100, 1.0e-100]) +def test_grouped_situ_invalid_caps(name, value): + with pytest.raises(ValueError, match=name): + grouped_situglu(jnp.ones((3, 16)), jnp.array([3], jnp.int32), **{name: value}) + + +@pytest.mark.parametrize( + "shape,counts,error", + [ + ((3, 15), [3], ValueError), + ((3, 0), [3], ValueError), + ((2, 3, 16), [3], ValueError), + ((3, 16), [], ValueError), + ], +) +def test_grouped_situ_invalid_shapes(shape, counts, error): + with pytest.raises(error): + grouped_situglu(jnp.ones(shape), jnp.asarray(counts, jnp.int32)) diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index adf261017bf..8e738e81fe2 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -573,6 +573,31 @@ def _quantization_recipe(quantization): return MXFP8BlockScaling() +def test_situ_uses_native_grouped_kernels(mesh, monkeypatch, make_block): + from transformer_engine.jax import cpp_extensions as tex + + calls = [] + forward, backward = tex.grouped_situglu_fwd, tex.grouped_situglu_bwd + + def checked_forward(x, counts, beta1, beta2): + assert counts.ndim == 1 and counts.dtype == jnp.int32 + calls.append("forward") + return forward(x, counts, beta1, beta2) + + def checked_backward(grad, x, counts, beta1, beta2): + assert counts.ndim == 1 and counts.dtype == jnp.int32 + calls.append("backward") + return backward(grad, x, counts, beta1, beta2) + + monkeypatch.setattr(tex, "grouped_situglu_fwd", checked_forward) + monkeypatch.setattr(tex, "grouped_situglu_bwd", checked_backward) + block = make_block(activation_type=SiTUActivation(3.0, 11.0)) + x = _make_inputs(jax.random.PRNGKey(88)) + variables, _, _, _ = _init_apply(block, mesh, x, jax.random.PRNGKey(89)) + _grad_step(block, variables, mesh, x) + assert "forward" in calls and "backward" in calls + + # ----------------------------------------------------------------------------- # Tests # ----------------------------------------------------------------------------- diff --git a/transformer_engine/common/CMakeLists.txt b/transformer_engine/common/CMakeLists.txt index 39bd2083d9a..409edeb4418 100644 --- a/transformer_engine/common/CMakeLists.txt +++ b/transformer_engine/common/CMakeLists.txt @@ -355,6 +355,7 @@ list(APPEND transformer_engine_cuda_arch_specific_sources activation/scaled_swiglu.cu activation/scaled_swiglu_grouped.cu activation/swiglu.cu + activation/situglu_grouped.cu activation/swiglu_dbias.cu activation/swiglu_grouped.cu activation/swiglu_grouped_bwd.cu diff --git a/transformer_engine/common/activation/situglu_grouped.cu b/transformer_engine/common/activation/situglu_grouped.cu new file mode 100644 index 00000000000..06e7d97b57c --- /dev/null +++ b/transformer_engine/common/activation/situglu_grouped.cu @@ -0,0 +1,141 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +#include +#include + +#include "../common.h" +#include "../util/cuda_runtime.h" +#include "../util/math.h" +#include "../utils.cuh" +#include "transformer_engine/activation.h" + +namespace transformer_engine { +namespace { + +constexpr int kThreads = 256; + +template +__global__ void grouped_situglu_kernel(const T *input, const T *grad, T *output, + const int64_t *group_sizes, size_t groups, size_t capacity, + size_t hidden, SiTUGLUParam params) { + __shared__ int64_t counts[kThreads]; + int64_t count = 0; + if (group_sizes != nullptr) { + for (size_t group = threadIdx.x; group < groups; group += blockDim.x) { + count += group_sizes[group]; + } + } + counts[threadIdx.x] = count; + __syncthreads(); + for (int stride = kThreads / 2; stride > 0; stride /= 2) { + if (threadIdx.x < stride) counts[threadIdx.x] += counts[threadIdx.x + stride]; + __syncthreads(); + } + const size_t active_rows = groups == 0 ? 0 : (group_sizes == nullptr ? capacity : counts[0]); + for (size_t index = blockIdx.x * blockDim.x + threadIdx.x; index < capacity * hidden; + index += static_cast(gridDim.x) * blockDim.x) { + const size_t row = index / hidden; + const size_t col = index % hidden; + const size_t gate_index = row * 2 * hidden + col; + const size_t up_index = gate_index + hidden; + float gate_value = 0.0f; + float up_value = 0.0f; + if (row < active_rows) { + const float gate = static_cast(input[gate_index]); + const float up = static_cast(input[up_index]); + if constexpr (Backward) { + const float dy = static_cast(grad[index]); + gate_value = + dy * dsitu_gate(gate, params) * situ_up(up, params); + up_value = dy * situ_gate(gate, params) * dsitu_up(up, params); + } else { + gate_value = situ_gate(gate, params) * situ_up(up, params); + } + } + if constexpr (Backward) { + output[gate_index] = static_cast(gate_value); + output[up_index] = static_cast(up_value); + } else { + output[index] = static_cast(gate_value); + } + } +} + +template +void grouped_situglu(const NVTEGroupedTensor input_handle, const NVTEGroupedTensor grad_handle, + NVTEGroupedTensor output_handle, float beta1, float beta2, + cudaStream_t stream) { + const auto *input = convertNVTEGroupedTensorCheck(input_handle); + auto *output = convertNVTEGroupedTensorCheck(output_handle); + NVTE_CHECK(std::isfinite(beta1) && beta1 > 0 && std::isfinite(beta2) && beta2 > 0, + "SiTU soft-cap parameters must be finite and positive."); + NVTE_CHECK(input->logical_shape.ndim == 2 && output->logical_shape.ndim == 2, + "Grouped SiTU requires rank-2 logical shapes."); + NVTE_CHECK(input->all_same_last_dim() && output->all_same_last_dim(), + "Grouped SiTU requires a uniform feature dimension."); + const size_t capacity = input->logical_shape.data[0]; + const size_t input_width = input->logical_shape.data[1]; + NVTE_CHECK(input_width > 0 && input_width % 2 == 0, + "Grouped SiTU input width must be positive and even."); + const size_t hidden = input_width / 2; + NVTE_CHECK(output->num_tensors == input->num_tensors && + output->logical_shape.data[0] == capacity && + output->logical_shape.data[1] == (Backward ? input_width : hidden), + "Grouped SiTU output shape or group count is inconsistent with input."); + const DType dtype = input->data.dtype; + NVTE_CHECK(dtype == DType::kFloat32 || dtype == DType::kFloat16 || dtype == DType::kBFloat16, + "Grouped SiTU supports float32, float16 and bfloat16."); + NVTE_CHECK(output->data.dtype == dtype, "Grouped SiTU input/output dtypes must match."); + NVTE_CHECK(input->data.numel() >= capacity * input_width && + output->data.numel() >= capacity * (Backward ? input_width : hidden), + "Grouped SiTU input/output storage is smaller than its logical shape."); + if (input->first_dims.has_data()) { + NVTE_CHECK( + input->first_dims.dtype == DType::kInt64 && input->first_dims.numel() == input->num_tensors, + "Grouped SiTU first_dims must contain one int64 count per group."); + } + const auto *grad = Backward ? convertNVTEGroupedTensorCheck(grad_handle) : nullptr; + if constexpr (Backward) { + NVTE_CHECK(grad->logical_shape.ndim == 2 && grad->all_same_last_dim() && + grad->logical_shape.data[0] == capacity && + grad->logical_shape.data[1] == hidden && + grad->num_tensors == input->num_tensors && grad->data.dtype == dtype, + "Grouped SiTU gradient shape, dtype or group count is inconsistent with input."); + NVTE_CHECK(grad->data.numel() >= capacity * hidden, + "Grouped SiTU gradient storage is smaller than its logical shape."); + } + if (capacity == 0) return; + NVTE_CHECK(input->data.has_data() && output->data.has_data(), + "Grouped SiTU input/output data must be allocated."); + if constexpr (Backward) NVTE_CHECK(grad->data.has_data(), "Grouped SiTU grad is not allocated."); + const auto *sizes = static_cast(input->first_dims.dptr); + const int blocks = std::min((capacity * hidden + kThreads - 1) / kThreads, 1024); + const SiTUGLUParam params{beta1, beta2}; + TRANSFORMER_ENGINE_TYPE_SWITCH_INPUT( + dtype, T, + grouped_situglu_kernel<<>>( + static_cast(input->data.dptr), + Backward ? static_cast(grad->data.dptr) : nullptr, + static_cast(output->data.dptr), sizes, input->num_tensors, capacity, hidden, + params);); + NVTE_CHECK_CUDA(cudaGetLastError()); +} + +} // namespace +} // namespace transformer_engine + +void nvte_group_situglu(const NVTEGroupedTensor input, NVTEGroupedTensor output, float beta1, + float beta2, cudaStream_t stream) { + NVTE_API_CALL(nvte_group_situglu); + transformer_engine::grouped_situglu(input, nullptr, output, beta1, beta2, stream); +} + +void nvte_group_dsituglu(const NVTEGroupedTensor grad, const NVTEGroupedTensor input, + NVTEGroupedTensor output, float beta1, float beta2, cudaStream_t stream) { + NVTE_API_CALL(nvte_group_dsituglu); + transformer_engine::grouped_situglu(input, grad, output, beta1, beta2, stream); +} diff --git a/transformer_engine/common/include/transformer_engine/activation.h b/transformer_engine/common/include/transformer_engine/activation.h index 31429cf89cf..cbb8d78d935 100644 --- a/transformer_engine/common/include/transformer_engine/activation.h +++ b/transformer_engine/common/include/transformer_engine/activation.h @@ -381,6 +381,23 @@ void nvte_swiglu(const NVTETensor input, NVTETensor output, cudaStream_t stream) void nvte_situglu(const NVTETensor input, NVTETensor output, float beta1, float beta2, cudaStream_t stream); +/*! \brief Grouped SiTU-GLU for FP32/FP16/BF16 tensors with uniform feature width. + * Input [capacity, 2H] stores contiguous gate/up halves. Output is [capacity, H]. + * Optional int64 first_dims contains nonnegative device row counts whose sum + * must not exceed capacity. Rows beyond that sum are zero-filled without reading + * their input. No row alignment is required. Quantized output is not supported. + * beta1 and beta2 are positive finite soft-cap parameters. + */ +void nvte_group_situglu(const NVTEGroupedTensor input, NVTEGroupedTensor output, float beta1, + float beta2, cudaStream_t stream); + +/*! \brief Grouped SiTU-GLU backward with the same metadata contract as forward. + * grad is [capacity, H], input and output are [capacity, 2H]. Inactive rows + * are zero-filled without reading their input or upstream gradient. + */ +void nvte_group_dsituglu(const NVTEGroupedTensor grad, const NVTEGroupedTensor input, + NVTEGroupedTensor output, float beta1, float beta2, cudaStream_t stream); + /*! \brief Computes the gated Swish activation of the input used in GPT OSS. * * \deprecated This function has been deprecated in favor of nvte_clamped_swiglu_v2, diff --git a/transformer_engine/jax/__init__.py b/transformer_engine/jax/__init__.py index 45379dbf0d9..ff354a4abff 100644 --- a/transformer_engine/jax/__init__.py +++ b/transformer_engine/jax/__init__.py @@ -48,6 +48,7 @@ from .quantize import NVTE_FP8_COLLECTION_NAME from .sharding import MeshResource +from .activation import grouped_situglu from ..common.utils import deprecate_wrapper from ..common.utils import DeprecatedEnum @@ -61,4 +62,5 @@ "MeshResource", "flax", "quantize", + "grouped_situglu", ] diff --git a/transformer_engine/jax/activation.py b/transformer_engine/jax/activation.py index b2b90a10c97..414f66827ba 100644 --- a/transformer_engine/jax/activation.py +++ b/transformer_engine/jax/activation.py @@ -107,3 +107,31 @@ def _activation_bwd_rule(activation_type, act_params, ctx, g): _activation.defvjp(_activation_fwd_rule, _activation_bwd_rule) + + +@partial(jax.custom_vjp, nondiff_argnums=(2, 3)) +def grouped_situglu(x, group_sizes, beta1=4.0, beta2=25.0): + """Apply native SiTU to contiguous expert groups in ``x[capacity, 2H]``. + + The halves of each row contain gate and up, respectively. ``group_sizes`` + is a device int32 vector of nonnegative counts whose sum must not exceed + capacity. Empty groups and arbitrary row counts are supported. Inactive + rows return zeros and do not read input values, in forward or backward. + + Inputs and output use FP32, FP16 or BF16; arithmetic uses FP32. Parameters + are static positive floats. For MXFP8, apply grouped quantization afterward. + In distributed expert code, call inside ``shard_map`` with local counts. + """ + return tex.grouped_situglu_fwd(x, group_sizes, beta1, beta2) + + +def _grouped_situglu_fwd(x, group_sizes, beta1, beta2): + return tex.grouped_situglu_fwd(x, group_sizes, beta1, beta2), (x, group_sizes) + + +def _grouped_situglu_bwd(beta1, beta2, residual, grad): + x, group_sizes = residual + return tex.grouped_situglu_bwd(grad, x, group_sizes, beta1, beta2), None + + +grouped_situglu.defvjp(_grouped_situglu_fwd, _grouped_situglu_bwd) diff --git a/transformer_engine/jax/cpp_extensions/activation.py b/transformer_engine/jax/cpp_extensions/activation.py index 5058192c3f3..b556fcda451 100644 --- a/transformer_engine/jax/cpp_extensions/activation.py +++ b/transformer_engine/jax/cpp_extensions/activation.py @@ -6,6 +6,7 @@ import operator from functools import reduce, partial from dataclasses import dataclass +import math import jax import jax.numpy as jnp @@ -38,7 +39,54 @@ ) -__all__ = ["act_lu", "dact_lu", "quantize_dact_dbias"] +__all__ = ["act_lu", "dact_lu", "quantize_dact_dbias", "grouped_situglu_fwd", "grouped_situglu_bwd"] + + +def _grouped_situglu_call(x, grad, group_sizes, beta1, beta2, *, backward): + if x.ndim != 2 or x.shape[1] == 0 or x.shape[1] % 2: + raise ValueError(f"Expected x[capacity, 2H] with positive H, got {x.shape}") + if x.dtype not in (jnp.float32, jnp.float16, jnp.bfloat16): + raise TypeError(f"Grouped SiTU requires float32, float16 or bfloat16, got {x.dtype}") + if group_sizes.ndim != 1 or group_sizes.dtype != jnp.int32: + raise TypeError("group_sizes must be a rank-1 int32 array") + if group_sizes.size == 0: + raise ValueError("group_sizes must contain at least one group") + for name, value in (("beta1", beta1), ("beta2", beta2)): + if not math.isfinite(value) or value <= 0 or value > float(np.finfo(np.float32).max): + raise ValueError(f"{name} must be finite, positive and representable in float32") + if np.float32(value) == 0: + raise ValueError(f"{name} is too small for float32") + output_shape = x.shape if backward else (x.shape[0], x.shape[1] // 2) + if backward and (grad.shape != (x.shape[0], x.shape[1] // 2) or grad.dtype != x.dtype): + raise ValueError("Grouped SiTU upstream gradient shape or dtype does not match x") + result, _ = ffi.ffi_call( + "te_grouped_situglu_ffi", + ( + jax.ShapeDtypeStruct(output_shape, x.dtype), + jax.ShapeDtypeStruct((group_sizes.size * 8,), jnp.uint8), + ), + vmap_method="sequential", + )( + x, + grad, + group_sizes, + beta1=np.float32(beta1), + beta2=np.float32(beta2), + backward=backward, + ) + return result + + +def grouped_situglu_fwd(x, group_sizes, beta1=4.0, beta2=25.0): + """Native grouped SiTU; inactive capacity rows are zero without reading x.""" + return _grouped_situglu_call( + x, jnp.empty((0,), dtype=x.dtype), group_sizes, beta1, beta2, backward=False + ) + + +def grouped_situglu_bwd(grad, x, group_sizes, beta1=4.0, beta2=25.0): + """Native grouped SiTU gradient; inactive rows do not read x or grad.""" + return _grouped_situglu_call(x, grad, group_sizes, beta1, beta2, backward=True) ActivationEnum = { diff --git a/transformer_engine/jax/csrc/extensions.h b/transformer_engine/jax/csrc/extensions.h index 893374b3de3..f9853f5e509 100644 --- a/transformer_engine/jax/csrc/extensions.h +++ b/transformer_engine/jax/csrc/extensions.h @@ -89,6 +89,7 @@ inline bool use_fp8(DType type) { return type == DType::kFloat8E4M3 || type == D XLA_FFI_DECLARE_HANDLER_SYMBOL(ActLuHandler); XLA_FFI_DECLARE_HANDLER_SYMBOL(ActLuInitializeHandler); +XLA_FFI_DECLARE_HANDLER_SYMBOL(GroupedSiTUGLUHandler); XLA_FFI_DECLARE_HANDLER_SYMBOL(DActLuDBiasQuantizeHandler); XLA_FFI_DECLARE_HANDLER_SYMBOL(DActLuDBiasQuantizeInitializeHandler); diff --git a/transformer_engine/jax/csrc/extensions/activation.cpp b/transformer_engine/jax/csrc/extensions/activation.cpp index 6325a700d15..4f0b1351802 100644 --- a/transformer_engine/jax/csrc/extensions/activation.cpp +++ b/transformer_engine/jax/csrc/extensions/activation.cpp @@ -9,11 +9,77 @@ #include "../extensions.h" #include "transformer_engine/cast.h" +#include "transformer_engine/gemm.h" #include "xla/ffi/api/c_api.h" namespace transformer_engine { namespace jax { +Error_Type GroupedSiTUGLUFFI(cudaStream_t stream, Buffer_Type input_buf, Buffer_Type grad_buf, + Buffer_Type sizes_buf, Result_Type output_buf, + Result_Type workspace_buf, float beta1, float beta2, bool backward) { + const auto shape = input_buf.dimensions(); + NVTE_CHECK(shape.size() == 2 && shape[1] > 0 && shape[1] % 2 == 0, + "Grouped SiTU input must have shape [capacity, 2H]."); + NVTE_CHECK( + sizes_buf.dimensions().size() == 1 && sizes_buf.element_type() == xla::ffi::DataType::S32, + "Grouped SiTU group_sizes must be rank-1 int32."); + const size_t groups = sizes_buf.dimensions()[0]; + NVTE_CHECK(product(workspace_buf->dimensions()) >= groups * sizeof(int64_t), + "Grouped SiTU metadata workspace is too small."); + const auto dtype = convert_ffi_datatype_to_te_dtype(input_buf.element_type()); + NVTE_CHECK(convert_ffi_datatype_to_te_dtype(output_buf->element_type()) == dtype, + "Grouped SiTU input/output dtypes must match."); + const auto out_shape = output_buf->dimensions(); + NVTE_CHECK(out_shape.size() == 2 && out_shape[0] == shape[0] && + out_shape[1] == (backward ? shape[1] : shape[1] / 2), + "Grouped SiTU output shape is invalid."); + auto *sizes = reinterpret_cast(workspace_buf->untyped_data()); + if (groups > 0) { + nvte_convert_int32_to_int64_with_multiplier( + static_cast(sizes_buf.untyped_data()), sizes, groups, 1, stream); + } + const auto scaling = get_nvte_scaling_mode(JAXX_Scaling_Mode::NO_SCALING); + const std::vector input_shape{static_cast(shape[0]), + static_cast(shape[1])}; + const std::vector output_shape{static_cast(out_shape[0]), + static_cast(out_shape[1])}; + const std::vector grad_shape{static_cast(shape[0]), + static_cast(shape[1] / 2)}; + const std::vector sizes_shape{groups}; + GroupedTensorWrapper input(groups, input_shape, scaling); + input.set_rowwise_data(input_buf.untyped_data(), dtype, input_shape) + .set_first_dims(sizes, DType::kInt64, sizes_shape); + GroupedTensorWrapper output(groups, output_shape, scaling); + output.set_rowwise_data(output_buf->untyped_data(), dtype, output_shape); + if (backward) { + const auto actual_grad_shape = grad_buf.dimensions(); + NVTE_CHECK(actual_grad_shape.size() == 2 && actual_grad_shape[0] == shape[0] && + actual_grad_shape[1] == shape[1] / 2 && + convert_ffi_datatype_to_te_dtype(grad_buf.element_type()) == dtype, + "Grouped SiTU upstream gradient shape or dtype is invalid."); + GroupedTensorWrapper grad(groups, grad_shape, scaling); + grad.set_rowwise_data(grad_buf.untyped_data(), dtype, grad_shape); + nvte_group_dsituglu(grad.data(), input.data(), output.data(), beta1, beta2, stream); + } else { + nvte_group_situglu(input.data(), output.data(), beta1, beta2, stream); + } + return ffi_with_cuda_error_check(); +} + +XLA_FFI_DEFINE_HANDLER_SYMBOL(GroupedSiTUGLUHandler, GroupedSiTUGLUFFI, + FFI::Bind() + .Ctx() + .Arg() + .Arg() + .Arg() + .Ret() + .Ret() + .Attr("beta1") + .Attr("beta2") + .Attr("backward"), + FFI_CudaGraph_Traits); + Error_Type ActLuFFI(cudaStream_t stream, Buffer_Type input_buf, Buffer_Type scale_buf, Buffer_Type amax_buf, Result_Type output_buf, Result_Type colwise_output_buf, Result_Type scale_inv_buf, Result_Type colwise_scale_inv_buf, diff --git a/transformer_engine/jax/csrc/extensions/pybind.cpp b/transformer_engine/jax/csrc/extensions/pybind.cpp index d82d109f650..5e748dc7cd2 100644 --- a/transformer_engine/jax/csrc/extensions/pybind.cpp +++ b/transformer_engine/jax/csrc/extensions/pybind.cpp @@ -24,6 +24,7 @@ pybind11::dict Registrations() { pybind11::dict dict; // Activation + dict["te_grouped_situglu_ffi"] = EncapsulateFFI(GroupedSiTUGLUHandler); dict["te_act_lu_ffi"] = pybind11::dict(pybind11::arg("initialize") = EncapsulateFFI(ActLuInitializeHandler), pybind11::arg("execute") = EncapsulateFFI(ActLuHandler)); diff --git a/transformer_engine/jax/flax/moe.py b/transformer_engine/jax/flax/moe.py index db7557b1e41..030bedd4ae5 100644 --- a/transformer_engine/jax/flax/moe.py +++ b/transformer_engine/jax/flax/moe.py @@ -77,7 +77,7 @@ class _MoEBlock(TransformerEngineBase): Activation between ``layer_w0 @ wi_0`` and the elementwise product with ``layer_w0 @ wi_1``. Default ``"silu"``. Pass :class:`~transformer_engine.jax.moe.SiTUActivation` to soft-cap both - branches in FP32 using the unfused grouped-GEMM path. + branches in FP32 using native grouped SiTU and separate grouped GEMMs. score_function : Union[str, ScoreFunction] ``"softmax"`` (default), ``"sigmoid"``, or ``"sqrtsoftplus"`` for the routing scores. diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index d0523c2fb7d..b94711185f8 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -89,8 +89,8 @@ class SiTUActivation: The defaults match Kimi-K3. Both soft-cap parameters must be finite and positive. Pass an instance as ``moe(..., activation_type=...)``. - This activation uses the unfused grouped-GEMM path for BF16 and MXFP8; - requesting cuDNN fusion retains its normal warning and fallback behavior. + BF16 and MXFP8 use native grouped SiTU forward/backward with separate GEMMs. + Requesting cuDNN fusion retains its normal warning and fallback behavior. """ gate_beta: float = 4.0 @@ -104,16 +104,14 @@ def __post_init__(self): object.__setattr__(self, name, value) -def _apply_ffn_activation(activation_type, gate, up): +def _apply_ffn_activation(activation_type, gate, up, group_sizes): if isinstance(activation_type, SiTUActivation): - gate_fp32, up_fp32 = gate.astype(jnp.float32), up.astype(jnp.float32) - gated = ( - activation_type.gate_beta - * jnp.tanh(gate_fp32 / activation_type.gate_beta) - * jax.nn.sigmoid(gate_fp32) + return tex.grouped_situglu_fwd( + jnp.concatenate([gate, up], axis=-1), + group_sizes, + activation_type.gate_beta, + activation_type.up_beta, ) - linear = activation_type.up_beta * jnp.tanh(up_fp32 / activation_type.up_beta) - return (gated * linear).astype(gate.dtype) return _convert_to_activation_function(activation_type)(gate) * up @@ -1089,7 +1087,9 @@ def _ffn_fwd_per_shard( # Keep projection residuals and activation output in the GEMM dtype; # SiTU evaluates its internal arithmetic in FP32 before casting back. if not use_cudnn_jax_fusion: - intermediate = _apply_ffn_activation(activation_type, gate_proj_out, up_proj_out) + intermediate = _apply_ffn_activation( + activation_type, gate_proj_out, up_proj_out, group_sizes + ) if apply_topk_weights_early and not use_cudnn_jax_fusion: # Fold the per-token combine weights into the FFN intermediate; # the downstream wo GEMM is linear so this is equivalent to the @@ -1257,7 +1257,7 @@ def _ffn_bwd_per_shard( # Masking is not required as grouped GEMMs consume only group-size rows. w_b = recv_w_flat[:, None].astype(d_intermediate.dtype) intermediate_unweighted = _apply_ffn_activation( - activation_type, gate_proj_out, up_proj_out + activation_type, gate_proj_out, up_proj_out, group_sizes ) d_recv_w_from_intermediate = jnp.sum( d_intermediate * intermediate_unweighted, @@ -1268,16 +1268,19 @@ def _ffn_bwd_per_shard( d_recv_w_from_intermediate = jnp.zeros_like(recv_w_flat) if isinstance(activation_type, SiTUActivation): - _, dact_pullback = jax.vjp( - partial(_apply_ffn_activation, activation_type), gate_proj_out, up_proj_out + d_combined_for_bias = tex.grouped_situglu_bwd( + d_intermediate, + jnp.concatenate([gate_proj_out, up_proj_out], axis=-1), + group_sizes, + activation_type.gate_beta, + activation_type.up_beta, ) - d_gate_proj_out, d_up_proj_out = dact_pullback(d_intermediate) else: act_fn = _convert_to_activation_function(activation_type) act_gp, dact_pullback = jax.vjp(act_fn, gate_proj_out) d_up_proj_out = d_intermediate * act_gp (d_gate_proj_out,) = dact_pullback(d_intermediate * up_proj_out) - d_combined_for_bias = jnp.concatenate([d_gate_proj_out, d_up_proj_out], axis=-1) + d_combined_for_bias = jnp.concatenate([d_gate_proj_out, d_up_proj_out], axis=-1) casted_d_combined = tex.grouped_quantize( d_combined_for_bias, fc1_quantizer_set.dgrad, @@ -2199,9 +2202,9 @@ def moe( ---------- activation_type : Union[str, SiTUActivation] Gate activation name (default ``"silu"``), or :class:`SiTUActivation` - for FP32 soft-capping of both gate and up branches. SiTU uses unfused - grouped GEMMs with BF16 or MXFP8 quantizer sets; cuDNN GLU/dGLU does - not implement it and follows the existing fallback path. + for FP32 soft-capping of both gate and up branches. SiTU uses native + grouped activation kernels and separate grouped GEMMs with BF16 or + MXFP8 quantizer sets. cuDNN GLU/dGLU uses the existing fallback path. router_info : Union[RouterComputationInfo, RoutingMapInfo] :class:`RouterComputationInfo` makes TE compute the gate and top-k from the given router parameters. :class:`RoutingMapInfo` supplies From ba409e6c24a4f0a95073868fdefee32dcc20e093 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 12 Nov 2026 14:37:21 +0100 Subject: [PATCH 4/7] [JAX] Reuse shared GLU templates for grouped SiTU Instantiate the existing vectorized gated activation and derivative kernels with SiTUGLUParam and optional device group counts. Preserve their alignment dispatch and skip inactive input and gradient loads while zeroing inactive output rows. Keep ordinary GLU instantiations on the compile-time ungrouped path. Remove the separate grouped SiTU CUDA kernel and translation unit; retain the C and JAX APIs. Cover aligned, uniformly unaligned and differently aligned storage, counts, caps, FP64 derivatives and NaN tails in common C++ tests. Extend PyTorch SiTU coverage to a width outside the TMA path. Signed-off-by: Pawel Gadzinski --- tests/cpp/operator/test_act.cu | 124 +++++++++++++++ tests/pytorch/test_fusible_ops.py | 7 +- transformer_engine/common/CMakeLists.txt | 1 - .../common/activation/situglu_grouped.cu | 141 ------------------ .../common/activation/swiglu.cu | 22 +++ .../common/cast/dispatch/gated.cuh | 61 ++++++++ .../common/util/vectorized_pointwise.h | 103 ++++++++++--- 7 files changed, 290 insertions(+), 169 deletions(-) delete mode 100644 transformer_engine/common/activation/situglu_grouped.cu diff --git a/tests/cpp/operator/test_act.cu b/tests/cpp/operator/test_act.cu index 6edc6bd63b0..3bf85cbabce 100644 --- a/tests/cpp/operator/test_act.cu +++ b/tests/cpp/operator/test_act.cu @@ -4,9 +4,11 @@ * See LICENSE for license information. ************************************************************************/ +#include #include #include #include +#include #include #include #include @@ -424,3 +426,125 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(test::all_fp_types), ::testing::Values(std::pair{768, 2816})), test_name_generator); + +namespace { + +template +void test_grouped_situglu(const size_t hidden, const int alignment_mode, const int count_mode, + const std::pair caps) { + constexpr size_t rows = 13; + const size_t input_offset = alignment_mode == 1 ? 1 : 0; + const size_t output_offset = alignment_mode == 0 ? 0 : 1; + const size_t active = count_mode == 0 ? 8 : (count_mode == 1 ? rows : 0); + const DType dtype = test::TypeInfo::dtype; + const std::vector input_shape{rows, 2 * hidden}, output_shape{rows, hidden}; + test::Tensor input("group_situ_input", std::vector{rows * 2 * hidden + input_offset + 1}, + dtype); + test::Tensor grad("group_situ_grad", std::vector{rows * hidden + input_offset + 1}, + dtype); + test::Tensor output("group_situ_output", std::vector{rows * hidden + output_offset + 1}, + dtype); + test::Tensor dx("group_situ_dx", std::vector{rows * 2 * hidden + output_offset + 1}, + dtype); + test::Tensor counts("group_situ_counts", std::vector{3}, DType::kInt64); + for (test::Tensor *tensor : {&input, &grad, &output, &dx}) { + std::fill_n(tensor->rowwise_cpu_dptr(), test::product(tensor->shape()), + static_cast(std::numeric_limits::quiet_NaN())); + } + auto *x = input.rowwise_cpu_dptr() + input_offset; + auto *dy = grad.rowwise_cpu_dptr() + input_offset; + for (size_t row = 0; row < active; ++row) { + for (size_t col = 0; col < hidden; ++col) { + x[row * 2 * hidden + col] = static_cast(static_cast(col % 11) - 5.25f); + x[row * 2 * hidden + hidden + col] = static_cast(static_cast(row) - 4.5f); + dy[row * hidden + col] = static_cast(0.125f * static_cast(col % 7) - 0.25f); + } + } + auto *sizes = counts.rowwise_cpu_dptr(); + sizes[0] = count_mode == 2 ? 0 : 3; + sizes[1] = 0; + sizes[2] = count_mode == 2 ? 0 : 5; + for (test::Tensor *tensor : {&input, &grad, &output, &dx, &counts}) tensor->from_cpu(); + + GroupedTensorWrapper in_group(3, input_shape), grad_group(3, output_shape); + GroupedTensorWrapper out_group(3, output_shape), dx_group(3, input_shape); + in_group.set_rowwise_data(static_cast(input.rowwise_dptr()) + input_offset, dtype, + input_shape); + grad_group.set_rowwise_data(static_cast(grad.rowwise_dptr()) + input_offset, dtype, + output_shape); + out_group.set_rowwise_data(static_cast(output.rowwise_dptr()) + output_offset, dtype, + output_shape); + dx_group.set_rowwise_data(static_cast(dx.rowwise_dptr()) + output_offset, dtype, + input_shape); + if (count_mode != 1) { + in_group.set_first_dims(counts.rowwise_dptr(), DType::kInt64, std::vector{3}); + } + nvte_group_situglu(in_group.data(), out_group.data(), caps.first, caps.second, nullptr); + nvte_group_dsituglu(grad_group.data(), in_group.data(), dx_group.data(), caps.first, caps.second, + nullptr); + ASSERT_EQ(cudaDeviceSynchronize(), cudaSuccess); + output.to_cpu(); + dx.to_cpu(); + const auto *y = output.rowwise_cpu_dptr() + output_offset; + const auto *actual_dx = dx.rowwise_cpu_dptr() + output_offset; + const double tolerance = + dtype == DType::kFloat32 ? 2e-5 : (dtype == DType::kFloat16 ? .003 : .02); + for (size_t row = 0; row < rows; ++row) { + for (size_t col = 0; col < hidden; ++col) { + double expected_y = 0, expected_gate = 0, expected_up = 0; + if (row < active) { + const double gate = static_cast(x[row * 2 * hidden + col]); + const double up = static_cast(x[row * 2 * hidden + hidden + col]); + const double upstream = static_cast(dy[row * hidden + col]); + const double sigmoid = 1 / (1 + std::exp(-gate)); + const double tg = std::tanh(gate / caps.first), tu = std::tanh(up / caps.second); + expected_y = caps.first * tg * sigmoid * caps.second * tu; + expected_gate = upstream * + ((1 - tg * tg) * sigmoid + caps.first * tg * sigmoid * (1 - sigmoid)) * + caps.second * tu; + expected_up = upstream * caps.first * tg * sigmoid * (1 - tu * tu); + } + EXPECT_NEAR(static_cast(y[row * hidden + col]), expected_y, + tolerance * (1 + std::abs(expected_y))); + EXPECT_NEAR(static_cast(actual_dx[row * 2 * hidden + col]), expected_gate, + tolerance * (1 + std::abs(expected_gate))); + EXPECT_NEAR(static_cast(actual_dx[row * 2 * hidden + hidden + col]), expected_up, + tolerance * (1 + std::abs(expected_up))); + } + } + for (test::Tensor *tensor : {&output, &dx}) { + const auto *data = tensor->rowwise_cpu_dptr(); + if (output_offset) EXPECT_TRUE(std::isnan(static_cast(data[0]))); + EXPECT_TRUE(std::isnan(static_cast(data[test::product(tensor->shape()) - 1]))); + } +} + +class GroupedSiTUGLUTest : public ::testing::TestWithParam< + std::tuple>> {}; + +TEST_P(GroupedSiTUGLUTest, AlignmentAndInactiveRows) { + const auto [dtype, hidden, alignment_mode, count_mode, caps] = GetParam(); + switch (dtype) { + case DType::kFloat32: + test_grouped_situglu(hidden, alignment_mode, count_mode, caps); + break; + case DType::kFloat16: + test_grouped_situglu(hidden, alignment_mode, count_mode, caps); + break; + case DType::kBFloat16: + test_grouped_situglu(hidden, alignment_mode, count_mode, caps); + break; + default: + FAIL() << "Unexpected test dtype"; + } +} + +INSTANTIATE_TEST_SUITE_P(Operator, GroupedSiTUGLUTest, + ::testing::Combine(::testing::Values(DType::kFloat32, DType::kFloat16, + DType::kBFloat16), + ::testing::Values(size_t{3}, size_t{16}, size_t{32}), + ::testing::Values(0, 1, 2), ::testing::Values(0, 1, 2), + ::testing::Values(std::pair{4, 25}, + std::pair{2, 8}))); + +} // namespace diff --git a/tests/pytorch/test_fusible_ops.py b/tests/pytorch/test_fusible_ops.py index 1ebf35acb27..da270dc9667 100644 --- a/tests/pytorch/test_fusible_ops.py +++ b/tests/pytorch/test_fusible_ops.py @@ -2799,19 +2799,20 @@ def test_interleaved_swiglu(self): ) @pytest.mark.parametrize("dtype", _dtypes) - @pytest.mark.parametrize("glu_interleave_size", (None, 32)) + @pytest.mark.parametrize("hidden,glu_interleave_size", ((13, None), (96, None), (96, 32))) @pytest.mark.parametrize("betas", ((4.0, 25.0), (3.0, 12.0))) def test_situglu( self, *, dtype: torch.dtype, + hidden: int, glu_interleave_size: Optional[int], betas: tuple[float, float], device: torch.device = "cuda", ) -> None: """SiTU-GLU forward and backward.""" - in_shape = (17, 192) - out_shape = (17, 96) + in_shape = (17, 2 * hidden) + out_shape = (17, hidden) beta1, beta2 = betas x_ref, x_test = make_reference_and_test_tensors( in_shape, diff --git a/transformer_engine/common/CMakeLists.txt b/transformer_engine/common/CMakeLists.txt index 409edeb4418..39bd2083d9a 100644 --- a/transformer_engine/common/CMakeLists.txt +++ b/transformer_engine/common/CMakeLists.txt @@ -355,7 +355,6 @@ list(APPEND transformer_engine_cuda_arch_specific_sources activation/scaled_swiglu.cu activation/scaled_swiglu_grouped.cu activation/swiglu.cu - activation/situglu_grouped.cu activation/swiglu_dbias.cu activation/swiglu_grouped.cu activation/swiglu_grouped_bwd.cu diff --git a/transformer_engine/common/activation/situglu_grouped.cu b/transformer_engine/common/activation/situglu_grouped.cu deleted file mode 100644 index 06e7d97b57c..00000000000 --- a/transformer_engine/common/activation/situglu_grouped.cu +++ /dev/null @@ -1,141 +0,0 @@ -/************************************************************************* - * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * - * See LICENSE for license information. - ************************************************************************/ - -#include -#include - -#include "../common.h" -#include "../util/cuda_runtime.h" -#include "../util/math.h" -#include "../utils.cuh" -#include "transformer_engine/activation.h" - -namespace transformer_engine { -namespace { - -constexpr int kThreads = 256; - -template -__global__ void grouped_situglu_kernel(const T *input, const T *grad, T *output, - const int64_t *group_sizes, size_t groups, size_t capacity, - size_t hidden, SiTUGLUParam params) { - __shared__ int64_t counts[kThreads]; - int64_t count = 0; - if (group_sizes != nullptr) { - for (size_t group = threadIdx.x; group < groups; group += blockDim.x) { - count += group_sizes[group]; - } - } - counts[threadIdx.x] = count; - __syncthreads(); - for (int stride = kThreads / 2; stride > 0; stride /= 2) { - if (threadIdx.x < stride) counts[threadIdx.x] += counts[threadIdx.x + stride]; - __syncthreads(); - } - const size_t active_rows = groups == 0 ? 0 : (group_sizes == nullptr ? capacity : counts[0]); - for (size_t index = blockIdx.x * blockDim.x + threadIdx.x; index < capacity * hidden; - index += static_cast(gridDim.x) * blockDim.x) { - const size_t row = index / hidden; - const size_t col = index % hidden; - const size_t gate_index = row * 2 * hidden + col; - const size_t up_index = gate_index + hidden; - float gate_value = 0.0f; - float up_value = 0.0f; - if (row < active_rows) { - const float gate = static_cast(input[gate_index]); - const float up = static_cast(input[up_index]); - if constexpr (Backward) { - const float dy = static_cast(grad[index]); - gate_value = - dy * dsitu_gate(gate, params) * situ_up(up, params); - up_value = dy * situ_gate(gate, params) * dsitu_up(up, params); - } else { - gate_value = situ_gate(gate, params) * situ_up(up, params); - } - } - if constexpr (Backward) { - output[gate_index] = static_cast(gate_value); - output[up_index] = static_cast(up_value); - } else { - output[index] = static_cast(gate_value); - } - } -} - -template -void grouped_situglu(const NVTEGroupedTensor input_handle, const NVTEGroupedTensor grad_handle, - NVTEGroupedTensor output_handle, float beta1, float beta2, - cudaStream_t stream) { - const auto *input = convertNVTEGroupedTensorCheck(input_handle); - auto *output = convertNVTEGroupedTensorCheck(output_handle); - NVTE_CHECK(std::isfinite(beta1) && beta1 > 0 && std::isfinite(beta2) && beta2 > 0, - "SiTU soft-cap parameters must be finite and positive."); - NVTE_CHECK(input->logical_shape.ndim == 2 && output->logical_shape.ndim == 2, - "Grouped SiTU requires rank-2 logical shapes."); - NVTE_CHECK(input->all_same_last_dim() && output->all_same_last_dim(), - "Grouped SiTU requires a uniform feature dimension."); - const size_t capacity = input->logical_shape.data[0]; - const size_t input_width = input->logical_shape.data[1]; - NVTE_CHECK(input_width > 0 && input_width % 2 == 0, - "Grouped SiTU input width must be positive and even."); - const size_t hidden = input_width / 2; - NVTE_CHECK(output->num_tensors == input->num_tensors && - output->logical_shape.data[0] == capacity && - output->logical_shape.data[1] == (Backward ? input_width : hidden), - "Grouped SiTU output shape or group count is inconsistent with input."); - const DType dtype = input->data.dtype; - NVTE_CHECK(dtype == DType::kFloat32 || dtype == DType::kFloat16 || dtype == DType::kBFloat16, - "Grouped SiTU supports float32, float16 and bfloat16."); - NVTE_CHECK(output->data.dtype == dtype, "Grouped SiTU input/output dtypes must match."); - NVTE_CHECK(input->data.numel() >= capacity * input_width && - output->data.numel() >= capacity * (Backward ? input_width : hidden), - "Grouped SiTU input/output storage is smaller than its logical shape."); - if (input->first_dims.has_data()) { - NVTE_CHECK( - input->first_dims.dtype == DType::kInt64 && input->first_dims.numel() == input->num_tensors, - "Grouped SiTU first_dims must contain one int64 count per group."); - } - const auto *grad = Backward ? convertNVTEGroupedTensorCheck(grad_handle) : nullptr; - if constexpr (Backward) { - NVTE_CHECK(grad->logical_shape.ndim == 2 && grad->all_same_last_dim() && - grad->logical_shape.data[0] == capacity && - grad->logical_shape.data[1] == hidden && - grad->num_tensors == input->num_tensors && grad->data.dtype == dtype, - "Grouped SiTU gradient shape, dtype or group count is inconsistent with input."); - NVTE_CHECK(grad->data.numel() >= capacity * hidden, - "Grouped SiTU gradient storage is smaller than its logical shape."); - } - if (capacity == 0) return; - NVTE_CHECK(input->data.has_data() && output->data.has_data(), - "Grouped SiTU input/output data must be allocated."); - if constexpr (Backward) NVTE_CHECK(grad->data.has_data(), "Grouped SiTU grad is not allocated."); - const auto *sizes = static_cast(input->first_dims.dptr); - const int blocks = std::min((capacity * hidden + kThreads - 1) / kThreads, 1024); - const SiTUGLUParam params{beta1, beta2}; - TRANSFORMER_ENGINE_TYPE_SWITCH_INPUT( - dtype, T, - grouped_situglu_kernel<<>>( - static_cast(input->data.dptr), - Backward ? static_cast(grad->data.dptr) : nullptr, - static_cast(output->data.dptr), sizes, input->num_tensors, capacity, hidden, - params);); - NVTE_CHECK_CUDA(cudaGetLastError()); -} - -} // namespace -} // namespace transformer_engine - -void nvte_group_situglu(const NVTEGroupedTensor input, NVTEGroupedTensor output, float beta1, - float beta2, cudaStream_t stream) { - NVTE_API_CALL(nvte_group_situglu); - transformer_engine::grouped_situglu(input, nullptr, output, beta1, beta2, stream); -} - -void nvte_group_dsituglu(const NVTEGroupedTensor grad, const NVTEGroupedTensor input, - NVTEGroupedTensor output, float beta1, float beta2, cudaStream_t stream) { - NVTE_API_CALL(nvte_group_dsituglu); - transformer_engine::grouped_situglu(input, grad, output, beta1, beta2, stream); -} diff --git a/transformer_engine/common/activation/swiglu.cu b/transformer_engine/common/activation/swiglu.cu index 0300ba1bd46..ec9c05920c5 100644 --- a/transformer_engine/common/activation/swiglu.cu +++ b/transformer_engine/common/activation/swiglu.cu @@ -58,6 +58,28 @@ void nvte_dsituglu(const NVTETensor grad, const NVTETensor input, NVTETensor out grad, input, output, param, stream); } +void nvte_group_situglu(const NVTEGroupedTensor input, NVTEGroupedTensor output, float beta1, + float beta2, cudaStream_t stream) { + NVTE_API_CALL(nvte_group_situglu); + using namespace transformer_engine; + NVTE_CHECK(beta1 > 0.0f && std::isfinite(beta1), "beta1 must be finite and positive."); + NVTE_CHECK(beta2 > 0.0f && std::isfinite(beta2), "beta2 must be finite and positive."); + const SiTUGLUParam params{beta1, beta2}; + dispatch::group_gated_act_helper>( + input, nullptr, output, params, stream); +} + +void nvte_group_dsituglu(const NVTEGroupedTensor grad, const NVTEGroupedTensor input, + NVTEGroupedTensor output, float beta1, float beta2, cudaStream_t stream) { + NVTE_API_CALL(nvte_group_dsituglu); + using namespace transformer_engine; + NVTE_CHECK(beta1 > 0.0f && std::isfinite(beta1), "beta1 must be finite and positive."); + NVTE_CHECK(beta2 > 0.0f && std::isfinite(beta2), "beta2 must be finite and positive."); + const SiTUGLUParam params{beta1, beta2}; + dispatch::group_gated_act_helper, + dsitu_gate>(input, grad, output, params, stream); +} + void nvte_clamped_swiglu(const NVTETensor input, NVTETensor output, float limit, float alpha, cudaStream_t stream) { NVTE_API_CALL(nvte_clamped_swiglu); diff --git a/transformer_engine/common/cast/dispatch/gated.cuh b/transformer_engine/common/cast/dispatch/gated.cuh index 0c1761264a1..5106df2784f 100644 --- a/transformer_engine/common/cast/dispatch/gated.cuh +++ b/transformer_engine/common/cast/dispatch/gated.cuh @@ -190,6 +190,67 @@ void quantize_gated_bwd_helper(const NVTETensor nvte_grad, const NVTETensor nvte NVTE_ERROR("Not supported scaling mode: " + to_string(output->scaling_mode) + "."); } } + +template +void group_gated_act_helper(const NVTEGroupedTensor input_handle, + const NVTEGroupedTensor grad_handle, NVTEGroupedTensor output_handle, + const ParamOP ¶ms, cudaStream_t stream) { + const auto *input = convertNVTEGroupedTensorCheck(input_handle); + auto *output = convertNVTEGroupedTensorCheck(output_handle); + NVTE_CHECK(input->logical_shape.ndim == 2 && output->logical_shape.ndim == 2, + "Grouped GLU requires rank-2 logical shapes."); + NVTE_CHECK(input->all_same_last_dim() && output->all_same_last_dim(), + "Grouped GLU requires a uniform feature dimension."); + const size_t capacity = input->logical_shape.data[0]; + const size_t input_width = input->logical_shape.data[1]; + NVTE_CHECK(input_width > 0 && input_width % 2 == 0, + "Grouped GLU input width must be positive and even."); + const size_t hidden = input_width / 2; + NVTE_CHECK(output->num_tensors == input->num_tensors && + output->logical_shape.data[0] == capacity && + output->logical_shape.data[1] == (Backward ? input_width : hidden), + "Grouped GLU output shape or group count is inconsistent with input."); + const DType dtype = input->data.dtype; + NVTE_CHECK(dtype == DType::kFloat32 || dtype == DType::kFloat16 || dtype == DType::kBFloat16, + "Grouped GLU supports float32, float16 and bfloat16."); + NVTE_CHECK(output->data.dtype == dtype, "Grouped GLU input/output dtypes must match."); + NVTE_CHECK(input->data.numel() >= capacity * input_width && + output->data.numel() >= capacity * (Backward ? input_width : hidden), + "Grouped GLU input/output storage is smaller than its logical shape."); + if (input->first_dims.has_data()) { + NVTE_CHECK( + input->first_dims.dtype == DType::kInt64 && input->first_dims.numel() == input->num_tensors, + "Grouped GLU first_dims must contain one int64 count per group."); + } + const auto *grad = Backward ? convertNVTEGroupedTensorCheck(grad_handle) : nullptr; + if constexpr (Backward) { + NVTE_CHECK(grad->logical_shape.ndim == 2 && grad->all_same_last_dim() && + grad->logical_shape.data[0] == capacity && + grad->logical_shape.data[1] == hidden && + grad->num_tensors == input->num_tensors && grad->data.dtype == dtype, + "Grouped GLU gradient shape, dtype or group count is inconsistent with input."); + NVTE_CHECK(grad->data.numel() >= capacity * hidden, + "Grouped GLU gradient storage is smaller than its logical shape."); + } + if (capacity == 0) return; + NVTE_CHECK(input->data.has_data() && output->data.has_data(), + "Grouped GLU input/output data must be allocated."); + if constexpr (Backward) NVTE_CHECK(grad->data.has_data(), "Grouped GLU grad is not allocated."); + const auto *sizes = static_cast(input->first_dims.dptr); + TRANSFORMER_ENGINE_TYPE_SWITCH_INPUT( + dtype, T, constexpr int nvec = 32 / sizeof(T); if constexpr (Backward) { + DGatedActivationKernelLauncher( + static_cast(grad->data.dptr), static_cast(input->data.dptr), + static_cast(output->data.dptr), nullptr, nullptr, nullptr, capacity, hidden, + params, stream, sizes, input->num_tensors); + } else { + GatedActivationKernelLauncher( + static_cast(input->data.dptr), static_cast(output->data.dptr), nullptr, + nullptr, nullptr, capacity, hidden, params, stream, sizes, input->num_tensors); + }); +} + } // namespace dispatch } // namespace transformer_engine diff --git a/transformer_engine/common/util/vectorized_pointwise.h b/transformer_engine/common/util/vectorized_pointwise.h index e5f855152cd..936870a1110 100644 --- a/transformer_engine/common/util/vectorized_pointwise.h +++ b/transformer_engine/common/util/vectorized_pointwise.h @@ -403,14 +403,42 @@ void VectorizedUnaryGradKernelLauncher(const InputTypeGrad *grad, const InputTyp } } +template +__device__ __forceinline__ size_t gated_active_rows(const size_t capacity, + const int64_t *group_sizes, + const size_t num_groups) { + if constexpr (!Grouped) { + return capacity; + } else { + __shared__ size_t active_rows; + if (threadIdx.x < THREADS_PER_WARP) { + int64_t rows = 0; + if (group_sizes != nullptr) { + for (size_t group = threadIdx.x; group < num_groups; group += THREADS_PER_WARP) { + rows += group_sizes[group]; + } + } + rows = warp_allreduce_sum(rows); + if (threadIdx.x == 0) { + active_rows = num_groups == 0 ? 0 : (group_sizes == nullptr ? capacity : rows); + } + } + __syncthreads(); + return active_rows; + } +} + template + ComputeType (*Activation)(const ComputeType, const Param &), bool Grouped = false, + typename InputType, typename OutputType> __launch_bounds__(unary_kernel_threads) __global__ void gated_act_kernel(const InputType *input, OutputType *output, const ComputeType *scale, ComputeType *amax, ComputeType *scale_inv, const size_t m, const size_t n, - const Param p, const size_t num_aligned_elements) { + const Param p, const size_t num_aligned_elements, + const int64_t *group_sizes, const size_t num_groups) { const size_t M = num_aligned_elements * m; + [[maybe_unused]] const size_t active_rows = + gated_active_rows(m, group_sizes, num_groups); ComputeType max = 0; ComputeType s = 1; const bool requires_amax = (amax != nullptr); @@ -426,6 +454,14 @@ __launch_bounds__(unary_kernel_threads) __global__ VectorizedLoader loader1(input + id_y * n * 2 + n, n); VectorizedStorer storer(output + id_y * n, n); + if constexpr (Grouped) { + if (id_y >= active_rows) { +#pragma unroll + for (int i = 0; i < nvec; ++i) storer.separate()[i] = static_cast(0); + storer.store(id_x, n); + continue; + } + } loader0.load(id_x, n); loader1.load(id_x, n); #pragma unroll @@ -470,11 +506,13 @@ __launch_bounds__(unary_kernel_threads) __global__ } template + ComputeType (*Activation)(const ComputeType, const Param &), bool Grouped = false, + typename InputType, typename OutputType> void GatedActivationKernelLauncher(const InputType *input, OutputType *output, const fp32 *scale, fp32 *amax, fp32 *scale_inv, const size_t m, const size_t n, - const Param &p, cudaStream_t stream) { + const Param &p, cudaStream_t stream, + const int64_t *group_sizes = nullptr, + const size_t num_groups = 0) { if (m != 0 && n != 0) { size_t num_aligned_elements = get_num_aligned_elements(input, n, nvec, sizeof(InputType)); constexpr size_t threads = unary_kernel_threads; @@ -484,19 +522,20 @@ void GatedActivationKernelLauncher(const InputType *input, OutputType *output, c switch (auto align = CheckAlignment(n, nvec, input, input + n, output)) { case Alignment::SAME_ALIGNED: - gated_act_kernel + gated_act_kernel <<>>(input, output, scale, amax, scale_inv, m, n, p, - num_aligned_elements); + num_aligned_elements, group_sizes, num_groups); break; case Alignment::SAME_UNALIGNED: - gated_act_kernel + gated_act_kernel <<>>(input, output, scale, amax, scale_inv, m, n, p, - num_aligned_elements); + num_aligned_elements, group_sizes, num_groups); break; case Alignment::DIFFERENT: { // If the pointers are aligned differently we cannot vectorize - gated_act_kernel<1, true, ComputeType, Param, Activation> - <<>>(input, output, scale, amax, scale_inv, m, n, p, n); + gated_act_kernel<1, true, ComputeType, Param, Activation, Grouped> + <<>>(input, output, scale, amax, scale_inv, m, n, p, n, + group_sizes, num_groups); break; } } @@ -506,14 +545,17 @@ void GatedActivationKernelLauncher(const InputType *input, OutputType *output, c template + ComputeType (*Dactivation)(const ComputeType, const Param &), bool Grouped = false, + typename InputType, typename OutputType> __launch_bounds__(unary_kernel_threads) __global__ void dgated_act_kernel(const InputType *grad, const InputType *input, OutputType *output, const ComputeType *scale, ComputeType *amax, ComputeType *scale_inv, const size_t m, const size_t n, const Param p, - const size_t num_aligned_elements) { + const size_t num_aligned_elements, const int64_t *group_sizes, + const size_t num_groups) { const size_t M = num_aligned_elements * m; + [[maybe_unused]] const size_t active_rows = + gated_active_rows(m, group_sizes, num_groups); ComputeType max = 0; ComputeType s = 1; const bool requires_amax = (amax != nullptr); @@ -531,6 +573,18 @@ __launch_bounds__(unary_kernel_threads) __global__ VectorizedStorer storer0(output + id_y * n * 2, n); VectorizedStorer storer1(output + id_y * n * 2 + n, n); + if constexpr (Grouped) { + if (id_y >= active_rows) { +#pragma unroll + for (int i = 0; i < nvec; ++i) { + storer0.separate()[i] = static_cast(0); + storer1.separate()[i] = static_cast(0); + } + storer0.store(id_x, n); + storer1.store(id_x, n); + continue; + } + } grad_loader.load(id_x, n); input_loader0.load(id_x, n); input_loader1.load(id_x, n); @@ -590,12 +644,13 @@ __launch_bounds__(unary_kernel_threads) __global__ template + ComputeType (*Dactivation)(const ComputeType, const Param &), bool Grouped = false, + typename InputType, typename OutputType> void DGatedActivationKernelLauncher(const InputType *grad, const InputType *input, OutputType *output, const fp32 *scale, fp32 *amax, fp32 *scale_inv, const size_t m, const size_t n, const Param &p, - cudaStream_t stream) { + cudaStream_t stream, const int64_t *group_sizes = nullptr, + const size_t num_groups = 0) { if (m != 0 && n != 0) { size_t num_aligned_elements = get_num_aligned_elements(grad, n, nvec, sizeof(InputType)); constexpr size_t threads = unary_kernel_threads; @@ -605,20 +660,20 @@ void DGatedActivationKernelLauncher(const InputType *grad, const InputType *inpu switch (auto align = CheckAlignment(n, nvec, input, input + n, output, output + n)) { case Alignment::SAME_ALIGNED: - dgated_act_kernel + dgated_act_kernel <<>>(grad, input, output, scale, amax, scale_inv, m, n, - p, num_aligned_elements); + p, num_aligned_elements, group_sizes, num_groups); break; case Alignment::SAME_UNALIGNED: - dgated_act_kernel + dgated_act_kernel <<>>(grad, input, output, scale, amax, scale_inv, m, n, - p, num_aligned_elements); + p, num_aligned_elements, group_sizes, num_groups); break; case Alignment::DIFFERENT: { // If the pointers are aligned differently we cannot vectorize - dgated_act_kernel<1, true, ComputeType, Param, Activation, Dactivation> + dgated_act_kernel<1, true, ComputeType, Param, Activation, Dactivation, Grouped> <<>>(grad, input, output, scale, amax, scale_inv, m, n, - p, n); + p, n, group_sizes, num_groups); break; } } From 043b1e2da4de7e42247f5e201666b1814cfc9278 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 9 Nov 2026 11:27:54 +0100 Subject: [PATCH 5/7] [JAX] Expose Quantile Balancing router statistics Bind the existing CUDA QB router with functional histogram aliasing and expose per-token-shard histogram contributions from MoEBlock. Leave global-step bias updates to the training framework. Runtime validation is deferred at the author request. [skip ci] Signed-off-by: Pawel Gadzinski --- docs/api/jax.rst | 39 +++++ tests/jax/moe/test_moe_resource_api.py | 34 ++++ tests/jax/test_distributed_router.py | 48 +++++ tests/jax/test_fused_router.py | 165 ++++++++++++++++++ tests/jax/test_te_ep_moe.py | 68 ++++++++ .../jax/cpp_extensions/router.py | 48 ++++- transformer_engine/jax/csrc/extensions.h | 1 + .../jax/csrc/extensions/pybind.cpp | 2 + .../jax/csrc/extensions/router.cpp | 87 +++++++++ transformer_engine/jax/flax/moe.py | 21 ++- transformer_engine/jax/moe.py | 87 ++++++++- transformer_engine/jax/router.py | 163 +++++++++++++++++ 12 files changed, 751 insertions(+), 12 deletions(-) diff --git a/docs/api/jax.rst b/docs/api/jax.rst index da9a3506961..90bed3c33ef 100644 --- a/docs/api/jax.rst +++ b/docs/api/jax.rst @@ -69,6 +69,8 @@ Router, routing kernels, and grouped dense for MoE layers. See .. autoapifunction:: transformer_engine.jax.router.fused_topk_with_score_function +.. autoapifunction:: transformer_engine.jax.router.fused_topk_with_score_function_qb + .. autoapifunction:: transformer_engine.jax.router.fused_moe_aux_loss .. autoapifunction:: transformer_engine.jax.permutation.sort_chunks_by_index @@ -101,6 +103,43 @@ The custom VJP returns gradients to both inputs, including when the latent is a differentiable projection of the router input. The default ``router_input=None`` preserves routing on the expert input. +Quantile Balancing statistics +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Set ``RouterComputationInfo.qb_bin_bounds`` to an FP32 ``[lower, upper]`` array +to collect QB statistics during MoE routing. Sigmoid routing, an FP32 expert +bias, no routing groups and zero auxiliary loss are required. The existing CUDA +QB kernels provide ``fused_atomic`` and ``two_kernel`` modes, selected by +``qb_histogram_mode``. The number of bins defaults to 1000. + +The bounds must be finite and ordered; their values are read on device without +host validation. Kimi uses ``[min(bias)-1, max(bias)+1]``. Keep both bias and +bounds unchanged throughout every microbatch in a training step. All token rows +must be valid: neither mode excludes sequence-padding tokens. + +With QB enabled, ``moe`` returns a fifth result, an int32 histogram array of +shape ``[token_shards, experts, bins]``. Its leading axis is sharded across the +compound DP/FSDP/EP token axes. The forward does not reduce these histograms +or update biases. Accumulate per-shard counts across microbatches and only then +sum the leading axis to obtain each expert's global-batch distribution. The +training framework derives and mean-centers the new bias from that distribution, +applying it on the next step. Default calls keep the four-result API. + +The standalone QB router returns ``(probs, routing_map, histogram)`` and accepts +an optional initial histogram for functional accumulation. Its FFI declares +input/output aliasing for the CUDA kernel's in-place histogram writes. For SPMD, +place the standalone operation inside ``shard_map`` with the expert axis +replicated and expose local histograms as a sharded leading axis, as MoE does. +Neither histogram collection nor bias/bounds inputs have a gradient. The router +reuses the ordinary sigmoid routing backward for gradients through its weights. +Use fresh forward statistics or explicit scan carry state; an external mutable +histogram would be counted twice when activation checkpointing recomputes it. + +The Flax wrapper accepts ``qb_bin_bounds`` on each call and exposes +``qb_num_bins`` and ``qb_histogram_mode`` as module configuration. It returns +the same optional fifth result. Step-level bias updates, checkpointing that +bias state, and TPU implementations belong to the training framework. + .. autoapifunction:: transformer_engine.jax.ep.ep_bootstrap .. autoapifunction:: transformer_engine.jax.ep.ep_finalize diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index 587fc22dd1c..4cee3a9804e 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -122,6 +122,40 @@ def test_gate_kernel_uses_router_width(): ) +@pytest.mark.parametrize( + "overrides,match", + [ + ({"score_function": "softmax"}, "requires sigmoid"), + ({"num_groups": 2, "group_topk": 1}, "ungrouped"), + ({"aux_loss_coeff": 0.1}, "aux_loss_coeff"), + ({"aux_loss_coeff": -0.1}, "aux_loss_coeff"), + ({"use_pre_softmax": True}, "use_pre_softmax"), + ({"qb_num_bins": 0}, "num_bins"), + ({"expert_bias": None}, "expert_bias"), + ({"qb_histogram_mode": "invalid"}, "histogram_mode"), + ], +) +def test_qb_configuration_rejected_before_ep(overrides, match): + module = importlib.import_module("transformer_engine.jax.moe") + kwargs = dict( + gate_kernel=jnp.ones((4, 2)), + num_experts_per_tok=1, + score_function="sigmoid", + expert_bias=jnp.zeros(2, jnp.float32), + qb_bin_bounds=jnp.asarray([-1.0, 1.0], jnp.float32), + ) + kwargs.update(overrides) + with pytest.raises(ValueError, match=match): + module.moe( + jnp.ones((1, 1, 4)), + RouterComputationInfo(**kwargs), + None, + None, + num_experts=2, + mesh_resource=MeshResource(ep_resource="ep"), + ) + + def test_explicit_resource_overrides_global_and_is_snapshotted(): explicit = MeshResource(dp_resource="dp", fsdp_resource="fsdp", ep_resource="ep") enclosing = MeshResource(ep_resource="other") diff --git a/tests/jax/test_distributed_router.py b/tests/jax/test_distributed_router.py index c6c8a11a844..0b091ded580 100644 --- a/tests/jax/test_distributed_router.py +++ b/tests/jax/test_distributed_router.py @@ -53,6 +53,7 @@ def _inject_router(): jax.config.update("jax_use_shardy_partitioner", True) from test_fused_router import ( + reference_qb_histogram, reference_topk_with_score_function, reference_compute_scores_for_aux_loss, reference_aux_loss, @@ -79,6 +80,53 @@ def _inject_router(): } +@pytest.mark.parametrize("device_count,mesh_shape,mesh_axes,mesh_resource", generate_configs()) +@pytest.mark.parametrize("histogram_mode", ["two_kernel", "fused_atomic"]) +def test_qb_shard_histograms(device_count, mesh_shape, mesh_axes, mesh_resource, histogram_mode): + from jax.experimental.shard_map import shard_map + from transformer_engine.jax.router import fused_topk_with_score_function_qb + + del mesh_resource + mesh = Mesh(np.asarray(jax.devices()[:device_count]).reshape(mesh_shape), mesh_axes) + token_axes = tuple(mesh_axes) + logits = jax.random.normal(jax.random.PRNGKey(831), (128 * device_count, 32)) + bias = jnp.linspace(-0.1, 0.1, 32, dtype=jnp.float32) + bounds = jnp.asarray([-1.1, 1.1], jnp.float32) + + def route(local_logits, expert_bias, limits): + probs, routing, histogram = fused_topk_with_score_function_qb( + local_logits, + 4, + expert_bias, + limits, + num_bins=64, + histogram_mode=histogram_mode, + ) + return probs, routing, histogram[None] + + with mesh: + sharded_logits = jax.device_put( + logits, NamedSharding(mesh, PartitionSpec(token_axes, None)) + ) + _, _, partial_histograms = jax.jit( + shard_map( + route, + mesh=mesh, + in_specs=(PartitionSpec(token_axes, None), PartitionSpec(), PartitionSpec()), + out_specs=( + PartitionSpec(token_axes, None), + PartitionSpec(token_axes, None), + PartitionSpec(token_axes, None, None), + ), + check_rep=False, + ) + )(sharded_logits, bias, bounds) + histogram = jax.jit(lambda h: h.sum(axis=0))(partial_histograms) + assert partial_histograms.shape == (device_count, 32, 64) + np.testing.assert_array_equal(histogram, reference_qb_histogram(logits, bias, bounds, 4, 64)) + np.testing.assert_array_equal(np.asarray(histogram).sum(axis=-1), np.full(32, logits.shape[0])) + + class TestDistributedFusedTopk: """Test distributed execution of fused_topk_with_score_function. diff --git a/tests/jax/test_fused_router.py b/tests/jax/test_fused_router.py index a0e46ca3012..2a26c8a36fa 100644 --- a/tests/jax/test_fused_router.py +++ b/tests/jax/test_fused_router.py @@ -10,6 +10,7 @@ import jax import jax.numpy as jnp +import numpy as np import pytest from utils import pytest_parametrize_wrapper @@ -111,6 +112,170 @@ def _inject_router(): SEED = 42 +def reference_qb_histogram(logits, expert_bias, bounds, topk, num_bins): + scores = np.asarray(jax.nn.sigmoid(jnp.asarray(logits, jnp.float32))) + selected, _ = jax.lax.top_k(jnp.asarray(scores + np.asarray(expert_bias)), topk + 1) + required_bias = np.asarray(selected)[:, -1:] - scores + lower, upper = np.asarray(bounds) + bins = np.floor((required_bias - lower) * (num_bins / (upper - lower))).astype(np.int32) + bins = np.clip(bins, 0, num_bins - 1) + histogram = np.zeros((scores.shape[-1], num_bins), np.int32) + np.add.at(histogram, (np.arange(scores.shape[-1])[None, :], bins), 1) + return histogram + + +@pytest.mark.parametrize("histogram_mode", ["two_kernel", "fused_atomic"]) +@pytest.mark.parametrize("routing_map_format", ["bytemap", "bitmap_u8"]) +@pytest.mark.parametrize("topk", [1, 4]) +def test_qb_router_forward_backward_and_remat(histogram_mode, routing_map_format, topk): + from transformer_engine.jax.router import fused_topk_with_score_function_qb + + logits = jax.random.normal(jax.random.PRNGKey(801), (128, 32)) + bias = jax.random.uniform(jax.random.PRNGKey(802), (32,), minval=-0.1, maxval=0.1) + bounds = jnp.asarray([-1.1, 1.1], jnp.float32) + coefficients = jax.random.normal(jax.random.PRNGKey(803), logits.shape) + + def loss(x, b, limits): + probs, routing, histogram = fused_topk_with_score_function_qb( + x, + topk, + b, + limits, + num_bins=64, + scaling_factor=1.5, + histogram_mode=histogram_mode, + routing_map_format=routing_map_format, + ) + return jnp.sum(probs * coefficients), (probs, routing, histogram) + + (_, (probs, routing, histogram)), (dx, db, dbounds) = jax.jit( + jax.value_and_grad(jax.checkpoint(loss), argnums=(0, 1, 2), has_aux=True) + )(logits, bias, bounds) + + def reference(x): + probs, _ = reference_topk_with_score_function( + x, + topk, + score_function="sigmoid", + expert_bias=bias, + scaling_factor=1.5, + ) + return jnp.sum(probs * coefficients), probs + + (_, expected_probs), expected_dx = jax.value_and_grad(reference, has_aux=True)(logits) + np.testing.assert_allclose(probs, expected_probs, rtol=2e-5, atol=2e-6) + np.testing.assert_allclose(dx, expected_dx, rtol=2e-4, atol=2e-6) + np.testing.assert_array_equal(db, jnp.zeros_like(bias)) + np.testing.assert_array_equal(dbounds, jnp.zeros_like(bounds)) + np.testing.assert_array_equal(histogram, reference_qb_histogram(logits, bias, bounds, topk, 64)) + expected_map = np.asarray(expected_probs != 0) + if routing_map_format == "bitmap_u8": + expected_map = np.packbits(expected_map, axis=-1, bitorder="little") + np.testing.assert_array_equal(routing, expected_map) + + +@pytest.mark.parametrize("histogram_mode", ["two_kernel", "fused_atomic"]) +def test_qb_scan_accumulation_preserves_initial_histogram(histogram_mode): + from transformer_engine.jax.router import fused_topk_with_score_function_qb + + logits = jax.random.normal(jax.random.PRNGKey(811), (3, 64, 32)) + bias = jnp.linspace(-0.1, 0.1, 32, dtype=jnp.float32) + bounds = jnp.asarray([-1.1, 1.1], jnp.float32) + initial = jnp.full((32, 64), 7, jnp.int32) + + def accumulate(histogram, x): + _, _, updated = fused_topk_with_score_function_qb( + x, + 4, + bias, + bounds, + num_bins=64, + histogram=histogram, + histogram_mode=histogram_mode, + ) + return updated, None + + @jax.jit + def run(histogram, x): + total, _ = jax.lax.scan(accumulate, histogram, x) + return total, histogram + 1 + + actual, still_live = run(initial, logits) + expected = np.full((32, 64), 7, np.int32) + for microbatch in np.asarray(logits): + expected += reference_qb_histogram(microbatch, bias, bounds, 4, 64) + np.testing.assert_array_equal(actual, expected) + np.testing.assert_array_equal(still_live, np.full((32, 64), 8, np.int32)) + np.testing.assert_array_equal(initial, np.full((32, 64), 7, np.int32)) + + +@pytest.mark.parametrize("histogram_mode", ["two_kernel", "fused_atomic"]) +def test_qb_bounds_change_and_clamp(histogram_mode): + from transformer_engine.jax.router import fused_topk_with_score_function_qb + + logits = jax.random.normal(jax.random.PRNGKey(821), (64, 32)) + bias = jnp.linspace(-0.2, 0.2, 32, dtype=jnp.float32) + route = jax.jit( + partial( + fused_topk_with_score_function_qb, + topk=4, + expert_bias=bias, + num_bins=32, + histogram_mode=histogram_mode, + ) + ) + results = [] + for limits in ([-1.2, 1.2], [-0.025, 0.025]): + bounds = jnp.asarray(limits, jnp.float32) + _, _, histogram = route(logits, bin_bounds=bounds) + np.testing.assert_array_equal( + histogram, reference_qb_histogram(logits, bias, bounds, 4, 32) + ) + np.testing.assert_array_equal(np.asarray(histogram).sum(axis=-1), np.full(32, 64)) + results.append(np.asarray(histogram)) + assert not np.array_equal(*results) + + +@pytest.mark.parametrize("histogram_mode", ["two_kernel", "fused_atomic"]) +def test_qb_tied_cutoff_counts_each_token_once(histogram_mode): + from transformer_engine.jax.router import fused_topk_with_score_function_qb + + _, routing, histogram = jax.jit( + partial( + fused_topk_with_score_function_qb, + topk=4, + num_bins=32, + histogram_mode=histogram_mode, + ) + )(jnp.zeros((64, 32)), expert_bias=jnp.zeros(32), bin_bounds=jnp.asarray([-1.0, 1.0])) + np.testing.assert_array_equal(np.asarray(routing).sum(axis=-1), np.full(64, 4)) + expected = np.zeros((32, 32), np.int32) + expected[:, 16] = 64 + np.testing.assert_array_equal(histogram, expected) + + +@pytest.mark.parametrize( + "overrides,match", + [ + ({"topk": 32}, "topk"), + ({"expert_bias": None}, "expert_bias"), + ({"num_bins": 0}, "num_bins"), + ({"histogram_mode": "unknown"}, "histogram_mode"), + ({"bin_bounds": jnp.zeros(3, jnp.float32)}, "bin_bounds"), + ({"histogram": jnp.zeros((32, 64), jnp.float32)}, "histogram"), + ], +) +def test_qb_router_rejects_invalid_inputs(overrides, match): + from transformer_engine.jax.router import fused_topk_with_score_function_qb + + kwargs = dict( + topk=4, expert_bias=jnp.zeros(32), bin_bounds=jnp.asarray([-1.0, 1.0]), num_bins=64 + ) + kwargs.update(overrides) + with pytest.raises(ValueError, match=match): + fused_topk_with_score_function_qb(jnp.zeros((64, 32)), **kwargs) + + # ============================================================================= # Reference Implementations # ============================================================================= diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index 8e738e81fe2..5f99b058365 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -863,6 +863,74 @@ def ref_loss(params, h, p): ) +@pytest.mark.parametrize("histogram_mode", ["two_kernel", "fused_atomic"]) +def test_qb_moe_histograms_and_remat_gradients(mesh, make_block, histogram_mode): + from test_fused_router import reference_qb_histogram + + def bias_init(key, shape, dtype): + del key + return jnp.linspace(-0.1, 0.1, shape[0], dtype=dtype) + + block = make_block( + score_function="sigmoid", + use_expert_routing_bias=True, + expert_bias_init=bias_init, + ).clone(qb_num_bins=64, qb_histogram_mode=histogram_mode) + bounds = jnp.asarray([-1.1, 1.1], jnp.float32) + x = jax.random.normal(jax.random.PRNGKey(841), (BATCH, SEQ, HIDDEN), DTYPE) + + with _ctx(mesh): + x_sharded = _shard_inputs(x, mesh) + variables = jax.jit(block.init)(jax.random.PRNGKey(842), x_sharded) + + def loss(v, activations, limits): + result = block.apply(v, activations, qb_bin_bounds=limits) + return jnp.mean(result[0].astype(jnp.float32) ** 2), result + + (_, baseline), (baseline_dv, baseline_dx) = jax.jit( + jax.value_and_grad( + partial(loss, limits=None), + argnums=(0, 1), + has_aux=True, + ) + )(variables, x_sharded) + (_, candidate), (dv, dx) = jax.jit( + jax.value_and_grad( + jax.checkpoint(loss), + argnums=(0, 1), + has_aux=True, + ) + )(variables, x_sharded, bounds) + + assert len(baseline) == 4 and len(candidate) == 5 + histogram_parts = _to_global_numpy(candidate[4], mesh) + assert histogram_parts.shape == (mesh.size, NUM_EXPERTS, 64) + params = _params_global_numpy(variables, mesh) + logits = (x @ jnp.asarray(params["gate_kernel"], DTYPE)).astype(jnp.float32) + expected_histogram = reference_qb_histogram( + logits.reshape(-1, NUM_EXPERTS), + params["expert_bias"], + bounds, + TOPK, + 64, + ) + np.testing.assert_array_equal(histogram_parts.sum(axis=0), expected_histogram) + for actual, expected in ((candidate[0], baseline[0]), (dx, baseline_dx)): + np.testing.assert_allclose( + _to_global_numpy(actual, mesh).astype(np.float32), + _to_global_numpy(expected, mesh).astype(np.float32), + rtol=2e-2, + atol=2e-2, + ) + for actual, expected in zip(jax.tree.leaves(dv), jax.tree.leaves(baseline_dv)): + np.testing.assert_allclose( + _to_global_numpy(_unwrap(actual), mesh).astype(np.float32), + _to_global_numpy(_unwrap(expected), mesh).astype(np.float32), + rtol=2e-2, + atol=2e-2, + ) + + def _reference_kwargs_from_config(config, params_np): """Pick out the reference-relevant pieces of a parametrize config.""" return dict( diff --git a/transformer_engine/jax/cpp_extensions/router.py b/transformer_engine/jax/cpp_extensions/router.py index ad49c7a7d1b..c97e46cdf97 100644 --- a/transformer_engine/jax/cpp_extensions/router.py +++ b/transformer_engine/jax/cpp_extensions/router.py @@ -6,7 +6,7 @@ from enum import IntEnum import jax.numpy as jnp -from jax import dtypes, ffi +from jax import ShapeDtypeStruct, dtypes, ffi from jax.sharding import NamedSharding, PartitionSpec from transformer_engine_jax import JAXX_Routing_Map_Format, JAXX_Score_Function @@ -18,6 +18,7 @@ "RoutingMapFormat", "fused_topk_with_score_function_fwd", "fused_topk_with_score_function_bwd", + "fused_topk_with_score_function_qb_fwd", "fused_moe_aux_loss_fwd", "fused_moe_aux_loss_bwd", ] @@ -732,6 +733,51 @@ def fused_topk_with_score_function_fwd( ) +def fused_topk_with_score_function_qb_fwd( + logits, + expert_bias, + bin_bounds, + histogram, + *, + topk, + scaling_factor, + histogram_mode, + routing_map_format=int(RoutingMapFormat.BYTEMAP), +): + """Shard-local QB forward; histogram input aliases the updated output.""" + num_tokens, num_experts = logits.shape + map_shape = ( + (num_tokens, (num_experts + 7) // 8) + if int(routing_map_format) == int(RoutingMapFormat.BITMAP_U8) + else logits.shape + ) + map_dtype = ( + jnp.uint8 if int(routing_map_format) == int(RoutingMapFormat.BITMAP_U8) else jnp.bool_ + ) + outputs = ( + ShapeDtypeStruct(logits.shape, jnp.float32), + ShapeDtypeStruct(map_shape, map_dtype), + ShapeDtypeStruct(logits.shape, jnp.float32), + ShapeDtypeStruct((num_tokens,), jnp.float32), + ShapeDtypeStruct(histogram.shape, jnp.int32), + ) + return ffi.ffi_call( + "te_fused_topk_with_score_function_qb_forward_ffi", + outputs, + input_output_aliases={3: 4}, + vmap_method="sequential", + )( + logits, + expert_bias, + bin_bounds, + histogram, + topk=int(topk), + scaling_factor=float(scaling_factor), + histogram_mode=int(histogram_mode), + routing_map_format=int(routing_map_format), + ) + + def fused_topk_with_score_function_bwd( routing_map: jnp.ndarray, saved_scores: jnp.ndarray, diff --git a/transformer_engine/jax/csrc/extensions.h b/transformer_engine/jax/csrc/extensions.h index f9853f5e509..9d2d85479a6 100644 --- a/transformer_engine/jax/csrc/extensions.h +++ b/transformer_engine/jax/csrc/extensions.h @@ -204,6 +204,7 @@ XLA_FFI_DECLARE_HANDLER_SYMBOL(CublasHandleInitHandler); // Router XLA_FFI_DECLARE_HANDLER_SYMBOL(FusedTopkWithScoreFunctionForwardHandler); +XLA_FFI_DECLARE_HANDLER_SYMBOL(FusedTopkWithScoreFunctionQBForwardHandler); XLA_FFI_DECLARE_HANDLER_SYMBOL(FusedTopkWithScoreFunctionBackwardHandler); XLA_FFI_DECLARE_HANDLER_SYMBOL(FusedMoEAuxLossForwardHandler); XLA_FFI_DECLARE_HANDLER_SYMBOL(FusedMoEAuxLossBackwardHandler); diff --git a/transformer_engine/jax/csrc/extensions/pybind.cpp b/transformer_engine/jax/csrc/extensions/pybind.cpp index 5e748dc7cd2..adbb2fe318e 100644 --- a/transformer_engine/jax/csrc/extensions/pybind.cpp +++ b/transformer_engine/jax/csrc/extensions/pybind.cpp @@ -115,6 +115,8 @@ pybind11::dict Registrations() { // Router dict["te_fused_topk_with_score_function_forward_ffi"] = EncapsulateFFI(FusedTopkWithScoreFunctionForwardHandler); + dict["te_fused_topk_with_score_function_qb_forward_ffi"] = + EncapsulateFFI(FusedTopkWithScoreFunctionQBForwardHandler); dict["te_fused_topk_with_score_function_backward_ffi"] = EncapsulateFFI(FusedTopkWithScoreFunctionBackwardHandler); dict["te_fused_moe_aux_loss_forward_ffi"] = EncapsulateFFI(FusedMoEAuxLossForwardHandler); diff --git a/transformer_engine/jax/csrc/extensions/router.cpp b/transformer_engine/jax/csrc/extensions/router.cpp index a1d0731b6db..9200afb6a14 100644 --- a/transformer_engine/jax/csrc/extensions/router.cpp +++ b/transformer_engine/jax/csrc/extensions/router.cpp @@ -102,6 +102,93 @@ XLA_FFI_DEFINE_HANDLER_SYMBOL(FusedTopkWithScoreFunctionForwardHandler, .Attr("routing_map_format"), FFI_CudaGraph_Traits); +Error_Type FusedTopkWithScoreFunctionQBForwardFFI( + cudaStream_t stream, Buffer_Type logits_buf, Buffer_Type expert_bias_buf, + Buffer_Type bin_bounds_buf, Buffer_Type histogram_buf, Result_Type probs_buf, + Result_Type routing_map_buf, Result_Type intermediate_buf, Result_Type cutoff_buf, + Result_Type histogram_out_buf, int64_t topk, double scaling_factor, int64_t histogram_mode, + JAXX_Routing_Map_Format routing_map_format) { + auto dims = logits_buf.dimensions(); + auto bias_dims = expert_bias_buf.dimensions(); + auto bounds_dims = bin_bounds_buf.dimensions(); + auto hist_dims = histogram_buf.dimensions(); + NVTE_CHECK(dims.size() == 2 && dims[0] > 0 && dims[1] > 0, + "QB logits must have shape [tokens, experts] with positive dimensions."); + const auto num_tokens = static_cast(dims[0]); + const auto num_experts = static_cast(dims[1]); + NVTE_CHECK(topk > 0 && topk < num_experts, "QB requires 0 < topk < num_experts."); + NVTE_CHECK(bias_dims.size() == 1 && bias_dims[0] == num_experts, + "QB expert_bias must have shape [experts]."); + NVTE_CHECK(bounds_dims.size() == 1 && bounds_dims[0] == 2, "QB bin_bounds must have shape [2]."); + NVTE_CHECK(hist_dims.size() == 2 && hist_dims[0] == num_experts && hist_dims[1] > 0, + "QB histogram must have shape [experts, bins]."); + NVTE_CHECK( + convert_ffi_datatype_to_te_dtype(logits_buf.element_type()) == DType::kFloat32 && + convert_ffi_datatype_to_te_dtype(expert_bias_buf.element_type()) == DType::kFloat32 && + convert_ffi_datatype_to_te_dtype(bin_bounds_buf.element_type()) == DType::kFloat32, + "QB logits, expert_bias and bin_bounds must be float32."); + NVTE_CHECK(convert_ffi_datatype_to_te_dtype(histogram_buf.element_type()) == DType::kInt32, + "QB histogram must be int32."); + NVTE_CHECK(histogram_buf.untyped_data() == histogram_out_buf->untyped_data(), + "QB histogram input and output must alias."); + NVTE_CHECK(histogram_mode == NVTE_QB_HISTOGRAM_TWO_KERNEL || + histogram_mode == NVTE_QB_HISTOGRAM_FUSED_ATOMIC, + "Invalid QB histogram mode."); + + const std::vector shape{static_cast(num_tokens), + static_cast(num_experts)}; + const auto format = static_cast(routing_map_format); + const size_t map_width = format == NVTE_ROUTING_MAP_FORMAT_BITMAP_U8 + ? static_cast((num_experts + 7) / 8) + : static_cast(num_experts); + auto logits = TensorWrapper(logits_buf.untyped_data(), shape, DType::kFloat32); + auto bias = TensorWrapper(expert_bias_buf.untyped_data(), + std::vector{static_cast(num_experts)}, DType::kFloat32); + auto bounds = + TensorWrapper(bin_bounds_buf.untyped_data(), std::vector{2}, DType::kFloat32); + auto probs = TensorWrapper(probs_buf->untyped_data(), shape, DType::kFloat32); + auto routing_map = + TensorWrapper(routing_map_buf->untyped_data(), + std::vector{static_cast(num_tokens), map_width}, DType::kByte); + auto intermediate = TensorWrapper(intermediate_buf->untyped_data(), shape, DType::kFloat32); + auto cutoff = + TensorWrapper(cutoff_buf->untyped_data(), + std::vector{static_cast(num_tokens)}, DType::kFloat32); + auto histogram = TensorWrapper( + histogram_out_buf->untyped_data(), + std::vector{static_cast(num_experts), static_cast(hist_dims[1])}, + DType::kInt32); + const auto mode = static_cast(histogram_mode); + nvte_fused_topk_with_score_function_forward_qb_v2( + logits.data(), num_tokens, num_experts, static_cast(topk), + static_cast(scaling_factor), bias.data(), probs.data(), routing_map.data(), format, + intermediate.data(), cutoff.data(), histogram.data(), bounds.data(), mode, stream); + if (mode == NVTE_QB_HISTOGRAM_TWO_KERNEL) { + nvte_qb_histogram_accumulate(intermediate.data(), cutoff.data(), bounds.data(), + histogram.data(), stream); + } + return ffi_with_cuda_error_check(); +} + +XLA_FFI_DEFINE_HANDLER_SYMBOL(FusedTopkWithScoreFunctionQBForwardHandler, + FusedTopkWithScoreFunctionQBForwardFFI, + FFI::Bind() + .Ctx() + .Arg() // logits + .Arg() // expert_bias + .Arg() // bin_bounds + .Arg() // histogram + .Ret() // probs + .Ret() // routing_map + .Ret() // intermediate + .Ret() // cutoff + .Ret() // updated histogram + .Attr("topk") + .Attr("scaling_factor") + .Attr("histogram_mode") + .Attr("routing_map_format"), + FFI_CudaGraph_Traits); + // ============================================================================ // Fused Top-K with Score Function - Backward // ============================================================================ diff --git a/transformer_engine/jax/flax/moe.py b/transformer_engine/jax/flax/moe.py index 030bedd4ae5..77b761326d4 100644 --- a/transformer_engine/jax/flax/moe.py +++ b/transformer_engine/jax/flax/moe.py @@ -176,6 +176,8 @@ class _MoEBlock(TransformerEngineBase): recv_capacity_per_rank: Optional[int] = None dispatch_checkpoint_name: Optional[str] = None collect_expert_counts: bool = True + qb_num_bins: int = 1000 + qb_histogram_mode: str = "fused_atomic" # Dtypes / init / misc dtype: DType = jnp.float32 @@ -198,8 +200,12 @@ def __post_init__(self): @nn.compact def __call__( - self, inputs: Array, *, router_input: Optional[Array] = None - ) -> Tuple[Array, Optional[Array], Array, Array]: + self, + inputs: Array, + *, + router_input: Optional[Array] = None, + qb_bin_bounds: Optional[Array] = None, + ) -> tuple: """Run the MoE forward pass. Parameters @@ -210,6 +216,10 @@ def __call__( Separate gate input ``[batch, sequence, router_hidden]``. ``None`` uses ``inputs``. Only the gate kernel uses ``router_hidden``; expert kernels and EP buffers retain ``hidden``. + qb_bin_bounds : Optional[jnp.ndarray] + FP32 ``[lower, upper]`` enables QB histogram collection. Requires + sigmoid routing, expert bias, no groups and zero auxiliary loss. + All input rows must be valid tokens; padding is not masked. Returns ------- @@ -223,6 +233,10 @@ def __call__( overflow when ``drop_on_overflow`` is set at ep_bootstrap. expert_counts : jnp.ndarray Global assignment counts for each expert. + qb_histograms : jnp.ndarray, optional + Fifth result only when ``qb_bin_bounds`` is supplied; int32 + ``[token_shards, experts, qb_num_bins]``. Reduce over token shards + and microbatches before updating the bias in the training loop. """ mesh_resource, quant_before_fsdp_ag = _resolve_moe_mesh_resource( self.mesh_resource, @@ -340,6 +354,9 @@ def make_grouped_quantizer_set(postfix): gate_kernel_axes=self.gate_kernel_axes, router_input=router_input, router_input_axes=self.router_input_axes, + qb_bin_bounds=qb_bin_bounds, + qb_num_bins=self.qb_num_bins, + qb_histogram_mode=self.qb_histogram_mode, ) return moe( inputs, diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index b94711185f8..67036599554 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -68,7 +68,7 @@ from .quantize.dequantizer import _unswizzle_mxfp8_grouped_scale from .cpp_extensions.gemm import swizzled_scale from .flax.module import _convert_to_activation_function -from .router import ScoreFunction, _validate_score_function +from .router import ScoreFunction, _validate_score_function, _validate_qb_router from .sharding import MeshResource, _get_mesh, global_mesh_resource, global_shard_guard __all__ = [ @@ -159,6 +159,13 @@ class RouterComputationInfo: router_input_axes : Tuple[Optional[str], ...] Logical sharding axes of ``router_input``. Its physical token layout follows the same DP/FSDP/EP placement as ``x``. + qb_bin_bounds : Optional[jnp.ndarray] + FP32 ``[lower, upper]`` enables QB statistics with sigmoid, no groups and + no auxiliary loss. All token rows must be valid; padding is not masked. + Bounds must be finite and ordered, and remain fixed throughout a step. + qb_num_bins, qb_histogram_mode : int, str + Histogram size and ``"fused_atomic"`` or ``"two_kernel"`` implementation. + The caller owns histogram reduction and bias updates. """ gate_kernel: jnp.ndarray @@ -175,6 +182,9 @@ class RouterComputationInfo: gate_kernel_axes: Tuple[Optional[str], ...] = flax.struct.field(pytree_node=False, default=()) router_input: Optional[jnp.ndarray] = None router_input_axes: Tuple[Optional[str], ...] = flax.struct.field(pytree_node=False, default=()) + qb_bin_bounds: Optional[jnp.ndarray] = None + qb_num_bins: int = flax.struct.field(pytree_node=False, default=1000) + qb_histogram_mode: str = flax.struct.field(pytree_node=False, default="fused_atomic") @flax.struct.dataclass @@ -1370,7 +1380,7 @@ def _moe_fwd_rule( ): """Forward: [gate -> route ->] ep_dispatch -> FFN -> ep_combine. - Returns ``(output, aux_loss, total_recv_tokens, expert_counts)``. + Returns ``(output, aux_loss, total_recv_tokens, expert_counts, qb_histograms)``. ``aux_loss`` is a zero scalar unless the router computes it and ``aux_loss_coeff`` > 0. """ with global_shard_guard(mesh_resource): @@ -1492,6 +1502,7 @@ def _moe_fwd_rule( # ------------------------ Routing ------------------------ + qb_histograms = jnp.zeros((0, 0, 0), dtype=jnp.int32) if router is None: # ---------------- Caller-supplied routing ---------------- routing_indices = jax.lax.with_sharding_constraint( @@ -1535,9 +1546,33 @@ def _moe_fwd_rule( "expert_bias": eb_arg, "compute_aux_scores": False, } - sparse_probs, routing_map, saved_scores = tex.fused_topk_with_score_function_fwd( - logits_2d, **topk_kwargs - ) + if router.qb_bin_bounds is None: + sparse_probs, routing_map, saved_scores = tex.fused_topk_with_score_function_fwd( + logits_2d, **topk_kwargs + ) + else: + mode = {"two_kernel": 0, "fused_atomic": 1}[router.qb_histogram_mode] + + def route_and_observe(local_logits, bias, bounds): + histogram = jnp.zeros((num_experts, router.qb_num_bins), dtype=jnp.int32) + probs, rmap, scores, _, histogram = tex.fused_topk_with_score_function_qb_fwd( + local_logits, + bias, + bounds, + histogram, + topk=K, + scaling_factor=router.scaling_factor, + histogram_mode=mode, + ) + return probs, rmap, scores, histogram[None] + + sparse_probs, routing_map, saved_scores, qb_histograms = shard_map( + route_and_observe, + mesh=mesh, + in_specs=(ep2_spec, P(None), P(None)), + out_specs=(ep2_spec, ep2_spec, ep2_spec, P(batch_pspec_axis, None, None)), + check_rep=False, + )(logits_2d, router.expert_bias, router.qb_bin_bounds) sparse_probs = sparse_probs.astype(dtype) # argsort on a bool tensor places True last (False=0 < True=1), so the # last K indices are the selected expert IDs. @@ -1772,7 +1807,7 @@ def _ffn_fwd_body(*args): "cudnn_native_weight_layout": cudnn_native_weight_layout, } # The integer metadata outputs are non-differentiable (see moe()). - return (output, aux_loss, total_recv_tokens, expert_counts), (ctx, static) + return (output, aux_loss, total_recv_tokens, expert_counts, qb_histograms), (ctx, static) def _moe_bwd_rule( @@ -1815,7 +1850,7 @@ def _moe_bwd_rule( from jax.experimental.shard_map import shard_map # The integer metadata outputs are non-differentiable; their cotangents are unused. - d_output, d_aux_loss, _d_total_recv_tokens, _d_expert_counts = cotangents + d_output, d_aux_loss, _d_total_recv_tokens, _d_expert_counts, _d_qb_histograms = cotangents ctx, static = residuals router = ctx.router @@ -2031,6 +2066,9 @@ def _ffn_bwd_body(*args): d_router_info = router.replace( gate_kernel=d_gate_kernel, router_input=d_router_input, + qb_bin_bounds=( + None if router.qb_bin_bounds is None else jnp.zeros_like(router.qb_bin_bounds) + ), expert_bias=( None if router.expert_bias is None else jnp.zeros_like(router.expert_bias) ), @@ -2187,7 +2225,7 @@ def moe( dispatch_checkpoint_name: Optional[str] = None, use_cudnn_fusion: bool = True, cudnn_native_weight_layout: Optional[bool] = None, -) -> Tuple[jnp.ndarray, Optional[jnp.ndarray], jnp.ndarray, jnp.ndarray]: +) -> tuple: """Run a full MoE block under a single fused custom_vjp on the TE EP path. Returns ``(output, aux_loss, total_recv_tokens, expert_counts)``. ``aux_loss`` @@ -2198,6 +2236,13 @@ def moe( ``[num_experts]`` containing global top-k assignment counts (or zeros when ``collect_expert_counts=False``). + With ``router_info.qb_bin_bounds`` set, a fifth result contains int32 QB + histograms ``[token_shards, experts, bins]``. The leading axis is sharded over + the compound DP/FSDP/EP token axes. Each token contributes exactly once; + there is no histogram collective in the forward. Accumulate over microbatches, + then sum the leading axis before deriving the next step's bias. The default + call keeps its four-result contract. QB statistics are non-differentiable. + Parameters ---------- activation_type : Union[str, SiTUActivation] @@ -2310,6 +2355,28 @@ def moe( aux_loss_coeff=float(router_info.aux_loss_coeff), ) has_aux_loss = router_info.aux_loss_coeff > 0.0 + if router_info.qb_bin_bounds is not None: + if ( + router_info.score_function != ScoreFunction.SIGMOID + or router_info.use_pre_softmax + or router_info.num_groups not in (None, -1, 0) + or router_info.group_topk not in (None, -1, 0) + or router_info.aux_loss_coeff != 0.0 + ): + raise ValueError( + "MoE QB requires sigmoid, ungrouped routing, use_pre_softmax=False " + "and aux_loss_coeff=0." + ) + _validate_qb_router( + jax.ShapeDtypeStruct((x.shape[0] * x.shape[1], num_experts), jnp.float32), + router_info.expert_bias, + router_info.qb_bin_bounds, + None, + router_info.num_experts_per_tok, + router_info.qb_num_bins, + router_info.scaling_factor, + router_info.qb_histogram_mode, + ) elif isinstance(router_info, RoutingMapInfo): routing_indices = router_info.routing_indices routing_weights = router_info.routing_weights @@ -2367,7 +2434,7 @@ def moe( ) use_cudnn_jax_fusion = _select_cudnn_jax_fusion(rejection_reasons) - output, aux_loss, total_recv_tokens, expert_counts = _moe( + output, aux_loss, total_recv_tokens, expert_counts, qb_histograms = _moe( x, router_info, wi, @@ -2400,4 +2467,6 @@ def moe( assert ( output.dtype == x.dtype ), f"moe() output dtype {output.dtype} != input dtype {x.dtype}" + if isinstance(router_info, RouterComputationInfo) and router_info.qb_bin_bounds is not None: + return output, aux_loss, total_recv_tokens, expert_counts, qb_histograms return output, aux_loss, total_recv_tokens, expert_counts diff --git a/transformer_engine/jax/router.py b/transformer_engine/jax/router.py index 0778155e143..1a7cd2f0beb 100644 --- a/transformer_engine/jax/router.py +++ b/transformer_engine/jax/router.py @@ -17,9 +17,14 @@ fused_moe_aux_loss: Compute the MoE auxiliary load-balancing loss scalar. + + fused_topk_with_score_function_qb: + Sigmoid Top-(k+1) routing with functional QB histogram accumulation. """ from functools import partial +import math +import operator from typing import Optional, Tuple, Union import jax @@ -30,6 +35,7 @@ RoutingMapFormat, fused_topk_with_score_function_fwd, fused_topk_with_score_function_bwd, + fused_topk_with_score_function_qb_fwd, fused_moe_aux_loss_fwd, fused_moe_aux_loss_bwd, ) @@ -38,6 +44,7 @@ "ScoreFunction", "RoutingMapFormat", "fused_topk_with_score_function", + "fused_topk_with_score_function_qb", "fused_moe_aux_loss", ] @@ -292,6 +299,162 @@ def _fused_topk_with_score_function_bwd( ) +# ============================================================================= +# Quantile Balancing router +# ============================================================================= + + +def _validate_qb_router( + logits, expert_bias, bin_bounds, histogram, topk, num_bins, scaling_factor, histogram_mode +): + if logits.ndim != 2 or min(logits.shape) <= 0: + raise ValueError("QB logits must have shape [tokens, experts] with positive dimensions.") + if logits.dtype != jnp.float32: + raise TypeError("QB logits must be float32.") + if expert_bias is None or expert_bias.shape != (logits.shape[-1],): + raise ValueError("QB expert_bias must have shape [experts].") + if expert_bias.dtype != jnp.float32: + raise TypeError("QB expert_bias must be float32.") + if bin_bounds is None or bin_bounds.shape != (2,) or bin_bounds.dtype != jnp.float32: + raise ValueError("QB bin_bounds must be float32 with shape [2].") + topk, num_bins = operator.index(topk), operator.index(num_bins) + if not 0 < topk < logits.shape[-1]: + raise ValueError("QB requires 0 < topk < num_experts.") + if num_bins <= 0: + raise ValueError("QB num_bins must be positive.") + if histogram is not None: + if histogram.shape != (logits.shape[-1], num_bins) or histogram.dtype != jnp.int32: + raise ValueError("QB histogram must be int32 with shape [experts, num_bins].") + if not isinstance(scaling_factor, (int, float)) or not math.isfinite(scaling_factor): + raise ValueError("QB scaling_factor must be a finite Python number.") + modes = {"two_kernel": 0, "fused_atomic": 1} + if histogram_mode not in modes: + raise ValueError("QB histogram_mode must be 'two_kernel' or 'fused_atomic'.") + return modes[histogram_mode] + + +def fused_topk_with_score_function_qb( + logits: jnp.ndarray, + topk: int, + expert_bias: jnp.ndarray, + bin_bounds: jnp.ndarray, + *, + num_bins: int = 1000, + histogram: Optional[jnp.ndarray] = None, + scaling_factor: float = 1.0, + histogram_mode: str = "fused_atomic", + routing_map_format: Union[str, RoutingMapFormat, int] = RoutingMapFormat.BYTEMAP, +) -> Tuple[jnp.ndarray, jnp.ndarray, jnp.ndarray]: + """Route with sigmoid Top-(k+1) and collect Quantile Balancing statistics. + + ``logits`` is FP32 ``[tokens, experts]``; ``expert_bias`` is FP32 ``[experts]``. + Selection uses biased scores, while the returned sparse weights use unbiased + sigmoid scores. The extra selected score is the cutoff for QB, not another route. + + ``bin_bounds`` is FP32 ``[lower, upper]``. The caller must supply finite values + with ``lower < upper``; bounds values are not synchronized to the host for + validation. Kimi uses ``[min(bias)-1, max(bias)+1]`` throughout a training step. + Each row contributes to every expert's histogram of ``cutoff - sigmoid(logits)``. + All input rows must be valid tokens: this kernel has no padding mask. + + Returns ``(probs, routing_map, updated_histogram)``. The histogram is int32 + ``[experts, num_bins]`` and starts at zero when ``histogram=None``. Otherwise + it functionally adds to the supplied histogram, with FFI input/output aliasing + allowing buffer reuse. Only ``probs`` has a gradient, through ``logits``. + + This is a shard-local operation. Use ``shard_map`` with the full expert axis + replicated to retain local histograms in SPMD; sum contributions across token + shards and microbatches before deriving the next bias. ``two_kernel`` uses a + separate histogram kernel, while ``fused_atomic`` accumulates in the router. + Neither mode updates biases or performs collectives. Grouped Top-k and auxiliary + scores are not supported. Automatic SPMD without ``shard_map`` may replicate + the call rather than distribute token rows. + """ + mode = _validate_qb_router( + logits, expert_bias, bin_bounds, histogram, topk, num_bins, scaling_factor, histogram_mode + ) + routing_map_format = _validate_routing_map_format(routing_map_format) + if histogram is None: + histogram = jnp.zeros((logits.shape[-1], num_bins), dtype=jnp.int32) + return _fused_topk_with_score_function_qb( + logits, + expert_bias, + bin_bounds, + histogram, + int(topk), + float(scaling_factor), + mode, + int(routing_map_format), + ) + + +@partial(jax.custom_vjp, nondiff_argnums=(4, 5, 6, 7)) +def _fused_topk_with_score_function_qb( + logits, + expert_bias, + bin_bounds, + histogram, + topk, + scaling_factor, + histogram_mode, + routing_map_format, +): + result, _ = _qb_router_fwd( + logits, + expert_bias, + bin_bounds, + histogram, + topk, + scaling_factor, + histogram_mode, + routing_map_format, + ) + return result + + +def _qb_router_fwd( + logits, + expert_bias, + bin_bounds, + histogram, + topk, + scaling_factor, + histogram_mode, + routing_map_format, +): + probs, routing_map, saved_scores, _, updated_histogram = fused_topk_with_score_function_qb_fwd( + logits, + expert_bias, + bin_bounds, + histogram, + topk=topk, + scaling_factor=scaling_factor, + histogram_mode=histogram_mode, + routing_map_format=routing_map_format, + ) + return (probs, routing_map, updated_histogram), (routing_map, saved_scores) + + +def _qb_router_bwd(topk, scaling_factor, histogram_mode, routing_map_format, residuals, cotangents): + del histogram_mode + routing_map, saved_scores = residuals + d_logits = fused_topk_with_score_function_bwd( + routing_map, + saved_scores, + cotangents[0], + topk, + False, + scaling_factor, + ScoreFunction.SIGMOID, + False, + routing_map_format, + ) + return d_logits, None, None, None + + +_fused_topk_with_score_function_qb.defvjp(_qb_router_fwd, _qb_router_bwd) + + # ============================================================================= # Fused MoE Aux Loss # ============================================================================= From e351c1bbd3b82eab330ac6399407ef488387babc Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Tue, 10 Nov 2026 11:33:16 +0100 Subject: [PATCH 6/7] [JAX] Exclude padding from QB histogram statistics Keep CUDA routing and cutoff generation while building masked histograms from saved scores in JAX. Preserve the existing CUDA histogram fast path without a mask. Runtime validation remains deferred. [skip ci] Signed-off-by: Pawel Gadzinski --- docs/api/jax.rst | 7 ++-- tests/jax/test_fused_router.py | 32 +++++++++++++++++++ tests/jax/test_te_ep_moe.py | 17 ++++++++-- .../jax/cpp_extensions/router.py | 14 +++++++- .../jax/csrc/extensions/router.cpp | 8 +++-- transformer_engine/jax/flax/moe.py | 5 ++- transformer_engine/jax/moe.py | 29 ++++++++++++++--- transformer_engine/jax/router.py | 18 +++++++++-- 8 files changed, 113 insertions(+), 17 deletions(-) diff --git a/docs/api/jax.rst b/docs/api/jax.rst index 90bed3c33ef..5b554d18fd3 100644 --- a/docs/api/jax.rst +++ b/docs/api/jax.rst @@ -114,8 +114,11 @@ QB kernels provide ``fused_atomic`` and ``two_kernel`` modes, selected by The bounds must be finite and ordered; their values are read on device without host validation. Kimi uses ``[min(bias)-1, max(bias)+1]``. Keep both bias and -bounds unchanged throughout every microbatch in a training step. All token rows -must be valid: neither mode excludes sequence-padding tokens. +bounds unchanged throughout every microbatch in a training step. With no mask, all token rows must be valid. Optional ``qb_token_mask`` +(``[batch, sequence]``) excludes padding: CUDA computes scores and cutoff, +while a JAX histogram counts only valid rows. This masked path does not use +CUDA histogram accumulation. The standalone router accepts ``token_mask`` +with shape ``[tokens]`` for the same behavior. With QB enabled, ``moe`` returns a fifth result, an int32 histogram array of shape ``[token_shards, experts, bins]``. Its leading axis is sharded across the diff --git a/tests/jax/test_fused_router.py b/tests/jax/test_fused_router.py index 2a26c8a36fa..bd08bffe81d 100644 --- a/tests/jax/test_fused_router.py +++ b/tests/jax/test_fused_router.py @@ -254,6 +254,38 @@ def test_qb_tied_cutoff_counts_each_token_once(histogram_mode): np.testing.assert_array_equal(histogram, expected) +@pytest.mark.parametrize("histogram_mode", ["two_kernel", "fused_atomic"]) +@pytest.mark.parametrize("valid_rows", [0, 17, 64]) +def test_qb_masked_histogram_and_gradients(histogram_mode, valid_rows): + from transformer_engine.jax.router import fused_topk_with_score_function_qb + + logits = jax.random.normal(jax.random.PRNGKey(851), (64, 32)) + bias = jnp.linspace(-0.1, 0.1, 32) + bounds = jnp.asarray([-1.1, 1.1], jnp.float32) + mask = jnp.arange(64) < valid_rows + + def loss(x): + probs, _, counts = fused_topk_with_score_function_qb( + x, + 4, + bias, + bounds, + token_mask=mask, + num_bins=64, + histogram_mode=histogram_mode, + ) + return jnp.sum(probs * x), counts + + (_, counts), gradient = jax.jit(jax.value_and_grad(jax.checkpoint(loss), has_aux=True))(logits) + if valid_rows: + expected = reference_qb_histogram(logits[:valid_rows], bias, bounds, 4, 64) + else: + expected = np.zeros((32, 64), np.int32) + np.testing.assert_array_equal(counts, expected) + np.testing.assert_array_equal(np.asarray(counts).sum(axis=-1), np.full(32, valid_rows)) + assert np.isfinite(np.asarray(gradient)).all() + + @pytest.mark.parametrize( "overrides,match", [ diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index 5f99b058365..ddfed8e6fa0 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -864,7 +864,8 @@ def ref_loss(params, h, p): @pytest.mark.parametrize("histogram_mode", ["two_kernel", "fused_atomic"]) -def test_qb_moe_histograms_and_remat_gradients(mesh, make_block, histogram_mode): +@pytest.mark.parametrize("masked", [False, True]) +def test_qb_moe_histograms_and_remat_gradients(mesh, make_block, histogram_mode, masked): from test_fused_router import reference_qb_histogram def bias_init(key, shape, dtype): @@ -879,12 +880,19 @@ def bias_init(key, shape, dtype): bounds = jnp.asarray([-1.1, 1.1], jnp.float32) x = jax.random.normal(jax.random.PRNGKey(841), (BATCH, SEQ, HIDDEN), DTYPE) + token_mask = (jnp.arange(BATCH * SEQ).reshape(BATCH, SEQ) % 2 == 0) if masked else None + with _ctx(mesh): x_sharded = _shard_inputs(x, mesh) variables = jax.jit(block.init)(jax.random.PRNGKey(842), x_sharded) def loss(v, activations, limits): - result = block.apply(v, activations, qb_bin_bounds=limits) + result = block.apply( + v, + activations, + qb_bin_bounds=limits, + qb_token_mask=token_mask if limits is not None else None, + ) return jnp.mean(result[0].astype(jnp.float32) ** 2), result (_, baseline), (baseline_dv, baseline_dx) = jax.jit( @@ -907,8 +915,11 @@ def loss(v, activations, limits): assert histogram_parts.shape == (mesh.size, NUM_EXPERTS, 64) params = _params_global_numpy(variables, mesh) logits = (x @ jnp.asarray(params["gate_kernel"], DTYPE)).astype(jnp.float32) + histogram_logits = logits.reshape(-1, NUM_EXPERTS) + if token_mask is not None: + histogram_logits = histogram_logits[np.asarray(token_mask).reshape(-1)] expected_histogram = reference_qb_histogram( - logits.reshape(-1, NUM_EXPERTS), + histogram_logits, params["expert_bias"], bounds, TOPK, diff --git a/transformer_engine/jax/cpp_extensions/router.py b/transformer_engine/jax/cpp_extensions/router.py index c97e46cdf97..32a0d7c6725 100644 --- a/transformer_engine/jax/cpp_extensions/router.py +++ b/transformer_engine/jax/cpp_extensions/router.py @@ -743,6 +743,7 @@ def fused_topk_with_score_function_qb_fwd( scaling_factor, histogram_mode, routing_map_format=int(RoutingMapFormat.BYTEMAP), + token_mask=None, ): """Shard-local QB forward; histogram input aliases the updated output.""" num_tokens, num_experts = logits.shape @@ -761,7 +762,7 @@ def fused_topk_with_score_function_qb_fwd( ShapeDtypeStruct((num_tokens,), jnp.float32), ShapeDtypeStruct(histogram.shape, jnp.int32), ) - return ffi.ffi_call( + result = ffi.ffi_call( "te_fused_topk_with_score_function_qb_forward_ffi", outputs, input_output_aliases={3: 4}, @@ -774,8 +775,19 @@ def fused_topk_with_score_function_qb_fwd( topk=int(topk), scaling_factor=float(scaling_factor), histogram_mode=int(histogram_mode), + collect_histogram=int(token_mask is None), routing_map_format=int(routing_map_format), ) + if token_mask is not None: + probs, routing_map, scores, cutoff, updated_histogram = result + lower, upper = bin_bounds[0], bin_bounds[1] + num_bins = histogram.shape[1] + bins = jnp.floor((cutoff[:, None] - scores - lower) * (num_bins / (upper - lower))) + bins = jnp.clip(bins.astype(jnp.int32), 0, num_bins - 1) + counts = token_mask.astype(jnp.int32)[:, None] + updated_histogram = updated_histogram.at[jnp.arange(num_experts)[None, :], bins].add(counts) + result = probs, routing_map, scores, cutoff, updated_histogram + return result def fused_topk_with_score_function_bwd( diff --git a/transformer_engine/jax/csrc/extensions/router.cpp b/transformer_engine/jax/csrc/extensions/router.cpp index 9200afb6a14..3916ed1d0c5 100644 --- a/transformer_engine/jax/csrc/extensions/router.cpp +++ b/transformer_engine/jax/csrc/extensions/router.cpp @@ -107,7 +107,7 @@ Error_Type FusedTopkWithScoreFunctionQBForwardFFI( Buffer_Type bin_bounds_buf, Buffer_Type histogram_buf, Result_Type probs_buf, Result_Type routing_map_buf, Result_Type intermediate_buf, Result_Type cutoff_buf, Result_Type histogram_out_buf, int64_t topk, double scaling_factor, int64_t histogram_mode, - JAXX_Routing_Map_Format routing_map_format) { + int64_t collect_histogram, JAXX_Routing_Map_Format routing_map_format) { auto dims = logits_buf.dimensions(); auto bias_dims = expert_bias_buf.dimensions(); auto bounds_dims = bin_bounds_buf.dimensions(); @@ -158,12 +158,13 @@ Error_Type FusedTopkWithScoreFunctionQBForwardFFI( histogram_out_buf->untyped_data(), std::vector{static_cast(num_experts), static_cast(hist_dims[1])}, DType::kInt32); - const auto mode = static_cast(histogram_mode); + const auto mode = collect_histogram ? static_cast(histogram_mode) + : NVTE_QB_HISTOGRAM_TWO_KERNEL; nvte_fused_topk_with_score_function_forward_qb_v2( logits.data(), num_tokens, num_experts, static_cast(topk), static_cast(scaling_factor), bias.data(), probs.data(), routing_map.data(), format, intermediate.data(), cutoff.data(), histogram.data(), bounds.data(), mode, stream); - if (mode == NVTE_QB_HISTOGRAM_TWO_KERNEL) { + if (collect_histogram && mode == NVTE_QB_HISTOGRAM_TWO_KERNEL) { nvte_qb_histogram_accumulate(intermediate.data(), cutoff.data(), bounds.data(), histogram.data(), stream); } @@ -186,6 +187,7 @@ XLA_FFI_DEFINE_HANDLER_SYMBOL(FusedTopkWithScoreFunctionQBForwardHandler, .Attr("topk") .Attr("scaling_factor") .Attr("histogram_mode") + .Attr("collect_histogram") .Attr("routing_map_format"), FFI_CudaGraph_Traits); diff --git a/transformer_engine/jax/flax/moe.py b/transformer_engine/jax/flax/moe.py index 77b761326d4..283902756ef 100644 --- a/transformer_engine/jax/flax/moe.py +++ b/transformer_engine/jax/flax/moe.py @@ -205,6 +205,7 @@ def __call__( *, router_input: Optional[Array] = None, qb_bin_bounds: Optional[Array] = None, + qb_token_mask: Optional[Array] = None, ) -> tuple: """Run the MoE forward pass. @@ -219,7 +220,8 @@ def __call__( qb_bin_bounds : Optional[jnp.ndarray] FP32 ``[lower, upper]`` enables QB histogram collection. Requires sigmoid routing, expert bias, no groups and zero auxiliary loss. - All input rows must be valid tokens; padding is not masked. + qb_token_mask : Optional[jnp.ndarray] + Boolean ``[batch, sequence]`` mask excluding padding from QB statistics. Returns ------- @@ -355,6 +357,7 @@ def make_grouped_quantizer_set(postfix): router_input=router_input, router_input_axes=self.router_input_axes, qb_bin_bounds=qb_bin_bounds, + qb_token_mask=qb_token_mask, qb_num_bins=self.qb_num_bins, qb_histogram_mode=self.qb_histogram_mode, ) diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index 67036599554..44d37a3f3e1 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -161,8 +161,11 @@ class RouterComputationInfo: follows the same DP/FSDP/EP placement as ``x``. qb_bin_bounds : Optional[jnp.ndarray] FP32 ``[lower, upper]`` enables QB statistics with sigmoid, no groups and - no auxiliary loss. All token rows must be valid; padding is not masked. + no auxiliary loss. Set ``qb_token_mask`` to exclude sequence padding. Bounds must be finite and ordered, and remain fixed throughout a step. + qb_token_mask : Optional[jnp.ndarray] + Boolean ``[batch, sequence]`` valid-token mask. With a mask, JAX builds + the histogram from CUDA router scores/cutoffs; without it CUDA collects it. qb_num_bins, qb_histogram_mode : int, str Histogram size and ``"fused_atomic"`` or ``"two_kernel"`` implementation. The caller owns histogram reduction and bias updates. @@ -183,6 +186,7 @@ class RouterComputationInfo: router_input: Optional[jnp.ndarray] = None router_input_axes: Tuple[Optional[str], ...] = flax.struct.field(pytree_node=False, default=()) qb_bin_bounds: Optional[jnp.ndarray] = None + qb_token_mask: Optional[jnp.ndarray] = None qb_num_bins: int = flax.struct.field(pytree_node=False, default=1000) qb_histogram_mode: str = flax.struct.field(pytree_node=False, default="fused_atomic") @@ -1553,7 +1557,7 @@ def _moe_fwd_rule( else: mode = {"two_kernel": 0, "fused_atomic": 1}[router.qb_histogram_mode] - def route_and_observe(local_logits, bias, bounds): + def route_and_observe(local_logits, bias, bounds, *mask): histogram = jnp.zeros((num_experts, router.qb_num_bins), dtype=jnp.int32) probs, rmap, scores, _, histogram = tex.fused_topk_with_score_function_qb_fwd( local_logits, @@ -1563,16 +1567,27 @@ def route_and_observe(local_logits, bias, bounds): topk=K, scaling_factor=router.scaling_factor, histogram_mode=mode, + token_mask=mask[0] if mask else None, ) return probs, rmap, scores, histogram[None] sparse_probs, routing_map, saved_scores, qb_histograms = shard_map( route_and_observe, mesh=mesh, - in_specs=(ep2_spec, P(None), P(None)), + in_specs=(ep2_spec, P(None), P(None)) + + ((P(batch_pspec_axis),) if router.qb_token_mask is not None else ()), out_specs=(ep2_spec, ep2_spec, ep2_spec, P(batch_pspec_axis, None, None)), check_rep=False, - )(logits_2d, router.expert_bias, router.qb_bin_bounds) + )( + logits_2d, + router.expert_bias, + router.qb_bin_bounds, + *( + (router.qb_token_mask.reshape(-1),) + if router.qb_token_mask is not None + else () + ), + ) sparse_probs = sparse_probs.astype(dtype) # argsort on a bool tensor places True last (False=0 < True=1), so the # last K indices are the selected expert IDs. @@ -2069,6 +2084,7 @@ def _ffn_bwd_body(*args): qb_bin_bounds=( None if router.qb_bin_bounds is None else jnp.zeros_like(router.qb_bin_bounds) ), + qb_token_mask=None, expert_bias=( None if router.expert_bias is None else jnp.zeros_like(router.expert_bias) ), @@ -2356,6 +2372,11 @@ def moe( ) has_aux_loss = router_info.aux_loss_coeff > 0.0 if router_info.qb_bin_bounds is not None: + if router_info.qb_token_mask is not None and ( + router_info.qb_token_mask.shape != x.shape[:2] + or router_info.qb_token_mask.dtype != jnp.bool_ + ): + raise ValueError("MoE qb_token_mask must be bool with shape [batch, sequence].") if ( router_info.score_function != ScoreFunction.SIGMOID or router_info.use_pre_softmax diff --git a/transformer_engine/jax/router.py b/transformer_engine/jax/router.py index 1a7cd2f0beb..e47a21cb3a3 100644 --- a/transformer_engine/jax/router.py +++ b/transformer_engine/jax/router.py @@ -341,6 +341,7 @@ def fused_topk_with_score_function_qb( *, num_bins: int = 1000, histogram: Optional[jnp.ndarray] = None, + token_mask: Optional[jnp.ndarray] = None, scaling_factor: float = 1.0, histogram_mode: str = "fused_atomic", routing_map_format: Union[str, RoutingMapFormat, int] = RoutingMapFormat.BYTEMAP, @@ -355,7 +356,9 @@ def fused_topk_with_score_function_qb( with ``lower < upper``; bounds values are not synchronized to the host for validation. Kimi uses ``[min(bias)-1, max(bias)+1]`` throughout a training step. Each row contributes to every expert's histogram of ``cutoff - sigmoid(logits)``. - All input rows must be valid tokens: this kernel has no padding mask. + ``token_mask`` optionally selects valid rows. With a mask, CUDA computes the + routing and cutoff and JAX builds the masked histogram from the saved scores. + Without a mask, the existing CUDA histogram modes run unchanged. Returns ``(probs, routing_map, updated_histogram)``. The histogram is int32 ``[experts, num_bins]`` and starts at zero when ``histogram=None``. Otherwise @@ -374,6 +377,10 @@ def fused_topk_with_score_function_qb( logits, expert_bias, bin_bounds, histogram, topk, num_bins, scaling_factor, histogram_mode ) routing_map_format = _validate_routing_map_format(routing_map_format) + if token_mask is not None and ( + token_mask.shape != (logits.shape[0],) or token_mask.dtype != jnp.bool_ + ): + raise ValueError("QB token_mask must be bool with shape [tokens].") if histogram is None: histogram = jnp.zeros((logits.shape[-1], num_bins), dtype=jnp.int32) return _fused_topk_with_score_function_qb( @@ -381,6 +388,7 @@ def fused_topk_with_score_function_qb( expert_bias, bin_bounds, histogram, + token_mask, int(topk), float(scaling_factor), mode, @@ -388,12 +396,13 @@ def fused_topk_with_score_function_qb( ) -@partial(jax.custom_vjp, nondiff_argnums=(4, 5, 6, 7)) +@partial(jax.custom_vjp, nondiff_argnums=(5, 6, 7, 8)) def _fused_topk_with_score_function_qb( logits, expert_bias, bin_bounds, histogram, + token_mask, topk, scaling_factor, histogram_mode, @@ -404,6 +413,7 @@ def _fused_topk_with_score_function_qb( expert_bias, bin_bounds, histogram, + token_mask, topk, scaling_factor, histogram_mode, @@ -417,6 +427,7 @@ def _qb_router_fwd( expert_bias, bin_bounds, histogram, + token_mask, topk, scaling_factor, histogram_mode, @@ -431,6 +442,7 @@ def _qb_router_fwd( scaling_factor=scaling_factor, histogram_mode=histogram_mode, routing_map_format=routing_map_format, + token_mask=token_mask, ) return (probs, routing_map, updated_histogram), (routing_map, saved_scores) @@ -449,7 +461,7 @@ def _qb_router_bwd(topk, scaling_factor, histogram_mode, routing_map_format, res False, routing_map_format, ) - return d_logits, None, None, None + return d_logits, None, None, None, None _fused_topk_with_score_function_qb.defvjp(_qb_router_fwd, _qb_router_bwd) From 1c974178859ab2e23e325b9d9921469955fadad6 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 12 Nov 2026 14:05:40 +0100 Subject: [PATCH 7/7] [JAX] Update MoE API test for the QB return contract The internal MoE custom VJP always returns a fifth histogram result, even when QB is disabled. Keep the API test stub consistent while retaining the public four-result contract without QB. Signed-off-by: Pawel Gadzinski --- tests/jax/moe/test_moe_resource_api.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index 4cee3a9804e..dbed5d96adf 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -251,7 +251,13 @@ def test_public_api_uses_selected_resource(monkeypatch, quantize): def fake_vjp(*args): captured.update(signature.bind(*args).arguments) assert global_mesh_resource().ep_resource == "ep" - return args[0], None, jnp.zeros((1,), jnp.int32), jnp.zeros((2,), jnp.int32) + return ( + args[0], + None, + jnp.zeros((1,), jnp.int32), + jnp.zeros((2,), jnp.int32), + jnp.zeros((0, 0, 0), jnp.int32), + ) monkeypatch.setattr(module, "_moe", fake_vjp) mesh = Mesh(np.asarray(jax.devices()[:1]).reshape(1, 1, 1), ("dp", "fsdp", "ep"))