diff --git a/docs/api/jax.rst b/docs/api/jax.rst index 99282be422c..a6a5fd07e93 100644 --- a/docs/api/jax.rst +++ b/docs/api/jax.rst @@ -76,6 +76,10 @@ Router, routing kernels, and grouped dense for MoE layers. See .. autoapifunction:: transformer_engine.jax.moe.moe +.. autoapiclass:: transformer_engine.jax.moe.RouterComputationInfo + +.. autoapiclass:: transformer_engine.jax.moe.RoutingMapInfo + .. autoapifunction:: transformer_engine.jax.ep.ep_bootstrap .. autoapifunction:: transformer_engine.jax.ep.ep_finalize diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index ab6c02932d6..661f424a013 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -36,6 +36,39 @@ def test_resource_required_for_function_and_block(): _MoEBlock().init(jax.random.PRNGKey(0), jnp.ones((1, 1, 4))) +@pytest.mark.parametrize("shape", [(1, 4), (2, 1, 8), (1, 2, 8)]) +def test_router_input_shape_is_validated(shape): + module = importlib.import_module("transformer_engine.jax.moe") + router = RouterComputationInfo( + gate_kernel=jnp.ones((8, 2)), num_experts_per_tok=1, router_input=jnp.ones(shape) + ) + with pytest.raises(ValueError, match="router_input must have shape"): + module.moe( + jnp.ones((1, 1, 4)), + router, + None, + None, + num_experts=2, + mesh_resource=MeshResource(ep_resource="ep"), + ) + + +def test_gate_kernel_uses_router_width(): + module = importlib.import_module("transformer_engine.jax.moe") + router = RouterComputationInfo( + gate_kernel=jnp.ones((4, 2)), num_experts_per_tok=1, router_input=jnp.ones((1, 1, 8)) + ) + with pytest.raises(ValueError, match="gate_kernel must have shape"): + module.moe( + jnp.ones((1, 1, 4)), + router, + None, + None, + num_experts=2, + mesh_resource=MeshResource(ep_resource="ep"), + ) + + def test_explicit_resource_overrides_global_and_is_snapshotted(): explicit = MeshResource(dp_resource="dp", fsdp_resource="fsdp", ep_resource="ep") enclosing = MeshResource(ep_resource="other") diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index ad5c897b8d5..7c8f19ccdc5 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -292,6 +292,7 @@ def _pure_jax_moe_reference( expert_bias=None, hash_input_ids=None, hash_tid2eid=None, + router_input=None, *, num_experts, num_experts_per_tok, @@ -304,8 +305,9 @@ def _pure_jax_moe_reference( K = num_experts_per_tok x_2d = x.reshape(T, H) - gate_kernel_cast = gate_kernel.astype(x.dtype) - logits = (x_2d @ gate_kernel_cast).astype(jnp.float32) # [T, E] + gate_input = x if router_input is None else router_input + gate_kernel_cast = gate_kernel.astype(gate_input.dtype) + logits = (gate_input.reshape(T, -1) @ gate_kernel_cast).astype(jnp.float32) # [T, E] if hash_input_ids is not None: if score_function != "sqrtsoftplus" or hash_tid2eid is None: @@ -724,6 +726,92 @@ def loss_fn(variables, inputs): ) +@pytest.mark.parametrize("quantization", _QUANTIZATION_CASES) +@pytest.mark.parametrize("early_weighting", [False, True]) +@pytest.mark.parametrize("router_dtype", [DTYPE, jnp.float32]) +def test_separate_router_input_latent_projection( + mesh, quantization, early_weighting, router_dtype, make_block +): + block = make_block( + score_function="sigmoid", + use_expert_routing_bias=True, + expert_bias_init=_strong_expert_bias_init, + aux_loss_coeff=1e-2, + apply_topk_weights_early=early_weighting, + quantization_recipe=_quantization_recipe(quantization), + ) + hidden = jax.random.normal(jax.random.PRNGKey(71), (BATCH, SEQ, 2 * HIDDEN), router_dtype) + projection = jax.random.normal(jax.random.PRNGKey(72), (2 * HIDDEN, HIDDEN), DTYPE) + projection = projection / jnp.sqrt(jnp.asarray(2 * HIDDEN, DTYPE)) + with _ctx(mesh): + hidden_sh = _shard_inputs(hidden, mesh) + variables = jax.jit(block.init)( + jax.random.PRNGKey(73), (hidden_sh @ projection).astype(DTYPE), router_input=hidden_sh + ) + + def loss_fn(v, h, p): + output, aux, _, _ = block.apply(v, (h @ p).astype(DTYPE), router_input=h) + return jnp.mean(output.astype(jnp.float32) ** 2) + aux + + grads, dh, dp = jax.jit(jax.grad(jax.checkpoint(loss_fn), argnums=(0, 1, 2)))( + variables, hidden_sh, projection + ) + output, aux, _, _ = jax.jit(block.apply)( + variables, (hidden_sh @ projection).astype(DTYPE), router_input=hidden_sh + ) + params_np = _params_global_numpy(variables, mesh) + assert params_np["gate_kernel"].shape == (2 * HIDDEN, NUM_EXPERTS) + assert params_np["wi"].shape == (NUM_EXPERTS, HIDDEN, 2 * INTER) + + def reference(params, h, p): + return _pure_jax_moe_reference( + (h @ p).astype(DTYPE), + params["gate_kernel"], + params["wi"], + params["wo"], + params["expert_bias"], + router_input=h, + num_experts=NUM_EXPERTS, + num_experts_per_tok=TOPK, + score_function="sigmoid", + aux_loss_coeff=1e-2, + ) + + def ref_loss(params, h, p): + out, aux_loss = reference(params, h, p) + return jnp.mean(out.astype(jnp.float32) ** 2) + aux_loss + + ref_params = {k: jnp.asarray(v) for k, v in params_np.items()} + out_ref, aux_ref = reference(ref_params, hidden, projection) + grads_ref, dh_ref, dp_ref = jax.jit(jax.grad(ref_loss, argnums=(0, 1, 2)))( + ref_params, hidden, projection + ) + np.testing.assert_allclose( + _to_global_numpy(output, mesh).astype(np.float32), + np.asarray(out_ref).astype(np.float32), + **FWD_TOLERANCE[quantization], + ) + np.testing.assert_allclose( + _to_global_numpy(aux, mesh).astype(np.float32), + np.asarray(aux_ref).astype(np.float32), + **AUX_TOLERANCE, + ) + for name in ("gate_kernel", "wi", "wo"): + tolerance = GRAD_GATE_TOLERANCE if name == "gate_kernel" else GRAD_FFN_TOLERANCE + np.testing.assert_allclose( + _to_global_numpy(_unwrap(grads["params"][name]), mesh).astype(np.float32), + np.asarray(grads_ref[name]).astype(np.float32), + **tolerance[quantization], + ) + for actual, expected, primal in ((dh, dh_ref, hidden), (dp, dp_ref, projection)): + assert actual.shape == primal.shape and actual.dtype == primal.dtype + np.testing.assert_allclose( + _to_global_numpy(actual, mesh).astype(np.float32), + np.asarray(expected).astype(np.float32), + **GRAD_FFN_TOLERANCE[quantization], + ) + + def _reference_kwargs_from_config(config, params_np): """Pick out the reference-relevant pieces of a parametrize config.""" return dict( diff --git a/transformer_engine/jax/flax/moe.py b/transformer_engine/jax/flax/moe.py index 19b1a41778f..4cd83c2bbfa 100644 --- a/transformer_engine/jax/flax/moe.py +++ b/transformer_engine/jax/flax/moe.py @@ -88,7 +88,7 @@ class _MoEBlock(TransformerEngineBase): If ``> 0``, return the MoE auxiliary load-balancing loss scalar in addition to the main output. - gate_kernel_axes, wi_kernel_axes, wo_kernel_axes, input_axes : + gate_kernel_axes, wi_kernel_axes, wo_kernel_axes, input_axes, router_input_axes : Logical sharding axis tuples (consumed by Flax's :func:`with_logical_partitioning` and our internal :func:`with_sharding_constraint_by_logical_axes`). @@ -156,6 +156,7 @@ class _MoEBlock(TransformerEngineBase): wi_kernel_axes: Tuple[Optional[str], ...] = ("exp", "embed", "mlp") wo_kernel_axes: Tuple[Optional[str], ...] = ("exp", "mlp", "embed") input_axes: Tuple[Optional[str], ...] = () + router_input_axes: Tuple[Optional[str], ...] = () # Parallelism mesh_resource: Optional[MeshResource] = None @@ -188,13 +189,19 @@ def __post_init__(self): super().__post_init__() @nn.compact - def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array]: + def __call__( + self, inputs: Array, *, router_input: Optional[Array] = None + ) -> Tuple[Array, Optional[Array], Array, Array]: """Run the MoE forward pass. Parameters ---------- inputs : jnp.ndarray ``[batch, sequence, hidden]``. + router_input : Optional[jnp.ndarray] + Separate gate input ``[batch, sequence, router_hidden]``. ``None`` + uses ``inputs``. Only the gate kernel uses ``router_hidden``; + expert kernels and EP buffers retain ``hidden``. Returns ------- @@ -218,6 +225,11 @@ def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array] inputs.ndim == 3 ), f"_MoEBlock expects [batch, sequence, hidden] input, got shape {inputs.shape}" _, _, hidden_size = inputs.shape + if router_input is not None and ( + router_input.ndim != 3 or router_input.shape[:2] != inputs.shape[:2] + ): + raise ValueError("router_input must match the batch and sequence dimensions of inputs.") + router_hidden_size = hidden_size if router_input is None else router_input.shape[-1] # Param registrations -- must run OUTSIDE any JAX transform that # alters the variable scope (e.g. shard_map). The functional @@ -226,7 +238,7 @@ def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array] gate_kernel = self.param( "gate_kernel", nn.with_logical_partitioning(self.kernel_init, self.gate_kernel_axes), - (hidden_size, self.num_experts), + (router_hidden_size, self.num_experts), self.dtype, ) # FC1 is stored as one gated-SwiGLU kernel. Keeping its two @@ -318,6 +330,8 @@ def make_grouped_quantizer_set(postfix): scaling_factor=self.scaling_factor, aux_loss_coeff=self.aux_loss_coeff, gate_kernel_axes=self.gate_kernel_axes, + router_input=router_input, + router_input_axes=self.router_input_axes, ) return moe( inputs, diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index 4e9bfe4aa8d..6de45f1a85e 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -88,7 +88,8 @@ class RouterComputationInfo: Parameters ---------- gate_kernel : jnp.ndarray - ``[hidden, num_experts]`` router projection. + ``[router_hidden, num_experts]`` router projection; ``router_hidden`` + is the width of ``router_input``, or ``x`` when no separate input is given. num_experts_per_tok : int Top-k value for routing. expert_bias : Optional[jnp.ndarray] @@ -113,6 +114,15 @@ class RouterComputationInfo: view; this lives off the dispatch critical path. gate_kernel_axes : Tuple[Optional[str], ...] Logical sharding axes of ``gate_kernel``. + router_input : Optional[jnp.ndarray] + Separate ``[batch, sequence, router_hidden]`` activations for the gate. + ``None`` uses the expert input ``x``. Batch and sequence dimensions must + match ``x``; hidden dimensions may differ. Gradients flow to this input + independently of the expert-input gradient and retain its dtype. + The gate kernel is cast to this input's dtype for the projection. + router_input_axes : Tuple[Optional[str], ...] + Logical sharding axes of ``router_input``. Its physical token layout + follows the same DP/FSDP/EP placement as ``x``. """ gate_kernel: jnp.ndarray @@ -127,6 +137,8 @@ class RouterComputationInfo: scaling_factor: float = flax.struct.field(pytree_node=False, default=1.0) aux_loss_coeff: float = flax.struct.field(pytree_node=False, default=0.0) gate_kernel_axes: Tuple[Optional[str], ...] = flax.struct.field(pytree_node=False, default=()) + router_input: Optional[jnp.ndarray] = None + router_input_axes: Tuple[Optional[str], ...] = flax.struct.field(pytree_node=False, default=()) @flax.struct.dataclass @@ -1325,6 +1337,12 @@ def _moe_fwd_rule( compute_aux_loss = router is not None and router.aux_loss_coeff > 0.0 x = with_sharding_constraint_by_logical_axes(x, input_axes) + if router is not None and router.router_input is not None: + router = router.replace( + router_input=with_sharding_constraint_by_logical_axes( + router.router_input, router.router_input_axes + ) + ) mesh = _get_mesh() if mesh is None or mesh.empty: @@ -1451,8 +1469,10 @@ def _moe_fwd_rule( # accumulate at low precision and silently produce NaNs on tokens # whose normalised weights underflow. Cast to fp32 here to stay in # the validated regime. - gate_kernel_cast = router.gate_kernel.astype(x.dtype) - gate_logits = jnp.einsum("bsh,he->bse", x, gate_kernel_cast) + gate_input = x if router.router_input is None else router.router_input + gate_input = jax.lax.with_sharding_constraint(gate_input, NamedSharding(mesh, ep3_spec)) + gate_kernel_cast = router.gate_kernel.astype(gate_input.dtype) + gate_logits = jnp.einsum("bsh,he->bse", gate_input, gate_kernel_cast) logits_2d = gate_logits.reshape(-1, num_experts).astype(jnp.float32) # ---------------- Routing (global view) ---------------- @@ -1945,19 +1965,28 @@ def _ffn_bwd_body(*args): # ---------------- Gate bwd (global view) ---------------- d_gate_logits = d_logits_2d.reshape(B, S, num_experts) - gate_kernel_cast = router.gate_kernel.astype(ctx.x.dtype) - d_x_from_gate = jnp.einsum("bse,he->bsh", d_gate_logits, gate_kernel_cast) - d_gate_kernel = jnp.einsum("bsh,bse->he", ctx.x, d_gate_logits).astype( + gate_input = ctx.x if router.router_input is None else router.router_input + gate_kernel_cast = router.gate_kernel.astype(gate_input.dtype) + d_gate_input = jnp.einsum("bse,he->bsh", d_gate_logits, gate_kernel_cast) + d_gate_kernel = jnp.einsum("bsh,bse->he", gate_input, d_gate_logits).astype( router.gate_kernel.dtype ) d_gate_kernel = with_sharding_constraint_by_logical_axes( d_gate_kernel, router.gate_kernel_axes ) - d_x = d_x_from_gate + d_x_from_dispatch + if router.router_input is None: + d_x = d_gate_input + d_x_from_dispatch + d_router_input = None + else: + d_x = d_x_from_dispatch + d_router_input = with_sharding_constraint_by_logical_axes( + d_gate_input.astype(router.router_input.dtype), router.router_input_axes + ) # expert_bias has no learnable bwd path through fused_topk: the # primitive's bwd returns None for the bias slot, so its cotangent is zero. d_router_info = router.replace( gate_kernel=d_gate_kernel, + router_input=d_router_input, expert_bias=( None if router.expert_bias is None else jnp.zeros_like(router.expert_bias) ), @@ -2216,6 +2245,17 @@ def moe( ep_axis, data_parallelism_axes = _moe_mesh_axes(mesh_resource) if isinstance(router_info, RouterComputationInfo): + gate_input = x if router_info.router_input is None else router_info.router_input + if gate_input.ndim != 3 or gate_input.shape[:2] != x.shape[:2]: + raise ValueError( + "router_input must have shape [batch, sequence, router_hidden] matching x, " + f"got x={x.shape}, router_input={gate_input.shape}." + ) + if router_info.gate_kernel.shape != (gate_input.shape[-1], num_experts): + raise ValueError( + "gate_kernel must have shape [router_hidden, num_experts], got " + f"{router_info.gate_kernel.shape} for router_input={gate_input.shape}." + ) router_info = router_info.replace( score_function=_validate_score_function(router_info.score_function), aux_loss_coeff=float(router_info.aux_loss_coeff), @@ -2250,6 +2290,15 @@ def moe( # so ep MUST be innermost in the partition spec. Soft re-pin: free if # upstream already matches, single reshard otherwise. x = _prepare_moe_input(x, ep_axis, data_parallelism_axes, caller="moe") + if isinstance(router_info, RouterComputationInfo) and router_info.router_input is not None: + router_info = router_info.replace( + router_input=_prepare_moe_input( + router_info.router_input, + ep_axis, + data_parallelism_axes, + caller="moe router_input", + ) + ) cudnn_native_weight_layout = _resolve_cudnn_native_weight_layout( wi, x.shape[-1], cudnn_native_weight_layout