Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/api/jax.rst
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,10 @@ Router, routing kernels, and grouped dense for MoE layers. See

.. autoapifunction:: transformer_engine.jax.moe.moe

.. autoapiclass:: transformer_engine.jax.moe.RouterComputationInfo

.. autoapiclass:: transformer_engine.jax.moe.RoutingMapInfo

.. autoapifunction:: transformer_engine.jax.ep.ep_bootstrap

.. autoapifunction:: transformer_engine.jax.ep.ep_finalize
Expand Down
33 changes: 33 additions & 0 deletions tests/jax/moe/test_moe_resource_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,39 @@ def test_resource_required_for_function_and_block():
_MoEBlock().init(jax.random.PRNGKey(0), jnp.ones((1, 1, 4)))


@pytest.mark.parametrize("shape", [(1, 4), (2, 1, 8), (1, 2, 8)])
def test_router_input_shape_is_validated(shape):
module = importlib.import_module("transformer_engine.jax.moe")
router = RouterComputationInfo(
gate_kernel=jnp.ones((8, 2)), num_experts_per_tok=1, router_input=jnp.ones(shape)
)
with pytest.raises(ValueError, match="router_input must have shape"):
module.moe(
jnp.ones((1, 1, 4)),
router,
None,
None,
num_experts=2,
mesh_resource=MeshResource(ep_resource="ep"),
)


def test_gate_kernel_uses_router_width():
module = importlib.import_module("transformer_engine.jax.moe")
router = RouterComputationInfo(
gate_kernel=jnp.ones((4, 2)), num_experts_per_tok=1, router_input=jnp.ones((1, 1, 8))
)
with pytest.raises(ValueError, match="gate_kernel must have shape"):
module.moe(
jnp.ones((1, 1, 4)),
router,
None,
None,
num_experts=2,
mesh_resource=MeshResource(ep_resource="ep"),
)


def test_explicit_resource_overrides_global_and_is_snapshotted():
explicit = MeshResource(dp_resource="dp", fsdp_resource="fsdp", ep_resource="ep")
enclosing = MeshResource(ep_resource="other")
Expand Down
92 changes: 90 additions & 2 deletions tests/jax/test_te_ep_moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,7 @@ def _pure_jax_moe_reference(
expert_bias=None,
hash_input_ids=None,
hash_tid2eid=None,
router_input=None,
*,
num_experts,
num_experts_per_tok,
Expand All @@ -304,8 +305,9 @@ def _pure_jax_moe_reference(
K = num_experts_per_tok
x_2d = x.reshape(T, H)

gate_kernel_cast = gate_kernel.astype(x.dtype)
logits = (x_2d @ gate_kernel_cast).astype(jnp.float32) # [T, E]
gate_input = x if router_input is None else router_input
gate_kernel_cast = gate_kernel.astype(gate_input.dtype)
logits = (gate_input.reshape(T, -1) @ gate_kernel_cast).astype(jnp.float32) # [T, E]

if hash_input_ids is not None:
if score_function != "sqrtsoftplus" or hash_tid2eid is None:
Expand Down Expand Up @@ -724,6 +726,92 @@ def loss_fn(variables, inputs):
)


@pytest.mark.parametrize("quantization", _QUANTIZATION_CASES)
@pytest.mark.parametrize("early_weighting", [False, True])
@pytest.mark.parametrize("router_dtype", [DTYPE, jnp.float32])
def test_separate_router_input_latent_projection(
mesh, quantization, early_weighting, router_dtype, make_block
):
block = make_block(
score_function="sigmoid",
use_expert_routing_bias=True,
expert_bias_init=_strong_expert_bias_init,
aux_loss_coeff=1e-2,
apply_topk_weights_early=early_weighting,
quantization_recipe=_quantization_recipe(quantization),
)
hidden = jax.random.normal(jax.random.PRNGKey(71), (BATCH, SEQ, 2 * HIDDEN), router_dtype)
projection = jax.random.normal(jax.random.PRNGKey(72), (2 * HIDDEN, HIDDEN), DTYPE)
projection = projection / jnp.sqrt(jnp.asarray(2 * HIDDEN, DTYPE))
with _ctx(mesh):
hidden_sh = _shard_inputs(hidden, mesh)
variables = jax.jit(block.init)(
jax.random.PRNGKey(73), (hidden_sh @ projection).astype(DTYPE), router_input=hidden_sh
)

def loss_fn(v, h, p):
output, aux, _, _ = block.apply(v, (h @ p).astype(DTYPE), router_input=h)
return jnp.mean(output.astype(jnp.float32) ** 2) + aux

grads, dh, dp = jax.jit(jax.grad(jax.checkpoint(loss_fn), argnums=(0, 1, 2)))(
variables, hidden_sh, projection
)
output, aux, _, _ = jax.jit(block.apply)(
variables, (hidden_sh @ projection).astype(DTYPE), router_input=hidden_sh
)
params_np = _params_global_numpy(variables, mesh)
assert params_np["gate_kernel"].shape == (2 * HIDDEN, NUM_EXPERTS)
assert params_np["wi"].shape == (NUM_EXPERTS, HIDDEN, 2 * INTER)

def reference(params, h, p):
return _pure_jax_moe_reference(
(h @ p).astype(DTYPE),
params["gate_kernel"],
params["wi"],
params["wo"],
params["expert_bias"],
router_input=h,
num_experts=NUM_EXPERTS,
num_experts_per_tok=TOPK,
score_function="sigmoid",
aux_loss_coeff=1e-2,
)

def ref_loss(params, h, p):
out, aux_loss = reference(params, h, p)
return jnp.mean(out.astype(jnp.float32) ** 2) + aux_loss

ref_params = {k: jnp.asarray(v) for k, v in params_np.items()}
out_ref, aux_ref = reference(ref_params, hidden, projection)
grads_ref, dh_ref, dp_ref = jax.jit(jax.grad(ref_loss, argnums=(0, 1, 2)))(
ref_params, hidden, projection
)
np.testing.assert_allclose(
_to_global_numpy(output, mesh).astype(np.float32),
np.asarray(out_ref).astype(np.float32),
**FWD_TOLERANCE[quantization],
)
np.testing.assert_allclose(
_to_global_numpy(aux, mesh).astype(np.float32),
np.asarray(aux_ref).astype(np.float32),
**AUX_TOLERANCE,
)
for name in ("gate_kernel", "wi", "wo"):
tolerance = GRAD_GATE_TOLERANCE if name == "gate_kernel" else GRAD_FFN_TOLERANCE
np.testing.assert_allclose(
_to_global_numpy(_unwrap(grads["params"][name]), mesh).astype(np.float32),
np.asarray(grads_ref[name]).astype(np.float32),
**tolerance[quantization],
)
for actual, expected, primal in ((dh, dh_ref, hidden), (dp, dp_ref, projection)):
assert actual.shape == primal.shape and actual.dtype == primal.dtype
np.testing.assert_allclose(
_to_global_numpy(actual, mesh).astype(np.float32),
np.asarray(expected).astype(np.float32),
**GRAD_FFN_TOLERANCE[quantization],
)


def _reference_kwargs_from_config(config, params_np):
"""Pick out the reference-relevant pieces of a parametrize config."""
return dict(
Expand Down
20 changes: 17 additions & 3 deletions transformer_engine/jax/flax/moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ class _MoEBlock(TransformerEngineBase):
If ``> 0``, return the MoE auxiliary load-balancing loss scalar
in addition to the main output.

gate_kernel_axes, wi_kernel_axes, wo_kernel_axes, input_axes :
gate_kernel_axes, wi_kernel_axes, wo_kernel_axes, input_axes, router_input_axes :
Logical sharding axis tuples (consumed by Flax's
:func:`with_logical_partitioning` and our internal
:func:`with_sharding_constraint_by_logical_axes`).
Expand Down Expand Up @@ -156,6 +156,7 @@ class _MoEBlock(TransformerEngineBase):
wi_kernel_axes: Tuple[Optional[str], ...] = ("exp", "embed", "mlp")
wo_kernel_axes: Tuple[Optional[str], ...] = ("exp", "mlp", "embed")
input_axes: Tuple[Optional[str], ...] = ()
router_input_axes: Tuple[Optional[str], ...] = ()

# Parallelism
mesh_resource: Optional[MeshResource] = None
Expand Down Expand Up @@ -188,13 +189,19 @@ def __post_init__(self):
super().__post_init__()

@nn.compact
def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array]:
def __call__(
self, inputs: Array, *, router_input: Optional[Array] = None
) -> Tuple[Array, Optional[Array], Array, Array]:
"""Run the MoE forward pass.

Parameters
----------
inputs : jnp.ndarray
``[batch, sequence, hidden]``.
router_input : Optional[jnp.ndarray]
Separate gate input ``[batch, sequence, router_hidden]``. ``None``
uses ``inputs``. Only the gate kernel uses ``router_hidden``;
expert kernels and EP buffers retain ``hidden``.

Returns
-------
Expand All @@ -218,6 +225,11 @@ def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array]
inputs.ndim == 3
), f"_MoEBlock expects [batch, sequence, hidden] input, got shape {inputs.shape}"
_, _, hidden_size = inputs.shape
if router_input is not None and (
router_input.ndim != 3 or router_input.shape[:2] != inputs.shape[:2]
):
raise ValueError("router_input must match the batch and sequence dimensions of inputs.")
router_hidden_size = hidden_size if router_input is None else router_input.shape[-1]

# Param registrations -- must run OUTSIDE any JAX transform that
# alters the variable scope (e.g. shard_map). The functional
Expand All @@ -226,7 +238,7 @@ def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array]
gate_kernel = self.param(
"gate_kernel",
nn.with_logical_partitioning(self.kernel_init, self.gate_kernel_axes),
(hidden_size, self.num_experts),
(router_hidden_size, self.num_experts),
self.dtype,
)
# FC1 is stored as one gated-SwiGLU kernel. Keeping its two
Expand Down Expand Up @@ -318,6 +330,8 @@ def make_grouped_quantizer_set(postfix):
scaling_factor=self.scaling_factor,
aux_loss_coeff=self.aux_loss_coeff,
gate_kernel_axes=self.gate_kernel_axes,
router_input=router_input,
router_input_axes=self.router_input_axes,
)
return moe(
inputs,
Expand Down
63 changes: 56 additions & 7 deletions transformer_engine/jax/moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,8 @@ class RouterComputationInfo:
Parameters
----------
gate_kernel : jnp.ndarray
``[hidden, num_experts]`` router projection.
``[router_hidden, num_experts]`` router projection; ``router_hidden``
is the width of ``router_input``, or ``x`` when no separate input is given.
num_experts_per_tok : int
Top-k value for routing.
expert_bias : Optional[jnp.ndarray]
Expand All @@ -113,6 +114,15 @@ class RouterComputationInfo:
view; this lives off the dispatch critical path.
gate_kernel_axes : Tuple[Optional[str], ...]
Logical sharding axes of ``gate_kernel``.
router_input : Optional[jnp.ndarray]
Separate ``[batch, sequence, router_hidden]`` activations for the gate.
``None`` uses the expert input ``x``. Batch and sequence dimensions must
match ``x``; hidden dimensions may differ. Gradients flow to this input
independently of the expert-input gradient and retain its dtype.
The gate kernel is cast to this input's dtype for the projection.
router_input_axes : Tuple[Optional[str], ...]
Logical sharding axes of ``router_input``. Its physical token layout
follows the same DP/FSDP/EP placement as ``x``.
"""

gate_kernel: jnp.ndarray
Expand All @@ -127,6 +137,8 @@ class RouterComputationInfo:
scaling_factor: float = flax.struct.field(pytree_node=False, default=1.0)
aux_loss_coeff: float = flax.struct.field(pytree_node=False, default=0.0)
gate_kernel_axes: Tuple[Optional[str], ...] = flax.struct.field(pytree_node=False, default=())
router_input: Optional[jnp.ndarray] = None
router_input_axes: Tuple[Optional[str], ...] = flax.struct.field(pytree_node=False, default=())


@flax.struct.dataclass
Expand Down Expand Up @@ -1325,6 +1337,12 @@ def _moe_fwd_rule(
compute_aux_loss = router is not None and router.aux_loss_coeff > 0.0

x = with_sharding_constraint_by_logical_axes(x, input_axes)
if router is not None and router.router_input is not None:
router = router.replace(
router_input=with_sharding_constraint_by_logical_axes(
router.router_input, router.router_input_axes
)
)

mesh = _get_mesh()
if mesh is None or mesh.empty:
Expand Down Expand Up @@ -1451,8 +1469,10 @@ def _moe_fwd_rule(
# accumulate at low precision and silently produce NaNs on tokens
# whose normalised weights underflow. Cast to fp32 here to stay in
# the validated regime.
gate_kernel_cast = router.gate_kernel.astype(x.dtype)
gate_logits = jnp.einsum("bsh,he->bse", x, gate_kernel_cast)
gate_input = x if router.router_input is None else router.router_input
gate_input = jax.lax.with_sharding_constraint(gate_input, NamedSharding(mesh, ep3_spec))
gate_kernel_cast = router.gate_kernel.astype(gate_input.dtype)
gate_logits = jnp.einsum("bsh,he->bse", gate_input, gate_kernel_cast)
logits_2d = gate_logits.reshape(-1, num_experts).astype(jnp.float32)

# ---------------- Routing (global view) ----------------
Expand Down Expand Up @@ -1945,19 +1965,28 @@ def _ffn_bwd_body(*args):

# ---------------- Gate bwd (global view) ----------------
d_gate_logits = d_logits_2d.reshape(B, S, num_experts)
gate_kernel_cast = router.gate_kernel.astype(ctx.x.dtype)
d_x_from_gate = jnp.einsum("bse,he->bsh", d_gate_logits, gate_kernel_cast)
d_gate_kernel = jnp.einsum("bsh,bse->he", ctx.x, d_gate_logits).astype(
gate_input = ctx.x if router.router_input is None else router.router_input
gate_kernel_cast = router.gate_kernel.astype(gate_input.dtype)
d_gate_input = jnp.einsum("bse,he->bsh", d_gate_logits, gate_kernel_cast)
d_gate_kernel = jnp.einsum("bsh,bse->he", gate_input, d_gate_logits).astype(
router.gate_kernel.dtype
)
d_gate_kernel = with_sharding_constraint_by_logical_axes(
d_gate_kernel, router.gate_kernel_axes
)
d_x = d_x_from_gate + d_x_from_dispatch
if router.router_input is None:
d_x = d_gate_input + d_x_from_dispatch
d_router_input = None
else:
d_x = d_x_from_dispatch
d_router_input = with_sharding_constraint_by_logical_axes(
d_gate_input.astype(router.router_input.dtype), router.router_input_axes
)
# expert_bias has no learnable bwd path through fused_topk: the
# primitive's bwd returns None for the bias slot, so its cotangent is zero.
d_router_info = router.replace(
gate_kernel=d_gate_kernel,
router_input=d_router_input,
expert_bias=(
None if router.expert_bias is None else jnp.zeros_like(router.expert_bias)
),
Expand Down Expand Up @@ -2216,6 +2245,17 @@ def moe(
ep_axis, data_parallelism_axes = _moe_mesh_axes(mesh_resource)

if isinstance(router_info, RouterComputationInfo):
gate_input = x if router_info.router_input is None else router_info.router_input
if gate_input.ndim != 3 or gate_input.shape[:2] != x.shape[:2]:
raise ValueError(
"router_input must have shape [batch, sequence, router_hidden] matching x, "
f"got x={x.shape}, router_input={gate_input.shape}."
)
if router_info.gate_kernel.shape != (gate_input.shape[-1], num_experts):
raise ValueError(
"gate_kernel must have shape [router_hidden, num_experts], got "
f"{router_info.gate_kernel.shape} for router_input={gate_input.shape}."
)
router_info = router_info.replace(
score_function=_validate_score_function(router_info.score_function),
aux_loss_coeff=float(router_info.aux_loss_coeff),
Expand Down Expand Up @@ -2250,6 +2290,15 @@ def moe(
# so ep MUST be innermost in the partition spec. Soft re-pin: free if
# upstream already matches, single reshard otherwise.
x = _prepare_moe_input(x, ep_axis, data_parallelism_axes, caller="moe")
if isinstance(router_info, RouterComputationInfo) and router_info.router_input is not None:
router_info = router_info.replace(
router_input=_prepare_moe_input(
router_info.router_input,
ep_axis,
data_parallelism_axes,
caller="moe router_input",
)
)

cudnn_native_weight_layout = _resolve_cudnn_native_weight_layout(
wi, x.shape[-1], cudnn_native_weight_layout
Expand Down