diff --git a/docs/api/jax.rst b/docs/api/jax.rst index 99282be422c..5b554d18fd3 100644 --- a/docs/api/jax.rst +++ b/docs/api/jax.rst @@ -32,6 +32,7 @@ Modules .. autoapifunction:: transformer_engine.jax.fp8_autocast .. autoapifunction:: transformer_engine.jax.autocast .. autoapifunction:: transformer_engine.jax.update_collections +.. autoapifunction:: transformer_engine.jax.grouped_situglu .. autoapiclass:: transformer_engine.jax.flax.LayerNorm(epsilon=1e-6, layernorm_type='layernorm', **kwargs) @@ -68,14 +69,80 @@ Router, routing kernels, and grouped dense for MoE layers. See .. autoapifunction:: transformer_engine.jax.router.fused_topk_with_score_function +.. autoapifunction:: transformer_engine.jax.router.fused_topk_with_score_function_qb + .. autoapifunction:: transformer_engine.jax.router.fused_moe_aux_loss .. autoapifunction:: transformer_engine.jax.permutation.sort_chunks_by_index .. autoapifunction:: transformer_engine.jax.dense.grouped_dense +.. autoapiclass:: transformer_engine.jax.moe.SiTUActivation + +``SiTUActivation`` uses native grouped activation kernels between the expert +GEMMs, including their backward pass. Device group counts bound active rows; +unused receive capacity is zero-filled without evaluating the activation there. +FP32, FP16 and BF16 activation outputs are supported. MXFP8 expert execution +uses grouped quantization after the activation. The standalone +``grouped_situglu(x, group_sizes, beta1=4.0, beta2=25.0)`` function has the same +math and automatic differentiation support. It expects contiguous gate/up +halves in each row of ``x[capacity, 2H]``. Counts must be nonnegative device +int32 values with a sum no greater than capacity. Distributed expert callers +should pass local arrays and counts inside ``shard_map``. + .. autoapifunction:: transformer_engine.jax.moe.moe +.. autoapiclass:: transformer_engine.jax.moe.RouterComputationInfo + +Latent MoE models can pass full-width hidden states through +``RouterComputationInfo(router_input=hidden_states, ...)`` while calling +``moe(latent_states, router_info, ...)`` with narrower expert activations. +Batch and sequence dimensions must agree. Router parameters use the router +width; expert parameters and EP bootstrap buffers use the latent width. +The custom VJP returns gradients to both inputs, including when the latent +is a differentiable projection of the router input. The default +``router_input=None`` preserves routing on the expert input. + +Quantile Balancing statistics +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Set ``RouterComputationInfo.qb_bin_bounds`` to an FP32 ``[lower, upper]`` array +to collect QB statistics during MoE routing. Sigmoid routing, an FP32 expert +bias, no routing groups and zero auxiliary loss are required. The existing CUDA +QB kernels provide ``fused_atomic`` and ``two_kernel`` modes, selected by +``qb_histogram_mode``. The number of bins defaults to 1000. + +The bounds must be finite and ordered; their values are read on device without +host validation. Kimi uses ``[min(bias)-1, max(bias)+1]``. Keep both bias and +bounds unchanged throughout every microbatch in a training step. With no mask, all token rows must be valid. Optional ``qb_token_mask`` +(``[batch, sequence]``) excludes padding: CUDA computes scores and cutoff, +while a JAX histogram counts only valid rows. This masked path does not use +CUDA histogram accumulation. The standalone router accepts ``token_mask`` +with shape ``[tokens]`` for the same behavior. + +With QB enabled, ``moe`` returns a fifth result, an int32 histogram array of +shape ``[token_shards, experts, bins]``. Its leading axis is sharded across the +compound DP/FSDP/EP token axes. The forward does not reduce these histograms +or update biases. Accumulate per-shard counts across microbatches and only then +sum the leading axis to obtain each expert's global-batch distribution. The +training framework derives and mean-centers the new bias from that distribution, +applying it on the next step. Default calls keep the four-result API. + +The standalone QB router returns ``(probs, routing_map, histogram)`` and accepts +an optional initial histogram for functional accumulation. Its FFI declares +input/output aliasing for the CUDA kernel's in-place histogram writes. For SPMD, +place the standalone operation inside ``shard_map`` with the expert axis +replicated and expose local histograms as a sharded leading axis, as MoE does. +Neither histogram collection nor bias/bounds inputs have a gradient. The router +reuses the ordinary sigmoid routing backward for gradients through its weights. +Use fresh forward statistics or explicit scan carry state; an external mutable +histogram would be counted twice when activation checkpointing recomputes it. + +The Flax wrapper accepts ``qb_bin_bounds`` on each call and exposes +``qb_num_bins`` and ``qb_histogram_mode`` as module configuration. It returns +the same optional fifth result. Step-level bias updates, checkpointing that +bias state, and TPU implementations belong to the training framework. + .. autoapifunction:: transformer_engine.jax.ep.ep_bootstrap .. autoapifunction:: transformer_engine.jax.ep.ep_finalize diff --git a/tests/cpp/operator/test_act.cu b/tests/cpp/operator/test_act.cu index 6edc6bd63b0..3bf85cbabce 100644 --- a/tests/cpp/operator/test_act.cu +++ b/tests/cpp/operator/test_act.cu @@ -4,9 +4,11 @@ * See LICENSE for license information. ************************************************************************/ +#include #include #include #include +#include #include #include #include @@ -424,3 +426,125 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(test::all_fp_types), ::testing::Values(std::pair{768, 2816})), test_name_generator); + +namespace { + +template +void test_grouped_situglu(const size_t hidden, const int alignment_mode, const int count_mode, + const std::pair caps) { + constexpr size_t rows = 13; + const size_t input_offset = alignment_mode == 1 ? 1 : 0; + const size_t output_offset = alignment_mode == 0 ? 0 : 1; + const size_t active = count_mode == 0 ? 8 : (count_mode == 1 ? rows : 0); + const DType dtype = test::TypeInfo::dtype; + const std::vector input_shape{rows, 2 * hidden}, output_shape{rows, hidden}; + test::Tensor input("group_situ_input", std::vector{rows * 2 * hidden + input_offset + 1}, + dtype); + test::Tensor grad("group_situ_grad", std::vector{rows * hidden + input_offset + 1}, + dtype); + test::Tensor output("group_situ_output", std::vector{rows * hidden + output_offset + 1}, + dtype); + test::Tensor dx("group_situ_dx", std::vector{rows * 2 * hidden + output_offset + 1}, + dtype); + test::Tensor counts("group_situ_counts", std::vector{3}, DType::kInt64); + for (test::Tensor *tensor : {&input, &grad, &output, &dx}) { + std::fill_n(tensor->rowwise_cpu_dptr(), test::product(tensor->shape()), + static_cast(std::numeric_limits::quiet_NaN())); + } + auto *x = input.rowwise_cpu_dptr() + input_offset; + auto *dy = grad.rowwise_cpu_dptr() + input_offset; + for (size_t row = 0; row < active; ++row) { + for (size_t col = 0; col < hidden; ++col) { + x[row * 2 * hidden + col] = static_cast(static_cast(col % 11) - 5.25f); + x[row * 2 * hidden + hidden + col] = static_cast(static_cast(row) - 4.5f); + dy[row * hidden + col] = static_cast(0.125f * static_cast(col % 7) - 0.25f); + } + } + auto *sizes = counts.rowwise_cpu_dptr(); + sizes[0] = count_mode == 2 ? 0 : 3; + sizes[1] = 0; + sizes[2] = count_mode == 2 ? 0 : 5; + for (test::Tensor *tensor : {&input, &grad, &output, &dx, &counts}) tensor->from_cpu(); + + GroupedTensorWrapper in_group(3, input_shape), grad_group(3, output_shape); + GroupedTensorWrapper out_group(3, output_shape), dx_group(3, input_shape); + in_group.set_rowwise_data(static_cast(input.rowwise_dptr()) + input_offset, dtype, + input_shape); + grad_group.set_rowwise_data(static_cast(grad.rowwise_dptr()) + input_offset, dtype, + output_shape); + out_group.set_rowwise_data(static_cast(output.rowwise_dptr()) + output_offset, dtype, + output_shape); + dx_group.set_rowwise_data(static_cast(dx.rowwise_dptr()) + output_offset, dtype, + input_shape); + if (count_mode != 1) { + in_group.set_first_dims(counts.rowwise_dptr(), DType::kInt64, std::vector{3}); + } + nvte_group_situglu(in_group.data(), out_group.data(), caps.first, caps.second, nullptr); + nvte_group_dsituglu(grad_group.data(), in_group.data(), dx_group.data(), caps.first, caps.second, + nullptr); + ASSERT_EQ(cudaDeviceSynchronize(), cudaSuccess); + output.to_cpu(); + dx.to_cpu(); + const auto *y = output.rowwise_cpu_dptr() + output_offset; + const auto *actual_dx = dx.rowwise_cpu_dptr() + output_offset; + const double tolerance = + dtype == DType::kFloat32 ? 2e-5 : (dtype == DType::kFloat16 ? .003 : .02); + for (size_t row = 0; row < rows; ++row) { + for (size_t col = 0; col < hidden; ++col) { + double expected_y = 0, expected_gate = 0, expected_up = 0; + if (row < active) { + const double gate = static_cast(x[row * 2 * hidden + col]); + const double up = static_cast(x[row * 2 * hidden + hidden + col]); + const double upstream = static_cast(dy[row * hidden + col]); + const double sigmoid = 1 / (1 + std::exp(-gate)); + const double tg = std::tanh(gate / caps.first), tu = std::tanh(up / caps.second); + expected_y = caps.first * tg * sigmoid * caps.second * tu; + expected_gate = upstream * + ((1 - tg * tg) * sigmoid + caps.first * tg * sigmoid * (1 - sigmoid)) * + caps.second * tu; + expected_up = upstream * caps.first * tg * sigmoid * (1 - tu * tu); + } + EXPECT_NEAR(static_cast(y[row * hidden + col]), expected_y, + tolerance * (1 + std::abs(expected_y))); + EXPECT_NEAR(static_cast(actual_dx[row * 2 * hidden + col]), expected_gate, + tolerance * (1 + std::abs(expected_gate))); + EXPECT_NEAR(static_cast(actual_dx[row * 2 * hidden + hidden + col]), expected_up, + tolerance * (1 + std::abs(expected_up))); + } + } + for (test::Tensor *tensor : {&output, &dx}) { + const auto *data = tensor->rowwise_cpu_dptr(); + if (output_offset) EXPECT_TRUE(std::isnan(static_cast(data[0]))); + EXPECT_TRUE(std::isnan(static_cast(data[test::product(tensor->shape()) - 1]))); + } +} + +class GroupedSiTUGLUTest : public ::testing::TestWithParam< + std::tuple>> {}; + +TEST_P(GroupedSiTUGLUTest, AlignmentAndInactiveRows) { + const auto [dtype, hidden, alignment_mode, count_mode, caps] = GetParam(); + switch (dtype) { + case DType::kFloat32: + test_grouped_situglu(hidden, alignment_mode, count_mode, caps); + break; + case DType::kFloat16: + test_grouped_situglu(hidden, alignment_mode, count_mode, caps); + break; + case DType::kBFloat16: + test_grouped_situglu(hidden, alignment_mode, count_mode, caps); + break; + default: + FAIL() << "Unexpected test dtype"; + } +} + +INSTANTIATE_TEST_SUITE_P(Operator, GroupedSiTUGLUTest, + ::testing::Combine(::testing::Values(DType::kFloat32, DType::kFloat16, + DType::kBFloat16), + ::testing::Values(size_t{3}, size_t{16}, size_t{32}), + ::testing::Values(0, 1, 2), ::testing::Values(0, 1, 2), + ::testing::Values(std::pair{4, 25}, + std::pair{2, 8}))); + +} // namespace diff --git a/tests/jax/moe/test_moe_resource_api.py b/tests/jax/moe/test_moe_resource_api.py index ab6c02932d6..dbed5d96adf 100644 --- a/tests/jax/moe/test_moe_resource_api.py +++ b/tests/jax/moe/test_moe_resource_api.py @@ -2,7 +2,7 @@ # # See LICENSE for license information. -"""MoE resource resolution and MeshResource-only API without EP kernels.""" +"""MoE resource and activation API contracts without EP kernels.""" import importlib import inspect @@ -13,15 +13,68 @@ import pytest from jax.sharding import Mesh +from transformer_engine.jax import grouped_situglu from transformer_engine.jax.flax import _MoEBlock from transformer_engine.jax.moe import ( RouterComputationInfo, + SiTUActivation, + _apply_ffn_activation, _moe_mesh_axes, _resolve_moe_mesh_resource, ) from transformer_engine.jax.sharding import MeshResource, global_mesh_resource, global_shard_guard +@pytest.mark.parametrize("betas", [(4.0, 25.0), (2.0, 7.0)]) +def test_situ_value_and_both_branch_derivatives(betas): + gate_beta, up_beta = betas + gate = jnp.asarray([-10000.0, -12.0, -0.5, 0.0, 0.5, 12.0, 10000.0]) + up = jnp.asarray([10000.0, 40.0, -3.0, 2.0, 3.0, -40.0, -10000.0]) + activation = SiTUActivation(gate_beta, up_beta) + counts = jnp.asarray([1], jnp.int32) + + def activate(g, u): + pair = jnp.concatenate([g, u])[None, :] + return grouped_situglu(pair, counts, gate_beta, up_beta)[0] + + output, pullback = jax.vjp(activate, gate, up) + dg, du = pullback(jnp.ones_like(output)) + np.testing.assert_array_equal( + output, _apply_ffn_activation(activation, gate[None, :], up[None, :], counts)[0] + ) + + g, u = np.asarray(gate, np.float64), np.asarray(up, np.float64) + sigmoid = 0.5 + 0.5 * np.tanh(g / 2) + tg, tu = np.tanh(g / gate_beta), np.tanh(u / up_beta) + capped_gate, capped_up = gate_beta * tg * sigmoid, up_beta * tu + derivative_gate = (1 - tg**2) * sigmoid + gate_beta * tg * sigmoid * (1 - sigmoid) + for actual, expected in ( + (output, capped_gate * capped_up), + (dg, derivative_gate * capped_up), + (du, capped_gate * (1 - tu**2)), + ): + np.testing.assert_allclose(actual, expected, atol=2e-5, rtol=2e-5) + + +def test_situ_bf16_uses_fp32_before_final_cast(): + gate = jnp.asarray([0.25, 1.5, -3.0, 12.0, 64.0], dtype=jnp.bfloat16) + up = jnp.asarray([0.75, -2.0, 32.0, 64.0, -128.0], dtype=jnp.bfloat16) + actual = _apply_ffn_activation( + SiTUActivation(), gate[None, :], up[None, :], jnp.asarray([1], jnp.int32) + )[0] + g, u = np.asarray(gate, np.float64), np.asarray(up, np.float64) + expected = jnp.asarray(100 * np.tanh(g / 4) / (1 + np.exp(-g)) * np.tanh(u / 25), jnp.bfloat16) + assert actual.dtype == jnp.bfloat16 + np.testing.assert_array_equal(actual, expected) + + +@pytest.mark.parametrize("value", [0, -1, float("inf"), float("nan")]) +@pytest.mark.parametrize("field", ["gate_beta", "up_beta"]) +def test_situ_rejects_invalid_soft_caps(field, value): + with pytest.raises(ValueError, match=field): + SiTUActivation(**{field: value}) + + @pytest.fixture(autouse=True) def no_global_resource(): with global_shard_guard(None): @@ -36,6 +89,73 @@ def test_resource_required_for_function_and_block(): _MoEBlock().init(jax.random.PRNGKey(0), jnp.ones((1, 1, 4))) +@pytest.mark.parametrize("shape", [(1, 4), (2, 1, 8), (1, 2, 8)]) +def test_router_input_shape_is_validated(shape): + module = importlib.import_module("transformer_engine.jax.moe") + router = RouterComputationInfo( + gate_kernel=jnp.ones((8, 2)), num_experts_per_tok=1, router_input=jnp.ones(shape) + ) + with pytest.raises(ValueError, match="router_input must have shape"): + module.moe( + jnp.ones((1, 1, 4)), + router, + None, + None, + num_experts=2, + mesh_resource=MeshResource(ep_resource="ep"), + ) + + +def test_gate_kernel_uses_router_width(): + module = importlib.import_module("transformer_engine.jax.moe") + router = RouterComputationInfo( + gate_kernel=jnp.ones((4, 2)), num_experts_per_tok=1, router_input=jnp.ones((1, 1, 8)) + ) + with pytest.raises(ValueError, match="gate_kernel must have shape"): + module.moe( + jnp.ones((1, 1, 4)), + router, + None, + None, + num_experts=2, + mesh_resource=MeshResource(ep_resource="ep"), + ) + + +@pytest.mark.parametrize( + "overrides,match", + [ + ({"score_function": "softmax"}, "requires sigmoid"), + ({"num_groups": 2, "group_topk": 1}, "ungrouped"), + ({"aux_loss_coeff": 0.1}, "aux_loss_coeff"), + ({"aux_loss_coeff": -0.1}, "aux_loss_coeff"), + ({"use_pre_softmax": True}, "use_pre_softmax"), + ({"qb_num_bins": 0}, "num_bins"), + ({"expert_bias": None}, "expert_bias"), + ({"qb_histogram_mode": "invalid"}, "histogram_mode"), + ], +) +def test_qb_configuration_rejected_before_ep(overrides, match): + module = importlib.import_module("transformer_engine.jax.moe") + kwargs = dict( + gate_kernel=jnp.ones((4, 2)), + num_experts_per_tok=1, + score_function="sigmoid", + expert_bias=jnp.zeros(2, jnp.float32), + qb_bin_bounds=jnp.asarray([-1.0, 1.0], jnp.float32), + ) + kwargs.update(overrides) + with pytest.raises(ValueError, match=match): + module.moe( + jnp.ones((1, 1, 4)), + RouterComputationInfo(**kwargs), + None, + None, + num_experts=2, + mesh_resource=MeshResource(ep_resource="ep"), + ) + + def test_explicit_resource_overrides_global_and_is_snapshotted(): explicit = MeshResource(dp_resource="dp", fsdp_resource="fsdp", ep_resource="ep") enclosing = MeshResource(ep_resource="other") @@ -131,7 +251,13 @@ def test_public_api_uses_selected_resource(monkeypatch, quantize): def fake_vjp(*args): captured.update(signature.bind(*args).arguments) assert global_mesh_resource().ep_resource == "ep" - return args[0], None, jnp.zeros((1,), jnp.int32), jnp.zeros((2,), jnp.int32) + return ( + args[0], + None, + jnp.zeros((1,), jnp.int32), + jnp.zeros((2,), jnp.int32), + jnp.zeros((0, 0, 0), jnp.int32), + ) monkeypatch.setattr(module, "_moe", fake_vjp) mesh = Mesh(np.asarray(jax.devices()[:1]).reshape(1, 1, 1), ("dp", "fsdp", "ep")) diff --git a/tests/jax/test_distributed_router.py b/tests/jax/test_distributed_router.py index c6c8a11a844..0b091ded580 100644 --- a/tests/jax/test_distributed_router.py +++ b/tests/jax/test_distributed_router.py @@ -53,6 +53,7 @@ def _inject_router(): jax.config.update("jax_use_shardy_partitioner", True) from test_fused_router import ( + reference_qb_histogram, reference_topk_with_score_function, reference_compute_scores_for_aux_loss, reference_aux_loss, @@ -79,6 +80,53 @@ def _inject_router(): } +@pytest.mark.parametrize("device_count,mesh_shape,mesh_axes,mesh_resource", generate_configs()) +@pytest.mark.parametrize("histogram_mode", ["two_kernel", "fused_atomic"]) +def test_qb_shard_histograms(device_count, mesh_shape, mesh_axes, mesh_resource, histogram_mode): + from jax.experimental.shard_map import shard_map + from transformer_engine.jax.router import fused_topk_with_score_function_qb + + del mesh_resource + mesh = Mesh(np.asarray(jax.devices()[:device_count]).reshape(mesh_shape), mesh_axes) + token_axes = tuple(mesh_axes) + logits = jax.random.normal(jax.random.PRNGKey(831), (128 * device_count, 32)) + bias = jnp.linspace(-0.1, 0.1, 32, dtype=jnp.float32) + bounds = jnp.asarray([-1.1, 1.1], jnp.float32) + + def route(local_logits, expert_bias, limits): + probs, routing, histogram = fused_topk_with_score_function_qb( + local_logits, + 4, + expert_bias, + limits, + num_bins=64, + histogram_mode=histogram_mode, + ) + return probs, routing, histogram[None] + + with mesh: + sharded_logits = jax.device_put( + logits, NamedSharding(mesh, PartitionSpec(token_axes, None)) + ) + _, _, partial_histograms = jax.jit( + shard_map( + route, + mesh=mesh, + in_specs=(PartitionSpec(token_axes, None), PartitionSpec(), PartitionSpec()), + out_specs=( + PartitionSpec(token_axes, None), + PartitionSpec(token_axes, None), + PartitionSpec(token_axes, None, None), + ), + check_rep=False, + ) + )(sharded_logits, bias, bounds) + histogram = jax.jit(lambda h: h.sum(axis=0))(partial_histograms) + assert partial_histograms.shape == (device_count, 32, 64) + np.testing.assert_array_equal(histogram, reference_qb_histogram(logits, bias, bounds, 4, 64)) + np.testing.assert_array_equal(np.asarray(histogram).sum(axis=-1), np.full(32, logits.shape[0])) + + class TestDistributedFusedTopk: """Test distributed execution of fused_topk_with_score_function. diff --git a/tests/jax/test_fused_router.py b/tests/jax/test_fused_router.py index a0e46ca3012..bd08bffe81d 100644 --- a/tests/jax/test_fused_router.py +++ b/tests/jax/test_fused_router.py @@ -10,6 +10,7 @@ import jax import jax.numpy as jnp +import numpy as np import pytest from utils import pytest_parametrize_wrapper @@ -111,6 +112,202 @@ def _inject_router(): SEED = 42 +def reference_qb_histogram(logits, expert_bias, bounds, topk, num_bins): + scores = np.asarray(jax.nn.sigmoid(jnp.asarray(logits, jnp.float32))) + selected, _ = jax.lax.top_k(jnp.asarray(scores + np.asarray(expert_bias)), topk + 1) + required_bias = np.asarray(selected)[:, -1:] - scores + lower, upper = np.asarray(bounds) + bins = np.floor((required_bias - lower) * (num_bins / (upper - lower))).astype(np.int32) + bins = np.clip(bins, 0, num_bins - 1) + histogram = np.zeros((scores.shape[-1], num_bins), np.int32) + np.add.at(histogram, (np.arange(scores.shape[-1])[None, :], bins), 1) + return histogram + + +@pytest.mark.parametrize("histogram_mode", ["two_kernel", "fused_atomic"]) +@pytest.mark.parametrize("routing_map_format", ["bytemap", "bitmap_u8"]) +@pytest.mark.parametrize("topk", [1, 4]) +def test_qb_router_forward_backward_and_remat(histogram_mode, routing_map_format, topk): + from transformer_engine.jax.router import fused_topk_with_score_function_qb + + logits = jax.random.normal(jax.random.PRNGKey(801), (128, 32)) + bias = jax.random.uniform(jax.random.PRNGKey(802), (32,), minval=-0.1, maxval=0.1) + bounds = jnp.asarray([-1.1, 1.1], jnp.float32) + coefficients = jax.random.normal(jax.random.PRNGKey(803), logits.shape) + + def loss(x, b, limits): + probs, routing, histogram = fused_topk_with_score_function_qb( + x, + topk, + b, + limits, + num_bins=64, + scaling_factor=1.5, + histogram_mode=histogram_mode, + routing_map_format=routing_map_format, + ) + return jnp.sum(probs * coefficients), (probs, routing, histogram) + + (_, (probs, routing, histogram)), (dx, db, dbounds) = jax.jit( + jax.value_and_grad(jax.checkpoint(loss), argnums=(0, 1, 2), has_aux=True) + )(logits, bias, bounds) + + def reference(x): + probs, _ = reference_topk_with_score_function( + x, + topk, + score_function="sigmoid", + expert_bias=bias, + scaling_factor=1.5, + ) + return jnp.sum(probs * coefficients), probs + + (_, expected_probs), expected_dx = jax.value_and_grad(reference, has_aux=True)(logits) + np.testing.assert_allclose(probs, expected_probs, rtol=2e-5, atol=2e-6) + np.testing.assert_allclose(dx, expected_dx, rtol=2e-4, atol=2e-6) + np.testing.assert_array_equal(db, jnp.zeros_like(bias)) + np.testing.assert_array_equal(dbounds, jnp.zeros_like(bounds)) + np.testing.assert_array_equal(histogram, reference_qb_histogram(logits, bias, bounds, topk, 64)) + expected_map = np.asarray(expected_probs != 0) + if routing_map_format == "bitmap_u8": + expected_map = np.packbits(expected_map, axis=-1, bitorder="little") + np.testing.assert_array_equal(routing, expected_map) + + +@pytest.mark.parametrize("histogram_mode", ["two_kernel", "fused_atomic"]) +def test_qb_scan_accumulation_preserves_initial_histogram(histogram_mode): + from transformer_engine.jax.router import fused_topk_with_score_function_qb + + logits = jax.random.normal(jax.random.PRNGKey(811), (3, 64, 32)) + bias = jnp.linspace(-0.1, 0.1, 32, dtype=jnp.float32) + bounds = jnp.asarray([-1.1, 1.1], jnp.float32) + initial = jnp.full((32, 64), 7, jnp.int32) + + def accumulate(histogram, x): + _, _, updated = fused_topk_with_score_function_qb( + x, + 4, + bias, + bounds, + num_bins=64, + histogram=histogram, + histogram_mode=histogram_mode, + ) + return updated, None + + @jax.jit + def run(histogram, x): + total, _ = jax.lax.scan(accumulate, histogram, x) + return total, histogram + 1 + + actual, still_live = run(initial, logits) + expected = np.full((32, 64), 7, np.int32) + for microbatch in np.asarray(logits): + expected += reference_qb_histogram(microbatch, bias, bounds, 4, 64) + np.testing.assert_array_equal(actual, expected) + np.testing.assert_array_equal(still_live, np.full((32, 64), 8, np.int32)) + np.testing.assert_array_equal(initial, np.full((32, 64), 7, np.int32)) + + +@pytest.mark.parametrize("histogram_mode", ["two_kernel", "fused_atomic"]) +def test_qb_bounds_change_and_clamp(histogram_mode): + from transformer_engine.jax.router import fused_topk_with_score_function_qb + + logits = jax.random.normal(jax.random.PRNGKey(821), (64, 32)) + bias = jnp.linspace(-0.2, 0.2, 32, dtype=jnp.float32) + route = jax.jit( + partial( + fused_topk_with_score_function_qb, + topk=4, + expert_bias=bias, + num_bins=32, + histogram_mode=histogram_mode, + ) + ) + results = [] + for limits in ([-1.2, 1.2], [-0.025, 0.025]): + bounds = jnp.asarray(limits, jnp.float32) + _, _, histogram = route(logits, bin_bounds=bounds) + np.testing.assert_array_equal( + histogram, reference_qb_histogram(logits, bias, bounds, 4, 32) + ) + np.testing.assert_array_equal(np.asarray(histogram).sum(axis=-1), np.full(32, 64)) + results.append(np.asarray(histogram)) + assert not np.array_equal(*results) + + +@pytest.mark.parametrize("histogram_mode", ["two_kernel", "fused_atomic"]) +def test_qb_tied_cutoff_counts_each_token_once(histogram_mode): + from transformer_engine.jax.router import fused_topk_with_score_function_qb + + _, routing, histogram = jax.jit( + partial( + fused_topk_with_score_function_qb, + topk=4, + num_bins=32, + histogram_mode=histogram_mode, + ) + )(jnp.zeros((64, 32)), expert_bias=jnp.zeros(32), bin_bounds=jnp.asarray([-1.0, 1.0])) + np.testing.assert_array_equal(np.asarray(routing).sum(axis=-1), np.full(64, 4)) + expected = np.zeros((32, 32), np.int32) + expected[:, 16] = 64 + np.testing.assert_array_equal(histogram, expected) + + +@pytest.mark.parametrize("histogram_mode", ["two_kernel", "fused_atomic"]) +@pytest.mark.parametrize("valid_rows", [0, 17, 64]) +def test_qb_masked_histogram_and_gradients(histogram_mode, valid_rows): + from transformer_engine.jax.router import fused_topk_with_score_function_qb + + logits = jax.random.normal(jax.random.PRNGKey(851), (64, 32)) + bias = jnp.linspace(-0.1, 0.1, 32) + bounds = jnp.asarray([-1.1, 1.1], jnp.float32) + mask = jnp.arange(64) < valid_rows + + def loss(x): + probs, _, counts = fused_topk_with_score_function_qb( + x, + 4, + bias, + bounds, + token_mask=mask, + num_bins=64, + histogram_mode=histogram_mode, + ) + return jnp.sum(probs * x), counts + + (_, counts), gradient = jax.jit(jax.value_and_grad(jax.checkpoint(loss), has_aux=True))(logits) + if valid_rows: + expected = reference_qb_histogram(logits[:valid_rows], bias, bounds, 4, 64) + else: + expected = np.zeros((32, 64), np.int32) + np.testing.assert_array_equal(counts, expected) + np.testing.assert_array_equal(np.asarray(counts).sum(axis=-1), np.full(32, valid_rows)) + assert np.isfinite(np.asarray(gradient)).all() + + +@pytest.mark.parametrize( + "overrides,match", + [ + ({"topk": 32}, "topk"), + ({"expert_bias": None}, "expert_bias"), + ({"num_bins": 0}, "num_bins"), + ({"histogram_mode": "unknown"}, "histogram_mode"), + ({"bin_bounds": jnp.zeros(3, jnp.float32)}, "bin_bounds"), + ({"histogram": jnp.zeros((32, 64), jnp.float32)}, "histogram"), + ], +) +def test_qb_router_rejects_invalid_inputs(overrides, match): + from transformer_engine.jax.router import fused_topk_with_score_function_qb + + kwargs = dict( + topk=4, expert_bias=jnp.zeros(32), bin_bounds=jnp.asarray([-1.0, 1.0]), num_bins=64 + ) + kwargs.update(overrides) + with pytest.raises(ValueError, match=match): + fused_topk_with_score_function_qb(jnp.zeros((64, 32)), **kwargs) + + # ============================================================================= # Reference Implementations # ============================================================================= diff --git a/tests/jax/test_grouped_situglu.py b/tests/jax/test_grouped_situglu.py new file mode 100644 index 00000000000..8bc3c5e5d78 --- /dev/null +++ b/tests/jax/test_grouped_situglu.py @@ -0,0 +1,134 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Native grouped SiTU values, derivatives and inactive-capacity isolation.""" + +from functools import partial + +import jax +import jax.numpy as jnp +import numpy as np +import pytest + +from transformer_engine.jax import grouped_situglu + + +def reference(x, counts, beta1, beta2): + active = jnp.arange(x.shape[0]) < counts.sum() + safe_x = jnp.where(active[:, None], x, 0).astype(jnp.float32) + gate, up = jnp.split(safe_x, 2, axis=-1) + result = beta1 * jnp.tanh(gate / beta1) * jax.nn.sigmoid(gate) + result *= beta2 * jnp.tanh(up / beta2) + return jnp.where(active[:, None], result.astype(x.dtype), 0) + + +def reference_backward_fp64(x, dy, active, beta1, beta2): + gate, up = np.split(np.asarray(x, np.float64)[:active], 2, axis=-1) + upstream = np.asarray(dy, np.float64)[:active] + tg, tu = np.tanh(gate / beta1), np.tanh(up / beta2) + sigmoid = 1 / (1 + np.exp(-gate)) + dx = np.zeros(x.shape, np.float64) + hidden = x.shape[1] // 2 + dx[:active, :hidden] = ( + upstream * ((1 - tg**2) * sigmoid + beta1 * tg * sigmoid * (1 - sigmoid)) * beta2 * tu + ) + dx[:active, hidden:] = upstream * beta1 * tg * sigmoid * (1 - tu**2) + return dx + + +@pytest.mark.parametrize("dtype", [jnp.float32, jnp.float16, jnp.bfloat16]) +@pytest.mark.parametrize("caps", [(4.0, 25.0), (2.0, 8.0)]) +@pytest.mark.parametrize( + "capacity,hidden,counts", + [ + (13, 6, [3, 0, 5]), + (256, 128, [128, 128]), + (513, 3584, [128, 0, 129]), + (7, 3, [0] * 257 + [3, 4]), + (9, 10, [0, 0]), + (0, 32, [0, 0]), + ], +) +def test_grouped_situ_forward_and_backward(dtype, caps, capacity, hidden, counts): + rng = np.random.default_rng(72) + host = rng.uniform(-30, 30, (capacity, 2 * hidden)).astype(np.float32) + active = sum(counts) + host[active:] = np.nan + x = jnp.asarray(host, dtype) + counts = jnp.asarray(counts, jnp.int32) + upstream = rng.normal(size=(capacity, hidden)).astype(np.float32) + upstream[active:] = np.nan + dy = jnp.asarray(upstream, dtype) + fn = partial(grouped_situglu, beta1=caps[0], beta2=caps[1]) + ref = partial(reference, beta1=caps[0], beta2=caps[1]) + y = jax.jit(fn)(x, counts) + _, pullback = jax.vjp(lambda inputs: fn(inputs, counts), x) + dx = jax.jit(pullback)(dy)[0] + expected_dx = reference_backward_fp64(x, dy, active, *caps) + tolerance = {jnp.float32: 2.0e-5, jnp.float16: 3.0e-3, jnp.bfloat16: 2.0e-2}[dtype] + np.testing.assert_allclose( + y.astype(jnp.float32), ref(x, counts).astype(jnp.float32), rtol=tolerance, atol=tolerance + ) + np.testing.assert_allclose(dx.astype(jnp.float32), expected_dx, rtol=tolerance, atol=tolerance) + assert np.isfinite(np.asarray(y, dtype=np.float32)).all() + assert np.isfinite(np.asarray(dx, dtype=np.float32)).all() + np.testing.assert_array_equal(np.asarray(y[active:], dtype=np.float32), 0) + np.testing.assert_array_equal(np.asarray(dx[active:], dtype=np.float32), 0) + + +def test_grouped_situ_dynamic_counts_replay_and_checkpoint(): + x = jnp.arange(13 * 24, dtype=jnp.float32).reshape(13, 24) / 30 - 5 + compiled = jax.jit( + grouped_situglu, compiler_options={"xla_gpu_enable_command_buffer": "FUSION,CUSTOM_CALL"} + ) + counts = jnp.array([3, 0, 5], jnp.int32) + baseline = compiled(x, counts) + poisoned = x.at[8:].set(jnp.nan) + np.testing.assert_array_equal(baseline, compiled(poisoned, counts)) + changed = jnp.array([1, 2, 2], jnp.int32) + np.testing.assert_allclose( + compiled(x, changed), reference(x, changed, 4.0, 25.0), rtol=2.0e-5, atol=2.0e-5 + ) + loss = lambda inputs: jnp.sum(grouped_situglu(inputs, counts)) + np.testing.assert_array_equal( + jax.jit(jax.grad(jax.checkpoint(loss)))(poisoned), jax.jit(jax.grad(loss))(poisoned) + ) + hlo = compiled.lower(x, counts).compile().as_text() + assert "te_grouped_situglu_ffi" in hlo + + +def test_grouped_situ_vmap(): + x = jnp.arange(2 * 9 * 16, dtype=jnp.float32).reshape(2, 9, 16) / 20 - 5 + counts = jnp.array([[2, 0, 3], [1, 1, 7]], jnp.int32) + actual = jax.jit(jax.vmap(grouped_situglu))(x, counts) + expected = jax.vmap(partial(reference, beta1=4.0, beta2=25.0))(x, counts) + np.testing.assert_allclose(actual, expected, rtol=2.0e-5, atol=2.0e-5) + actual_grad = jax.jit(jax.grad(lambda inputs: jax.vmap(grouped_situglu)(inputs, counts).sum()))( + x + ) + expected_grad = jax.grad( + lambda inputs: jax.vmap(partial(reference, beta1=4.0, beta2=25.0))(inputs, counts).sum() + )(x) + np.testing.assert_allclose(actual_grad, expected_grad, rtol=2.0e-5, atol=2.0e-5) + + +@pytest.mark.parametrize("name", ["beta1", "beta2"]) +@pytest.mark.parametrize("value", [0.0, -1.0, float("nan"), float("inf"), 1.0e100, 1.0e-100]) +def test_grouped_situ_invalid_caps(name, value): + with pytest.raises(ValueError, match=name): + grouped_situglu(jnp.ones((3, 16)), jnp.array([3], jnp.int32), **{name: value}) + + +@pytest.mark.parametrize( + "shape,counts,error", + [ + ((3, 15), [3], ValueError), + ((3, 0), [3], ValueError), + ((2, 3, 16), [3], ValueError), + ((3, 16), [], ValueError), + ], +) +def test_grouped_situ_invalid_shapes(shape, counts, error): + with pytest.raises(error): + grouped_situglu(jnp.ones(shape), jnp.asarray(counts, jnp.int32)) diff --git a/tests/jax/test_te_ep_moe.py b/tests/jax/test_te_ep_moe.py index ad5c897b8d5..ddfed8e6fa0 100644 --- a/tests/jax/test_te_ep_moe.py +++ b/tests/jax/test_te_ep_moe.py @@ -124,6 +124,7 @@ def _read_mp_options(): from transformer_engine.jax.moe import ( RouterComputationInfo, RoutingMapInfo, + SiTUActivation, get_moe_recv_capacity_per_rank, moe, record_ep_bootstrap_signature_for_moe, @@ -282,6 +283,7 @@ def mesh(use_cudnn_fusion): "num_experts_per_tok", "aux_loss_coeff", "score_function", + "activation_type", ), ) def _pure_jax_moe_reference( @@ -292,20 +294,23 @@ def _pure_jax_moe_reference( expert_bias=None, hash_input_ids=None, hash_tid2eid=None, + router_input=None, *, num_experts, num_experts_per_tok, aux_loss_coeff: float = 0.0, score_function: str = "softmax", scaling_factor: float = 1.0, + activation_type="silu", ): B, S, H = x.shape T = B * S K = num_experts_per_tok x_2d = x.reshape(T, H) - gate_kernel_cast = gate_kernel.astype(x.dtype) - logits = (x_2d @ gate_kernel_cast).astype(jnp.float32) # [T, E] + gate_input = x if router_input is None else router_input + gate_kernel_cast = gate_kernel.astype(gate_input.dtype) + logits = (gate_input.reshape(T, -1) @ gate_kernel_cast).astype(jnp.float32) # [T, E] if hash_input_ids is not None: if score_function != "sqrtsoftplus" or hash_tid2eid is None: @@ -342,10 +347,18 @@ def _pure_jax_moe_reference( wi_0, wi_1 = jnp.split(wi, 2, axis=-1) layer_w0 = jnp.einsum("th,ehm->tem", x_2d, wi_0) layer_w1 = jnp.einsum("th,ehm->tem", x_2d, wi_1) - # Activation runs in x.dtype (typically bf16) to mirror the impl -- - # the impl keeps silu+multiply in the wi GEMM output dtype because - # storing higher precision than the consumer (wo) GEMM buys nothing. - intermediate = jax.nn.silu(layer_w0) * layer_w1 + # SiTU evaluates both branches in FP32 before the final cast. + if isinstance(activation_type, SiTUActivation): + gate, up = layer_w0.astype(jnp.float32), layer_w1.astype(jnp.float32) + intermediate = ( + activation_type.gate_beta + * jnp.tanh(gate / activation_type.gate_beta) + * jax.nn.sigmoid(gate) + * activation_type.up_beta + * jnp.tanh(up / activation_type.up_beta) + ).astype(layer_w0.dtype) + else: + intermediate = jax.nn.silu(layer_w0) * layer_w1 expert_out = jnp.einsum("tem,emh->teh", intermediate, wo) # [T, E, H] output_2d = jnp.einsum("te,teh->th", routing_weights_full.astype(x.dtype), expert_out) output = output_2d.reshape(B, S, H).astype(x.dtype) @@ -384,6 +397,7 @@ def _pure_jax_moe_reference( def _make_block( *, + activation_type="silu", apply_topk_weights_early=False, aux_loss_coeff=0.0, use_expert_routing_bias=False, @@ -401,6 +415,7 @@ def _make_block( num_experts=NUM_EXPERTS, num_experts_per_tok=TOPK, intermediate_size=INTER, + activation_type=activation_type, mesh_resource=mesh_resource, apply_topk_weights_early=apply_topk_weights_early, aux_loss_coeff=aux_loss_coeff, @@ -558,6 +573,31 @@ def _quantization_recipe(quantization): return MXFP8BlockScaling() +def test_situ_uses_native_grouped_kernels(mesh, monkeypatch, make_block): + from transformer_engine.jax import cpp_extensions as tex + + calls = [] + forward, backward = tex.grouped_situglu_fwd, tex.grouped_situglu_bwd + + def checked_forward(x, counts, beta1, beta2): + assert counts.ndim == 1 and counts.dtype == jnp.int32 + calls.append("forward") + return forward(x, counts, beta1, beta2) + + def checked_backward(grad, x, counts, beta1, beta2): + assert counts.ndim == 1 and counts.dtype == jnp.int32 + calls.append("backward") + return backward(grad, x, counts, beta1, beta2) + + monkeypatch.setattr(tex, "grouped_situglu_fwd", checked_forward) + monkeypatch.setattr(tex, "grouped_situglu_bwd", checked_backward) + block = make_block(activation_type=SiTUActivation(3.0, 11.0)) + x = _make_inputs(jax.random.PRNGKey(88)) + variables, _, _, _ = _init_apply(block, mesh, x, jax.random.PRNGKey(89)) + _grad_step(block, variables, mesh, x) + assert "forward" in calls and "backward" in calls + + # ----------------------------------------------------------------------------- # Tests # ----------------------------------------------------------------------------- @@ -572,6 +612,19 @@ def _quantization_recipe(quantization): # ----------------------------------------------------------------------------- _CONFIGS = [ + *[ + pytest.param( + dict( + activation_type=SiTUActivation(), + score_function="sigmoid", + use_expert_routing_bias=True, + expert_bias_init=_strong_expert_bias_init, + apply_topk_weights_early=early, + ), + id=f"situ-{'early' if early else 'late'}-weighting", + ) + for early in (False, True) + ], pytest.param( dict(score_function="softmax"), id="softmax", @@ -724,9 +777,175 @@ def loss_fn(variables, inputs): ) +@pytest.mark.parametrize("quantization", _QUANTIZATION_CASES) +@pytest.mark.parametrize("early_weighting", [False, True]) +@pytest.mark.parametrize("router_dtype", [DTYPE, jnp.float32]) +def test_separate_router_input_latent_projection( + mesh, quantization, early_weighting, router_dtype, make_block +): + block = make_block( + score_function="sigmoid", + use_expert_routing_bias=True, + expert_bias_init=_strong_expert_bias_init, + aux_loss_coeff=1e-2, + apply_topk_weights_early=early_weighting, + quantization_recipe=_quantization_recipe(quantization), + ) + hidden = jax.random.normal(jax.random.PRNGKey(71), (BATCH, SEQ, 2 * HIDDEN), router_dtype) + projection = jax.random.normal(jax.random.PRNGKey(72), (2 * HIDDEN, HIDDEN), DTYPE) + projection = projection / jnp.sqrt(jnp.asarray(2 * HIDDEN, DTYPE)) + with _ctx(mesh): + hidden_sh = _shard_inputs(hidden, mesh) + variables = jax.jit(block.init)( + jax.random.PRNGKey(73), (hidden_sh @ projection).astype(DTYPE), router_input=hidden_sh + ) + + def loss_fn(v, h, p): + output, aux, _, _ = block.apply(v, (h @ p).astype(DTYPE), router_input=h) + return jnp.mean(output.astype(jnp.float32) ** 2) + aux + + grads, dh, dp = jax.jit(jax.grad(jax.checkpoint(loss_fn), argnums=(0, 1, 2)))( + variables, hidden_sh, projection + ) + output, aux, _, _ = jax.jit(block.apply)( + variables, (hidden_sh @ projection).astype(DTYPE), router_input=hidden_sh + ) + params_np = _params_global_numpy(variables, mesh) + assert params_np["gate_kernel"].shape == (2 * HIDDEN, NUM_EXPERTS) + assert params_np["wi"].shape == (NUM_EXPERTS, HIDDEN, 2 * INTER) + + def reference(params, h, p): + return _pure_jax_moe_reference( + (h @ p).astype(DTYPE), + params["gate_kernel"], + params["wi"], + params["wo"], + params["expert_bias"], + router_input=h, + num_experts=NUM_EXPERTS, + num_experts_per_tok=TOPK, + score_function="sigmoid", + aux_loss_coeff=1e-2, + ) + + def ref_loss(params, h, p): + out, aux_loss = reference(params, h, p) + return jnp.mean(out.astype(jnp.float32) ** 2) + aux_loss + + ref_params = {k: jnp.asarray(v) for k, v in params_np.items()} + out_ref, aux_ref = reference(ref_params, hidden, projection) + grads_ref, dh_ref, dp_ref = jax.jit(jax.grad(ref_loss, argnums=(0, 1, 2)))( + ref_params, hidden, projection + ) + np.testing.assert_allclose( + _to_global_numpy(output, mesh).astype(np.float32), + np.asarray(out_ref).astype(np.float32), + **FWD_TOLERANCE[quantization], + ) + np.testing.assert_allclose( + _to_global_numpy(aux, mesh).astype(np.float32), + np.asarray(aux_ref).astype(np.float32), + **AUX_TOLERANCE, + ) + for name in ("gate_kernel", "wi", "wo"): + tolerance = GRAD_GATE_TOLERANCE if name == "gate_kernel" else GRAD_FFN_TOLERANCE + np.testing.assert_allclose( + _to_global_numpy(_unwrap(grads["params"][name]), mesh).astype(np.float32), + np.asarray(grads_ref[name]).astype(np.float32), + **tolerance[quantization], + ) + for actual, expected, primal in ((dh, dh_ref, hidden), (dp, dp_ref, projection)): + assert actual.shape == primal.shape and actual.dtype == primal.dtype + np.testing.assert_allclose( + _to_global_numpy(actual, mesh).astype(np.float32), + np.asarray(expected).astype(np.float32), + **GRAD_FFN_TOLERANCE[quantization], + ) + + +@pytest.mark.parametrize("histogram_mode", ["two_kernel", "fused_atomic"]) +@pytest.mark.parametrize("masked", [False, True]) +def test_qb_moe_histograms_and_remat_gradients(mesh, make_block, histogram_mode, masked): + from test_fused_router import reference_qb_histogram + + def bias_init(key, shape, dtype): + del key + return jnp.linspace(-0.1, 0.1, shape[0], dtype=dtype) + + block = make_block( + score_function="sigmoid", + use_expert_routing_bias=True, + expert_bias_init=bias_init, + ).clone(qb_num_bins=64, qb_histogram_mode=histogram_mode) + bounds = jnp.asarray([-1.1, 1.1], jnp.float32) + x = jax.random.normal(jax.random.PRNGKey(841), (BATCH, SEQ, HIDDEN), DTYPE) + + token_mask = (jnp.arange(BATCH * SEQ).reshape(BATCH, SEQ) % 2 == 0) if masked else None + + with _ctx(mesh): + x_sharded = _shard_inputs(x, mesh) + variables = jax.jit(block.init)(jax.random.PRNGKey(842), x_sharded) + + def loss(v, activations, limits): + result = block.apply( + v, + activations, + qb_bin_bounds=limits, + qb_token_mask=token_mask if limits is not None else None, + ) + return jnp.mean(result[0].astype(jnp.float32) ** 2), result + + (_, baseline), (baseline_dv, baseline_dx) = jax.jit( + jax.value_and_grad( + partial(loss, limits=None), + argnums=(0, 1), + has_aux=True, + ) + )(variables, x_sharded) + (_, candidate), (dv, dx) = jax.jit( + jax.value_and_grad( + jax.checkpoint(loss), + argnums=(0, 1), + has_aux=True, + ) + )(variables, x_sharded, bounds) + + assert len(baseline) == 4 and len(candidate) == 5 + histogram_parts = _to_global_numpy(candidate[4], mesh) + assert histogram_parts.shape == (mesh.size, NUM_EXPERTS, 64) + params = _params_global_numpy(variables, mesh) + logits = (x @ jnp.asarray(params["gate_kernel"], DTYPE)).astype(jnp.float32) + histogram_logits = logits.reshape(-1, NUM_EXPERTS) + if token_mask is not None: + histogram_logits = histogram_logits[np.asarray(token_mask).reshape(-1)] + expected_histogram = reference_qb_histogram( + histogram_logits, + params["expert_bias"], + bounds, + TOPK, + 64, + ) + np.testing.assert_array_equal(histogram_parts.sum(axis=0), expected_histogram) + for actual, expected in ((candidate[0], baseline[0]), (dx, baseline_dx)): + np.testing.assert_allclose( + _to_global_numpy(actual, mesh).astype(np.float32), + _to_global_numpy(expected, mesh).astype(np.float32), + rtol=2e-2, + atol=2e-2, + ) + for actual, expected in zip(jax.tree.leaves(dv), jax.tree.leaves(baseline_dv)): + np.testing.assert_allclose( + _to_global_numpy(_unwrap(actual), mesh).astype(np.float32), + _to_global_numpy(_unwrap(expected), mesh).astype(np.float32), + rtol=2e-2, + atol=2e-2, + ) + + def _reference_kwargs_from_config(config, params_np): """Pick out the reference-relevant pieces of a parametrize config.""" return dict( + activation_type=config.get("activation_type", "silu"), score_function=config.get("score_function", "softmax"), expert_bias=( jnp.asarray(params_np["expert_bias"]) diff --git a/tests/pytorch/test_fusible_ops.py b/tests/pytorch/test_fusible_ops.py index 1ebf35acb27..da270dc9667 100644 --- a/tests/pytorch/test_fusible_ops.py +++ b/tests/pytorch/test_fusible_ops.py @@ -2799,19 +2799,20 @@ def test_interleaved_swiglu(self): ) @pytest.mark.parametrize("dtype", _dtypes) - @pytest.mark.parametrize("glu_interleave_size", (None, 32)) + @pytest.mark.parametrize("hidden,glu_interleave_size", ((13, None), (96, None), (96, 32))) @pytest.mark.parametrize("betas", ((4.0, 25.0), (3.0, 12.0))) def test_situglu( self, *, dtype: torch.dtype, + hidden: int, glu_interleave_size: Optional[int], betas: tuple[float, float], device: torch.device = "cuda", ) -> None: """SiTU-GLU forward and backward.""" - in_shape = (17, 192) - out_shape = (17, 96) + in_shape = (17, 2 * hidden) + out_shape = (17, hidden) beta1, beta2 = betas x_ref, x_test = make_reference_and_test_tensors( in_shape, diff --git a/transformer_engine/common/activation/swiglu.cu b/transformer_engine/common/activation/swiglu.cu index 0300ba1bd46..ec9c05920c5 100644 --- a/transformer_engine/common/activation/swiglu.cu +++ b/transformer_engine/common/activation/swiglu.cu @@ -58,6 +58,28 @@ void nvte_dsituglu(const NVTETensor grad, const NVTETensor input, NVTETensor out grad, input, output, param, stream); } +void nvte_group_situglu(const NVTEGroupedTensor input, NVTEGroupedTensor output, float beta1, + float beta2, cudaStream_t stream) { + NVTE_API_CALL(nvte_group_situglu); + using namespace transformer_engine; + NVTE_CHECK(beta1 > 0.0f && std::isfinite(beta1), "beta1 must be finite and positive."); + NVTE_CHECK(beta2 > 0.0f && std::isfinite(beta2), "beta2 must be finite and positive."); + const SiTUGLUParam params{beta1, beta2}; + dispatch::group_gated_act_helper>( + input, nullptr, output, params, stream); +} + +void nvte_group_dsituglu(const NVTEGroupedTensor grad, const NVTEGroupedTensor input, + NVTEGroupedTensor output, float beta1, float beta2, cudaStream_t stream) { + NVTE_API_CALL(nvte_group_dsituglu); + using namespace transformer_engine; + NVTE_CHECK(beta1 > 0.0f && std::isfinite(beta1), "beta1 must be finite and positive."); + NVTE_CHECK(beta2 > 0.0f && std::isfinite(beta2), "beta2 must be finite and positive."); + const SiTUGLUParam params{beta1, beta2}; + dispatch::group_gated_act_helper, + dsitu_gate>(input, grad, output, params, stream); +} + void nvte_clamped_swiglu(const NVTETensor input, NVTETensor output, float limit, float alpha, cudaStream_t stream) { NVTE_API_CALL(nvte_clamped_swiglu); diff --git a/transformer_engine/common/cast/dispatch/gated.cuh b/transformer_engine/common/cast/dispatch/gated.cuh index 0c1761264a1..5106df2784f 100644 --- a/transformer_engine/common/cast/dispatch/gated.cuh +++ b/transformer_engine/common/cast/dispatch/gated.cuh @@ -190,6 +190,67 @@ void quantize_gated_bwd_helper(const NVTETensor nvte_grad, const NVTETensor nvte NVTE_ERROR("Not supported scaling mode: " + to_string(output->scaling_mode) + "."); } } + +template +void group_gated_act_helper(const NVTEGroupedTensor input_handle, + const NVTEGroupedTensor grad_handle, NVTEGroupedTensor output_handle, + const ParamOP ¶ms, cudaStream_t stream) { + const auto *input = convertNVTEGroupedTensorCheck(input_handle); + auto *output = convertNVTEGroupedTensorCheck(output_handle); + NVTE_CHECK(input->logical_shape.ndim == 2 && output->logical_shape.ndim == 2, + "Grouped GLU requires rank-2 logical shapes."); + NVTE_CHECK(input->all_same_last_dim() && output->all_same_last_dim(), + "Grouped GLU requires a uniform feature dimension."); + const size_t capacity = input->logical_shape.data[0]; + const size_t input_width = input->logical_shape.data[1]; + NVTE_CHECK(input_width > 0 && input_width % 2 == 0, + "Grouped GLU input width must be positive and even."); + const size_t hidden = input_width / 2; + NVTE_CHECK(output->num_tensors == input->num_tensors && + output->logical_shape.data[0] == capacity && + output->logical_shape.data[1] == (Backward ? input_width : hidden), + "Grouped GLU output shape or group count is inconsistent with input."); + const DType dtype = input->data.dtype; + NVTE_CHECK(dtype == DType::kFloat32 || dtype == DType::kFloat16 || dtype == DType::kBFloat16, + "Grouped GLU supports float32, float16 and bfloat16."); + NVTE_CHECK(output->data.dtype == dtype, "Grouped GLU input/output dtypes must match."); + NVTE_CHECK(input->data.numel() >= capacity * input_width && + output->data.numel() >= capacity * (Backward ? input_width : hidden), + "Grouped GLU input/output storage is smaller than its logical shape."); + if (input->first_dims.has_data()) { + NVTE_CHECK( + input->first_dims.dtype == DType::kInt64 && input->first_dims.numel() == input->num_tensors, + "Grouped GLU first_dims must contain one int64 count per group."); + } + const auto *grad = Backward ? convertNVTEGroupedTensorCheck(grad_handle) : nullptr; + if constexpr (Backward) { + NVTE_CHECK(grad->logical_shape.ndim == 2 && grad->all_same_last_dim() && + grad->logical_shape.data[0] == capacity && + grad->logical_shape.data[1] == hidden && + grad->num_tensors == input->num_tensors && grad->data.dtype == dtype, + "Grouped GLU gradient shape, dtype or group count is inconsistent with input."); + NVTE_CHECK(grad->data.numel() >= capacity * hidden, + "Grouped GLU gradient storage is smaller than its logical shape."); + } + if (capacity == 0) return; + NVTE_CHECK(input->data.has_data() && output->data.has_data(), + "Grouped GLU input/output data must be allocated."); + if constexpr (Backward) NVTE_CHECK(grad->data.has_data(), "Grouped GLU grad is not allocated."); + const auto *sizes = static_cast(input->first_dims.dptr); + TRANSFORMER_ENGINE_TYPE_SWITCH_INPUT( + dtype, T, constexpr int nvec = 32 / sizeof(T); if constexpr (Backward) { + DGatedActivationKernelLauncher( + static_cast(grad->data.dptr), static_cast(input->data.dptr), + static_cast(output->data.dptr), nullptr, nullptr, nullptr, capacity, hidden, + params, stream, sizes, input->num_tensors); + } else { + GatedActivationKernelLauncher( + static_cast(input->data.dptr), static_cast(output->data.dptr), nullptr, + nullptr, nullptr, capacity, hidden, params, stream, sizes, input->num_tensors); + }); +} + } // namespace dispatch } // namespace transformer_engine diff --git a/transformer_engine/common/include/transformer_engine/activation.h b/transformer_engine/common/include/transformer_engine/activation.h index 31429cf89cf..cbb8d78d935 100644 --- a/transformer_engine/common/include/transformer_engine/activation.h +++ b/transformer_engine/common/include/transformer_engine/activation.h @@ -381,6 +381,23 @@ void nvte_swiglu(const NVTETensor input, NVTETensor output, cudaStream_t stream) void nvte_situglu(const NVTETensor input, NVTETensor output, float beta1, float beta2, cudaStream_t stream); +/*! \brief Grouped SiTU-GLU for FP32/FP16/BF16 tensors with uniform feature width. + * Input [capacity, 2H] stores contiguous gate/up halves. Output is [capacity, H]. + * Optional int64 first_dims contains nonnegative device row counts whose sum + * must not exceed capacity. Rows beyond that sum are zero-filled without reading + * their input. No row alignment is required. Quantized output is not supported. + * beta1 and beta2 are positive finite soft-cap parameters. + */ +void nvte_group_situglu(const NVTEGroupedTensor input, NVTEGroupedTensor output, float beta1, + float beta2, cudaStream_t stream); + +/*! \brief Grouped SiTU-GLU backward with the same metadata contract as forward. + * grad is [capacity, H], input and output are [capacity, 2H]. Inactive rows + * are zero-filled without reading their input or upstream gradient. + */ +void nvte_group_dsituglu(const NVTEGroupedTensor grad, const NVTEGroupedTensor input, + NVTEGroupedTensor output, float beta1, float beta2, cudaStream_t stream); + /*! \brief Computes the gated Swish activation of the input used in GPT OSS. * * \deprecated This function has been deprecated in favor of nvte_clamped_swiglu_v2, diff --git a/transformer_engine/common/util/vectorized_pointwise.h b/transformer_engine/common/util/vectorized_pointwise.h index e5f855152cd..936870a1110 100644 --- a/transformer_engine/common/util/vectorized_pointwise.h +++ b/transformer_engine/common/util/vectorized_pointwise.h @@ -403,14 +403,42 @@ void VectorizedUnaryGradKernelLauncher(const InputTypeGrad *grad, const InputTyp } } +template +__device__ __forceinline__ size_t gated_active_rows(const size_t capacity, + const int64_t *group_sizes, + const size_t num_groups) { + if constexpr (!Grouped) { + return capacity; + } else { + __shared__ size_t active_rows; + if (threadIdx.x < THREADS_PER_WARP) { + int64_t rows = 0; + if (group_sizes != nullptr) { + for (size_t group = threadIdx.x; group < num_groups; group += THREADS_PER_WARP) { + rows += group_sizes[group]; + } + } + rows = warp_allreduce_sum(rows); + if (threadIdx.x == 0) { + active_rows = num_groups == 0 ? 0 : (group_sizes == nullptr ? capacity : rows); + } + } + __syncthreads(); + return active_rows; + } +} + template + ComputeType (*Activation)(const ComputeType, const Param &), bool Grouped = false, + typename InputType, typename OutputType> __launch_bounds__(unary_kernel_threads) __global__ void gated_act_kernel(const InputType *input, OutputType *output, const ComputeType *scale, ComputeType *amax, ComputeType *scale_inv, const size_t m, const size_t n, - const Param p, const size_t num_aligned_elements) { + const Param p, const size_t num_aligned_elements, + const int64_t *group_sizes, const size_t num_groups) { const size_t M = num_aligned_elements * m; + [[maybe_unused]] const size_t active_rows = + gated_active_rows(m, group_sizes, num_groups); ComputeType max = 0; ComputeType s = 1; const bool requires_amax = (amax != nullptr); @@ -426,6 +454,14 @@ __launch_bounds__(unary_kernel_threads) __global__ VectorizedLoader loader1(input + id_y * n * 2 + n, n); VectorizedStorer storer(output + id_y * n, n); + if constexpr (Grouped) { + if (id_y >= active_rows) { +#pragma unroll + for (int i = 0; i < nvec; ++i) storer.separate()[i] = static_cast(0); + storer.store(id_x, n); + continue; + } + } loader0.load(id_x, n); loader1.load(id_x, n); #pragma unroll @@ -470,11 +506,13 @@ __launch_bounds__(unary_kernel_threads) __global__ } template + ComputeType (*Activation)(const ComputeType, const Param &), bool Grouped = false, + typename InputType, typename OutputType> void GatedActivationKernelLauncher(const InputType *input, OutputType *output, const fp32 *scale, fp32 *amax, fp32 *scale_inv, const size_t m, const size_t n, - const Param &p, cudaStream_t stream) { + const Param &p, cudaStream_t stream, + const int64_t *group_sizes = nullptr, + const size_t num_groups = 0) { if (m != 0 && n != 0) { size_t num_aligned_elements = get_num_aligned_elements(input, n, nvec, sizeof(InputType)); constexpr size_t threads = unary_kernel_threads; @@ -484,19 +522,20 @@ void GatedActivationKernelLauncher(const InputType *input, OutputType *output, c switch (auto align = CheckAlignment(n, nvec, input, input + n, output)) { case Alignment::SAME_ALIGNED: - gated_act_kernel + gated_act_kernel <<>>(input, output, scale, amax, scale_inv, m, n, p, - num_aligned_elements); + num_aligned_elements, group_sizes, num_groups); break; case Alignment::SAME_UNALIGNED: - gated_act_kernel + gated_act_kernel <<>>(input, output, scale, amax, scale_inv, m, n, p, - num_aligned_elements); + num_aligned_elements, group_sizes, num_groups); break; case Alignment::DIFFERENT: { // If the pointers are aligned differently we cannot vectorize - gated_act_kernel<1, true, ComputeType, Param, Activation> - <<>>(input, output, scale, amax, scale_inv, m, n, p, n); + gated_act_kernel<1, true, ComputeType, Param, Activation, Grouped> + <<>>(input, output, scale, amax, scale_inv, m, n, p, n, + group_sizes, num_groups); break; } } @@ -506,14 +545,17 @@ void GatedActivationKernelLauncher(const InputType *input, OutputType *output, c template + ComputeType (*Dactivation)(const ComputeType, const Param &), bool Grouped = false, + typename InputType, typename OutputType> __launch_bounds__(unary_kernel_threads) __global__ void dgated_act_kernel(const InputType *grad, const InputType *input, OutputType *output, const ComputeType *scale, ComputeType *amax, ComputeType *scale_inv, const size_t m, const size_t n, const Param p, - const size_t num_aligned_elements) { + const size_t num_aligned_elements, const int64_t *group_sizes, + const size_t num_groups) { const size_t M = num_aligned_elements * m; + [[maybe_unused]] const size_t active_rows = + gated_active_rows(m, group_sizes, num_groups); ComputeType max = 0; ComputeType s = 1; const bool requires_amax = (amax != nullptr); @@ -531,6 +573,18 @@ __launch_bounds__(unary_kernel_threads) __global__ VectorizedStorer storer0(output + id_y * n * 2, n); VectorizedStorer storer1(output + id_y * n * 2 + n, n); + if constexpr (Grouped) { + if (id_y >= active_rows) { +#pragma unroll + for (int i = 0; i < nvec; ++i) { + storer0.separate()[i] = static_cast(0); + storer1.separate()[i] = static_cast(0); + } + storer0.store(id_x, n); + storer1.store(id_x, n); + continue; + } + } grad_loader.load(id_x, n); input_loader0.load(id_x, n); input_loader1.load(id_x, n); @@ -590,12 +644,13 @@ __launch_bounds__(unary_kernel_threads) __global__ template + ComputeType (*Dactivation)(const ComputeType, const Param &), bool Grouped = false, + typename InputType, typename OutputType> void DGatedActivationKernelLauncher(const InputType *grad, const InputType *input, OutputType *output, const fp32 *scale, fp32 *amax, fp32 *scale_inv, const size_t m, const size_t n, const Param &p, - cudaStream_t stream) { + cudaStream_t stream, const int64_t *group_sizes = nullptr, + const size_t num_groups = 0) { if (m != 0 && n != 0) { size_t num_aligned_elements = get_num_aligned_elements(grad, n, nvec, sizeof(InputType)); constexpr size_t threads = unary_kernel_threads; @@ -605,20 +660,20 @@ void DGatedActivationKernelLauncher(const InputType *grad, const InputType *inpu switch (auto align = CheckAlignment(n, nvec, input, input + n, output, output + n)) { case Alignment::SAME_ALIGNED: - dgated_act_kernel + dgated_act_kernel <<>>(grad, input, output, scale, amax, scale_inv, m, n, - p, num_aligned_elements); + p, num_aligned_elements, group_sizes, num_groups); break; case Alignment::SAME_UNALIGNED: - dgated_act_kernel + dgated_act_kernel <<>>(grad, input, output, scale, amax, scale_inv, m, n, - p, num_aligned_elements); + p, num_aligned_elements, group_sizes, num_groups); break; case Alignment::DIFFERENT: { // If the pointers are aligned differently we cannot vectorize - dgated_act_kernel<1, true, ComputeType, Param, Activation, Dactivation> + dgated_act_kernel<1, true, ComputeType, Param, Activation, Dactivation, Grouped> <<>>(grad, input, output, scale, amax, scale_inv, m, n, - p, n); + p, n, group_sizes, num_groups); break; } } diff --git a/transformer_engine/jax/__init__.py b/transformer_engine/jax/__init__.py index 45379dbf0d9..ff354a4abff 100644 --- a/transformer_engine/jax/__init__.py +++ b/transformer_engine/jax/__init__.py @@ -48,6 +48,7 @@ from .quantize import NVTE_FP8_COLLECTION_NAME from .sharding import MeshResource +from .activation import grouped_situglu from ..common.utils import deprecate_wrapper from ..common.utils import DeprecatedEnum @@ -61,4 +62,5 @@ "MeshResource", "flax", "quantize", + "grouped_situglu", ] diff --git a/transformer_engine/jax/activation.py b/transformer_engine/jax/activation.py index b2b90a10c97..414f66827ba 100644 --- a/transformer_engine/jax/activation.py +++ b/transformer_engine/jax/activation.py @@ -107,3 +107,31 @@ def _activation_bwd_rule(activation_type, act_params, ctx, g): _activation.defvjp(_activation_fwd_rule, _activation_bwd_rule) + + +@partial(jax.custom_vjp, nondiff_argnums=(2, 3)) +def grouped_situglu(x, group_sizes, beta1=4.0, beta2=25.0): + """Apply native SiTU to contiguous expert groups in ``x[capacity, 2H]``. + + The halves of each row contain gate and up, respectively. ``group_sizes`` + is a device int32 vector of nonnegative counts whose sum must not exceed + capacity. Empty groups and arbitrary row counts are supported. Inactive + rows return zeros and do not read input values, in forward or backward. + + Inputs and output use FP32, FP16 or BF16; arithmetic uses FP32. Parameters + are static positive floats. For MXFP8, apply grouped quantization afterward. + In distributed expert code, call inside ``shard_map`` with local counts. + """ + return tex.grouped_situglu_fwd(x, group_sizes, beta1, beta2) + + +def _grouped_situglu_fwd(x, group_sizes, beta1, beta2): + return tex.grouped_situglu_fwd(x, group_sizes, beta1, beta2), (x, group_sizes) + + +def _grouped_situglu_bwd(beta1, beta2, residual, grad): + x, group_sizes = residual + return tex.grouped_situglu_bwd(grad, x, group_sizes, beta1, beta2), None + + +grouped_situglu.defvjp(_grouped_situglu_fwd, _grouped_situglu_bwd) diff --git a/transformer_engine/jax/cpp_extensions/activation.py b/transformer_engine/jax/cpp_extensions/activation.py index 5058192c3f3..b556fcda451 100644 --- a/transformer_engine/jax/cpp_extensions/activation.py +++ b/transformer_engine/jax/cpp_extensions/activation.py @@ -6,6 +6,7 @@ import operator from functools import reduce, partial from dataclasses import dataclass +import math import jax import jax.numpy as jnp @@ -38,7 +39,54 @@ ) -__all__ = ["act_lu", "dact_lu", "quantize_dact_dbias"] +__all__ = ["act_lu", "dact_lu", "quantize_dact_dbias", "grouped_situglu_fwd", "grouped_situglu_bwd"] + + +def _grouped_situglu_call(x, grad, group_sizes, beta1, beta2, *, backward): + if x.ndim != 2 or x.shape[1] == 0 or x.shape[1] % 2: + raise ValueError(f"Expected x[capacity, 2H] with positive H, got {x.shape}") + if x.dtype not in (jnp.float32, jnp.float16, jnp.bfloat16): + raise TypeError(f"Grouped SiTU requires float32, float16 or bfloat16, got {x.dtype}") + if group_sizes.ndim != 1 or group_sizes.dtype != jnp.int32: + raise TypeError("group_sizes must be a rank-1 int32 array") + if group_sizes.size == 0: + raise ValueError("group_sizes must contain at least one group") + for name, value in (("beta1", beta1), ("beta2", beta2)): + if not math.isfinite(value) or value <= 0 or value > float(np.finfo(np.float32).max): + raise ValueError(f"{name} must be finite, positive and representable in float32") + if np.float32(value) == 0: + raise ValueError(f"{name} is too small for float32") + output_shape = x.shape if backward else (x.shape[0], x.shape[1] // 2) + if backward and (grad.shape != (x.shape[0], x.shape[1] // 2) or grad.dtype != x.dtype): + raise ValueError("Grouped SiTU upstream gradient shape or dtype does not match x") + result, _ = ffi.ffi_call( + "te_grouped_situglu_ffi", + ( + jax.ShapeDtypeStruct(output_shape, x.dtype), + jax.ShapeDtypeStruct((group_sizes.size * 8,), jnp.uint8), + ), + vmap_method="sequential", + )( + x, + grad, + group_sizes, + beta1=np.float32(beta1), + beta2=np.float32(beta2), + backward=backward, + ) + return result + + +def grouped_situglu_fwd(x, group_sizes, beta1=4.0, beta2=25.0): + """Native grouped SiTU; inactive capacity rows are zero without reading x.""" + return _grouped_situglu_call( + x, jnp.empty((0,), dtype=x.dtype), group_sizes, beta1, beta2, backward=False + ) + + +def grouped_situglu_bwd(grad, x, group_sizes, beta1=4.0, beta2=25.0): + """Native grouped SiTU gradient; inactive rows do not read x or grad.""" + return _grouped_situglu_call(x, grad, group_sizes, beta1, beta2, backward=True) ActivationEnum = { diff --git a/transformer_engine/jax/cpp_extensions/router.py b/transformer_engine/jax/cpp_extensions/router.py index ad49c7a7d1b..32a0d7c6725 100644 --- a/transformer_engine/jax/cpp_extensions/router.py +++ b/transformer_engine/jax/cpp_extensions/router.py @@ -6,7 +6,7 @@ from enum import IntEnum import jax.numpy as jnp -from jax import dtypes, ffi +from jax import ShapeDtypeStruct, dtypes, ffi from jax.sharding import NamedSharding, PartitionSpec from transformer_engine_jax import JAXX_Routing_Map_Format, JAXX_Score_Function @@ -18,6 +18,7 @@ "RoutingMapFormat", "fused_topk_with_score_function_fwd", "fused_topk_with_score_function_bwd", + "fused_topk_with_score_function_qb_fwd", "fused_moe_aux_loss_fwd", "fused_moe_aux_loss_bwd", ] @@ -732,6 +733,63 @@ def fused_topk_with_score_function_fwd( ) +def fused_topk_with_score_function_qb_fwd( + logits, + expert_bias, + bin_bounds, + histogram, + *, + topk, + scaling_factor, + histogram_mode, + routing_map_format=int(RoutingMapFormat.BYTEMAP), + token_mask=None, +): + """Shard-local QB forward; histogram input aliases the updated output.""" + num_tokens, num_experts = logits.shape + map_shape = ( + (num_tokens, (num_experts + 7) // 8) + if int(routing_map_format) == int(RoutingMapFormat.BITMAP_U8) + else logits.shape + ) + map_dtype = ( + jnp.uint8 if int(routing_map_format) == int(RoutingMapFormat.BITMAP_U8) else jnp.bool_ + ) + outputs = ( + ShapeDtypeStruct(logits.shape, jnp.float32), + ShapeDtypeStruct(map_shape, map_dtype), + ShapeDtypeStruct(logits.shape, jnp.float32), + ShapeDtypeStruct((num_tokens,), jnp.float32), + ShapeDtypeStruct(histogram.shape, jnp.int32), + ) + result = ffi.ffi_call( + "te_fused_topk_with_score_function_qb_forward_ffi", + outputs, + input_output_aliases={3: 4}, + vmap_method="sequential", + )( + logits, + expert_bias, + bin_bounds, + histogram, + topk=int(topk), + scaling_factor=float(scaling_factor), + histogram_mode=int(histogram_mode), + collect_histogram=int(token_mask is None), + routing_map_format=int(routing_map_format), + ) + if token_mask is not None: + probs, routing_map, scores, cutoff, updated_histogram = result + lower, upper = bin_bounds[0], bin_bounds[1] + num_bins = histogram.shape[1] + bins = jnp.floor((cutoff[:, None] - scores - lower) * (num_bins / (upper - lower))) + bins = jnp.clip(bins.astype(jnp.int32), 0, num_bins - 1) + counts = token_mask.astype(jnp.int32)[:, None] + updated_histogram = updated_histogram.at[jnp.arange(num_experts)[None, :], bins].add(counts) + result = probs, routing_map, scores, cutoff, updated_histogram + return result + + def fused_topk_with_score_function_bwd( routing_map: jnp.ndarray, saved_scores: jnp.ndarray, diff --git a/transformer_engine/jax/csrc/extensions.h b/transformer_engine/jax/csrc/extensions.h index 893374b3de3..9d2d85479a6 100644 --- a/transformer_engine/jax/csrc/extensions.h +++ b/transformer_engine/jax/csrc/extensions.h @@ -89,6 +89,7 @@ inline bool use_fp8(DType type) { return type == DType::kFloat8E4M3 || type == D XLA_FFI_DECLARE_HANDLER_SYMBOL(ActLuHandler); XLA_FFI_DECLARE_HANDLER_SYMBOL(ActLuInitializeHandler); +XLA_FFI_DECLARE_HANDLER_SYMBOL(GroupedSiTUGLUHandler); XLA_FFI_DECLARE_HANDLER_SYMBOL(DActLuDBiasQuantizeHandler); XLA_FFI_DECLARE_HANDLER_SYMBOL(DActLuDBiasQuantizeInitializeHandler); @@ -203,6 +204,7 @@ XLA_FFI_DECLARE_HANDLER_SYMBOL(CublasHandleInitHandler); // Router XLA_FFI_DECLARE_HANDLER_SYMBOL(FusedTopkWithScoreFunctionForwardHandler); +XLA_FFI_DECLARE_HANDLER_SYMBOL(FusedTopkWithScoreFunctionQBForwardHandler); XLA_FFI_DECLARE_HANDLER_SYMBOL(FusedTopkWithScoreFunctionBackwardHandler); XLA_FFI_DECLARE_HANDLER_SYMBOL(FusedMoEAuxLossForwardHandler); XLA_FFI_DECLARE_HANDLER_SYMBOL(FusedMoEAuxLossBackwardHandler); diff --git a/transformer_engine/jax/csrc/extensions/activation.cpp b/transformer_engine/jax/csrc/extensions/activation.cpp index 6325a700d15..4f0b1351802 100644 --- a/transformer_engine/jax/csrc/extensions/activation.cpp +++ b/transformer_engine/jax/csrc/extensions/activation.cpp @@ -9,11 +9,77 @@ #include "../extensions.h" #include "transformer_engine/cast.h" +#include "transformer_engine/gemm.h" #include "xla/ffi/api/c_api.h" namespace transformer_engine { namespace jax { +Error_Type GroupedSiTUGLUFFI(cudaStream_t stream, Buffer_Type input_buf, Buffer_Type grad_buf, + Buffer_Type sizes_buf, Result_Type output_buf, + Result_Type workspace_buf, float beta1, float beta2, bool backward) { + const auto shape = input_buf.dimensions(); + NVTE_CHECK(shape.size() == 2 && shape[1] > 0 && shape[1] % 2 == 0, + "Grouped SiTU input must have shape [capacity, 2H]."); + NVTE_CHECK( + sizes_buf.dimensions().size() == 1 && sizes_buf.element_type() == xla::ffi::DataType::S32, + "Grouped SiTU group_sizes must be rank-1 int32."); + const size_t groups = sizes_buf.dimensions()[0]; + NVTE_CHECK(product(workspace_buf->dimensions()) >= groups * sizeof(int64_t), + "Grouped SiTU metadata workspace is too small."); + const auto dtype = convert_ffi_datatype_to_te_dtype(input_buf.element_type()); + NVTE_CHECK(convert_ffi_datatype_to_te_dtype(output_buf->element_type()) == dtype, + "Grouped SiTU input/output dtypes must match."); + const auto out_shape = output_buf->dimensions(); + NVTE_CHECK(out_shape.size() == 2 && out_shape[0] == shape[0] && + out_shape[1] == (backward ? shape[1] : shape[1] / 2), + "Grouped SiTU output shape is invalid."); + auto *sizes = reinterpret_cast(workspace_buf->untyped_data()); + if (groups > 0) { + nvte_convert_int32_to_int64_with_multiplier( + static_cast(sizes_buf.untyped_data()), sizes, groups, 1, stream); + } + const auto scaling = get_nvte_scaling_mode(JAXX_Scaling_Mode::NO_SCALING); + const std::vector input_shape{static_cast(shape[0]), + static_cast(shape[1])}; + const std::vector output_shape{static_cast(out_shape[0]), + static_cast(out_shape[1])}; + const std::vector grad_shape{static_cast(shape[0]), + static_cast(shape[1] / 2)}; + const std::vector sizes_shape{groups}; + GroupedTensorWrapper input(groups, input_shape, scaling); + input.set_rowwise_data(input_buf.untyped_data(), dtype, input_shape) + .set_first_dims(sizes, DType::kInt64, sizes_shape); + GroupedTensorWrapper output(groups, output_shape, scaling); + output.set_rowwise_data(output_buf->untyped_data(), dtype, output_shape); + if (backward) { + const auto actual_grad_shape = grad_buf.dimensions(); + NVTE_CHECK(actual_grad_shape.size() == 2 && actual_grad_shape[0] == shape[0] && + actual_grad_shape[1] == shape[1] / 2 && + convert_ffi_datatype_to_te_dtype(grad_buf.element_type()) == dtype, + "Grouped SiTU upstream gradient shape or dtype is invalid."); + GroupedTensorWrapper grad(groups, grad_shape, scaling); + grad.set_rowwise_data(grad_buf.untyped_data(), dtype, grad_shape); + nvte_group_dsituglu(grad.data(), input.data(), output.data(), beta1, beta2, stream); + } else { + nvte_group_situglu(input.data(), output.data(), beta1, beta2, stream); + } + return ffi_with_cuda_error_check(); +} + +XLA_FFI_DEFINE_HANDLER_SYMBOL(GroupedSiTUGLUHandler, GroupedSiTUGLUFFI, + FFI::Bind() + .Ctx() + .Arg() + .Arg() + .Arg() + .Ret() + .Ret() + .Attr("beta1") + .Attr("beta2") + .Attr("backward"), + FFI_CudaGraph_Traits); + Error_Type ActLuFFI(cudaStream_t stream, Buffer_Type input_buf, Buffer_Type scale_buf, Buffer_Type amax_buf, Result_Type output_buf, Result_Type colwise_output_buf, Result_Type scale_inv_buf, Result_Type colwise_scale_inv_buf, diff --git a/transformer_engine/jax/csrc/extensions/pybind.cpp b/transformer_engine/jax/csrc/extensions/pybind.cpp index d82d109f650..adbb2fe318e 100644 --- a/transformer_engine/jax/csrc/extensions/pybind.cpp +++ b/transformer_engine/jax/csrc/extensions/pybind.cpp @@ -24,6 +24,7 @@ pybind11::dict Registrations() { pybind11::dict dict; // Activation + dict["te_grouped_situglu_ffi"] = EncapsulateFFI(GroupedSiTUGLUHandler); dict["te_act_lu_ffi"] = pybind11::dict(pybind11::arg("initialize") = EncapsulateFFI(ActLuInitializeHandler), pybind11::arg("execute") = EncapsulateFFI(ActLuHandler)); @@ -114,6 +115,8 @@ pybind11::dict Registrations() { // Router dict["te_fused_topk_with_score_function_forward_ffi"] = EncapsulateFFI(FusedTopkWithScoreFunctionForwardHandler); + dict["te_fused_topk_with_score_function_qb_forward_ffi"] = + EncapsulateFFI(FusedTopkWithScoreFunctionQBForwardHandler); dict["te_fused_topk_with_score_function_backward_ffi"] = EncapsulateFFI(FusedTopkWithScoreFunctionBackwardHandler); dict["te_fused_moe_aux_loss_forward_ffi"] = EncapsulateFFI(FusedMoEAuxLossForwardHandler); diff --git a/transformer_engine/jax/csrc/extensions/router.cpp b/transformer_engine/jax/csrc/extensions/router.cpp index a1d0731b6db..3916ed1d0c5 100644 --- a/transformer_engine/jax/csrc/extensions/router.cpp +++ b/transformer_engine/jax/csrc/extensions/router.cpp @@ -102,6 +102,95 @@ XLA_FFI_DEFINE_HANDLER_SYMBOL(FusedTopkWithScoreFunctionForwardHandler, .Attr("routing_map_format"), FFI_CudaGraph_Traits); +Error_Type FusedTopkWithScoreFunctionQBForwardFFI( + cudaStream_t stream, Buffer_Type logits_buf, Buffer_Type expert_bias_buf, + Buffer_Type bin_bounds_buf, Buffer_Type histogram_buf, Result_Type probs_buf, + Result_Type routing_map_buf, Result_Type intermediate_buf, Result_Type cutoff_buf, + Result_Type histogram_out_buf, int64_t topk, double scaling_factor, int64_t histogram_mode, + int64_t collect_histogram, JAXX_Routing_Map_Format routing_map_format) { + auto dims = logits_buf.dimensions(); + auto bias_dims = expert_bias_buf.dimensions(); + auto bounds_dims = bin_bounds_buf.dimensions(); + auto hist_dims = histogram_buf.dimensions(); + NVTE_CHECK(dims.size() == 2 && dims[0] > 0 && dims[1] > 0, + "QB logits must have shape [tokens, experts] with positive dimensions."); + const auto num_tokens = static_cast(dims[0]); + const auto num_experts = static_cast(dims[1]); + NVTE_CHECK(topk > 0 && topk < num_experts, "QB requires 0 < topk < num_experts."); + NVTE_CHECK(bias_dims.size() == 1 && bias_dims[0] == num_experts, + "QB expert_bias must have shape [experts]."); + NVTE_CHECK(bounds_dims.size() == 1 && bounds_dims[0] == 2, "QB bin_bounds must have shape [2]."); + NVTE_CHECK(hist_dims.size() == 2 && hist_dims[0] == num_experts && hist_dims[1] > 0, + "QB histogram must have shape [experts, bins]."); + NVTE_CHECK( + convert_ffi_datatype_to_te_dtype(logits_buf.element_type()) == DType::kFloat32 && + convert_ffi_datatype_to_te_dtype(expert_bias_buf.element_type()) == DType::kFloat32 && + convert_ffi_datatype_to_te_dtype(bin_bounds_buf.element_type()) == DType::kFloat32, + "QB logits, expert_bias and bin_bounds must be float32."); + NVTE_CHECK(convert_ffi_datatype_to_te_dtype(histogram_buf.element_type()) == DType::kInt32, + "QB histogram must be int32."); + NVTE_CHECK(histogram_buf.untyped_data() == histogram_out_buf->untyped_data(), + "QB histogram input and output must alias."); + NVTE_CHECK(histogram_mode == NVTE_QB_HISTOGRAM_TWO_KERNEL || + histogram_mode == NVTE_QB_HISTOGRAM_FUSED_ATOMIC, + "Invalid QB histogram mode."); + + const std::vector shape{static_cast(num_tokens), + static_cast(num_experts)}; + const auto format = static_cast(routing_map_format); + const size_t map_width = format == NVTE_ROUTING_MAP_FORMAT_BITMAP_U8 + ? static_cast((num_experts + 7) / 8) + : static_cast(num_experts); + auto logits = TensorWrapper(logits_buf.untyped_data(), shape, DType::kFloat32); + auto bias = TensorWrapper(expert_bias_buf.untyped_data(), + std::vector{static_cast(num_experts)}, DType::kFloat32); + auto bounds = + TensorWrapper(bin_bounds_buf.untyped_data(), std::vector{2}, DType::kFloat32); + auto probs = TensorWrapper(probs_buf->untyped_data(), shape, DType::kFloat32); + auto routing_map = + TensorWrapper(routing_map_buf->untyped_data(), + std::vector{static_cast(num_tokens), map_width}, DType::kByte); + auto intermediate = TensorWrapper(intermediate_buf->untyped_data(), shape, DType::kFloat32); + auto cutoff = + TensorWrapper(cutoff_buf->untyped_data(), + std::vector{static_cast(num_tokens)}, DType::kFloat32); + auto histogram = TensorWrapper( + histogram_out_buf->untyped_data(), + std::vector{static_cast(num_experts), static_cast(hist_dims[1])}, + DType::kInt32); + const auto mode = collect_histogram ? static_cast(histogram_mode) + : NVTE_QB_HISTOGRAM_TWO_KERNEL; + nvte_fused_topk_with_score_function_forward_qb_v2( + logits.data(), num_tokens, num_experts, static_cast(topk), + static_cast(scaling_factor), bias.data(), probs.data(), routing_map.data(), format, + intermediate.data(), cutoff.data(), histogram.data(), bounds.data(), mode, stream); + if (collect_histogram && mode == NVTE_QB_HISTOGRAM_TWO_KERNEL) { + nvte_qb_histogram_accumulate(intermediate.data(), cutoff.data(), bounds.data(), + histogram.data(), stream); + } + return ffi_with_cuda_error_check(); +} + +XLA_FFI_DEFINE_HANDLER_SYMBOL(FusedTopkWithScoreFunctionQBForwardHandler, + FusedTopkWithScoreFunctionQBForwardFFI, + FFI::Bind() + .Ctx() + .Arg() // logits + .Arg() // expert_bias + .Arg() // bin_bounds + .Arg() // histogram + .Ret() // probs + .Ret() // routing_map + .Ret() // intermediate + .Ret() // cutoff + .Ret() // updated histogram + .Attr("topk") + .Attr("scaling_factor") + .Attr("histogram_mode") + .Attr("collect_histogram") + .Attr("routing_map_format"), + FFI_CudaGraph_Traits); + // ============================================================================ // Fused Top-K with Score Function - Backward // ============================================================================ diff --git a/transformer_engine/jax/flax/moe.py b/transformer_engine/jax/flax/moe.py index 19b1a41778f..283902756ef 100644 --- a/transformer_engine/jax/flax/moe.py +++ b/transformer_engine/jax/flax/moe.py @@ -34,7 +34,13 @@ from flax import linen as nn from transformer_engine.common.recipe import Recipe -from ..moe import RouterComputationInfo, _moe_mesh_axes, _resolve_moe_mesh_resource, moe +from ..moe import ( + RouterComputationInfo, + SiTUActivation, + _moe_mesh_axes, + _resolve_moe_mesh_resource, + moe, +) from ..quantize import QuantizerSet from ..router import ScoreFunction from ..sharding import MeshResource, _get_mesh, global_shard_guard @@ -67,9 +73,11 @@ class _MoEBlock(TransformerEngineBase): Top-k value for routing. intermediate_size : int Hidden dim of the per-expert FFN (the inner ``mlp`` axis). - activation_type : str + activation_type : Union[str, SiTUActivation] Activation between ``layer_w0 @ wi_0`` and the elementwise - product with ``layer_w0 @ wi_1``. Default ``"silu"``. + product with ``layer_w0 @ wi_1``. Default ``"silu"``. Pass + :class:`~transformer_engine.jax.moe.SiTUActivation` to soft-cap both + branches in FP32 using native grouped SiTU and separate grouped GEMMs. score_function : Union[str, ScoreFunction] ``"softmax"`` (default), ``"sigmoid"``, or ``"sqrtsoftplus"`` for the routing scores. @@ -88,7 +96,7 @@ class _MoEBlock(TransformerEngineBase): If ``> 0``, return the MoE auxiliary load-balancing loss scalar in addition to the main output. - gate_kernel_axes, wi_kernel_axes, wo_kernel_axes, input_axes : + gate_kernel_axes, wi_kernel_axes, wo_kernel_axes, input_axes, router_input_axes : Logical sharding axis tuples (consumed by Flax's :func:`with_logical_partitioning` and our internal :func:`with_sharding_constraint_by_logical_axes`). @@ -140,7 +148,7 @@ class _MoEBlock(TransformerEngineBase): num_experts: int = 8 num_experts_per_tok: int = 2 intermediate_size: int = 2048 - activation_type: str = "silu" + activation_type: Union[str, SiTUActivation] = "silu" # Routing score_function: Union[str, ScoreFunction] = "softmax" @@ -156,6 +164,7 @@ class _MoEBlock(TransformerEngineBase): wi_kernel_axes: Tuple[Optional[str], ...] = ("exp", "embed", "mlp") wo_kernel_axes: Tuple[Optional[str], ...] = ("exp", "mlp", "embed") input_axes: Tuple[Optional[str], ...] = () + router_input_axes: Tuple[Optional[str], ...] = () # Parallelism mesh_resource: Optional[MeshResource] = None @@ -167,6 +176,8 @@ class _MoEBlock(TransformerEngineBase): recv_capacity_per_rank: Optional[int] = None dispatch_checkpoint_name: Optional[str] = None collect_expert_counts: bool = True + qb_num_bins: int = 1000 + qb_histogram_mode: str = "fused_atomic" # Dtypes / init / misc dtype: DType = jnp.float32 @@ -188,13 +199,29 @@ def __post_init__(self): super().__post_init__() @nn.compact - def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array]: + def __call__( + self, + inputs: Array, + *, + router_input: Optional[Array] = None, + qb_bin_bounds: Optional[Array] = None, + qb_token_mask: Optional[Array] = None, + ) -> tuple: """Run the MoE forward pass. Parameters ---------- inputs : jnp.ndarray ``[batch, sequence, hidden]``. + router_input : Optional[jnp.ndarray] + Separate gate input ``[batch, sequence, router_hidden]``. ``None`` + uses ``inputs``. Only the gate kernel uses ``router_hidden``; + expert kernels and EP buffers retain ``hidden``. + qb_bin_bounds : Optional[jnp.ndarray] + FP32 ``[lower, upper]`` enables QB histogram collection. Requires + sigmoid routing, expert bias, no groups and zero auxiliary loss. + qb_token_mask : Optional[jnp.ndarray] + Boolean ``[batch, sequence]`` mask excluding padding from QB statistics. Returns ------- @@ -208,6 +235,10 @@ def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array] overflow when ``drop_on_overflow`` is set at ep_bootstrap. expert_counts : jnp.ndarray Global assignment counts for each expert. + qb_histograms : jnp.ndarray, optional + Fifth result only when ``qb_bin_bounds`` is supplied; int32 + ``[token_shards, experts, qb_num_bins]``. Reduce over token shards + and microbatches before updating the bias in the training loop. """ mesh_resource, quant_before_fsdp_ag = _resolve_moe_mesh_resource( self.mesh_resource, @@ -218,6 +249,11 @@ def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array] inputs.ndim == 3 ), f"_MoEBlock expects [batch, sequence, hidden] input, got shape {inputs.shape}" _, _, hidden_size = inputs.shape + if router_input is not None and ( + router_input.ndim != 3 or router_input.shape[:2] != inputs.shape[:2] + ): + raise ValueError("router_input must match the batch and sequence dimensions of inputs.") + router_hidden_size = hidden_size if router_input is None else router_input.shape[-1] # Param registrations -- must run OUTSIDE any JAX transform that # alters the variable scope (e.g. shard_map). The functional @@ -226,10 +262,10 @@ def __call__(self, inputs: Array) -> Tuple[Array, Optional[Array], Array, Array] gate_kernel = self.param( "gate_kernel", nn.with_logical_partitioning(self.kernel_init, self.gate_kernel_axes), - (hidden_size, self.num_experts), + (router_hidden_size, self.num_experts), self.dtype, ) - # FC1 is stored as one gated-SwiGLU kernel. Keeping its two + # FC1 is stored as one gated FFN kernel. Keeping its two # projections contiguous lets the functional MoE path quantize and # all-gather one FP8 data buffer (and one scale buffer), rather than # materializing a concatenate inside the custom-VJP. @@ -318,6 +354,12 @@ def make_grouped_quantizer_set(postfix): scaling_factor=self.scaling_factor, aux_loss_coeff=self.aux_loss_coeff, gate_kernel_axes=self.gate_kernel_axes, + router_input=router_input, + router_input_axes=self.router_input_axes, + qb_bin_bounds=qb_bin_bounds, + qb_token_mask=qb_token_mask, + qb_num_bins=self.qb_num_bins, + qb_histogram_mode=self.qb_histogram_mode, ) return moe( inputs, diff --git a/transformer_engine/jax/moe.py b/transformer_engine/jax/moe.py index 4e9bfe4aa8d..44d37a3f3e1 100644 --- a/transformer_engine/jax/moe.py +++ b/transformer_engine/jax/moe.py @@ -41,7 +41,7 @@ import math import warnings -from dataclasses import replace +from dataclasses import dataclass, replace from functools import partial from typing import Any, Optional, Tuple, Union @@ -68,10 +68,11 @@ from .quantize.dequantizer import _unswizzle_mxfp8_grouped_scale from .cpp_extensions.gemm import swizzled_scale from .flax.module import _convert_to_activation_function -from .router import ScoreFunction, _validate_score_function +from .router import ScoreFunction, _validate_score_function, _validate_qb_router from .sharding import MeshResource, _get_mesh, global_mesh_resource, global_shard_guard __all__ = [ + "SiTUActivation", "RouterComputationInfo", "RoutingMapInfo", "get_moe_recv_capacity_per_rank", @@ -79,6 +80,41 @@ ] +@dataclass(frozen=True) +class SiTUActivation: + """Two-branch SiTU-GLU for MoE experts. + + Computes ``gate_beta * tanh(gate / gate_beta) * sigmoid(gate)`` times + ``up_beta * tanh(up / up_beta)`` in FP32, then casts to the gate dtype. + The defaults match Kimi-K3. Both soft-cap parameters must be finite and + positive. Pass an instance as ``moe(..., activation_type=...)``. + + BF16 and MXFP8 use native grouped SiTU forward/backward with separate GEMMs. + Requesting cuDNN fusion retains its normal warning and fallback behavior. + """ + + gate_beta: float = 4.0 + up_beta: float = 25.0 + + def __post_init__(self): + for name in ("gate_beta", "up_beta"): + value = float(getattr(self, name)) + if not math.isfinite(value) or value <= 0: + raise ValueError(f"{name} must be finite and positive, got {value}") + object.__setattr__(self, name, value) + + +def _apply_ffn_activation(activation_type, gate, up, group_sizes): + if isinstance(activation_type, SiTUActivation): + return tex.grouped_situglu_fwd( + jnp.concatenate([gate, up], axis=-1), + group_sizes, + activation_type.gate_beta, + activation_type.up_beta, + ) + return _convert_to_activation_function(activation_type)(gate) * up + + @flax.struct.dataclass class RouterComputationInfo: """Router parameters and settings for :func:`moe` to compute routing itself. @@ -88,7 +124,8 @@ class RouterComputationInfo: Parameters ---------- gate_kernel : jnp.ndarray - ``[hidden, num_experts]`` router projection. + ``[router_hidden, num_experts]`` router projection; ``router_hidden`` + is the width of ``router_input``, or ``x`` when no separate input is given. num_experts_per_tok : int Top-k value for routing. expert_bias : Optional[jnp.ndarray] @@ -113,6 +150,25 @@ class RouterComputationInfo: view; this lives off the dispatch critical path. gate_kernel_axes : Tuple[Optional[str], ...] Logical sharding axes of ``gate_kernel``. + router_input : Optional[jnp.ndarray] + Separate ``[batch, sequence, router_hidden]`` activations for the gate. + ``None`` uses the expert input ``x``. Batch and sequence dimensions must + match ``x``; hidden dimensions may differ. Gradients flow to this input + independently of the expert-input gradient and retain its dtype. + The gate kernel is cast to this input's dtype for the projection. + router_input_axes : Tuple[Optional[str], ...] + Logical sharding axes of ``router_input``. Its physical token layout + follows the same DP/FSDP/EP placement as ``x``. + qb_bin_bounds : Optional[jnp.ndarray] + FP32 ``[lower, upper]`` enables QB statistics with sigmoid, no groups and + no auxiliary loss. Set ``qb_token_mask`` to exclude sequence padding. + Bounds must be finite and ordered, and remain fixed throughout a step. + qb_token_mask : Optional[jnp.ndarray] + Boolean ``[batch, sequence]`` valid-token mask. With a mask, JAX builds + the histogram from CUDA router scores/cutoffs; without it CUDA collects it. + qb_num_bins, qb_histogram_mode : int, str + Histogram size and ``"fused_atomic"`` or ``"two_kernel"`` implementation. + The caller owns histogram reduction and bias updates. """ gate_kernel: jnp.ndarray @@ -127,6 +183,12 @@ class RouterComputationInfo: scaling_factor: float = flax.struct.field(pytree_node=False, default=1.0) aux_loss_coeff: float = flax.struct.field(pytree_node=False, default=0.0) gate_kernel_axes: Tuple[Optional[str], ...] = flax.struct.field(pytree_node=False, default=()) + router_input: Optional[jnp.ndarray] = None + router_input_axes: Tuple[Optional[str], ...] = flax.struct.field(pytree_node=False, default=()) + qb_bin_bounds: Optional[jnp.ndarray] = None + qb_token_mask: Optional[jnp.ndarray] = None + qb_num_bins: int = flax.struct.field(pytree_node=False, default=1000) + qb_histogram_mode: str = flax.struct.field(pytree_node=False, default="fused_atomic") @flax.struct.dataclass @@ -890,7 +952,7 @@ def _ffn_fwd_per_shard( quantizer_sets: Tuple[QuantizerSet, QuantizerSet], *, num_local_experts: int, - activation_type: str, + activation_type: Union[str, SiTUActivation], apply_topk_weights_early: bool, use_cudnn_jax_fusion: bool, wi_0_checkpoint_name: Optional[str], @@ -1036,14 +1098,12 @@ def _ffn_fwd_per_shard( if wi_1_checkpoint_name is not None: up_proj_out = checkpoint_name(up_proj_out, wi_1_checkpoint_name) - # Activation inputs (gate_proj_out, up_proj_out) stay in the wi GEMM - # output dtype; the activation output (`intermediate`) stays in the - # dtype the wo GEMM / wo's quantized input consumes. For bf16 compute - # that's all bf16; for FP8/FP4 the downstream grouped_quantize is what - # transitions to the target precision. - act_fn = _convert_to_activation_function(activation_type) + # Keep projection residuals and activation output in the GEMM dtype; + # SiTU evaluates its internal arithmetic in FP32 before casting back. if not use_cudnn_jax_fusion: - intermediate = act_fn(gate_proj_out) * up_proj_out + intermediate = _apply_ffn_activation( + activation_type, gate_proj_out, up_proj_out, group_sizes + ) if apply_topk_weights_early and not use_cudnn_jax_fusion: # Fold the per-token combine weights into the FFN intermediate; # the downstream wo GEMM is linear so this is equivalent to the @@ -1102,7 +1162,7 @@ def _ffn_bwd_per_shard( recv_topk_weights_local: jnp.ndarray, quantizer_sets: Tuple[QuantizerSet, QuantizerSet], *, - activation_type: str, + activation_type: Union[str, SiTUActivation], apply_topk_weights_early: bool, has_bias: bool, use_cudnn_jax_fusion: bool, @@ -1206,12 +1266,13 @@ def _ffn_bwd_per_shard( casted_wo_rhs_trans, contracting_dims=((1,), (2,)), ) - act_fn = _convert_to_activation_function(activation_type) if apply_topk_weights_early: # intermediate' = intermediate * w. # Masking is not required as grouped GEMMs consume only group-size rows. w_b = recv_w_flat[:, None].astype(d_intermediate.dtype) - intermediate_unweighted = act_fn(gate_proj_out) * up_proj_out + intermediate_unweighted = _apply_ffn_activation( + activation_type, gate_proj_out, up_proj_out, group_sizes + ) d_recv_w_from_intermediate = jnp.sum( d_intermediate * intermediate_unweighted, axis=-1, @@ -1220,11 +1281,20 @@ def _ffn_bwd_per_shard( else: d_recv_w_from_intermediate = jnp.zeros_like(recv_w_flat) - # Activation bwd stays in the GEMM dtype, matching the forward path. - act_gp, dact_pullback = jax.vjp(act_fn, gate_proj_out) - d_up_proj_out = d_intermediate * act_gp - (d_gate_proj_out,) = dact_pullback(d_intermediate * up_proj_out) - d_combined_for_bias = jnp.concatenate([d_gate_proj_out, d_up_proj_out], axis=-1) + if isinstance(activation_type, SiTUActivation): + d_combined_for_bias = tex.grouped_situglu_bwd( + d_intermediate, + jnp.concatenate([gate_proj_out, up_proj_out], axis=-1), + group_sizes, + activation_type.gate_beta, + activation_type.up_beta, + ) + else: + act_fn = _convert_to_activation_function(activation_type) + act_gp, dact_pullback = jax.vjp(act_fn, gate_proj_out) + d_up_proj_out = d_intermediate * act_gp + (d_gate_proj_out,) = dact_pullback(d_intermediate * up_proj_out) + d_combined_for_bias = jnp.concatenate([d_gate_proj_out, d_up_proj_out], axis=-1) casted_d_combined = tex.grouped_quantize( d_combined_for_bias, fc1_quantizer_set.dgrad, @@ -1314,7 +1384,7 @@ def _moe_fwd_rule( ): """Forward: [gate -> route ->] ep_dispatch -> FFN -> ep_combine. - Returns ``(output, aux_loss, total_recv_tokens, expert_counts)``. + Returns ``(output, aux_loss, total_recv_tokens, expert_counts, qb_histograms)``. ``aux_loss`` is a zero scalar unless the router computes it and ``aux_loss_coeff`` > 0. """ with global_shard_guard(mesh_resource): @@ -1325,6 +1395,12 @@ def _moe_fwd_rule( compute_aux_loss = router is not None and router.aux_loss_coeff > 0.0 x = with_sharding_constraint_by_logical_axes(x, input_axes) + if router is not None and router.router_input is not None: + router = router.replace( + router_input=with_sharding_constraint_by_logical_axes( + router.router_input, router.router_input_axes + ) + ) mesh = _get_mesh() if mesh is None or mesh.empty: @@ -1430,6 +1506,7 @@ def _moe_fwd_rule( # ------------------------ Routing ------------------------ + qb_histograms = jnp.zeros((0, 0, 0), dtype=jnp.int32) if router is None: # ---------------- Caller-supplied routing ---------------- routing_indices = jax.lax.with_sharding_constraint( @@ -1451,8 +1528,10 @@ def _moe_fwd_rule( # accumulate at low precision and silently produce NaNs on tokens # whose normalised weights underflow. Cast to fp32 here to stay in # the validated regime. - gate_kernel_cast = router.gate_kernel.astype(x.dtype) - gate_logits = jnp.einsum("bsh,he->bse", x, gate_kernel_cast) + gate_input = x if router.router_input is None else router.router_input + gate_input = jax.lax.with_sharding_constraint(gate_input, NamedSharding(mesh, ep3_spec)) + gate_kernel_cast = router.gate_kernel.astype(gate_input.dtype) + gate_logits = jnp.einsum("bsh,he->bse", gate_input, gate_kernel_cast) logits_2d = gate_logits.reshape(-1, num_experts).astype(jnp.float32) # ---------------- Routing (global view) ---------------- @@ -1471,9 +1550,44 @@ def _moe_fwd_rule( "expert_bias": eb_arg, "compute_aux_scores": False, } - sparse_probs, routing_map, saved_scores = tex.fused_topk_with_score_function_fwd( - logits_2d, **topk_kwargs - ) + if router.qb_bin_bounds is None: + sparse_probs, routing_map, saved_scores = tex.fused_topk_with_score_function_fwd( + logits_2d, **topk_kwargs + ) + else: + mode = {"two_kernel": 0, "fused_atomic": 1}[router.qb_histogram_mode] + + def route_and_observe(local_logits, bias, bounds, *mask): + histogram = jnp.zeros((num_experts, router.qb_num_bins), dtype=jnp.int32) + probs, rmap, scores, _, histogram = tex.fused_topk_with_score_function_qb_fwd( + local_logits, + bias, + bounds, + histogram, + topk=K, + scaling_factor=router.scaling_factor, + histogram_mode=mode, + token_mask=mask[0] if mask else None, + ) + return probs, rmap, scores, histogram[None] + + sparse_probs, routing_map, saved_scores, qb_histograms = shard_map( + route_and_observe, + mesh=mesh, + in_specs=(ep2_spec, P(None), P(None)) + + ((P(batch_pspec_axis),) if router.qb_token_mask is not None else ()), + out_specs=(ep2_spec, ep2_spec, ep2_spec, P(batch_pspec_axis, None, None)), + check_rep=False, + )( + logits_2d, + router.expert_bias, + router.qb_bin_bounds, + *( + (router.qb_token_mask.reshape(-1),) + if router.qb_token_mask is not None + else () + ), + ) sparse_probs = sparse_probs.astype(dtype) # argsort on a bool tensor places True last (False=0 < True=1), so the # last K indices are the selected expert IDs. @@ -1708,7 +1822,7 @@ def _ffn_fwd_body(*args): "cudnn_native_weight_layout": cudnn_native_weight_layout, } # The integer metadata outputs are non-differentiable (see moe()). - return (output, aux_loss, total_recv_tokens, expert_counts), (ctx, static) + return (output, aux_loss, total_recv_tokens, expert_counts, qb_histograms), (ctx, static) def _moe_bwd_rule( @@ -1751,7 +1865,7 @@ def _moe_bwd_rule( from jax.experimental.shard_map import shard_map # The integer metadata outputs are non-differentiable; their cotangents are unused. - d_output, d_aux_loss, _d_total_recv_tokens, _d_expert_counts = cotangents + d_output, d_aux_loss, _d_total_recv_tokens, _d_expert_counts, _d_qb_histograms = cotangents ctx, static = residuals router = ctx.router @@ -1945,19 +2059,32 @@ def _ffn_bwd_body(*args): # ---------------- Gate bwd (global view) ---------------- d_gate_logits = d_logits_2d.reshape(B, S, num_experts) - gate_kernel_cast = router.gate_kernel.astype(ctx.x.dtype) - d_x_from_gate = jnp.einsum("bse,he->bsh", d_gate_logits, gate_kernel_cast) - d_gate_kernel = jnp.einsum("bsh,bse->he", ctx.x, d_gate_logits).astype( + gate_input = ctx.x if router.router_input is None else router.router_input + gate_kernel_cast = router.gate_kernel.astype(gate_input.dtype) + d_gate_input = jnp.einsum("bse,he->bsh", d_gate_logits, gate_kernel_cast) + d_gate_kernel = jnp.einsum("bsh,bse->he", gate_input, d_gate_logits).astype( router.gate_kernel.dtype ) d_gate_kernel = with_sharding_constraint_by_logical_axes( d_gate_kernel, router.gate_kernel_axes ) - d_x = d_x_from_gate + d_x_from_dispatch + if router.router_input is None: + d_x = d_gate_input + d_x_from_dispatch + d_router_input = None + else: + d_x = d_x_from_dispatch + d_router_input = with_sharding_constraint_by_logical_axes( + d_gate_input.astype(router.router_input.dtype), router.router_input_axes + ) # expert_bias has no learnable bwd path through fused_topk: the # primitive's bwd returns None for the bias slot, so its cotangent is zero. d_router_info = router.replace( gate_kernel=d_gate_kernel, + router_input=d_router_input, + qb_bin_bounds=( + None if router.qb_bin_bounds is None else jnp.zeros_like(router.qb_bin_bounds) + ), + qb_token_mask=None, expert_bias=( None if router.expert_bias is None else jnp.zeros_like(router.expert_bias) ), @@ -2094,7 +2221,7 @@ def moe( wo_bias: Optional[jnp.ndarray] = None, *, num_experts: int, - activation_type: str = "silu", + activation_type: Union[str, SiTUActivation] = "silu", apply_topk_weights_early: bool = False, quantizer_sets: Tuple[QuantizerSet, QuantizerSet] = ( noop_quantizer_set, @@ -2114,7 +2241,7 @@ def moe( dispatch_checkpoint_name: Optional[str] = None, use_cudnn_fusion: bool = True, cudnn_native_weight_layout: Optional[bool] = None, -) -> Tuple[jnp.ndarray, Optional[jnp.ndarray], jnp.ndarray, jnp.ndarray]: +) -> tuple: """Run a full MoE block under a single fused custom_vjp on the TE EP path. Returns ``(output, aux_loss, total_recv_tokens, expert_counts)``. ``aux_loss`` @@ -2125,8 +2252,20 @@ def moe( ``[num_experts]`` containing global top-k assignment counts (or zeros when ``collect_expert_counts=False``). + With ``router_info.qb_bin_bounds`` set, a fifth result contains int32 QB + histograms ``[token_shards, experts, bins]``. The leading axis is sharded over + the compound DP/FSDP/EP token axes. Each token contributes exactly once; + there is no histogram collective in the forward. Accumulate over microbatches, + then sum the leading axis before deriving the next step's bias. The default + call keeps its four-result contract. QB statistics are non-differentiable. + Parameters ---------- + activation_type : Union[str, SiTUActivation] + Gate activation name (default ``"silu"``), or :class:`SiTUActivation` + for FP32 soft-capping of both gate and up branches. SiTU uses native + grouped activation kernels and separate grouped GEMMs with BF16 or + MXFP8 quantizer sets. cuDNN GLU/dGLU uses the existing fallback path. router_info : Union[RouterComputationInfo, RoutingMapInfo] :class:`RouterComputationInfo` makes TE compute the gate and top-k from the given router parameters. :class:`RoutingMapInfo` supplies @@ -2216,11 +2355,49 @@ def moe( ep_axis, data_parallelism_axes = _moe_mesh_axes(mesh_resource) if isinstance(router_info, RouterComputationInfo): + gate_input = x if router_info.router_input is None else router_info.router_input + if gate_input.ndim != 3 or gate_input.shape[:2] != x.shape[:2]: + raise ValueError( + "router_input must have shape [batch, sequence, router_hidden] matching x, " + f"got x={x.shape}, router_input={gate_input.shape}." + ) + if router_info.gate_kernel.shape != (gate_input.shape[-1], num_experts): + raise ValueError( + "gate_kernel must have shape [router_hidden, num_experts], got " + f"{router_info.gate_kernel.shape} for router_input={gate_input.shape}." + ) router_info = router_info.replace( score_function=_validate_score_function(router_info.score_function), aux_loss_coeff=float(router_info.aux_loss_coeff), ) has_aux_loss = router_info.aux_loss_coeff > 0.0 + if router_info.qb_bin_bounds is not None: + if router_info.qb_token_mask is not None and ( + router_info.qb_token_mask.shape != x.shape[:2] + or router_info.qb_token_mask.dtype != jnp.bool_ + ): + raise ValueError("MoE qb_token_mask must be bool with shape [batch, sequence].") + if ( + router_info.score_function != ScoreFunction.SIGMOID + or router_info.use_pre_softmax + or router_info.num_groups not in (None, -1, 0) + or router_info.group_topk not in (None, -1, 0) + or router_info.aux_loss_coeff != 0.0 + ): + raise ValueError( + "MoE QB requires sigmoid, ungrouped routing, use_pre_softmax=False " + "and aux_loss_coeff=0." + ) + _validate_qb_router( + jax.ShapeDtypeStruct((x.shape[0] * x.shape[1], num_experts), jnp.float32), + router_info.expert_bias, + router_info.qb_bin_bounds, + None, + router_info.num_experts_per_tok, + router_info.qb_num_bins, + router_info.scaling_factor, + router_info.qb_histogram_mode, + ) elif isinstance(router_info, RoutingMapInfo): routing_indices = router_info.routing_indices routing_weights = router_info.routing_weights @@ -2250,6 +2427,15 @@ def moe( # so ep MUST be innermost in the partition spec. Soft re-pin: free if # upstream already matches, single reshard otherwise. x = _prepare_moe_input(x, ep_axis, data_parallelism_axes, caller="moe") + if isinstance(router_info, RouterComputationInfo) and router_info.router_input is not None: + router_info = router_info.replace( + router_input=_prepare_moe_input( + router_info.router_input, + ep_axis, + data_parallelism_axes, + caller="moe router_input", + ) + ) cudnn_native_weight_layout = _resolve_cudnn_native_weight_layout( wi, x.shape[-1], cudnn_native_weight_layout @@ -2269,7 +2455,7 @@ def moe( ) use_cudnn_jax_fusion = _select_cudnn_jax_fusion(rejection_reasons) - output, aux_loss, total_recv_tokens, expert_counts = _moe( + output, aux_loss, total_recv_tokens, expert_counts, qb_histograms = _moe( x, router_info, wi, @@ -2302,4 +2488,6 @@ def moe( assert ( output.dtype == x.dtype ), f"moe() output dtype {output.dtype} != input dtype {x.dtype}" + if isinstance(router_info, RouterComputationInfo) and router_info.qb_bin_bounds is not None: + return output, aux_loss, total_recv_tokens, expert_counts, qb_histograms return output, aux_loss, total_recv_tokens, expert_counts diff --git a/transformer_engine/jax/router.py b/transformer_engine/jax/router.py index 0778155e143..e47a21cb3a3 100644 --- a/transformer_engine/jax/router.py +++ b/transformer_engine/jax/router.py @@ -17,9 +17,14 @@ fused_moe_aux_loss: Compute the MoE auxiliary load-balancing loss scalar. + + fused_topk_with_score_function_qb: + Sigmoid Top-(k+1) routing with functional QB histogram accumulation. """ from functools import partial +import math +import operator from typing import Optional, Tuple, Union import jax @@ -30,6 +35,7 @@ RoutingMapFormat, fused_topk_with_score_function_fwd, fused_topk_with_score_function_bwd, + fused_topk_with_score_function_qb_fwd, fused_moe_aux_loss_fwd, fused_moe_aux_loss_bwd, ) @@ -38,6 +44,7 @@ "ScoreFunction", "RoutingMapFormat", "fused_topk_with_score_function", + "fused_topk_with_score_function_qb", "fused_moe_aux_loss", ] @@ -292,6 +299,174 @@ def _fused_topk_with_score_function_bwd( ) +# ============================================================================= +# Quantile Balancing router +# ============================================================================= + + +def _validate_qb_router( + logits, expert_bias, bin_bounds, histogram, topk, num_bins, scaling_factor, histogram_mode +): + if logits.ndim != 2 or min(logits.shape) <= 0: + raise ValueError("QB logits must have shape [tokens, experts] with positive dimensions.") + if logits.dtype != jnp.float32: + raise TypeError("QB logits must be float32.") + if expert_bias is None or expert_bias.shape != (logits.shape[-1],): + raise ValueError("QB expert_bias must have shape [experts].") + if expert_bias.dtype != jnp.float32: + raise TypeError("QB expert_bias must be float32.") + if bin_bounds is None or bin_bounds.shape != (2,) or bin_bounds.dtype != jnp.float32: + raise ValueError("QB bin_bounds must be float32 with shape [2].") + topk, num_bins = operator.index(topk), operator.index(num_bins) + if not 0 < topk < logits.shape[-1]: + raise ValueError("QB requires 0 < topk < num_experts.") + if num_bins <= 0: + raise ValueError("QB num_bins must be positive.") + if histogram is not None: + if histogram.shape != (logits.shape[-1], num_bins) or histogram.dtype != jnp.int32: + raise ValueError("QB histogram must be int32 with shape [experts, num_bins].") + if not isinstance(scaling_factor, (int, float)) or not math.isfinite(scaling_factor): + raise ValueError("QB scaling_factor must be a finite Python number.") + modes = {"two_kernel": 0, "fused_atomic": 1} + if histogram_mode not in modes: + raise ValueError("QB histogram_mode must be 'two_kernel' or 'fused_atomic'.") + return modes[histogram_mode] + + +def fused_topk_with_score_function_qb( + logits: jnp.ndarray, + topk: int, + expert_bias: jnp.ndarray, + bin_bounds: jnp.ndarray, + *, + num_bins: int = 1000, + histogram: Optional[jnp.ndarray] = None, + token_mask: Optional[jnp.ndarray] = None, + scaling_factor: float = 1.0, + histogram_mode: str = "fused_atomic", + routing_map_format: Union[str, RoutingMapFormat, int] = RoutingMapFormat.BYTEMAP, +) -> Tuple[jnp.ndarray, jnp.ndarray, jnp.ndarray]: + """Route with sigmoid Top-(k+1) and collect Quantile Balancing statistics. + + ``logits`` is FP32 ``[tokens, experts]``; ``expert_bias`` is FP32 ``[experts]``. + Selection uses biased scores, while the returned sparse weights use unbiased + sigmoid scores. The extra selected score is the cutoff for QB, not another route. + + ``bin_bounds`` is FP32 ``[lower, upper]``. The caller must supply finite values + with ``lower < upper``; bounds values are not synchronized to the host for + validation. Kimi uses ``[min(bias)-1, max(bias)+1]`` throughout a training step. + Each row contributes to every expert's histogram of ``cutoff - sigmoid(logits)``. + ``token_mask`` optionally selects valid rows. With a mask, CUDA computes the + routing and cutoff and JAX builds the masked histogram from the saved scores. + Without a mask, the existing CUDA histogram modes run unchanged. + + Returns ``(probs, routing_map, updated_histogram)``. The histogram is int32 + ``[experts, num_bins]`` and starts at zero when ``histogram=None``. Otherwise + it functionally adds to the supplied histogram, with FFI input/output aliasing + allowing buffer reuse. Only ``probs`` has a gradient, through ``logits``. + + This is a shard-local operation. Use ``shard_map`` with the full expert axis + replicated to retain local histograms in SPMD; sum contributions across token + shards and microbatches before deriving the next bias. ``two_kernel`` uses a + separate histogram kernel, while ``fused_atomic`` accumulates in the router. + Neither mode updates biases or performs collectives. Grouped Top-k and auxiliary + scores are not supported. Automatic SPMD without ``shard_map`` may replicate + the call rather than distribute token rows. + """ + mode = _validate_qb_router( + logits, expert_bias, bin_bounds, histogram, topk, num_bins, scaling_factor, histogram_mode + ) + routing_map_format = _validate_routing_map_format(routing_map_format) + if token_mask is not None and ( + token_mask.shape != (logits.shape[0],) or token_mask.dtype != jnp.bool_ + ): + raise ValueError("QB token_mask must be bool with shape [tokens].") + if histogram is None: + histogram = jnp.zeros((logits.shape[-1], num_bins), dtype=jnp.int32) + return _fused_topk_with_score_function_qb( + logits, + expert_bias, + bin_bounds, + histogram, + token_mask, + int(topk), + float(scaling_factor), + mode, + int(routing_map_format), + ) + + +@partial(jax.custom_vjp, nondiff_argnums=(5, 6, 7, 8)) +def _fused_topk_with_score_function_qb( + logits, + expert_bias, + bin_bounds, + histogram, + token_mask, + topk, + scaling_factor, + histogram_mode, + routing_map_format, +): + result, _ = _qb_router_fwd( + logits, + expert_bias, + bin_bounds, + histogram, + token_mask, + topk, + scaling_factor, + histogram_mode, + routing_map_format, + ) + return result + + +def _qb_router_fwd( + logits, + expert_bias, + bin_bounds, + histogram, + token_mask, + topk, + scaling_factor, + histogram_mode, + routing_map_format, +): + probs, routing_map, saved_scores, _, updated_histogram = fused_topk_with_score_function_qb_fwd( + logits, + expert_bias, + bin_bounds, + histogram, + topk=topk, + scaling_factor=scaling_factor, + histogram_mode=histogram_mode, + routing_map_format=routing_map_format, + token_mask=token_mask, + ) + return (probs, routing_map, updated_histogram), (routing_map, saved_scores) + + +def _qb_router_bwd(topk, scaling_factor, histogram_mode, routing_map_format, residuals, cotangents): + del histogram_mode + routing_map, saved_scores = residuals + d_logits = fused_topk_with_score_function_bwd( + routing_map, + saved_scores, + cotangents[0], + topk, + False, + scaling_factor, + ScoreFunction.SIGMOID, + False, + routing_map_format, + ) + return d_logits, None, None, None, None + + +_fused_topk_with_score_function_qb.defvjp(_qb_router_fwd, _qb_router_bwd) + + # ============================================================================= # Fused MoE Aux Loss # =============================================================================