From 7c92f4a92cadd8487b4c023c92cb1c87c59cfd02 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Fri, 20 Nov 2026 09:13:40 -0800 Subject: [PATCH] [Common][PyTorch] Add MoE EP communication/GEMM overlap ops (FusedMoE) with in-call weight gradients for BF16 and MXFP8 Signed-off-by: Phuong Nguyen --- 3rdparty/nccl-extensions | 2 +- docs/api/pytorch.rst | 11 +- docs/envvars.rst | 21 + tests/pytorch/distributed/run_ep.py | 1131 +++++++++++++++- tests/pytorch/distributed/run_test_ep.sh | 51 +- tests/pytorch/test_ep_dispatch_gemm.py | 429 +++++++ transformer_engine/common/CMakeLists.txt | 4 +- transformer_engine/common/CuTeDSL/__init__.py | 30 + .../common/CuTeDSL/ep_gemm/__init__.py | 13 + .../common/CuTeDSL/ep_gemm/_loader.py | 338 +++++ .../common/CuTeDSL/ep_gemm/combine_gemm.py | 215 ++++ .../common/CuTeDSL/ep_gemm/dispatch_gemm.py | 388 ++++++ .../common/CuTeDSL/ep_gemm/requantize.py | 331 +++++ .../common/CuTeDSL/ep_gemm/wgrad_gemm.py | 323 +++++ transformer_engine/common/CuTeDSL/utils.py | 6 + transformer_engine/common/ep/ep_api.cpp | 4 + transformer_engine/common/ep/ep_backend.cpp | 165 ++- transformer_engine/common/ep/ep_backend.h | 44 +- transformer_engine/common/ep/ep_gemm_api.cpp | 1034 +++++++++++++++ transformer_engine/common/ep/ep_gemm_config.h | 307 +++++ .../common/ep/ep_overlap_workspace.cu | 75 ++ .../common/ep/ep_overlap_workspace.h | 47 + .../common/ep/nccl_ep_provider.cpp | 12 + .../common/ep/nccl_ep_provider.h | 3 + .../common/include/transformer_engine/ep.h | 18 +- .../include/transformer_engine/ep_gemm.h | 223 ++++ transformer_engine/common/tvm_ffi_bridge.h | 6 + transformer_engine/common/util/cuda_runtime.h | 3 + transformer_engine/common/util/utils.cu | 51 + transformer_engine/pytorch/csrc/extensions.h | 4 +- .../pytorch/csrc/extensions/ep.cpp | 448 ++++++- transformer_engine/pytorch/ep.py | 139 +- transformer_engine/pytorch/ops/__init__.py | 1 + .../pytorch/ops/basic/combine.py | 2 + .../pytorch/ops/basic/dispatch.py | 26 +- .../pytorch/ops/fused/__init__.py | 5 + .../pytorch/ops/fused/moe_overlap.py | 1141 +++++++++++++++++ transformer_engine/pytorch/ops/fuser.py | 13 +- 38 files changed, 6985 insertions(+), 79 deletions(-) create mode 100644 tests/pytorch/test_ep_dispatch_gemm.py create mode 100644 transformer_engine/common/CuTeDSL/ep_gemm/__init__.py create mode 100644 transformer_engine/common/CuTeDSL/ep_gemm/_loader.py create mode 100644 transformer_engine/common/CuTeDSL/ep_gemm/combine_gemm.py create mode 100644 transformer_engine/common/CuTeDSL/ep_gemm/dispatch_gemm.py create mode 100644 transformer_engine/common/CuTeDSL/ep_gemm/requantize.py create mode 100644 transformer_engine/common/CuTeDSL/ep_gemm/wgrad_gemm.py create mode 100644 transformer_engine/common/ep/ep_gemm_api.cpp create mode 100644 transformer_engine/common/ep/ep_gemm_config.h create mode 100644 transformer_engine/common/ep/ep_overlap_workspace.cu create mode 100644 transformer_engine/common/ep/ep_overlap_workspace.h create mode 100644 transformer_engine/common/include/transformer_engine/ep_gemm.h create mode 100644 transformer_engine/pytorch/ops/fused/moe_overlap.py diff --git a/3rdparty/nccl-extensions b/3rdparty/nccl-extensions index 998d9223e00..d3ead7e49ea 160000 --- a/3rdparty/nccl-extensions +++ b/3rdparty/nccl-extensions @@ -1 +1 @@ -Subproject commit 998d9223e00c20ee515ceeccf4f0f086ee8038cd +Subproject commit d3ead7e49ea28fe47eaa04776f2784efd4b6d8f2 diff --git a/docs/api/pytorch.rst b/docs/api/pytorch.rst index 2d8cc87827e..ab7d539c61a 100644 --- a/docs/api/pytorch.rst +++ b/docs/api/pytorch.rst @@ -129,13 +129,20 @@ NCCL-based dispatch and combine for experts sharded across ranks. See ``MoeDispatch`` and ``MoeCombine`` can wrap the local expert operations in a ``transformer_engine.pytorch.ops.Sequential``. They share an ``EpConfig`` and -an ``EpBuffer``; the guide shows how to connect the per-expert counts and -routing weights through the sequence's extra-tensor channels. +an ``EpBuffer``. A buffer holds the routing state of one call until its backward has run, so +several calls in flight (for example pipeline microbatches) each need their own ``EpBuffer`` and +op instance. The guide shows how to connect the per-expert counts and routing weights through the +sequence's extra-tensor channels. ``moe_overlap_supported`` tells whether such a sequence runs as +one fused op. .. autoapiclass:: transformer_engine.pytorch.ops.MoeDispatch .. autoapiclass:: transformer_engine.pytorch.ops.MoeCombine +.. autoapifunction:: transformer_engine.pytorch.ops.moe_overlap_supported + +.. autoapifunction:: transformer_engine.pytorch.ops.moe_overlap_unsupported_reason + .. autoapiclass:: transformer_engine.pytorch.ep.EpConfig .. autoapifunction:: transformer_engine.pytorch.ep.ep_bootstrap diff --git a/docs/envvars.rst b/docs/envvars.rst index a0a64b3f637..2442abf0295 100644 --- a/docs/envvars.rst +++ b/docs/envvars.rst @@ -161,6 +161,27 @@ General ``WARN`` emits a warning explaining why CuTeDSL kernels are unavailable and falls back to CUDA kernels. ``ERROR`` raises an error and fails instead. Only effective when the CuTeDSL backend is enabled (see :envvar:`NVTE_ENABLE_CUTEDSL_BACKEND`). +.. envvar:: NVTE_MOE_OVERLAP + + :Type: ``str`` (directory) + :Default: unset (overlap off) + :Description: Directory of the CuTeDSL grouped GEMMs that overlap the NCCL EP dispatch and + combine with the expert GEMMs (``ep_gemm.h``, used by the fused + :class:`transformer_engine.pytorch.ops.MoeDispatch` expert MLP). The overlap is + off unless this is set together with ``NVTE_ENABLE_CUTEDSL_BACKEND=1``; then + ``ep_bootstrap`` defaults to 24 communication SMs unless ``max_num_sms`` is + given. Otherwise the unfused ops run. The directory is the ``gemm/`` tree of + the EPCompOverlap kernel sources (with ``blackwell/``, ``rubin/`` and + ``utils/``); its modules are imported by name and their version is not checked. + +.. envvar:: NVTE_EP_OVERLAP_COMBINE_CHUNK_COLS + + :Type: ``int`` + :Default: 1024 when it divides the hidden size, else the whole row + :Description: Hidden columns of the overlapped combine per readiness flag (see + :envvar:`NVTE_MOE_OVERLAP`). The hidden size or a multiple of 1024 dividing it. + Smaller values let the combine start earlier but cost more flags to set and poll. + .. envvar:: NVTE_GROUPED_TENSOR_HANDLE_POOL_SIZE_MB :Type: ``int`` (positive integer) diff --git a/tests/pytorch/distributed/run_ep.py b/tests/pytorch/distributed/run_ep.py index 6ab5f901f73..2f9a47f32be 100644 --- a/tests/pytorch/distributed/run_ep.py +++ b/tests/pytorch/distributed/run_ep.py @@ -5,9 +5,14 @@ from contextlib import nullcontext from dataclasses import replace +import contextlib +import itertools import os import sys +import tempfile import unittest +import warnings +from unittest import mock import numpy as np import torch @@ -32,6 +37,7 @@ is_symm_backed, _ep_combine_raw, _ep_dispatch_raw, + _moe_overlap_available, ) from transformer_engine.pytorch.tensor.mxfp8_tensor import MXFP8Tensor @@ -41,10 +47,22 @@ # Fused prepare+dispatch (caller-supplied recv buffers) is opt-in on the C++ side; the tests # that exercise it only run when the same env var is set so they match the active dispatch path. FUSED_COUNT = os.environ.get("NVTE_EP_FUSED_PREPARE_DISPATCH", "0") == "1" +# Overlapped MoE ops (ep_gemm.h): only the tests marked _overlap_test run, with NVTE_EP_MXFP8_PASS. +OVERLAP = os.environ.get("NVTE_EP_OVERLAP", "0") == "1" +# Per-expert alignment of the overlap tests: 256, 128 or 64 (BF16 only; MXFP8 cases skip). +OVERLAP_ALIGNMENT = int(os.environ.get("NVTE_EP_OVERLAP_ALIGNMENT", "256")) +# Below alignment 256 the unfused ops do not fuse into the grouped MLP, so single results of the +# fused op may round to other values than theirs. +UNFUSED_MLP_REFERENCE = OVERLAP_ALIGNMENT % 256 != 0 # Must come after the transformer_engine import so libtransformer_engine.so is loaded. import transformer_engine_torch as tex # noqa: F401 +# SM 10.7 has no BF16 grouped MLP fusion, so there the BF16 unfused ops never fuse into it. +UNFUSED_BF16_MLP_REFERENCE = UNFUSED_MLP_REFERENCE or ( + torch.cuda.is_available() and torch.cuda.get_device_capability() == (10, 7) +) + NUM_LOCAL_EXPERTS = 2 # MXFP8 dispatch needs HIDDEN_DIM % 512 == 0 and TOKENS_PER_RANK % 32 == 0. Defaults # satisfy both so the MXFP8 tests run by default; override via NVTE_EP_HIDDEN_DIM / @@ -52,6 +70,44 @@ HIDDEN_DIM = int(os.environ.get("NVTE_EP_HIDDEN_DIM", "512")) TOP_K = 2 TOKENS_PER_RANK = int(os.environ.get("NVTE_EP_TOKENS_PER_RANK", "32")) +OVERLAP_COMM_SMS = 16 # SMs the dispatch keeps; the overlapped GEMM runs on the rest +# SMs of the forward dispatch, forward combine, backward dispatch and backward combine +# (NVTE_EP_COMM_SMS=12,20,24,16); 0 inherits as ep_bootstrap documents. +COMM_SMS = tuple(int(sms) for sms in os.environ.get("NVTE_EP_COMM_SMS", "0,0,0,0").split(",")) +if len(COMM_SMS) != 4: + raise ValueError(f"NVTE_EP_COMM_SMS needs 4 comma-separated counts, got {COMM_SMS}") + + +def _comm_sms(): + """The four SM counts of the overlap pass, inherited from OVERLAP_COMM_SMS.""" + fwd = [sms or OVERLAP_COMM_SMS for sms in COMM_SMS[:2]] + return (*fwd, *(sms or f for sms, f in zip(COMM_SMS[2:], fwd))) + + +@contextlib.contextmanager +def _nccl_ep_report(lines): + """Run with NCCL_EP_ENV_VERBOSE=1 and add the lines written to file descriptor 2, where NCCL + EP reports its settings, to ``lines``; they are written through.""" + sys.stderr.flush() + saved = os.dup(2) + verbose = os.environ.get("NCCL_EP_ENV_VERBOSE") + os.environ["NCCL_EP_ENV_VERBOSE"] = "1" + with tempfile.TemporaryFile() as captured: + os.dup2(captured.fileno(), 2) + try: + yield + finally: + sys.stderr.flush() + os.dup2(saved, 2) + os.close(saved) + if verbose is None: + del os.environ["NCCL_EP_ENV_VERBOSE"] + else: + os.environ["NCCL_EP_ENV_VERBOSE"] = verbose + captured.seek(0) + text = captured.read().decode(errors="replace") + sys.stderr.write(text) + lines.extend(text.splitlines()) def _zero_copy_test_include(fn): @@ -84,6 +140,26 @@ def _mxfp8_align_test(fn): return fn +def _overlap_test(fn): + """Mark a test of the overlapped MoE ops; it runs only in the overlap pass.""" + fn._overlap_test = True + return fn + + +def _mxfp8_overlap_min_alignment() -> int: + """Smallest expert alignment of FusedMoE under MXFP8: SM 10.7 has no MXFP8 tail tiles.""" + return 256 if torch.cuda.get_device_capability() == (10, 7) else 128 + + +def _mxfp8_overlap() -> bool: + """Whether the overlap tests run their MXFP8 cases: shapes and alignment FusedMoE takes.""" + return ( + HIDDEN_DIM % 512 == 0 + and TOKENS_PER_RANK % 32 == 0 + and OVERLAP_ALIGNMENT >= _mxfp8_overlap_min_alignment() + ) + + class _StageToSymm(torch.autograd.Function): """Identity op that stages ``src`` into a symm-mem buffer; grad passes through. Lets a test feed a symm-mem-backed, autograd-tracked tensor into ep_combine. @@ -128,6 +204,22 @@ def _build_ep_group(): return dist.new_group(ranks=ranks, backend="nccl") +def _gate_up_rows(t): + """Per-expert FC1 rows [experts, 2 * ffn, ...] of the 32-wide interleaved GLU layout in the + gate-then-up layout (glu_interleave_size=None).""" + return t.reshape(t.shape[0], -1, 2, 32, *t.shape[2:]).transpose(1, 2).reshape(t.shape) + + +def _assert_close_in_norm(actual, expected, rtol): + """``actual`` is within ``rtol`` of ``expected`` in norm, per expert of a grouped weight.""" + rows = expected.shape[0] if expected.dim() == 3 else 1 + actual = actual.float().reshape(rows, -1) + expected = expected.float().reshape(rows, -1) + err, ref = (actual - expected).norm(dim=1), expected.norm(dim=1) + if not bool((err <= rtol * ref).all()): + raise AssertionError(f"error norms {err.tolist()} exceed {rtol} x {ref.tolist()}") + + def _make_identity_inputs(rank, ep_size, device="cuda"): """Per-rank identity routing + uniform weights so combine matches tokens.""" T = TOKENS_PER_RANK @@ -218,8 +310,9 @@ def _make_cfg() -> _Cfg: cfg.recv_capacity_per_rank = NUM_LOCAL_EXPERTS * max(T * cfg.ep_size * TOP_K, 16) * overconc * 2 if OVERFLOW: # Undersize recv capacity so identity routing overflows a rank's budget; - # HT requires capacity >= max_tokens_per_rank. - cfg.recv_capacity_per_rank = TOKENS_PER_RANK + # HT requires capacity >= max_tokens_per_rank. Each expert pads to 256 rows, so one slab + # fits and the second overflows. + cfg.recv_capacity_per_rank = 256 if OVERLAP else TOKENS_PER_RANK cfg.device = torch.device("cuda", torch.cuda.current_device()) return cfg @@ -229,38 +322,61 @@ class _EpTestCase(unittest.TestCase): cfg: _Cfg ep_group: dist.ProcessGroup + # Why this pass's EP group could not be created, if so; every test class then skips. + bootstrap_skip = None @classmethod def setUpClass(cls): # unittest invokes this fixture for each concrete subclass, but EP is process-global. if hasattr(_EpTestCase, "cfg"): + if _EpTestCase.bootstrap_skip: + raise unittest.SkipTest(_EpTestCase.bootstrap_skip) return if _device_sm() < 90: raise unittest.SkipTest(f"NCCL EP requires SM>=90 (got SM{_device_sm()})") _EpTestCase.cfg = _make_cfg() _EpTestCase.ep_group = _build_ep_group() - ep_bootstrap( - cls.ep_group, - num_experts=cls.cfg.num_experts, - max_tokens_per_rank=TOKENS_PER_RANK, - hidden_dim=HIDDEN_DIM, - num_topk=TOP_K, - # Omit recv_capacity_per_rank to select eager mode. - recv_capacity_per_rank=None if EAGER else cls.cfg.recv_capacity_per_rank, - zero_copy=ZERO_COPY, - drop_on_overflow=OVERFLOW, - ) + # The overlap pass keeps NCCL EP's report of the group it creates. + _EpTestCase.nccl_ep_report = [] + report = _nccl_ep_report(cls.nccl_ep_report) if OVERLAP else nullcontext() + with report: + try: + ep_bootstrap( + cls.ep_group, + num_experts=cls.cfg.num_experts, + max_tokens_per_rank=TOKENS_PER_RANK, + hidden_dim=HIDDEN_DIM, + num_topk=TOP_K, + # Omit recv_capacity_per_rank to select eager mode. + recv_capacity_per_rank=None if EAGER else cls.cfg.recv_capacity_per_rank, + zero_copy=ZERO_COPY, + drop_on_overflow=OVERFLOW, + max_num_sms=OVERLAP_COMM_SMS if OVERLAP else 0, + dispatch_fwd_num_sms=COMM_SMS[0], + combine_fwd_num_sms=COMM_SMS[1], + dispatch_bwd_num_sms=COMM_SMS[2], + combine_bwd_num_sms=COMM_SMS[3], + ) + except RuntimeError as err: + # A backward count other than the forward one needs NCCL EP backward budgets. + if "built against an NCCL EP with dispatch_bwd_num_sms" not in str(err): + raise + _EpTestCase.bootstrap_skip = "NCCL EP has no backward SM budgets" + raise unittest.SkipTest(_EpTestCase.bootstrap_skip) from err def setUp(self): # alignment=256 MXFP8 tests run only in the dedicated MXFP8 pass; everything else skips # there (and the MXFP8 tests skip outside it) since the backend pins one alignment/process. is_mxfp8_align = getattr(getattr(self, self._testMethodName), "_mxfp8_align_test", False) + is_overlap = getattr(getattr(self, self._testMethodName), "_overlap_test", False) + if OVERLAP != is_overlap: + self.skipTest("overlapped MoE tests run only in the overlap pass") if MXFP8_PASS and not is_mxfp8_align: self.skipTest("only alignment=256 MXFP8 tests run in the MXFP8 pass") if not MXFP8_PASS and is_mxfp8_align: self.skipTest("alignment=256 MXFP8 tests run in the dedicated MXFP8 pass") - # MXFP8 quantization requires Blackwell (SM 10.0) or newer. - if is_mxfp8_align and torch.cuda.get_device_capability() < (10, 0): + # MXFP8 quantization requires Blackwell (SM 10.0) or newer; the BF16 overlap tests do not. + if is_mxfp8_align and not is_overlap and torch.cuda.get_device_capability() < (10, 0): self.skipTest("MXFP8 EP tests require Blackwell (SM 10.0) or newer") # Only the zero-copy-capable tests run in the zero-copy pass. if ZERO_COPY and not getattr( @@ -299,6 +415,14 @@ def _make_config( combine_bwd_quant_recipe=combine_bwd_quant_recipe, ) + def _overlap_config(self, recipe=None): + """EpConfig of the overlapped MoE tests; ``recipe`` is the EP transport recipe.""" + return self._make_config( + alignment=OVERLAP_ALIGNMENT, + dispatch_fwd_quant_recipe=recipe, + combine_bwd_quant_recipe=recipe, + ) + def _make_buffer_from_config(self, config): """Build the NCCL EP buffer that a config describes, recipes included.""" return EpBuffer( @@ -334,6 +458,8 @@ def _require_mxfp8_shapes(self): "MXFP8 needs HIDDEN_DIM % 512 == 0 and TOKENS_PER_RANK % 32 == 0 " "(set NVTE_EP_HIDDEN_DIM / NVTE_EP_TOKENS_PER_RANK)" ) + if OVERLAP and OVERLAP_ALIGNMENT < _mxfp8_overlap_min_alignment(): + self.skipTest("FusedMoE runs MXFP8 with alignment 128 or 256 (256 on SM 10.7)") class TestEP(_EpTestCase): @@ -343,6 +469,24 @@ def test_bootstrap_accessors(self): self.assertIs(get_ep_group(), self.ep_group) self.assertEqual(get_ep_drop_on_overflow(), OVERFLOW) + @_eager_test_include + @_overflow_test_include + @_mxfp8_align_test + @_overlap_test + def test_comm_sms_reach_nccl_ep(self): + """NCCL EP runs the dispatch and combine, forward and backward, on the SM counts given to + ep_bootstrap, a count of 0 inherited from max_num_sms or the forward count.""" + if self.cfg.rank != 0: + self.skipTest("NCCL EP reports its settings on rank 0") + report = [line for line in self.nccl_ep_report if "SM budgets fwd/bwd:" in line] + if not report: + self.skipTest("this NCCL EP does not report its SM budgets") + dispatch_fwd, combine_fwd, dispatch_bwd, combine_bwd = _comm_sms() + self.assertIn( + f"dispatch {dispatch_fwd}/{dispatch_bwd} combine {combine_fwd}/{combine_bwd};", + report[-1], + ) + def _expert_out(self, expert_out): """Stage the combine input into symm-mem under zero-copy (combine requires it).""" if not ZERO_COPY: @@ -377,6 +521,59 @@ def _weighted(recv_tokens, recv_w): mask = (recv_w != 0).to(torch.float32).unsqueeze(-1) return (recv_tokens.float() * recv_w.unsqueeze(-1).float() * mask).to(recv_tokens.dtype) + def _require_overlap_kernels(self): + if not _moe_overlap_available(): + self.skipTest( + "the EP grouped GEMMs need SM 10.0, 10.3 or 10.7, NVTE_ENABLE_CUTEDSL_BACKEND=1 and" + " NVTE_MOE_OVERLAP" + ) + + def _moe_ops( + self, + config, + buffer, + glu_interleave_size=32, + accumulate_into_main_grad=False, + *, + ffn=HIDDEN_DIM, + activation=None, + fc_kwargs=({}, {}), + **linear_kwargs, + ): + """MoeDispatch, GroupedLinear, ``activation`` (default ScaledSwiGLU), GroupedLinear and + MoeCombine over ``buffer``, with the routing channels connected and fixed weights (BF16 + unless ``linear_kwargs`` change the GroupedLinear settings; ``fc_kwargs`` adds settings + of the first and of the second GroupedLinear).""" + device = self.cfg.device + gen = torch.Generator(device=device).manual_seed(5) + linear_kwargs = { + "bias": False, + "dtype": torch.bfloat16, + "single_grouped_weight": True, + "split_alignment": OVERLAP_ALIGNMENT, + "accumulate_into_main_grad": accumulate_into_main_grad, + **linear_kwargs, + } + dispatch = te_ops.MoeDispatch(config, buffer) + fc1 = te_ops.GroupedLinear( + NUM_LOCAL_EXPERTS, HIDDEN_DIM, 2 * ffn, device=device, **linear_kwargs, **fc_kwargs[0] + ) + fc2 = te_ops.GroupedLinear( + NUM_LOCAL_EXPERTS, ffn, HIDDEN_DIM, device=device, **linear_kwargs, **fc_kwargs[1] + ) + act = activation or te_ops.ScaledSwiGLU(glu_interleave_size=glu_interleave_size) + combine = te_ops.MoeCombine(config, buffer) + with torch.no_grad(): + for fc in (fc1, fc2): + for w in fc.parameters(): + w.copy_((torch.rand(w.shape, device=device, generator=gen) - 0.5) * 0.1) + dispatch.set_extra_output_channel(0, "tokens_per_expert", output_to_caller=False) + dispatch.set_extra_output_channel(1, "routing_weights", output_to_caller=False) + fc1.set_extra_input_channel(0, "tokens_per_expert") + act.set_extra_input_channel(0, "routing_weights") + fc2.set_extra_input_channel(0, "tokens_per_expert") + return dispatch, fc1, act, fc2, combine + def _moe_step(self, buffer, topk_idx, tokens, w): recv_t, recv_w_out, _tc = ep_dispatch(buffer, tokens, topk_idx, w) expert_out = self._weighted(recv_t, recv_w_out) @@ -491,6 +688,886 @@ def test_primitive_dispatch_combine_identity(self): torch.cuda.synchronize() torch.testing.assert_close(result.float(), tokens.float(), atol=5e-2, rtol=5e-2) + # Overlapped MoE + + @_eager_test_include + @_overflow_test_include + @_mxfp8_align_test + @_overlap_test + def test_moe_overlap_op(self): + """The fused MoE op matches the same ops run unfused: output, every gradient and the + receive peak, also when tokens overflow the capacity and are dropped. Where the + overlapped GEMMs cannot run, the op falls back to the unfused ops.""" + self._check_moe_overlap_op(None) + + @_eager_test_include + @_overflow_test_include + @_mxfp8_align_test + @_overlap_test + def test_moe_overlap_op_mxfp8(self): + """Under MXFP8 with MXFP8 EP transport the fused op matches the unfused ops, with weight + scales that differ across blocks in both directions, and its GEMM-swizzled weight scales + equal TE's per-tensor swizzle of each expert, rowwise and columnwise. The fused op + dispatches x and dY in place from the EP dispatch window. With BF16 EP transport it falls + back to the unfused ops.""" + self._require_mxfp8_shapes() + recipe = MXFP8BlockScaling() + with mock.patch.object( + tex, "ep_dispatch_gemm_fwd", wraps=tex.ep_dispatch_gemm_fwd + ) as fwd, mock.patch.object( + tex, "ep_combine_gemm_bwd", wraps=tex.ep_combine_gemm_bwd + ) as bwd: + self._check_moe_overlap_op(recipe, recipe) + for call in fwd.call_args_list + bwd.call_args_list: + self.assertEqual(call.kwargs["symm_input"], tex.ep_zero_copy_supported()) + config = self._overlap_config() + model = te_ops.Sequential(*self._moe_ops(config, self._make_buffer_from_config(config))) + topk_idx, tokens, w = _make_identity_inputs(self.cfg.rank, self.cfg.ep_size) + with self.assertWarnsRegex(UserWarning, "MXFP8 needs MXFP8 EP transport"), te.autocast( + recipe=recipe + ): + model(tokens, topk_idx, w, topk_idx) + self.assertFalse(self._moe_overlap_selected(model)) + + @_eager_test_include + @_overflow_test_include + @_mxfp8_align_test + @_overlap_test + def test_moe_overlap_op_variants(self): + """The fused op matches the unfused ops, in BF16 and, where the overlap tests run MXFP8, in + MXFP8, also: without input and routing-weight gradients (no dispatch backward), with + MXFP8 weight parameters, with accumulate_into_main_grad and both GLU layouts, with an FFN + size other than the hidden size, and with FP32 weight parameters cast by + ``Module.to(torch.bfloat16)``.""" + # (recipe, _check_moe_overlap_op settings) + cases = [ + (None, {"input_grad": False}), + # ffn != hidden, so a transposed weight layout cannot pass. + (None, {"ffn": HIDDEN_DIM // 2}), + (None, {"main_grad": True}), + (None, {"main_grad": True, "glu_interleave_size": None}), + (None, {"params_to_bf16": True}), + (None, {"params_to_bf16": True, "glu_interleave_size": None}), + (None, {"params_to_bf16": True, "main_grad": True}), + ] + if _mxfp8_overlap(): + recipe = MXFP8BlockScaling() + cases += [ + (recipe, {"input_grad": False}), + (recipe, {"params_to_bf16": True}), + (recipe, {"quantized_weights": True}), + ] + cases += [ + (recipe, {"quantized_weights": q, "glu_interleave_size": glu, "main_grad": True}) + for q, glu in itertools.product((False, True), (32, None)) + ] + for recipe, settings in cases: + with self.subTest(recipe=recipe, **settings): + self._check_moe_overlap_op(recipe, recipe, **settings) + + def _assert_glu_none_matches(self, got, want, got_grads, want_grads): + """The outputs and FC1, FC2 weight gradients of the fused op with glu_interleave_size=None + match those of the 32-wide layer, the FC1 rows mapped between the layouts.""" + for g, w in zip(got[:2], want[:2]): + self.assertTrue(torch.equal(g, w)) + if OVERLAP_ALIGNMENT >= 256 and HIDDEN_DIM <= 512: + self.assertTrue(torch.equal(got[2], want[2])) + else: + # The routing-weight gradient adds more than two column tiles, or tail tiles, in + # arrival order, which changes the FP32 sums by up to about 1e-6 of the largest one. + torch.testing.assert_close( + got[2], want[2], rtol=1.3e-6, atol=1e-6 * float(want[2].abs().amax()) + ) + fc1_shape = (NUM_LOCAL_EXPERTS, -1, HIDDEN_DIM) + # The rows of an expert may arrive in another order, which changes the FP32 wgrad sums by + # up to about 1e-6 of the expert's largest one, then by a BF16 rounding. + for g, w in ( + (got_grads[0].reshape(fc1_shape), _gate_up_rows(want_grads[0].reshape(fc1_shape))), + (got_grads[1], want_grads[1]), + ): + for ge, we in zip(g.reshape(NUM_LOCAL_EXPERTS, -1), w.reshape(NUM_LOCAL_EXPERTS, -1)): + torch.testing.assert_close(ge, we, rtol=1e-2, atol=1e-6 * float(we.abs().amax())) + + def _check_moe_overlap_op( + self, recipe, transport=None, quantized_weights=False, glu_interleave_size=32, + main_grad=False, input_grad=True, params_to_bf16=False, ffn=HIDDEN_DIM, + ): # fmt: skip + from transformer_engine.pytorch.ops.fused import moe_overlap + + device = self.cfg.device + config = self._overlap_config(transport) + with te.quantized_model_init(enabled=quantized_weights, recipe=recipe): + buffer = self._make_buffer_from_config(config) + ops = self._moe_ops( + config, + buffer, + glu_interleave_size, + main_grad, + ffn=ffn, + dtype=torch.float32 if params_to_bf16 else torch.bfloat16, + ) + dispatch = ops[0] + gen = torch.Generator(device=device).manual_seed(6) + if recipe is not None: + # Power-of-two factors per row and column give every scale block its own exponent. + with torch.no_grad(): + for fc in (ops[1], ops[3]): + for w in fc.parameters(): + rows = torch.randint( + -3, 4, (*w.shape[:-1], 1), device=device, generator=gen + ) + cols = torch.randint(-3, 4, w.shape[-1:], device=device, generator=gen) + values = torch.rand(w.shape, device=device, generator=gen) - 0.5 + w.copy_(values * 0.1 * torch.exp2((rows + cols).float())) + if glu_interleave_size is None: + # The layer of glu_interleave_size=32 with its FC1 rows mapped to gate then up. + with torch.no_grad(): + ops[1].weight.copy_(_gate_up_rows(ops[1].weight * 1)) + if params_to_bf16: + for op in ops: + op.to(torch.bfloat16) + topk_idx, tokens, w = _make_identity_inputs(self.cfg.rank, self.cfg.ep_size) + grad_out = torch.randn(tokens.shape, device=device, generator=gen).to(torch.bfloat16) + + def run(fused): + if not fused and params_to_bf16: + # The BF16 grouped MLP fusion reads the weight data as stored, so this reference + # runs the GroupedLinear ops, which cast it, and not FusedMoE. + ctx = contextlib.ExitStack() + for fusion in (moe_overlap.GroupedMLP_CuTeGEMMGLUBf16, moe_overlap.FusedMoE): + ctx.enter_context(mock.patch.object(fusion, "is_supported", return_value=False)) + elif not fused: + ctx = mock.patch.object(moe_overlap.FusedMoE, "is_supported", return_value=False) + elif input_grad: + ctx = contextlib.nullcontext() + else: + ctx = mock.patch.object( + moe_overlap.tex, "ep_dispatch_gemm_bwd", side_effect=AssertionError("called") + ) + dispatch.recv_tokens_peak = torch.zeros(1, dtype=torch.int64, device=device) + if fused: + # The fused op uses its own counts and recv total, not the reference's. + buffer.overlap_tokens_per_expert.fill_(-1) + buffer.overlap_total_recv_tokens.fill_(-1) + # Paged stashing pages the tensors that carry a token count. + stashable = [] + num_tokens = [] + + def pack(tensor): + if hasattr(tensor, "grouped_tensor_num_tokens"): + stashable.append((tensor.dtype, tensor.shape, tensor.nbytes)) + n = tensor.grouped_tensor_num_tokens + num_tokens.append(n.clone() if torch.is_tensor(n) else n) + return tensor + + with ctx, te.autocast( + enabled=recipe is not None, recipe=recipe + ), torch.autograd.graph.saved_tensors_hooks(pack, lambda tensor: tensor): + model = te_ops.Sequential(*ops) + x = tokens.clone().requires_grad_(input_grad) + topk_w = w.clone().requires_grad_(input_grad) + for p in model.parameters(): + p.grad = None + if main_grad: + p.main_grad = torch.full(p.shape, 0.5, dtype=torch.float32, device=device) + p.grad_added_to_main_grad = False + y = model(x, topk_idx, topk_w, topk_idx) + y.backward(grad_out) + torch.cuda.synchronize() + fused_ops = self._moe_overlap_selected(model) + if main_grad: + self.assertTrue(all(p.grad_added_to_main_grad for p in model.parameters())) + grads = [(p.main_grad if main_grad else p.grad).clone() for p in model.parameters()] + peak = int(dispatch.recv_tokens_peak.item()) + counts = [int(n) for n in num_tokens] + return y.detach(), x.grad, topk_w.grad, grads, bool(fused_ops), peak, stashable, counts + + ref = run(False) + out = run(True) + self.assertFalse(ref[4]) + expected = moe_overlap.FusedMoE.is_supported() and ( + recipe is None + or (moe_overlap.GroupedMLP_CuTeGEMMGLU.is_supported() and transport is not None) + ) + # SM 10.7 runs the MXFP8 weight gradients into main_grad only above 256 receive rows. + one_tile_main_grad = ( + main_grad + and recipe is not None + and torch.cuda.get_device_capability() == (10, 7) + and config.recv_capacity_per_rank == 256 + ) + self.assertEqual(out[4], expected and not one_tile_main_grad, "fusion selection") + if out[4]: + # The launcher requires this line from every rank of an overlap pass. + print(f"OVERLAP_EXECUTED rank={self.cfg.rank}", flush=True) + supported = te_ops.moe_overlap_supported( + config, ffn, params_dtype=torch.bfloat16, + glu_interleave_size=glu_interleave_size, single_grouped_weight=True, recipe=recipe, + primary_weights_in_fp8=quantized_weights, + ) # fmt: skip + self.assertEqual( + supported, out[4] or one_tile_main_grad, "moe_overlap_supported matches the fuser" + ) + self.assertEqual(out[5], ref[5]) + if out[4]: + self.assertEqual(len(out[6]), 4, "the fused op saves four stashable tensors") + # Their token count is the number of received rows kept. + kept = int(buffer.overlap_tokens_per_expert.sum()) + self.assertEqual(out[7], [kept] * 4, "stashed token counts") + if recipe is not None: + # The FC2 dgrad recomputes the SwiGLU output, so no [rows, ffn] tensor is saved: + # the recv tokens and scales, the FC1 output and the recv routing weights. + rows = next(shape[0] for dtype, shape, _ in out[6] if dtype == torch.float32) + saved = sum(nbytes for _, _, nbytes in out[6]) + ffn = ops[3].in_features + self.assertEqual( + saved, rows * (HIDDEN_DIM + HIDDEN_DIM // 32 + 2 * ffn * 2 + 4), "saved bytes" + ) + if OVERFLOW: + self.assertGreater(out[5], self.cfg.recv_capacity_per_rank) + # Where the unfused ops compute the SwiGLU output in their own kernel, single elements may + # round to another value, so those results are compared in norm. The unfused MXFP8 ops also + # round the FC2 dgrad and the SwiGLU backward output to BF16 before quantizing them, so + # outputs and weight gradients may differ by up to the E4M3 epsilon. + norm_rtol = 2e-2 if recipe is None else 1 / 32 + unfused_reference = glu_interleave_size is None or ( + UNFUSED_BF16_MLP_REFERENCE if recipe is None else UNFUSED_MLP_REFERENCE + ) + outputs_in_norm = unfused_reference or params_to_bf16 + outputs_rtol = 1 / 16 if recipe is not None and unfused_reference else norm_rtol + if not input_grad: + self.assertTrue(all(g is None for g in out[1:3] + ref[1:3])) + for got, want in zip(out[: 3 if input_grad else 1], ref[:3]): + if outputs_in_norm: + _assert_close_in_norm(got, want, rtol=outputs_rtol) + else: + torch.testing.assert_close(got, want, rtol=2e-2, atol=2e-2) + for g, g_ref in zip(out[3], ref[3]): + if ( + recipe is None + and glu_interleave_size is not None + and not (UNFUSED_BF16_MLP_REFERENCE or params_to_bf16) + ): + torch.testing.assert_close(g, g_ref, rtol=2e-2, atol=2e-2) + else: + _assert_close_in_norm(g, g_ref, rtol=outputs_rtol) + + @_mxfp8_align_test + @_overlap_test + def test_moe_overlap_weight_cache(self): + """The MXFP8 weights, and the FC1 rows reordered for glu_interleave_size=None, are + prepared once with is_first_microbatch=True and reused with False; an in-place weight + update prepares them again, and None prepares them at each call.""" + from transformer_engine.pytorch.ops.fused import moe_overlap + + topk_idx, tokens, w = _make_identity_inputs(self.cfg.rank, self.cfg.ep_size) + # recipe, quantized weights, glu_interleave_size, prepare function, calls per forward + cases = [(None, False, None, "_interleave_glu_rows", 1)] + if _mxfp8_overlap(): + cases += [ + (MXFP8BlockScaling(), quantized, glu, "_mxfp8_weight", 2) + for quantized, glu in ((False, 32), (True, 32), (False, None), (True, None)) + ] + for recipe, quantized_weights, glu, prepare, per_call in cases: + with self.subTest(recipe=recipe, quantized_weights=quantized_weights, glu=glu): + config = self._overlap_config(recipe) + with te.quantized_model_init(enabled=quantized_weights, recipe=recipe): + ops = self._moe_ops(config, self._make_buffer_from_config(config), glu) + model = te_ops.Sequential(*ops) + fc1, fc2 = ops[1], ops[3] + + def run(first): + kwargs = {"is_first_microbatch": first} + with mock.patch.object( + moe_overlap, prepare, wraps=getattr(moe_overlap, prepare) + ) as prep, te.autocast(enabled=recipe is not None, recipe=recipe): + x = tokens.clone().requires_grad_(True) + y = model( + x, topk_idx, w.clone().requires_grad_(True), topk_idx, + op_kwargs={fc1: kwargs, fc2: kwargs}, + ) # fmt: skip + y.sum().backward() + torch.cuda.synchronize() + return y.detach(), prep.call_count + + y_first, calls = run(True) + if not self._moe_overlap_selected(model): + self.skipTest("FusedMoE is not supported here") + self.assertEqual(calls, per_call) + y_next, calls = run(False) + self.assertEqual(calls, 0, "a later microbatch reuses the prepared weights") + # Equal weights; the EP layer itself is not bitwise reproducible across calls. + torch.testing.assert_close(y_next, y_first, rtol=2e-2, atol=2e-2) + with torch.no_grad(): + for fc in (fc1, fc2): + fc.weight.copy_(fc.weight * 2) + y_updated, calls = run(False) + self.assertEqual(calls, per_call, "an in-place weight update invalidates the cache") + y_fresh, calls = run(None) + self.assertEqual(calls, per_call) + torch.testing.assert_close(y_updated, y_fresh, rtol=2e-2, atol=2e-2) + self.assertFalse(torch.allclose(y_updated, y_first, rtol=2e-2, atol=2e-2)) + # An update of the stored data, which leaves the weight's own counter alone. + run(True) + with torch.no_grad(): + for fc in (fc1, fc2): + fc.weight.rowwise_data.zero_() + y_zeroed, calls = run(False) + self.assertEqual(calls, per_call, "an update of the stored data invalidates it") + y_fresh, _ = run(None) + torch.testing.assert_close(y_zeroed, y_fresh, rtol=2e-2, atol=2e-2) + + @_mxfp8_align_test + @_overlap_test + def test_moe_overlap_inference_mode(self): + """Weights created in inference mode, which have no version counter, run in the fused op + for every is_first_microbatch, with the FC1 rows reordered at each call.""" + self._require_overlap_kernels() + config = self._overlap_config() + with torch.inference_mode(): + ops = self._moe_ops(config, self._make_buffer_from_config(config), None) + model = te_ops.Sequential(*ops) + topk_idx, tokens, w = _make_identity_inputs(self.cfg.rank, self.cfg.ep_size) + for first in (True, False, None): + kwargs = {"is_first_microbatch": first} + model(tokens, topk_idx, w, topk_idx, op_kwargs={ops[1]: kwargs, ops[3]: kwargs}) + torch.cuda.synchronize() + self.assertTrue(self._moe_overlap_selected(model)) + + @_eager_test_include + @_mxfp8_align_test + @_overlap_test + def test_moe_overlap_cpu_offload(self): + """Under CPU offload the tensors the fused op saves come back bitwise, and its output and + gradients match those without offload.""" + from transformer_engine.pytorch import cpu_offload + + self._require_moe_overlap() + tokens, topk_idx, w, grad_out = self._moe_calls(1)[0] + for recipe in [None] + ([MXFP8BlockScaling()] if _mxfp8_overlap() else []): + config = self._overlap_config(recipe) + ops = self._moe_ops(config, self._make_buffer_from_config(config)) + results = [] + for offload in (False, True): + saved = [] + for p in ops[1].parameters(): + p.grad = None + for p in ops[3].parameters(): + p.grad = None + with te.autocast(enabled=recipe is not None, recipe=recipe): + model = te_ops.Sequential(*ops) + x = tokens.clone().requires_grad_(True) + topk_w = w.clone().requires_grad_(True) + if offload: + ctx, sync, controller = cpu_offload.get_cpu_offload_context( + enabled=True, model_layers=1, manual_synchronization=True + ) + sync_state = ctx.offload_synchronizer + push, pop = sync_state.push_tensor, sync_state.pop_tensor + + def push_tensor(tensor, push=push): + saved.append([tensor.detach().reshape(-1).view(torch.uint8).cpu()]) + return len(saved) - 1, push(tensor) + + def pop_tensor(packed, pop=pop): + tensor = pop(packed[1]) + saved[packed[0]].append(tensor.detach().reshape(-1).view(torch.uint8)) + return tensor + + with mock.patch.object( + sync_state, "push_tensor", push_tensor + ), mock.patch.object(sync_state, "pop_tensor", pop_tensor): + with ctx: + y = model(x, topk_idx, topk_w, topk_idx) + y = sync(y) + controller.start_offload_layer(0) + controller.release_activation_forward_gpu_memory(0) + controller.start_reload_layer(0) + else: + y = model(x, topk_idx, topk_w, topk_idx) + y.backward(grad_out) + torch.cuda.synchronize() + self.assertTrue(self._moe_overlap_selected(model)) + self.assertEqual(bool(saved), offload, "offloaded tensors") + for before, after in saved: + self.assertTrue(torch.equal(before, after.cpu()), "a saved tensor changed") + grads = [p.grad.clone() for p in model.parameters()] + results.append([y.detach(), x.grad, topk_w.grad, *grads]) + with self.subTest(recipe=recipe): + for i, (got, want) in enumerate(zip(*results)): + if recipe is not None and i >= 3: + _assert_close_in_norm(got, want, rtol=1 / 32) + else: + torch.testing.assert_close(got, want, rtol=2e-2, atol=2e-2) + + def _require_moe_overlap(self): + from transformer_engine.pytorch.ops.fused import moe_overlap + + if not moe_overlap.FusedMoE.is_supported(): + self.skipTest( + "FusedMoE needs SM 10.0, 10.3 or 10.7, NVTE_ENABLE_CUTEDSL_BACKEND=1," + " NVTE_MOE_OVERLAP and the BF16 cuDNN grouped GEMM kernels" + ) + + @staticmethod + def _moe_fusion(fused): + """Context in which the fuser may select FusedMoE, or keeps the MoE ops unfused.""" + from transformer_engine.pytorch.ops.fused import moe_overlap + + if fused: + return contextlib.nullcontext() + return mock.patch.object(moe_overlap.FusedMoE, "is_supported", return_value=False) + + @staticmethod + def _moe_overlap_selected(model): + """The FusedMoE ops of the plan ``model`` ran last.""" + from transformer_engine.pytorch.ops.fused import moe_overlap + + return [ + op + for group in model._module_groups + for op, _ in getattr(group, "_forward_ops", []) + if isinstance(op, moe_overlap.FusedMoE) + ] + + def _moe_calls(self, num_calls): + """(tokens, topk_idx, topk_weights, grad_output) of calls that each route differently.""" + gen = torch.Generator(device=self.cfg.device).manual_seed(8) + topk_idx, tokens, w = _make_identity_inputs(self.cfg.rank, self.cfg.ep_size) + return [ + ( + tokens.roll(i, 0), + (topk_idx + i) % self.cfg.num_experts, + w, + torch.randn(tokens.shape, device=self.cfg.device, generator=gen).to(torch.bfloat16), + ) + for i in range(num_calls) + ] + + @staticmethod + def _moe_call(model, tokens, topk_idx, w, grad, *, tokens_grad=True, weights_grad=True): + """Forward, then backward when ``grad`` is given: (output, tokens grad, topk_weights + grad). Parameter gradients accumulate on the parameters.""" + x = tokens.clone().requires_grad_(tokens_grad) + topk_w = w.clone().requires_grad_(weights_grad) + y = model(x, topk_idx, topk_w, topk_idx) + if grad is not None: + y.backward(grad) + return y.detach(), x.grad, topk_w.grad + + @staticmethod + def _param_grads(model): + return [None if p.grad is None else p.grad.clone() for p in model.parameters()] + + def _assert_moe_close(self, got, ref, in_norm=UNFUSED_BF16_MLP_REFERENCE, outputs_in_norm=True): + """Fused and unfused MoE results, nested in lists, agree within BF16 GEMM tolerances; + with in_norm the grouped weight gradients in norm, and with outputs_in_norm as well the + outputs and input gradients.""" + if isinstance(ref, (list, tuple)): + self.assertEqual(len(got), len(ref)) + for g, r in zip(got, ref): + self._assert_moe_close(g, r, in_norm, outputs_in_norm) + elif ref is None: + self.assertIsNone(got) + elif in_norm and (outputs_in_norm or ref.dim() == 3): + _assert_close_in_norm(got, ref, rtol=2e-2) + else: + torch.testing.assert_close(got, ref, rtol=2e-2, atol=2e-2) + + @_mxfp8_align_test + @_overlap_test + def test_moe_overlap_main_grad(self): + """With accumulate_into_main_grad, the fused op writes the weight gradients of two calls + into an FP32 or BF16 main_grad, accumulated or overwritten, as the unfused ops do.""" + self._require_moe_overlap() + config = self._overlap_config() + ops = self._moe_ops( + config, self._make_buffer_from_config(config), accumulate_into_main_grad=True + ) + weights = (ops[1].weight, ops[3].weight) + calls = self._moe_calls(2) + for dtype, overwrite in itertools.product((torch.float32, torch.bfloat16), (False, True)): + results = [] + for fused in (False, True): + for p in weights: + p.grad = None + p.main_grad = torch.full(p.size(), 0.5, dtype=dtype, device=self.cfg.device) + p.overwrite_main_grad = overwrite + p.grad_added_to_main_grad = False + with self._moe_fusion(fused): + model = te_ops.Sequential(*ops) + outs = [self._moe_call(model, *call) for call in calls] + torch.cuda.synchronize() + self.assertEqual(bool(self._moe_overlap_selected(model)), fused) + for p in weights: + self.assertTrue(p.grad_added_to_main_grad) + results.append((outs, [p.main_grad.clone() for p in weights])) + with self.subTest(dtype=dtype, overwrite=overwrite): + self._assert_moe_close(results[1], results[0]) + + @_mxfp8_align_test + @_overlap_test + def test_moe_overlap_delay_wgrad(self): + """With delay_wgrad_compute on one or both GroupedLinear ops, FusedMoE is not selected + and the unfused ops run with a warning: the delayed weight gradient waits for backward_dw, + the other is computed in the backward, and the results match the fusion disabled, into + main_grad or not.""" + self._require_moe_overlap() + config = self._overlap_config() + calls = self._moe_calls(2) + # delayed (FC1, FC2), accumulate_into_main_grad + cases = [ + ((True, True), False), + ((True, True), True), + ((True, False), False), + ((False, True), False), + ] + for delayed, main_grad in cases: + ops = self._moe_ops( + config, self._make_buffer_from_config(config), + accumulate_into_main_grad=main_grad, + fc_kwargs=tuple({"delay_wgrad_compute": d} for d in delayed), + ) # fmt: skip + fcs = (ops[1], ops[3]) + results = [] + for fused in (False, True): + for fc in fcs: + fc.weight.grad = None + if main_grad: + fc.weight.main_grad = torch.zeros( + fc.weight.size(), dtype=torch.float32, device=self.cfg.device + ) + fc.weight.grad_added_to_main_grad = False + with self._moe_fusion(fused), warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + model = te_ops.Sequential(*ops) + outs = [] + for i, call in enumerate(calls): + outs.append(self._moe_call(model, *call)) + if i == 0: + torch.cuda.synchronize() + for fc, is_delayed in zip(fcs, delayed): + if not is_delayed: + self.assertIsNotNone(fc.weight.grad) + elif main_grad: + self.assertEqual(int(fc.weight.main_grad.count_nonzero()), 0) + else: + self.assertIsNone(fc.weight.grad) + for fc, is_delayed in zip(fcs, delayed): + if is_delayed: + fc.backward_dw() + torch.cuda.synchronize() + self.assertFalse(self._moe_overlap_selected(model)) + if fused: + self.assertTrue(any("delay_wgrad_compute" in str(w.message) for w in caught)) + wgrads = [fc.weight.main_grad if main_grad else fc.weight.grad for fc in fcs] + results.append((outs, [g.clone() for g in wgrads])) + with self.subTest(delayed=delayed, main_grad=main_grad): + self._assert_moe_close(results[1], results[0]) + + @_eager_test_include + @_mxfp8_align_test + @_overlap_test + def test_moe_overlap_rank_without_tokens(self): + """Every token picks the first expert of ranks 0 and 1, so the other ranks and the other + experts of ranks 0 and 1 receive no tokens. The fused op matches the unfused ops, and + the weight gradients of the experts without tokens are zero. The fused op with + glu_interleave_size=None and the FC1 rows mapped matches it.""" + self._require_moe_overlap() + if self.cfg.ep_size < 3: + self.skipTest("needs a rank that receives no tokens") + config = self._overlap_config() + ops = self._moe_ops(config, self._make_buffer_from_config(config)) + tokens, _, w, grad = self._moe_calls(1)[0] + topk_idx = torch.tensor([0, NUM_LOCAL_EXPERTS], device=self.cfg.device).repeat( + TOKENS_PER_RANK, 1 + ) + results = [] + for fused in (False, True): + for p in ops[1].parameters(): + p.grad = None + for p in ops[3].parameters(): + p.grad = None + with self._moe_fusion(fused): + model = te_ops.Sequential(*ops) + out = self._moe_call(model, tokens, topk_idx, w, grad) + torch.cuda.synchronize() + self.assertEqual(bool(self._moe_overlap_selected(model)), fused) + results.append((out, self._param_grads(model))) + self._assert_moe_close(results[1], results[0]) + has_tokens = self.cfg.rank < 2 + for g in results[1][1]: + per_expert = g.view(NUM_LOCAL_EXPERTS, -1) + self.assertEqual(int(per_expert[int(has_tokens) :].count_nonzero()), 0) + if has_tokens: + self.assertGreater(int(per_expert[0].count_nonzero()), 0) + + ops_none = self._moe_ops(config, self._make_buffer_from_config(config), None) + shape = (NUM_LOCAL_EXPERTS, ops[1].out_features, ops[1].in_features) + with torch.no_grad(): + ops_none[1].weight.rowwise_data.view(shape).copy_( + _gate_up_rows(ops[1].weight.rowwise_data.view(shape)) + ) + model = te_ops.Sequential(*ops_none) + out = self._moe_call(model, tokens, topk_idx, w, grad) + torch.cuda.synchronize() + self.assertTrue(self._moe_overlap_selected(model)) + self._assert_glu_none_matches(out, results[1][0], self._param_grads(model), results[1][1]) + + @_mxfp8_align_test + @_overlap_test + def test_moe_overlap_cuda_graph(self): + """The fused forward and backward, captured in a CUDA graph after warmup and replayed + with new routing, tokens, topk_weights and output gradient, match the unfused ops.""" + self._require_moe_overlap() + config = self._overlap_config() + ops = self._moe_ops(config, self._make_buffer_from_config(config)) + ref_ops = self._moe_ops(config, self._make_buffer_from_config(config)) + gen = torch.Generator(device=self.cfg.device).manual_seed(9) + random_idx, random_tokens, random_w = _make_moe_inputs(self.cfg.rank, self.cfg.ep_size) + replays = [ + ( + random_tokens, + random_idx, + random_w, + torch.randn(random_tokens.shape, device=self.cfg.device, generator=gen).to( + torch.bfloat16 + ), + ), + self._moe_calls(1)[0], + ] + # Static inputs, holding the captured call's values first. + tokens, topk_idx, w, grad = (t.clone() for t in self._moe_calls(2)[1]) + x = tokens.requires_grad_(True) + topk_w = w.requires_grad_(True) + model = te_ops.Sequential(*ops) + params = list(model.parameters()) + out = {} + + def step(): + for t in (x, topk_w, *params): + t.grad = None + out["y"] = model(x, topk_idx, topk_w, topk_idx) + out["y"].backward(grad) + + graph = self._capture(step) + self.assertTrue(self._moe_overlap_selected(model)) + results = [] + for call in replays: + with torch.no_grad(): + for static, value in zip((x, topk_idx, topk_w, grad), call): + static.copy_(value) + graph.replay() + got = ( + (out["y"].detach().clone(), x.grad.clone(), topk_w.grad.clone()), + [p.grad.clone() for p in params], + ) + ref_model = te_ops.Sequential(*ref_ops) + for p in ref_model.parameters(): + p.grad = None + with self._moe_fusion(False): + ref = (self._moe_call(ref_model, *call), self._param_grads(ref_model)) + torch.cuda.synchronize() + results.append((got, ref)) + # Compare once every rank has run every replay, so a mismatch on one rank leaves no rank + # waiting in a later replay. + for got, ref in results: + self._assert_moe_close(got, ref) + + @_mxfp8_align_test + @_overlap_test + def test_moe_overlap_fallback(self): + """For each setting moe_overlap_supported rejects, the fuser warns with the query's + reason, keeps the ops unfused and gives the results of the unfused ops.""" + from transformer_engine.pytorch.ops.fused import moe_overlap + + self._require_moe_overlap() + config = self._overlap_config() + call = self._moe_calls(1)[0] + supported = { + "ffn_hidden_size": HIDDEN_DIM, + "params_dtype": torch.bfloat16, + "glu_interleave_size": 32, + "bias": False, + "single_grouped_weight": True, + } + cases = [ + {"single_grouped_weight": False}, + {"bias": True}, + {"params_dtype": torch.float32}, + {"ffn_hidden_size": HIDDEN_DIM + 32}, + ] + for change in cases: + settings = {**supported, **change} + ops = self._moe_ops( + config, self._make_buffer_from_config(config), ffn=settings["ffn_hidden_size"], + glu_interleave_size=settings["glu_interleave_size"], bias=settings["bias"], + dtype=settings["params_dtype"], + single_grouped_weight=settings["single_grouped_weight"], + ) # fmt: skip + # FP32 weights compute in BF16 under autocast. + use_autocast = settings["params_dtype"] != torch.bfloat16 + with self.subTest(change=change): + self.assertFalse(te_ops.moe_overlap_supported(config, **settings)) + reason = te_ops.moe_overlap_unsupported_reason(config, **settings) + self.assertIsNotNone(reason) + with warnings.catch_warnings(record=True) as caught, torch.autocast( + "cuda", dtype=torch.bfloat16, enabled=use_autocast + ): + warnings.simplefilter("always") + model = te_ops.Sequential(*ops) + out = (self._moe_call(model, *call), self._param_grads(model)) + self.assertFalse(self._moe_overlap_selected(model)) + expected = f"FusedMoE fusion skipped, running the unfused ops: {reason}." + self.assertIn(expected, [str(warning.message) for warning in caught]) + skipped = [w for w in caught if str(w.message) == expected] + self.assertEqual(skipped[0].filename, __file__, "warned at the caller's frame") + for p in model.parameters(): + p.grad = None + with self._moe_fusion(False), torch.autocast( + "cuda", dtype=torch.bfloat16, enabled=use_autocast + ): + ref_model = te_ops.Sequential(*ops) + ref = (self._moe_call(ref_model, *call), self._param_grads(ref_model)) + torch.cuda.synchronize() + self._assert_moe_close(out, ref, UNFUSED_MLP_REFERENCE, outputs_in_norm=False) + # Without the overlap enabled the unfused ops are expected, and nothing warns. + ops = self._moe_ops(config, self._make_buffer_from_config(config), bias=True) + with mock.patch.object( + moe_overlap, "_moe_overlap_available", return_value=False + ), warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self._moe_call(te_ops.Sequential(*ops), *call) + self.assertFalse([w for w in caught if "fusion skipped" in str(w.message)]) + + @_mxfp8_align_test + @_overlap_test + def test_moe_overlap_execution_fallback(self): + """Deterministic execution, an FP16 autocast and GLU activations other than SwiGLU leave + the unfused ops in place with a warning that names the reason, and moe_overlap_supported + gives the same answer.""" + from transformer_engine.pytorch.ops.fused import moe_overlap + + self._require_overlap_kernels() + config = self._overlap_config() + buffer = self._make_buffer_from_config(config) + settings = { + "params_dtype": torch.bfloat16, + "glu_interleave_size": 32, + "single_grouped_weight": True, + } + no_state = contextlib.nullcontext + deterministic = mock.patch.dict(os.environ, NVTE_ALLOW_NONDETERMINISTIC_ALGO="0") + fp16 = torch.autocast("cuda", dtype=torch.float16) + geglu = te_ops.ScaledClampedQGeGLU(32) + situ = te_ops.ScaledSiTUGLU(32) + cases = [ + # (name, activation, state, query settings, reason) + ("swiglu", None, no_state(), {}, None), + ("deterministic", None, deterministic, {}, "deterministic"), + ("fp16 autocast", None, fp16, {}, "BF16 autocast"), + ("clamped geglu", geglu, no_state(), {"activation": "clamped_geglu"}, "SwiGLU"), + ("situ", situ, no_state(), {"activation": "situ"}, "SwiGLU"), + ] + for name, activation, state, query, reason in cases: + with self.subTest(case=name), state, warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + ops = list(self._moe_ops(config, buffer, activation=activation)) + fused = moe_overlap.fuse_moe_overlap_ops(ops) + if reason is None: + self.assertEqual(len(fused), 1) + self.assertIsInstance(fused[0], moe_overlap.FusedMoE) + else: + self.assertEqual(fused, ops) + messages = [str(w.message) for w in caught if "FusedMoE" in str(w.message)] + self.assertEqual(len(messages), 1) + self.assertIn(reason, messages[0]) + self.assertEqual( + te_ops.moe_overlap_supported(config, HIDDEN_DIM, **settings, **query), + reason is None, + ) + + @_mxfp8_align_test + @_overlap_test + def test_moe_overlap_shared_buffer(self): + """Two FusedMoE instances, each with its own EpBuffer, interleave safely: the second + instance's forward and backward between the first instance's forward and backward leave + the first instance's output and gradients identical to running it alone. One buffer and op + instance per outstanding call.""" + self._require_moe_overlap() + config = self._overlap_config() + # Separate instances and buffers; _moe_ops seeds identical weights. + model_a = te_ops.Sequential(*self._moe_ops(config, self._make_buffer_from_config(config))) + model_b = te_ops.Sequential(*self._moe_ops(config, self._make_buffer_from_config(config))) + call_a, call_b = self._moe_calls(2) + + def forward(model, call): + for p in model.parameters(): + p.grad = None + x = call[0].clone().requires_grad_(True) + topk_w = call[2].clone().requires_grad_(True) + return x, topk_w, model(x, call[1], topk_w, call[1]) + + def backward(model, x, topk_w, y, grad): + y.backward(grad) + torch.cuda.synchronize() + return [y.detach(), x.grad, topk_w.grad] + self._param_grads(model) + + # Each call run alone, as the serial reference. + xa, wa, ya = forward(model_a, call_a) + alone_a = backward(model_a, xa, wa, ya, call_a[3]) + self.assertTrue(self._moe_overlap_selected(model_a)) + xb, wb, yb = forward(model_b, call_b) + alone_b = backward(model_b, xb, wb, yb, call_b[3]) + self.assertTrue(self._moe_overlap_selected(model_b)) + + # Interleaved: the second instance's whole call lands between the first's forward and + # backward; separate buffers keep each call's routing state intact. + xa, wa, ya = forward(model_a, call_a) + xb, wb, yb = forward(model_b, call_b) + inter_b = backward(model_b, xb, wb, yb, call_b[3]) + inter_a = backward(model_a, xa, wa, ya, call_a[3]) + self._assert_moe_close(inter_a, alone_a, UNFUSED_MLP_REFERENCE, outputs_in_norm=False) + self._assert_moe_close(inter_b, alone_b, UNFUSED_MLP_REFERENCE, outputs_in_norm=False) + + @_mxfp8_align_test + @_overlap_test + def test_moe_overlap_policy_change(self): + """One Sequential reselects its plan when deterministic mode or the autocast dtype + changes: FusedMoE runs, falls back while the setting holds and runs again after.""" + self._require_moe_overlap() + config = self._overlap_config() + ops = self._moe_ops(config, self._make_buffer_from_config(config)) + call = self._moe_calls(1)[0] + model = te_ops.Sequential(*ops) + deterministic = mock.patch.dict(os.environ, NVTE_ALLOW_NONDETERMINISTIC_ALGO="0") + fp16 = torch.autocast("cuda", dtype=torch.float16) + results = [] + for state, selected in ( + (contextlib.nullcontext(), True), + (deterministic, False), + (fp16, False), + (contextlib.nullcontext(), True), + ): + with state, warnings.catch_warnings(): + warnings.simplefilter("ignore") + if state is deterministic: + # The unfused grouped MLP, which only runs at alignment 256 or above, rejects + # the nondeterministic backward as well. + try: + out = self._moe_call(model, *call) + except RuntimeError as err: + self.assertIn("Deterministic execution", str(err)) + out = None + else: + out = self._moe_call(model, *call) + torch.cuda.synchronize() + self.assertEqual(bool(self._moe_overlap_selected(model)), selected) + if out is not None and state is not fp16: + results.append(out) + for got in results[1:]: + self._assert_moe_close(got, results[0]) + # Autograd @_zero_copy_test_include @@ -1252,6 +2329,30 @@ def _run_dispatch_combine_identity(self, *, mxfp8): torch.testing.assert_close(output, tokens, atol=5e-2, rtol=5e-2) self.assertEqual(tokens_per_expert.data_ptr(), buffer.tokens_per_expert.data_ptr()) + @_eager_test_include + @_mxfp8_align_test + @_overlap_test + def test_unfused_ops_on_overlap_group(self): + """On an overlap-pass group, MoeDispatch and MoeCombine run unfused and form an identity, + forward and backward.""" + config = self._overlap_config() + buffer = self._make_buffer_from_config(config) + dispatch = te_ops.MoeDispatch(config, buffer) + combine = te_ops.MoeCombine(config, buffer) + topk_idx, tokens, topk_weights = _make_identity_inputs(self.cfg.rank, self.cfg.ep_size) + x = tokens.clone().requires_grad_(True) + w = topk_weights.clone().requires_grad_(True) + recv_tokens, _tokens_per_expert, recv_weights = dispatch(x, topk_idx, w) + output = combine(recv_tokens * recv_weights.unsqueeze(-1).to(torch.bfloat16), topk_idx) + gen = torch.Generator(device=self.cfg.device).manual_seed(8) + grad_out = torch.randn(tokens.shape, device=self.cfg.device, generator=gen) + output.backward(grad_out.to(torch.bfloat16)) + torch.cuda.synchronize() + torch.testing.assert_close(output, tokens, atol=5e-2, rtol=5e-2) + torch.testing.assert_close(x.grad.float(), grad_out, atol=5e-2, rtol=5e-2) + grad_w_ref = (tokens.float() * grad_out).sum(-1, keepdim=True).expand_as(w) + torch.testing.assert_close(w.grad.float(), grad_w_ref, atol=5e-1, rtol=5e-2) + @_eager_test_include def test_dispatch_combine_identity_bf16(self): """MoeDispatch and MoeCombine basic ops form an identity in BF16.""" diff --git a/tests/pytorch/distributed/run_test_ep.sh b/tests/pytorch/distributed/run_test_ep.sh index b9c7746d4ad..470ca96096c 100755 --- a/tests/pytorch/distributed/run_test_ep.sh +++ b/tests/pytorch/distributed/run_test_ep.sh @@ -49,12 +49,16 @@ run_pass() { local overflow="${4:-0}" local mxfp8="${5:-0}" local fused="${6:-0}" + local overlap="${7:-0}" local log="stdout_ep_${label}.txt" - echo "=== Running ${SCRIPT} [${label}] on ${NUM_RANKS} GPUs (timeout=${TEST_TIMEOUT_S}s) ===" + # The overlap passes compile CuTeDSL GEMMs on first use and need a longer timeout. + local timeout_s="${TEST_TIMEOUT_S}" + if [ "${overlap}" = "1" ] && [ "${timeout_s}" -lt 600 ]; then timeout_s=600; fi + echo "=== Running ${SCRIPT} [${label}] on ${NUM_RANKS} GPUs (timeout=${timeout_s}s) ===" # setsid + kill-after so SIGKILL takes down the whole process group, not just torchrun. NVTE_EP_ZERO_COPY="${zc}" NVTE_EP_EAGER="${eager}" NVTE_EP_OVERFLOW="${overflow}" \ - NVTE_EP_MXFP8_PASS="${mxfp8}" NVTE_EP_FUSED_PREPARE_DISPATCH="${fused}" \ - setsid timeout --foreground --kill-after=10 --signal=TERM "${TEST_TIMEOUT_S}" \ + NVTE_EP_MXFP8_PASS="${mxfp8}" NVTE_EP_FUSED_PREPARE_DISPATCH="${fused}" NVTE_EP_OVERLAP="${overlap}" \ + setsid timeout --foreground --kill-after=10 --signal=TERM "${timeout_s}" \ torchrun --standalone --nnodes=1 --nproc-per-node="${NUM_RANKS}" \ "${SCRIPT}" 2>&1 | tee "${log}" local rc=${PIPESTATUS[0]} @@ -68,6 +72,20 @@ run_pass() { echo "[${label}] ERROR: no test summary — likely hang or early crash" RET=1 fi + # An overlap pass in which a rank never ran the fused ops covers nothing: tests that skip, or + # that fall back to the unfused ops, do not report. Every rank reports from a numerical test. + if [ "${overlap}" = "1" ]; then + local executed + executed=$(grep -oE "OVERLAP_EXECUTED rank=[0-9]+" "${log}" | sort -u | wc -l) + if grep -q "NCCL EP has no backward SM budgets" "${log}"; then + executed="${NUM_RANKS}" + fi + if [ "${executed}" -lt "${NUM_RANKS}" ]; then + echo "[${label}] ERROR: the fused overlap ops ran on ${executed} of ${NUM_RANKS} ranks;" \ + "set NVTE_ENABLE_CUTEDSL_BACKEND=1 and NVTE_MOE_OVERLAP" + RET=1 + fi + fi if [ -z "${KEEP_EP_LOGS:-}" ]; then rm -f "${log}"; fi } @@ -98,4 +116,31 @@ run_pass "fused" 0 0 0 0 1 run_pass "fused_overflow" 0 0 1 0 1 run_pass "fused_mxfp8" 0 0 0 1 1 +# Overlapped MoE ops (ep_gemm.h, te.ops FusedMoE): alignment 256, capacity and eager IO. Run only +# when the overlap is available (NVTE_ENABLE_CUTEDSL_BACKEND=1, NVTE_MOE_OVERLAP and SM 10.x). +OVERLAP_AVAILABLE=$(python3 -c \ + "import transformer_engine.pytorch.ep as ep; print(int(ep._moe_overlap_available()))" 2>/dev/null \ + || echo 0) +if [ "${OVERLAP_AVAILABLE}" != "1" ]; then + echo "Overlapped MoE ops unavailable (needs NVTE_ENABLE_CUTEDSL_BACKEND=1, NVTE_MOE_OVERLAP, SM 10.x); SKIPPING overlap passes." +else + run_pass "overlap" 0 0 0 1 0 1 + run_pass "overlap_eager" 0 1 0 1 0 1 + run_pass "overlap_overflow" 0 0 1 1 0 1 + # Per-direction SM counts (dispatch fwd, combine fwd, dispatch bwd, combine bwd). The backward + # split skips when Transformer Engine was built against an NCCL EP without backward SM budgets. + NVTE_EP_COMM_SMS=12,20,0,0 run_pass "overlap_split" 0 0 0 1 0 1 + NVTE_EP_COMM_SMS=12,20,24,16 run_pass "overlap_split_bwd" 0 0 0 1 0 1 + # A hidden size that 1024 does not divide: the combine waits on whole rows. + NVTE_EP_HIDDEN_DIM=1536 run_pass "overlap_h1536" 0 0 0 1 0 1 + # Expert alignments below 256 (MXFP8 cases skip at 64): an expert's last GEMM tile holds fewer + # rows. With 288 tokens per rank each expert also fills a whole tile; only the fused op tests run. + for align in 128 64; do + NVTE_EP_OVERLAP_ALIGNMENT=${align} run_pass "overlap_a${align}" 0 0 0 1 0 1 + NVTE_EP_OVERLAP_ALIGNMENT=${align} run_pass "overlap_a${align}_eager" 0 1 0 1 0 1 + NVTE_EP_OVERLAP_ALIGNMENT=${align} NVTE_EP_TOKENS_PER_RANK=288 \ + NVTE_EP_TEST_FILTER=test_moe_overlap run_pass "overlap_a${align}_t288" 0 0 0 1 0 1 + done +fi + exit $RET diff --git a/tests/pytorch/test_ep_dispatch_gemm.py b/tests/pytorch/test_ep_dispatch_gemm.py new file mode 100644 index 00000000000..8c0a96dce5d --- /dev/null +++ b/tests/pytorch/test_ep_dispatch_gemm.py @@ -0,0 +1,429 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Signal-aware grouped GEMM that follows the EP dispatch, loaded through TVM-FFI.""" + +# pylint: disable=wrong-import-position + +import ctypes +import os + +import pytest +import torch + +# The optional CuTeDSL dependencies are imported only when the overlap is requested, where a +# missing one fails the module. +if not os.environ.get("NVTE_MOE_OVERLAP"): + pytest.skip("NVTE_MOE_OVERLAP is not set", allow_module_level=True) + +import tvm_ffi + +from transformer_engine.common.CuTeDSL.ep_gemm import ( + combine_gemm, + dispatch_gemm, + requantize, + wgrad_gemm, +) + +pytestmark = [ + pytest.mark.skipif( + not torch.cuda.is_available() + or torch.cuda.get_device_capability() not in ((10, 0), (10, 3), (10, 7)), + reason="the kernels need SM 10.0, 10.3 or 10.7", + ), +] + +TILE = dispatch_gemm.ROWS_PER_TILE +# SM 10.7 has no MXFP8 tail tiles. +MXFP8_TAILS = torch.cuda.is_available() and torch.cuda.get_device_capability() != (10, 7) + + +def _unit_e8m0(count, device): + """``count`` unit E8M0 scales (127), which read the same in any scale layout.""" + return torch.full((count,), 127, dtype=torch.uint8, device=device).view(torch.float8_e8m0fnu) + + +def _run_gemm(rows_per_expert, rows_alignment, mxfp8, tail_clusters, device_layout): + """Launch the forward GEMM with every row tile ready, and with ``tail_clusters`` a second + launch behind it on the same workspace; return (a, b, alpha, prob, c, d, claims) with MXFP8 + operands at unit scales in float32, the MXFP8 ``d`` dequantized and the final claim counter. + ``device_layout``: the row ends follow expert 0's offset, as the dispatch writes them.""" + device = torch.device("cuda") + hidden, n, experts = 1024, 1024, len(rows_per_expert) + rows = sum(rows_per_expert) + TILE # spare tile: the capacity exceeds the rows in use + cfg = dispatch_gemm.DispatchGemmConfig( + rows, + hidden, + n, + experts, + num_clusters=8, + mxfp8=mxfp8, + rows_alignment=rows_alignment, + device_layout=device_layout, + ) + launches = [] + for num_clusters in (cfg.num_clusters, tail_clusters): + if num_clusters == 0: + continue + name = ( + f"test_ep_dispatch_gemm_{rows}_{rows_alignment}_{int(mxfp8)}_{num_clusters}" + f"_{int(device_layout)}" + ) + assert dispatch_gemm.get_ep_dispatch_gemm_function( + name, + rows, + hidden, + n, + experts, + num_clusters, + False, + mxfp8, + rows_alignment, + device_layout, + ) + launches.append(tvm_ffi.get_global_func(name)) + + gen = torch.Generator(device=device).manual_seed(0) + dtype = torch.float8_e4m3fn if mxfp8 else torch.bfloat16 + a = torch.rand(rows, hidden, device=device, generator=gen).to(dtype) + b = (torch.rand(experts, n, hidden, device=device, generator=gen) * 2 * hidden**-0.5).to(dtype) + # Powers of two: alpha and prob cost no tolerance. + alpha = torch.pow(2.0, (torch.arange(experts, device=device) % 5 - 2).float()) + prob = torch.pow(2.0, (torch.arange(rows, device=device) % 3 - 1).float()) + ends = torch.tensor(rows_per_expert, device=device).cumsum(0).to(torch.int32) + if device_layout: + # Offsets then the total: expert 0's offset is the word before the 16-byte aligned ends. + layout = torch.zeros(4 + experts, dtype=torch.int32, device=device) + layout[4:] = ends + ends = layout[4:] + d_cols = n // 2 + c = torch.zeros(rows, n, dtype=torch.bfloat16, device=device) + d = torch.zeros(rows, d_cols, dtype=dtype, device=device) + ws_bytes = dispatch_gemm.workspace_layout(cfg)[0] + workspace = torch.zeros(ws_bytes, dtype=torch.uint8, device=device) + _, counter_offset, signal_offset, _ = dispatch_gemm.workspace_layout(cfg) + words = workspace.view(torch.int32) + # Slot s is ready once signal[s] == s. + words[signal_offset // 4 :][: cfg.signal_capacity] = torch.arange( + cfg.signal_capacity, dtype=torch.int32, device=device + ) + counter = words[counter_offset // 4] + stream = ctypes.c_void_p(torch.cuda.current_stream().cuda_stream) + if not mxfp8: + for gemm in launches: + gemm(a, b, c, d, ends, alpha, prob, workspace, stream) + torch.cuda.synchronize() + return a.float(), b.float(), alpha, prob, c, d.float(), counter.item() + + # The forward takes no prob. + sfd = torch.zeros(rows * d_cols // 32, dtype=torch.uint8, device=device) + norm_const = torch.ones(1, dtype=torch.float32, device=device) + for gemm in launches: + gemm( + a, b, _unit_e8m0(experts * n * hidden // 32, device), + _unit_e8m0(rows * hidden // 32, device), c, d, + sfd.view(torch.float8_e8m0fnu), ends, alpha, norm_const, workspace, stream, + ) # fmt: skip + torch.cuda.synchronize() + # GEMM-swizzled scales of [rows, d_cols]: 128 x 4 atoms, rows 32 apart interleaved. + scale_cols = d_cols // 32 + scales = sfd.view(rows // 128, scale_cols // 4, 32, 4, 4).permute(0, 3, 2, 1, 4) + scales = torch.pow(2.0, scales.reshape(rows, scale_cols).float() - 127) + d_float = d.float() * scales.repeat_interleave(32, dim=1) + return a.float(), b.float(), alpha, torch.ones_like(prob), c, d_float, counter.item() + + +@pytest.mark.parametrize( + "rows_alignment,mxfp8,rows_per_expert", + [ + (TILE, False, [2 * TILE, TILE, 0, 3 * TILE]), + (TILE, True, [2 * TILE, TILE, 0, 3 * TILE]), + # Tail tiles: an expert of only a tail, a tail after full tiles, an empty expert. + (128, False, [128, 2 * TILE + 128, 0, TILE]), + pytest.param( + 128, + True, + [128, 2 * TILE + 128, 0, TILE], + marks=pytest.mark.skipif(not MXFP8_TAILS, reason="no MXFP8 tail tiles on SM 10.7"), + ), + # Every tail height, and the tails that run inside the last full tile of their expert. + (64, False, [64, 128, 192, 2 * TILE + 64, 2 * TILE + 128, 0, TILE]), + ], +) +@pytest.mark.parametrize("tail_clusters", [0, 4]) +@pytest.mark.parametrize("device_layout", [False, True]) +def test_dispatch_gemm_matches_reference( + rows_alignment, mxfp8, rows_per_expert, tail_clusters, device_layout +): + """Grouped GEMM over padded slabs matches a float32 reference, also with experts padded to + fewer rows than a tile, with a tail launch sharing the tiles and with the dispatch-written + layout; rows past the last expert stay 0.""" + a, b, alpha, prob, c, d, claims = _run_gemm( + rows_per_expert, rows_alignment, mxfp8, tail_clusters, device_layout + ) + if tail_clusters: + # The tail's schedulers claimed from the counter they share with the base's. + assert claims > _run_gemm(rows_per_expert, rows_alignment, mxfp8, 0, device_layout)[-1] + used = sum(rows_per_expert) + ref_c = torch.zeros(a.shape[0], b.shape[1], dtype=torch.float32, device=a.device) + start = 0 + for e, count in enumerate(rows_per_expert): + ref_c[start : start + count] = a[start : start + count] @ b[e].t() * alpha[e] + start += count + col = torch.arange(b.shape[1] // 2, device=a.device) + gate = 64 * (col // 32) + col % 32 + ref_d = ref_c[:, gate + 32] * torch.nn.functional.silu(ref_c[:, gate]) * prob[:, None] + # BF16 outputs round once; the MXFP8 d may round to either neighbor of its E4M3 value. + for got, ref, tol in ((c, ref_c, 2.0**-8), (d, ref_d, 2.0**-3 if mxfp8 else 2.0**-8)): + err = (got[:used].float() - ref[:used]).abs() / ref[:used].abs().clamp_min(1e-6) + assert err.max() <= tol + assert torch.count_nonzero(got[used:]) == 0 + + +def _run_combine(rows_per_expert, rows_alignment, mxfp8, backward): + """Launch the combine GEMM over padded slabs and return (a, b, alpha, prob, d, used) with + MXFP8 operands at unit scales in float32. The forward weight is [E, n, k], backward [E, k, n]; + ``d`` [rows, n] is BF16 in both recipes. Rows past the last expert stay 0.""" + device = torch.device("cuda") + k, n, experts = 1024, 1024, len(rows_per_expert) + rows = sum(rows_per_expert) + TILE # spare tile: the capacity exceeds the rows in use + name = f"test_ep_combine_gemm_{rows}_{rows_alignment}_{int(mxfp8)}_{int(backward)}" + assert combine_gemm.get_ep_combine_gemm_function( + name, rows, k, n, experts, 8, 0, False, backward, mxfp8, rows_alignment + ) + gemm = tvm_ffi.get_global_func(name) + + gen = torch.Generator(device=device).manual_seed(0) + dtype = torch.float8_e4m3fn if mxfp8 else torch.bfloat16 + w_shape = (experts, k, n) if backward else (experts, n, k) + a = torch.rand(rows, k, device=device, generator=gen).to(dtype) + b = (torch.rand(w_shape, device=device, generator=gen) * 2 * k**-0.5).to(dtype) + # Powers of two: alpha and prob cost no tolerance. + alpha = torch.pow(2.0, (torch.arange(experts, device=device) % 5 - 2).float()) + prob = torch.pow(2.0, (torch.arange(rows, device=device) % 3 - 1).float()) + ends = torch.tensor(rows_per_expert, device=device).cumsum(0).to(torch.int32) + d = torch.zeros(rows, n, dtype=torch.bfloat16, device=device) + ws_bytes = combine_gemm.get_ep_combine_gemm_workspace_layout( + rows, k, n, experts, 8, 0, False, backward, mxfp8, rows_alignment + )[0] + workspace = torch.zeros(ws_bytes, dtype=torch.uint8, device=device) + stream = ctypes.c_void_p(torch.cuda.current_stream().cuda_stream) + if mxfp8: + gemm( + a, b, _unit_e8m0(experts * n * k // 32, device), + _unit_e8m0(rows * k // 32, device), d, ends, alpha, prob, workspace, stream, + ) # fmt: skip + else: + gemm(a, b, d, ends, alpha, prob, workspace, stream) + torch.cuda.synchronize() + return a.float(), b.float(), alpha, prob, d, sum(rows_per_expert) + + +@pytest.mark.parametrize( + "backward,mxfp8,rows_alignment,rows_per_expert", + [ + (False, False, TILE, [2 * TILE, TILE, 0, 3 * TILE]), + (False, True, TILE, [2 * TILE, TILE, 0, 3 * TILE]), + (True, False, TILE, [2 * TILE, TILE, 0, 3 * TILE]), + (True, True, TILE, [2 * TILE, TILE, 0, 3 * TILE]), + # Tail tiles: an expert of only a tail, a tail after full tiles, an empty expert. + (False, False, 128, [128, 2 * TILE + 128, 0, TILE]), + ], +) +def test_combine_gemm_matches_reference(backward, mxfp8, rows_alignment, rows_per_expert): + """Grouped combine GEMM over padded slabs, per expert ``d = (a @ weight) * alpha * prob``, + matches a float32 reference; rows past the last expert stay 0.""" + a, b, alpha, prob, d, used = _run_combine(rows_per_expert, rows_alignment, mxfp8, backward) + ref = torch.zeros(a.shape[0], b.shape[2] if backward else b.shape[1], device=a.device) + start = 0 + for e, count in enumerate(rows_per_expert): + sl = slice(start, start + count) + weight = b[e] if backward else b[e].t() + ref[sl] = a[sl] @ weight * alpha[e] * prob[sl, None] + start += count + # BF16 output rounds once, whether the operands are BF16 or MXFP8. + err = (d[:used].float() - ref[:used]).abs() / ref[:used].abs().clamp_min(1e-6) + assert err.max() <= 2.0**-8 + assert torch.count_nonzero(d[used:]) == 0 + + +def _run_wgrad(rows_per_expert, mxfp8, fp32_out, accumulate): + """Launch the weight gradient over padded slabs and return (grad, x, init, weight_grad) with + MXFP8 operands at unit scales in float32. ``weight_grad[e] = grad[e]^T @ x[e]``, written or + added to ``init``.""" + device = torch.device("cuda") + m, n, experts = 256, 512, len(rows_per_expert) + rows = sum(rows_per_expert) + TILE # spare tile: the capacity exceeds the rows in use + routed = max(sum(rows_per_expert) // experts, 1) + args = (rows, m, n, experts, 8, fp32_out, accumulate, False, 0, mxfp8, False, TILE, routed) + name = f"test_ep_wgrad_{rows}_{int(mxfp8)}_{int(fp32_out)}_{int(accumulate)}" + assert wgrad_gemm.get_ep_wgrad_function(name, *args) + launches = [tvm_ffi.get_global_func(name)] + if mxfp8 and wgrad_gemm.get_ep_wgrad_reads_descriptors(*args): + # A descriptors-only launch writes the per-expert descriptors the GEMM launch reads. + desc = (rows, m, n, experts, 1, fp32_out, accumulate, False, 0, mxfp8, True, TILE, routed) + assert wgrad_gemm.get_ep_wgrad_function(name + "_desc", *desc) + launches.insert(0, tvm_ffi.get_global_func(name + "_desc")) + + gen = torch.Generator(device=device).manual_seed(0) + dtype = torch.float8_e4m3fn if mxfp8 else torch.bfloat16 + # Powers of two keep every product and sum exact in float32; signs on grad add cancellation. + signs = 1 - 2 * torch.randint(0, 2, (rows, m), device=device, generator=gen) + grad = ( + signs + * torch.pow(2.0, -torch.randint(0, 2, (rows, m), device=device, generator=gen).float()) + ).to(dtype) + x = torch.pow(2.0, -torch.randint(0, 2, (rows, n), device=device, generator=gen).float()).to( + dtype + ) + out_dtype = torch.float32 if fp32_out else torch.bfloat16 + init = torch.pow(2.0, (torch.arange(experts * m * n, device=device) % 3 - 1).float()).reshape( + experts, m, n + ) + weight_grad = (init if accumulate else torch.zeros_like(init)).to(out_dtype, copy=True) + ends = torch.tensor(rows_per_expert, device=device).cumsum(0).to(torch.int32) + ws_bytes = wgrad_gemm.get_ep_wgrad_workspace_layout(*args)[0] + workspace = torch.zeros(ws_bytes, dtype=torch.uint8, device=device) + stream = ctypes.c_void_p(torch.cuda.current_stream().cuda_stream) + for gemm in launches: + if mxfp8: + gemm( + grad, x, _unit_e8m0(rows * m // 32, device), _unit_e8m0(rows * n // 32, device), + weight_grad, ends, workspace, stream, + ) # fmt: skip + else: + gemm(grad, x, weight_grad, ends, workspace, stream) + torch.cuda.synchronize() + base = init if accumulate else torch.zeros_like(init) + return grad.float(), x.float(), base, weight_grad.float() + + +@pytest.mark.parametrize( + "mxfp8,fp32_out,accumulate,rows_per_expert", + [ + (False, False, False, [2 * TILE, TILE, 0, 3 * TILE]), # BF16 written + (False, True, False, [2 * TILE, TILE, 0, 3 * TILE]), # FP32 written + (False, False, True, [TILE, 0, 2 * TILE, TILE]), # BF16 accumulated + (True, False, False, [2 * TILE, TILE, 0, 3 * TILE]), # MXFP8 operands + (True, True, False, [2 * TILE, TILE, 0, 3 * TILE]), # MXFP8, FP32 written + (True, False, True, [TILE, 0, 2 * TILE, TILE]), # MXFP8, BF16 accumulated + (True, True, True, [TILE, 0, 2 * TILE, TILE]), # MXFP8, FP32 accumulated + ], +) +def test_wgrad_gemm_matches_reference(mxfp8, fp32_out, accumulate, rows_per_expert): + """Grouped weight gradient matches a float32 reference; empty experts keep their input.""" + grad, x, base, weight_grad = _run_wgrad(rows_per_expert, mxfp8, fp32_out, accumulate) + ref = base.clone() + start = 0 + for e, count in enumerate(rows_per_expert): + sl = slice(start, start + count) + ref[e] += grad[sl].t() @ x[sl] + start += count + # A written FP32 gradient is exact; a BF16 one rounds once. + tol = 2.0**-16 if fp32_out else 2.0**-8 + err = (weight_grad - ref).abs() / ref.abs().clamp_min(1e-6) + assert err.max() <= tol + + +@pytest.mark.skipif(MXFP8_TAILS, reason="SM 10.7 only") +def test_wgrad_gemm_rejects_one_tile_capacity(): + """SM 10.7 refuses an added or FP32 MXFP8 weight gradient over 256 receive rows or fewer.""" + cfg = wgrad_gemm.WgradConfig( + TILE, 256, 512, 2, 8, fp32_out=True, mxfp8=True, rows_alignment=TILE, routed_rows=TILE + ) + with pytest.raises(ValueError, match="above 256 rows"): + wgrad_gemm.build_kernel(cfg) + + +def _read_col_scales(words, rows_per_expert, cols, device): + """Columnwise E8M0 scale per row and column from the GEMM-swizzled per-expert scale words.""" + scale = torch.ones(sum(rows_per_expert), cols, device=device) + col = torch.arange(cols, device=device) + start = 0 + for count in rows_per_expert: + if count: + r = torch.arange(start, start + count, device=device) + group = r // 128 - start // 128 + index = ( + start * (cols // 128) + + (col // 128) * count + + group[:, None] * 128 + + (col % 32)[None, :] * 4 + + ((col // 32) % 4)[None, :] + ) + block = (r // 32) % 4 + e8m0 = (words[index] >> (8 * block[:, None])) & 0xFF + scale[start : start + count] = torch.pow(2.0, e8m0.float() - 127) + start += count + return scale + + +def _run_requant(rows_per_expert, cols, ready_slot_rows=0, nan_scale=None): + """Launch the MXFP8 rowwise-to-columnwise requant and return (codes, col_codes, scale words, + used). Row scales are unit, so the rowwise values are the codes themselves. With + ``ready_slot_rows``, the rows and then their dispatch flags arrive late, on another stream. + ``nan_scale`` is a (row, 32-column block) whose rowwise scale is the E8M0 NaN.""" + device = torch.device("cuda") + experts = len(rows_per_expert) + # Spare rows, a whole slot: the capacity exceeds the rows in use. + rows = sum(rows_per_expert) + max(128, ready_slot_rows) + name = f"test_ep_requant_{rows}_{cols}_{ready_slot_rows}" + assert requantize.get_ep_requant_function(name, rows, cols, experts, ready_slot_rows) + fn = tvm_ffi.get_global_func(name) + + gen = torch.Generator(device=device).manual_seed(0) + # Powers of two: the columnwise requant represents them exactly, so it costs no tolerance. + exps = torch.randint(-2, 2, (rows, cols), device=device, generator=gen).float() + signs = 1 - 2 * torch.randint(0, 2, (rows, cols), device=device, generator=gen) + codes = (signs * torch.pow(2.0, exps)).to(torch.float8_e4m3fn) + row_scales = _unit_e8m0(rows * cols // 32, device) + if nan_scale is not None: + row_scales.view(torch.uint8)[nan_scale[0] * (cols // 32) + nan_scale[1]] = 255 + col_codes = torch.zeros(rows, cols, dtype=torch.float8_e4m3fn, device=device) + col_scales = _unit_e8m0(rows * cols // 32, device) + ends = torch.tensor(rows_per_expert, device=device).cumsum(0).to(torch.int32) + stream = ctypes.c_void_p(torch.cuda.current_stream().cuda_stream) + if ready_slot_rows: + received = torch.zeros_like(codes) + signal = torch.full((rows // ready_slot_rows,), -1, dtype=torch.int32, device=device) + # Enqueued first, so a run that serializes kernels does not hang. + arrival = torch.cuda.Stream() + arrival.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(arrival): + torch.cuda._sleep(10**7) + received.copy_(codes) + signal.fill_(0) + fn(received, row_scales, col_codes, col_scales, ends, signal, stream) + else: + fn(codes, row_scales, col_codes, col_scales, ends, stream) + torch.cuda.synchronize() + words = col_scales.view(torch.uint8).view(torch.int32) + return codes.float(), col_codes.float(), words, sum(rows_per_expert) + + +@pytest.mark.parametrize("ready_slot_rows", [0, 128, 256]) +@pytest.mark.parametrize( + "cols,rows_per_expert", + [ + (256, [128, 256, 0, 128]), + (512, [256, 0, 128, 128]), + ], +) +def test_requantize_matches_reference(cols, rows_per_expert, ready_slot_rows): + """MXFP8 rowwise-to-columnwise requant reproduces the dequantized rows exactly for power-of-two + values; rows past the last expert stay 0. With ``ready_slot_rows``, it waits for rows that + arrive after its launch.""" + codes, col_codes, words, used = _run_requant(rows_per_expert, cols, ready_slot_rows) + scale = _read_col_scales(words, rows_per_expert, cols, codes.device) + assert torch.equal(col_codes[:used] * scale, codes[:used]) + assert torch.count_nonzero(col_codes[used:]) == 0 + + +def test_requantize_nan_scale(): + """A rowwise E8M0 scale of 255 makes its 32 values NaN and leaves every other value finite.""" + cols, row, block = 256, 5, 2 + _, col_codes, _, used = _run_requant([128], cols, nan_scale=(row, block)) + expected = torch.zeros(used, cols, dtype=torch.bool, device=col_codes.device) + expected[row, block * 32 : (block + 1) * 32] = True + assert torch.equal(col_codes[:used].isnan(), expected) diff --git a/transformer_engine/common/CMakeLists.txt b/transformer_engine/common/CMakeLists.txt index f90f74449c5..e94cbba1086 100644 --- a/transformer_engine/common/CMakeLists.txt +++ b/transformer_engine/common/CMakeLists.txt @@ -624,6 +624,8 @@ target_link_libraries(transformer_engine PUBLIC ${NCCL_LIB}) target_sources(transformer_engine PRIVATE ep/ep_backend.cpp ep/ep_api.cpp + ep/ep_gemm_api.cpp + ep/ep_overlap_workspace.cu ep/nccl_ep_provider.cpp) target_compile_definitions(transformer_engine PRIVATE NVTE_WITH_NCCL_EP) @@ -631,7 +633,7 @@ message(STATUS "NCCL EP enabled (runtime-loaded library): ${NCCL_EP_SHARED_LIB}" message(STATUS "NCCL EP include: ${NCCL_EP_INCLUDE_DIR}") else() # NCCL EP off: ep_api.cpp's #else branch exports throwing nvte_ep_* stubs. - target_sources(transformer_engine PRIVATE ep/ep_api.cpp) + target_sources(transformer_engine PRIVATE ep/ep_api.cpp ep/ep_gemm_api.cpp) message(STATUS "NCCL EP disabled (NVTE_WITH_NCCL_EP=OFF) - using nvte_ep_* stubs") endif() diff --git a/transformer_engine/common/CuTeDSL/__init__.py b/transformer_engine/common/CuTeDSL/__init__.py index e5a37354528..4243d6b418f 100644 --- a/transformer_engine/common/CuTeDSL/__init__.py +++ b/transformer_engine/common/CuTeDSL/__init__.py @@ -8,6 +8,8 @@ They should provide a string function name which can be used to retrieve the corresponding CuTeDSL kernel function via TVM-FFI. """ +import logging + import tvm_ffi from transformer_engine.common.CuTeDSL.cast.mxfp8.dequantize_mxfp8 import ( @@ -21,6 +23,33 @@ get_mxfp8_requantization_function, ) +logger = logging.getLogger("transformer_engine.cutedsl") + + +def _register_ep_backends(): + """Register the EP grouped GEMM backends; a failure leaves the other backends registered.""" + try: + from transformer_engine.common.CuTeDSL.ep_gemm import ( # pylint: disable=import-outside-toplevel + dispatch_gemm, + combine_gemm, + requantize, + wgrad_gemm, + ) + + for function in ( + dispatch_gemm.get_ep_dispatch_gemm_function, + dispatch_gemm.get_ep_dispatch_gemm_workspace_layout, + combine_gemm.get_ep_combine_gemm_function, + combine_gemm.get_ep_combine_gemm_workspace_layout, + wgrad_gemm.get_ep_wgrad_function, + wgrad_gemm.get_ep_wgrad_workspace_layout, + wgrad_gemm.get_ep_wgrad_reads_descriptors, + requantize.get_ep_requant_function, + ): + tvm_ffi.register_global_func(function.__name__, function, override=True) + except Exception as e: # pylint: disable=broad-exception-caught + logger.warning("EP grouped GEMM CuTeDSL backends are unavailable: %s", e) + def register_cutedsl_backends(): """Register all available CuTeDSL backends for on-demand compilation via TVM-FFI. @@ -33,3 +62,4 @@ def register_cutedsl_backends(): get_mxfp8_requantization_function, ): tvm_ffi.register_global_func(entrypoint.__name__, entrypoint, override=True) + _register_ep_backends() diff --git a/transformer_engine/common/CuTeDSL/ep_gemm/__init__.py b/transformer_engine/common/CuTeDSL/ep_gemm/__init__.py new file mode 100644 index 00000000000..2e9e21a1314 --- /dev/null +++ b/transformer_engine/common/CuTeDSL/ep_gemm/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Grouped GEMMs overlapped with EP communication, loaded through TVM-FFI. + +dispatch_gemm: reads row tiles as a dispatch delivers them (FC1 forward, FC2 data gradient). +combine_gemm: publishes column chunks for a combine to send (FC2 forward, FC1 data gradient). +wgrad_gemm: weight gradients chained behind the backward GEMMs. +requantize: columnwise MXFP8 operands of the weight gradients from received rowwise rows. +""" + +from . import dispatch_gemm, combine_gemm, requantize, wgrad_gemm diff --git a/transformer_engine/common/CuTeDSL/ep_gemm/_loader.py b/transformer_engine/common/CuTeDSL/ep_gemm/_loader.py new file mode 100644 index 00000000000..ce6d3ab7bc3 --- /dev/null +++ b/transformer_engine/common/CuTeDSL/ep_gemm/_loader.py @@ -0,0 +1,338 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Loading of the EP grouped GEMMs: kernel tree, fake operands and TVM-FFI registration. + +The kernels are loaded from ``NVTE_MOE_OVERLAP`` (a ``gemm/`` tree with ``blackwell/``, +``rubin/`` and ``utils/``), compiled once per configuration and registered under a name chosen by +C++. +""" + +import functools +import importlib +import logging +import os +import sys +from dataclasses import dataclass +from typing import Optional + +import tvm_ffi + +from ..utils import device_compute_capability, tvm_ffi_function + +logger = logging.getLogger(__name__) + +GEMM_PATH_ENV = "NVTE_MOE_OVERLAP" +MMA_TILER_MN = (256, 256) +CLUSTER_MN = (2, 1) +# Rows per M tile. A smaller expert alignment ends an expert in a tail tile. +ROWS_PER_TILE = MMA_TILER_MN[0] // 2 * CLUSTER_MN[0] +# Values per MXFP8 E8M0 scale. +SF_VEC_SIZE = 32 + + +@dataclass(frozen=True) +class GemmArch: + """The kernels of one architecture: their subdirectories of the GEMM path, searched in order, + and the BF16 kernels as ``module.Class``.""" + + subdirs: tuple + dispatch_fwd: str # FC1 + SwiGLU + dispatch_bwd: str # FC2 data gradient + SwiGLU backward + combine: str # FC2, FC1 data gradient + mxfp8_tail_tiles: bool # MXFP8 kernels for experts padded to fewer rows than a tile + mxfp8_modules: tuple # modules of the MXFP8 kernels + + +_SM100 = GemmArch( + subdirs=("blackwell",), + dispatch_fwd="moe_grouped_gemm_glu_sm100.MoEGroupedGemmGluBiasBf16Kernel", + dispatch_bwd="moe_grouped_gemm_dglu_dbias.MoEGroupedGemmDgluDbiasBf16Kernel", + combine="moe_grouped_gemm_sm100.MoEGroupedGemmBf16Kernel", + mxfp8_tail_tiles=True, + mxfp8_modules=( + "moe_blockscaled_grouped_gemm_quant_sm100", + "moe_blockscaled_grouped_gemm_dglu_dbias", + "moe_blockscaled_grouped_gemm_wgrad", + "moe_blockscaled_grouped_gemm_wgrad_dense", + "moe_blockscaled_grouped_gemm_wgrad_reuse", + ), +) +# The SM 10.7 kernels build on the SM 10.0 ones. +_SM107 = GemmArch( + subdirs=("rubin", "blackwell"), + dispatch_fwd="moe_grouped_gemm_glu_sm107.MoEGroupedGemmGluBiasBf16Kernel", + dispatch_bwd="moe_grouped_gemm_sm107.MoEGroupedGemmDgluDbiasBf16Kernel", + combine="moe_grouped_gemm_sm107.MoEGroupedGemmBf16Kernel", + mxfp8_tail_tiles=False, + mxfp8_modules=( + "moe_blockscaled_grouped_gemm_quant_rubin", + "moe_blockscaled_grouped_gemm_dglu_rubin", + "moe_blockscaled_grouped_gemm_glu_rubin", + "moe_blockscaled_grouped_gemm_wgrad_rubin", + "moe_blockscaled_grouped_gemm_wgrad_reuse_rubin", + ), +) +# Kernels by compute capability; other architectures run the unfused ops. +GEMM_ARCHS = {(10, 0): _SM100, (10, 3): _SM100, (10, 7): _SM107} + + +def gemm_arch(): + """The kernels of the current device (the SM 10.0 ones on an unsupported device).""" + return GEMM_ARCHS.get(device_compute_capability(), _SM100) + + +def is_sm107(): + """Whether the current device runs the SM 10.7 kernels.""" + return gemm_arch() is _SM107 + + +_LOADED_ROOT: list = [] + +# What TE reads from the kernel tree, module by module (dotted names are nested attributes). +_SHARED_INTERFACE = { + "moe_utils": ("MoEWeightMode.DENSE", "WGradInputOrder.Tensor2D", "WgradTileClaim.COUNTER"), + "moe_tail_tiles": ("FUSED_TAIL_MAX_ROWS",), + "moe_signal_helpers": ("signal_workspace_offsets",), + "moe_streaming_output_chunk_tile_scheduler": ( + "OutputNotifyConfig", + "OutputNotifyMode.N_CHUNK", + "output_notify_capacity", + "output_notify_offsets", + ), +} +_WGRAD_CLASSES = ("MoEGroupedGemmWgradBF16Kernel", "MoEGroupedGemmWgradWideBF16Kernel") + + +def _missing(obj, dotted_names): + """The names of ``dotted_names`` that ``obj`` lacks.""" + missing = [] + for dotted in dotted_names: + target = obj + for part in dotted.split("."): + target = getattr(target, part, None) + if target is None: + missing.append(dotted) + return missing + + +@functools.lru_cache(maxsize=None) +def _gemm_interface_reason(root, capability, mxfp8): + del root, capability # The cache key: the tree and the architecture the checks ran on. + problems = [] + try: + for name, attributes in _SHARED_INTERFACE.items(): + missing = _missing(import_gemm_modules(name)[0], attributes) + if missing: + problems.append(f"{name} lacks {', '.join(missing)}") + arch = gemm_arch() + for role in ("dispatch_fwd", "dispatch_bwd", "combine"): + module_name, class_name = getattr(arch, role).split(".") + kernel = getattr(import_gemm_modules(module_name)[0], class_name, None) + if kernel is None: + problems.append(f"{module_name} lacks {class_name}") + elif role == "dispatch_fwd" and not hasattr(kernel, "FC1_TAIL_AT_SLOTS_READY"): + problems.append(f"{class_name} lacks FC1_TAIL_AT_SLOTS_READY") + if mxfp8: + for module_name in arch.mxfp8_modules: + import_gemm_modules(module_name) + names = ( + ("moe_grouped_gemm_sm107",) * 2 + if is_sm107() + else ("moe_grouped_gemm_wgrad", "moe_grouped_gemm_wgrad_wide") + ) + for module_name, class_name in zip(names, _WGRAD_CLASSES): + if getattr(import_gemm_modules(module_name)[0], class_name, None) is None: + problems.append(f"{module_name} lacks {class_name}") + except ImportError as e: + problems.append(f"cannot import {e.name or e}") + if problems: + return "the EP grouped GEMM kernel tree does not match Transformer Engine: " + "; ".join( + problems + ) + return None + + +# TODO(phuonguyen): check the kernel tree's version (cudnn-frontend release or EPCompOverlap +# commit) against a minimum once the kernels ship in a versioned package; for now the tree is +# only checked for the names TE reads. +def gemm_interface_reason(mxfp8=False) -> Optional[str]: + """Why the kernel tree at ``NVTE_MOE_OVERLAP`` lacks what the EP grouped GEMMs read, or None; + ``mxfp8`` also checks the MXFP8 kernels. Checked once per tree, architecture and recipe; the + tree's version is not otherwise identified.""" + return _gemm_interface_reason(os.environ.get(GEMM_PATH_ENV), device_compute_capability(), mxfp8) + + +def import_gemm_modules(*names): + """Import the named modules of the kernel tree (and ``moe_utils`` last) from the GEMM path.""" + root = os.environ.get(GEMM_PATH_ENV) + if not root: + raise ModuleNotFoundError( + f"{GEMM_PATH_ENV} is not set: the EP grouped GEMMs are not available" + ) + if _LOADED_ROOT and _LOADED_ROOT[0] != root: + # The modules are imported by name, so those of the first tree stay in sys.modules. + raise RuntimeError( + f"{GEMM_PATH_ENV} changed from {_LOADED_ROOT[0]} to {root} after the EP grouped GEMMs" + " were loaded; restart the process to use the other kernel tree." + ) + _LOADED_ROOT[:] = [root] + for sub in reversed(("", *gemm_arch().subdirs, "utils")): + path = os.path.join(root, sub) if sub else root + if path not in sys.path: + sys.path.insert(0, path) + importlib.invalidate_caches() + return tuple(importlib.import_module(name) for name in (*names, "moe_utils")) + + +def import_bf16_kernel(role): + """The current device's BF16 kernel class of ``role`` (a ``GemmArch`` field) and + ``moe_utils``.""" + module_name, class_name = getattr(gemm_arch(), role).split(".") + module, moe_utils = import_gemm_modules(module_name) + return getattr(module, class_name), moe_utils + + +def unsupported_device_reason(what): + """Why ``what`` cannot run on the current device, or None.""" + if device_compute_capability() in GEMM_ARCHS: + return None + supported = ", ".join(f"{major}.{minor}" for major, minor in GEMM_ARCHS) + major, minor = device_compute_capability() + return f"The {what} needs SM {supported}, got SM {major}.{minor}." + + +def device_smem_optin(): + """Opt-in shared memory per block of the current CUDA device, in bytes.""" + from cuda.bindings import runtime as cudart # pylint: disable=import-outside-toplevel + + err, device = cudart.cudaGetDevice() + assert err == cudart.cudaError_t.cudaSuccess, err + err, smem = cudart.cudaDeviceGetAttribute( + cudart.cudaDeviceAttr.cudaDevAttrMaxSharedMemoryPerBlockOptin, device + ) + assert err == cudart.cudaError_t.cudaSuccess, err + return smem + + +def fake_tensor(dtype, shape, order, align): + """A compact fake gmem tensor to compile against.""" + from cutlass import cute # pylint: disable=import-outside-toplevel + + return cute.runtime.make_fake_compact_tensor( + dtype, shape, stride_order=order, memspace=cute.AddressSpace.gmem, assumed_align=align + ) + + +def rows_view(t, rows, cols): + """Row-major [rows, cols] operand as the kernel's (rows, cols, 1) tensor.""" + from cutlass import cute # pylint: disable=import-outside-toplevel + + return cute.make_tensor( + t.iterator, cute.make_layout((rows, cols, 1), stride=(cols, 1, rows * cols)) + ) + + +def weight_view(b, n, k, experts, n_major): + """Weight [experts, n, k], or N-major [experts, k, n], as the kernel's (n, k, experts) tensor.""" + from cutlass import cute # pylint: disable=import-outside-toplevel + + stride = (1, n, k * n) if n_major else (k, 1, n * k) + return cute.make_tensor(b.iterator, cute.make_layout((n, k, experts), stride=stride)) + + +def compile_launch(launch, *fakes): + """Compile ``launch`` with TVM-FFI against fake operands and a fake stream.""" + from cutlass import cute # pylint: disable=import-outside-toplevel + + return cute.compile(launch, *fakes, cute.runtime.make_fake_stream(), options="--enable-tvm-ffi") + + +def prob_view(prob, rows): + """[rows] probabilities as the kernel's (rows, 1, 1) tensor.""" + from cutlass import cute # pylint: disable=import-outside-toplevel + + return cute.make_tensor(prob.iterator, cute.make_layout((rows, 1, 1), stride=(1, 1, 1))) + + +def fp8_fake(shape, order): + """A fake E4M3 operand.""" + import cutlass # pylint: disable=import-outside-toplevel + + return fake_tensor(cutlass.Float8E4M3FN, shape, order, 16) + + +def scale_fake(rows, cols): + """Fake E8M0 scales of a [rows, cols] MXFP8 operand, compact or GEMM-swizzled.""" + import cutlass # pylint: disable=import-outside-toplevel + + return fake_tensor(cutlass.Float8E8M0FNU, (rows * cols // SF_VEC_SIZE,), (0,), 16) + + +def blockscaled_kwargs(num_experts): + """Constructor arguments shared by the MXFP8 kernels.""" + import cutlass # pylint: disable=import-outside-toplevel + + moe_utils = import_gemm_modules()[0] + return { + "sf_vec_size": SF_VEC_SIZE, + "acc_dtype": cutlass.Float32, + "use_2cta_instrs": True, + "mma_tiler_mn": MMA_TILER_MN, + "cluster_shape_mn": CLUSTER_MN, + "vectorized_f32": False, + "expert_cnt": num_experts, + "weight_mode": moe_utils.MoEWeightMode.DENSE, + "use_dynamic_sched": True, + } + + +def tail_tile_kwargs(rows_alignment, fused=False): + """Constructor arguments for experts padded to ``rows_alignment`` rows; ``fused`` runs the + smallest tails inside their expert's last full tile (BF16 forward GEMMs).""" + if rows_alignment >= ROWS_PER_TILE: + return {} + kwargs = {"tail_rows_alignment": rows_alignment} + if fused: + tail_tiles = import_gemm_modules("moe_tail_tiles")[0] + kwargs["fused_tail_max_rows"] = tail_tiles.FUSED_TAIL_MAX_ROWS + return kwargs + + +def mxfp8_tail_tile_kwargs(rows_alignment): + """``tail_tile_kwargs`` of the MXFP8 kernels.""" + if gemm_arch().mxfp8_tail_tiles: + return tail_tile_kwargs(rows_alignment) + if rows_alignment < ROWS_PER_TILE: + raise ValueError( + f"The MXFP8 kernels of this device need experts padded to {ROWS_PER_TILE} rows." + ) + return {} + + +def tail_fakes(rows, experts, ws_bytes): + """Fake row ends, alpha, prob and workspace operands shared by every launch.""" + import cutlass # pylint: disable=import-outside-toplevel + + return ( + fake_tensor(cutlass.Int32, (experts,), (0,), 16), + fake_tensor(cutlass.Float32, (experts,), (0,), 16), + fake_tensor(cutlass.Float32, (rows,), (0,), 16), + fake_tensor(cutlass.Uint8, (ws_bytes,), (0,), 128), + ) + + +def register_function(fn_name, compile_fn, what) -> bool: + """Compile and register ``fn_name`` unless registered; return whether it is.""" + try: + if tvm_ffi.get_global_func(fn_name, allow_missing=True) is not None: + return True + if (reason := unsupported_device_reason(what)) is not None: + logger.warning("%s", reason) + return False + tvm_ffi.register_global_func(fn_name, tvm_ffi_function(compile_fn()), override=True) + return True + except Exception as e: # pylint: disable=broad-exception-caught + logger.error("The %s is not available: %s", what, e) + return False diff --git a/transformer_engine/common/CuTeDSL/ep_gemm/combine_gemm.py b/transformer_engine/common/CuTeDSL/ep_gemm/combine_gemm.py new file mode 100644 index 00000000000..eadc28be005 --- /dev/null +++ b/transformer_engine/common/CuTeDSL/ep_gemm/combine_gemm.py @@ -0,0 +1,215 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Grouped GEMM that publishes its output to the EP combine (BF16 or MXFP8, SM 10.0, 10.3 and +10.7). + + mode a b (weight) d + forward [rows,k] [E,n,k] [rows,n] BF16, scaled per row by prob + backward [rows,k] [E,k,n] (the forward weight) [rows,n] BF16, scaled per row by prob + +BF16 operands, or E4M3 with GEMM-swizzled E8M0 scales ``sfa`` and ``sfb`` for MXFP8 (the backward +reads the columnwise forward weight). Common operands: ``expert_row_ends`` [E] int32, ``alpha`` +[E] FP32, ``prob`` [rows] FP32, ``workspace`` uint8, stream. ``signal[u] = u`` once columns +``[u * n_chunk, (u + 1) * n_chunk)`` of ``d`` are final; the caller sets ``signal[]`` to -1 before +each launch. With ``n_chunk`` 0 the GEMM publishes nothing and runs in the stock tile order; its +workspace ends in one flag word the caller sets. +""" + +import functools +from dataclasses import dataclass, field + +import cutlass +from cutlass import cute +from cutlass.cute.nvgpu import OperandMajorMode + +from ._loader import ( + device_compute_capability, + CLUSTER_MN, + MMA_TILER_MN, + ROWS_PER_TILE, + blockscaled_kwargs, + compile_launch, + fake_tensor, + fp8_fake, + import_bf16_kernel, + import_gemm_modules, + is_sm107, + mxfp8_tail_tile_kwargs, + prob_view, + register_function, + rows_view, + scale_fake, + tail_fakes, + tail_tile_kwargs, + weight_view, +) + + +@dataclass(frozen=True) +class CombineGemmConfig: + """Static configuration of one compiled EP grouped GEMM combine.""" + + rows: int # receive capacity in rows + k: int # reduction size (ffn) + n: int # output columns (hidden) + num_experts: int # local experts + num_clusters: int # persistent clusters beside the combine's SMs + n_chunk: int # columns per readiness flag, 0: no flags + dependents_at_entry: bool # the combine may start at the GEMM's entry (signaled combine only) + backward: bool = False + mxfp8: bool = False + rows_alignment: int = ROWS_PER_TILE # rows per expert slab are a multiple of it + grid_clusters: int = 0 # clusters launched (0: num_clusters), extras stop after the first chunk + # Kernels, workspace layouts and compiled functions are specific to the device architecture. + capability: tuple = field(default_factory=device_compute_capability) + + +def build_kernel(cfg: CombineGemmConfig): + """Kernel for ``cfg`` that publishes finished column chunks to ``signal[]`` (none: n_chunk 0).""" + comb, moe_utils = import_gemm_modules("moe_streaming_output_chunk_tile_scheduler") + notify = None + if cfg.n_chunk: + notify = comb.OutputNotifyConfig( + comb.OutputNotifyMode.N_CHUNK, + cfg.n_chunk, + comb.output_notify_capacity(cfg.n_chunk, cfg.n), + caller_reset=True, + clusters_beside_combine=cfg.num_clusters, + ) + if cfg.mxfp8: + rubin = is_sm107() + module = import_gemm_modules( + "moe_blockscaled_grouped_gemm_quant_rubin" + if rubin + else "moe_blockscaled_grouped_gemm_quant_sm100" + )[0] + return module.BlockScaledMoEGroupedGemmQuantKernel( + generate_sfd=False, + discrete_col_sfd=False, + enable_bias=False, + output_notify=notify, + launch_dependents_at_entry=cfg.dependents_at_entry, + **({"generate_c": False} if rubin else {}), + **blockscaled_kwargs(cfg.num_experts), + **mxfp8_tail_tile_kwargs(cfg.rows_alignment), + ) + kernel_class, _ = import_bf16_kernel("combine") + return kernel_class( + acc_dtype=cutlass.Float32, + use_2cta_instrs=True, + mma_tiler_mn=MMA_TILER_MN, + cluster_shape_mn=CLUSTER_MN, + vectorized_f32=False, + generate_c=False, + enable_bias=False, + expert_cnt=cfg.num_experts, + weight_mode=moe_utils.MoEWeightMode.DENSE, + use_dynamic_sched=True, + output_notify=notify, + launch_dependents_at_entry=cfg.dependents_at_entry, + **tail_tile_kwargs(cfg.rows_alignment, fused=not cfg.backward), + ) + + +@functools.lru_cache(maxsize=None) +def workspace_layout(cfg: CombineGemmConfig): + """(total bytes, byte offset of signal[], entries of signal[]) of ``cfg``'s workspace.""" + comb = import_gemm_modules("moe_streaming_output_chunk_tile_scheduler")[0] + if not cfg.n_chunk: + signal = -(-build_kernel(cfg).get_workspace_bytes() // 4) * 4 + return signal + 4, signal, 1 + offsets = comb.output_notify_offsets(build_kernel(cfg)) + return offsets["total_bytes"], offsets["signal"], offsets["capacity"] + + +def _compile(cfg: CombineGemmConfig): + """Compile the launch wrapper of ``cfg`` with TVM-FFI.""" + kernel = build_kernel(cfg) + rows, k, n, experts = cfg.rows, cfg.k, cfg.n, cfg.num_experts + num_clusters, backward = cfg.grid_clusters or cfg.num_clusters, cfg.backward + + @cute.jit + def launch(a, b, d, expert_row_ends, alpha, prob, workspace, stream): + weight = weight_view(b, n, k, experts, n_major=backward) + d3 = rows_view(d, rows, n) + kernel( + rows_view(a, rows, k), weight, cutlass.Int32(0), cutlass.Int32(0), cutlass.Int64(0), + OperandMajorMode.K, workspace.iterator, d3, d3, expert_row_ends, alpha, None, + prob_view(prob, rows), num_clusters, stream, + ) # fmt: skip + + return compile_launch( + launch, + fake_tensor(cutlass.BFloat16, (rows, k), (1, 0), 16), + fake_tensor( + cutlass.BFloat16, (experts, k, n) if backward else (experts, n, k), (2, 1, 0), 16 + ), + fake_tensor(cutlass.BFloat16, (rows, n), (1, 0), 16), + *tail_fakes(rows, experts, workspace_layout(cfg)[0]), + ) + + +def _compile_mxfp8(cfg: CombineGemmConfig): + """Compile the MXFP8 launch wrapper of ``cfg`` with TVM-FFI.""" + kernel = build_kernel(cfg) + rows, k, n, experts = cfg.rows, cfg.k, cfg.n, cfg.num_experts + num_clusters, backward = cfg.grid_clusters or cfg.num_clusters, cfg.backward + + @cute.jit + def launch(a, b, sfb, sfa, d, expert_row_ends, alpha, prob, workspace, stream): + weight = weight_view(b, n, k, experts, n_major=backward) + d3 = rows_view(d, rows, n) + kernel( + rows_view(a, rows, k), weight, sfb, cutlass.Int32(0), cutlass.Int32(0), + cutlass.Int64(0), OperandMajorMode.K, workspace.iterator, d3, d3, sfa, None, None, + None, None, expert_row_ends, alpha, None, None, prob_view(prob, rows), None, + num_clusters, stream, + ) # fmt: skip + + @cute.jit + def launch_rubin(a, b, sfb, sfa, d, expert_row_ends, alpha, prob, workspace, stream): + # The SM 10.7 kernel takes c (not stored) ahead of d, and the bias ahead of prob. + weight = weight_view(b, n, k, experts, n_major=backward) + d3 = rows_view(d, rows, n) + kernel( + rows_view(a, rows, k), weight, sfb, cutlass.Int32(0), cutlass.Int32(0), + cutlass.Int64(0), OperandMajorMode.K, workspace.iterator, d3, d3, d3, sfa, None, None, + None, None, expert_row_ends, alpha, None, prob_view(prob, rows), num_clusters, stream, + ) # fmt: skip + + return compile_launch( + launch_rubin if is_sm107() else launch, + fp8_fake((rows, k), (1, 0)), + fp8_fake((experts, k, n) if backward else (experts, n, k), (2, 1, 0)), + scale_fake(experts * n, k), + scale_fake(rows, k), + fake_tensor(cutlass.BFloat16, (rows, n), (1, 0), 16), + *tail_fakes(rows, experts, workspace_layout(cfg)[0]), + ) + + +def get_ep_combine_gemm_workspace_layout( + rows, k, n, num_experts, num_clusters, n_chunk, dependents_at_entry, backward, mxfp8=False, + rows_alignment=ROWS_PER_TILE, grid_clusters=0, +): # fmt: skip + """[total bytes, signal offset, signal entries] of the workspace of this configuration.""" + cfg = CombineGemmConfig( + rows, k, n, num_experts, num_clusters, n_chunk, bool(dependents_at_entry), bool(backward), + bool(mxfp8), rows_alignment, grid_clusters, + ) # fmt: skip + return list(workspace_layout(cfg)) + + +def get_ep_combine_gemm_function( + fn_name, rows, k, n, num_experts, num_clusters, n_chunk, dependents_at_entry, backward, + mxfp8=False, rows_alignment=ROWS_PER_TILE, grid_clusters=0, +) -> bool: # fmt: skip + """Compile and register the GEMM of this configuration as ``fn_name``; return whether it is.""" + cfg = CombineGemmConfig( + rows, k, n, num_experts, num_clusters, n_chunk, bool(dependents_at_entry), bool(backward), + bool(mxfp8), rows_alignment, grid_clusters, + ) # fmt: skip + compile_fn = _compile_mxfp8 if cfg.mxfp8 else _compile + return register_function(fn_name, lambda: compile_fn(cfg), "EP grouped GEMM combine") diff --git a/transformer_engine/common/CuTeDSL/ep_gemm/dispatch_gemm.py b/transformer_engine/common/CuTeDSL/ep_gemm/dispatch_gemm.py new file mode 100644 index 00000000000..c1ffa73a855 --- /dev/null +++ b/transformer_engine/common/CuTeDSL/ep_gemm/dispatch_gemm.py @@ -0,0 +1,388 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Signal-aware grouped GEMMs that follow an EP dispatch (BF16 or MXFP8, SM 10.0, 10.3 and 10.7). + +An M tile starts once each of its slots s has ``signal[s] == s``. Operands (rows = receive rows): + + mode a b (weight) outputs + fwd BF16 [rows,k] [E,n,k] c [rows,n]; d [rows,n/2] = silu(gate)*up*prob + bwd BF16 [rows,k] [E,k,n] d [rows,2n] = grad of c; dprob [rows] FP32 (added to) + fwd MXFP8 E4M3 E4M3 c BF16 [rows,n]; d E4M3 [rows,n/2] + swizzled sfd + bwd MXFP8 E4M3 columnwise E4M3 d rowwise + d_col columnwise, h [rows,n] columnwise + +Forward is FC1 (gate and up alternate in 32-column blocks of ``c``). Backward is the FC2 data +gradient with the SwiGLU backward in the epilogue; its ``c`` is the FC1 output [rows, 2n]. +Common operands: ``expert_row_ends`` [E] int32, ``alpha`` [E] FP32, ``prob`` [rows] FP32, +``workspace`` uint8, stream. With ``device_layout`` the forward also waits for the dispatch to +write ``expert_row_ends`` and the word before it (expert 0's offset), both -1 beforehand. The +kernel tree must wait until they are at least 0 (EPC fc1db0b8 or later). + +MXFP8 scales: ``sfa`` is compact (as the dispatch delivers it) and ``sfb`` swizzled. The forward +takes no ``prob`` and a ``norm_const`` [1] FP32 of 1. The backward writes ``sfd_row`` (swizzled), +``sfd_col`` and ``sfh_col`` (swizzled per expert). +""" + +import functools +from dataclasses import dataclass, field + +import cutlass +from cutlass import cute +from cutlass.cute.nvgpu import OperandMajorMode + +from ._loader import ( + device_compute_capability, + CLUSTER_MN, + MMA_TILER_MN, + ROWS_PER_TILE, + blockscaled_kwargs, + compile_launch, + device_smem_optin, + fake_tensor, + fp8_fake, + import_bf16_kernel, + import_gemm_modules, + is_sm107, + mxfp8_tail_tile_kwargs, + prob_view, + register_function, + rows_view, + scale_fake, + tail_fakes, + tail_tile_kwargs, + weight_view, +) + +# Activation of the FC1 GLU to the activation function of its backward epilogue. +_BACKWARD_ACTIVATIONS = {"swiglu": "dswiglu"} +# A/B stages of the SM 10.7 forward GEMMs, BF16 and MXFP8 with compact scales read by TMA. +_RUBIN_FC1_AB_STAGES = 8 + + +@dataclass(frozen=True) +class DispatchGemmConfig: + """Static configuration of one compiled dispatch grouped GEMM.""" + + rows: int # receive capacity in rows, a multiple of slot_rows + k: int # reduction size + n: int # GEMM columns: forward 2 * ffn (gate and up), backward ffn + num_experts: int # local experts + num_clusters: int # persistent clusters beside the dispatch's SMs + backward: bool = False + mxfp8: bool = False + rows_alignment: int = ROWS_PER_TILE # rows per expert slab are a multiple of it + device_layout: bool = False # forward: the dispatch writes the expert row ends + activation: str = "swiglu" + # Kernels, workspace layouts and compiled functions are specific to the device architecture. + capability: tuple = field(default_factory=device_compute_capability) + + def __post_init__(self): + if self.activation not in _BACKWARD_ACTIVATIONS: + raise NotImplementedError( + f"Activation {self.activation!r} is not supported by the dispatch grouped GEMMs;" + f" supported: {sorted(_BACKWARD_ACTIVATIONS)}." + ) + + @property + def slot_rows(self) -> int: + """Rows per ``signal[]`` entry: the expert alignment, at most one row tile.""" + return min(self.rows_alignment, ROWS_PER_TILE) + + @property + def signal_capacity(self) -> int: + """Entries of ``signal[]``, one per slot of the capacity.""" + return self.rows // self.slot_rows + + +def _tail_trigger(kernel_class) -> dict: + """PDL trigger of a forward kernel for its tail on the dispatch's SMs: at slots ready or at + entry, as the kernel class prefers.""" + if kernel_class.FC1_TAIL_AT_SLOTS_READY: + return {"launch_dependents_at_slots_ready": True} + return {"launch_dependents_at_entry": True} + + +def build_kernel(cfg: DispatchGemmConfig): + """Kernel for ``cfg``; row tiles wait on ``signal[]``. A forward one lets its tail launch at the + ``_tail_trigger`` point. The backward one lets the kernel behind it start at entry.""" + backward_activation = _BACKWARD_ACTIVATIONS[cfg.activation] + rubin = is_sm107() + if cfg.mxfp8: + # The TMA warp turns the received compact scales into the GEMM's layout per tile. + common = { + **blockscaled_kwargs(cfg.num_experts), + "signal_capacity": cfg.signal_capacity, + "compact_sfa_sf_warp": True, + "use_pdl": True, + **mxfp8_tail_tile_kwargs(cfg.rows_alignment), + } + if cfg.backward and rubin: + # 128-deep K tiles of one scale atom each: the TMA warp queues the compact scales ahead. + module = import_gemm_modules("moe_blockscaled_grouped_gemm_dglu_rubin")[0] + return module.BlockScaledMoEGroupedGemmDgluKernel( + discrete_col_sfd=True, + act_func=backward_activation, + fp8_k_tile=128, + compact_sfa_queue=True, + **common, + ) + if cfg.backward: + module = import_gemm_modules("moe_blockscaled_grouped_gemm_dglu_dbias")[0] + return module.BlockScaledMoEGroupedGemmDgluDbiasKernel( + discrete_col_sfd=True, + act_func=backward_activation, + **common, + ) + if rubin: + # The fused SwiGLU GEMM. Compact scale rows of whole TMA boxes load by TMA, others + # through registers with the opt-in shared memory. + module = import_gemm_modules("moe_blockscaled_grouped_gemm_glu_rubin")[0] + kernel_class = module.BlockScaledMoEGroupedGemmGluKernel + scale_rows_tma = cfg.k % 1024 == 0 + return kernel_class( + generate_sfd=True, + discrete_col_sfd=False, + act_func=cfg.activation, + enable_bias=False, + generate_c=True, + generate_sfd_col=False, + smem_capacity=None if scale_rows_tma else device_smem_optin(), + device_layout=cfg.device_layout, + max_ab_stages=_RUBIN_FC1_AB_STAGES if scale_rows_tma else None, + compact_sfa_rows=scale_rows_tma, + **_tail_trigger(kernel_class), + **{**common, "vectorized_f32": True}, + ) + module = import_gemm_modules("moe_blockscaled_grouped_gemm_quant_sm100")[0] + kernel_class = module.BlockScaledMoEGroupedGemmQuantKernel + return kernel_class( + generate_sfd=True, + discrete_col_sfd=False, + enable_bias=False, + fuse_swiglu=True, + device_layout=cfg.device_layout, + **_tail_trigger(kernel_class), + **common, + ) + common = { + "acc_dtype": cutlass.Float32, + "use_2cta_instrs": True, + "mma_tiler_mn": MMA_TILER_MN, + "cluster_shape_mn": CLUSTER_MN, + "vectorized_f32": False, + "expert_cnt": cfg.num_experts, + "use_dynamic_sched": True, + "signal_capacity": cfg.signal_capacity, + "use_pdl": True, + **tail_tile_kwargs(cfg.rows_alignment, fused=not cfg.backward), + } + if cfg.backward: + kernel_class, moe_utils = import_bf16_kernel("dispatch_bwd") + return kernel_class( + weight_mode=moe_utils.MoEWeightMode.DENSE, + act_func=backward_activation, + **common, + ) + kernel_class, moe_utils = import_bf16_kernel("dispatch_fwd") + return kernel_class( + weight_mode=moe_utils.MoEWeightMode.DENSE, + generate_c=True, + enable_bias=False, + device_layout=cfg.device_layout, + **({"max_ab_stages": _RUBIN_FC1_AB_STAGES} if rubin else {}), + **_tail_trigger(kernel_class), + **common, + ) + + +@functools.lru_cache(maxsize=None) +def workspace_layout(cfg: DispatchGemmConfig): + """Total bytes and byte offsets of the claim counter, ``signal[]`` and scheduler state.""" + helpers, _ = import_gemm_modules("moe_signal_helpers") + offsets = helpers.signal_workspace_offsets(build_kernel(cfg)) + return offsets["total_bytes"], offsets["counter"], offsets["signal"], offsets["state"] + + +def _prologue(cfg: DispatchGemmConfig): + """Kernel, (rows, k, n, experts, num_clusters) and workspace tail fakes shared by a compiler.""" + kernel = build_kernel(cfg) + dims = (cfg.rows, cfg.k, cfg.n, cfg.num_experts, cfg.num_clusters) + tail = tail_fakes(cfg.rows, cfg.num_experts, workspace_layout(cfg)[0]) + return kernel, dims, tail + + +def _compile_forward(cfg: DispatchGemmConfig): + """FC1: SwiGLU GEMM with the weight [experts, n, k], writing ``c`` and ``d``.""" + kernel, (rows, k, n, experts, num_clusters), tail = _prologue(cfg) + d_cols = n // 2 + + @cute.jit + def launch(a, b, c, d, expert_row_ends, alpha, prob, workspace, stream): + weight = weight_view(b, n, k, experts, n_major=False) + kernel( + rows_view(a, rows, k), weight, cutlass.Int32(0), cutlass.Int32(0), cutlass.Int64(0), + OperandMajorMode.K, workspace.iterator, rows_view(c, rows, n), + rows_view(d, rows, d_cols), expert_row_ends, alpha, prob_view(prob, rows), None, + num_clusters, stream, + ) # fmt: skip + + return compile_launch( + launch, + fake_tensor(cutlass.BFloat16, (rows, k), (1, 0), 16), + fake_tensor(cutlass.BFloat16, (experts, n, k), (2, 1, 0), 16), + fake_tensor(cutlass.BFloat16, (rows, n), (1, 0), 16), + fake_tensor(cutlass.BFloat16, (rows, d_cols), (1, 0), 16), + *tail, + ) + + +def _compile_backward(cfg: DispatchGemmConfig): + """FC2 data gradient with the forward weight [experts, k, n] read N-major, and SwiGLU backward + in the epilogue (unit alpha and beta).""" + kernel, (rows, k, n, experts, num_clusters), tail = _prologue(cfg) + + @cute.jit + def launch(a, b, c, d, dprob, expert_row_ends, alpha, prob, workspace, stream): + weight = weight_view(b, n, k, experts, n_major=True) + kernel( + rows_view(a, rows, k), weight, cutlass.Int32(0), cutlass.Int32(0), cutlass.Int64(0), + OperandMajorMode.K, workspace.iterator, rows_view(c, rows, 2 * n), + rows_view(d, rows, 2 * n), expert_row_ends, alpha, alpha, prob_view(prob, rows), + prob_view(dprob, rows), cutlass.Float32(0.0), None, num_clusters, stream, + ) # fmt: skip + + return compile_launch( + launch, + fake_tensor(cutlass.BFloat16, (rows, k), (1, 0), 16), + fake_tensor(cutlass.BFloat16, (experts, k, n), (2, 1, 0), 16), + fake_tensor(cutlass.BFloat16, (rows, 2 * n), (1, 0), 16), + fake_tensor(cutlass.BFloat16, (rows, 2 * n), (1, 0), 16), + fake_tensor(cutlass.Float32, (rows,), (0,), 16), + *tail, + ) + + +def _compile_forward_mxfp8(cfg: DispatchGemmConfig): + """MXFP8 FC1: SwiGLU GEMM writing the BF16 ``c`` and the E4M3 ``d`` with its scales.""" + kernel, (rows, k, n, experts, num_clusters), tail = _prologue(cfg) + d_cols = n // 2 + + @cute.jit + def launch(a, b, sfb, sfa, c, d, sfd, expert_row_ends, alpha, norm_const, workspace, stream): + weight = weight_view(b, n, k, experts, n_major=False) + d3 = rows_view(d, rows, d_cols) + kernel( + rows_view(a, rows, k), weight, sfb, cutlass.Int32(0), cutlass.Int32(0), + cutlass.Int64(0), OperandMajorMode.K, workspace.iterator, d3, d3, sfa, sfd, None, None, + norm_const, expert_row_ends, alpha, None, None, None, rows_view(c, rows, n), + num_clusters, stream, + ) # fmt: skip + + @cute.jit + def launch_rubin( + a, b, sfb, sfa, c, d, sfd, expert_row_ends, alpha, norm_const, workspace, stream + ): + # The SM 10.7 kernel takes c ahead of d and no row scale. + weight = weight_view(b, n, k, experts, n_major=False) + d3 = rows_view(d, rows, d_cols) + kernel( + rows_view(a, rows, k), weight, sfb, cutlass.Int32(0), cutlass.Int32(0), + cutlass.Int64(0), OperandMajorMode.K, workspace.iterator, rows_view(c, rows, n), d3, + d3, sfa, sfd, None, None, norm_const, expert_row_ends, alpha, None, None, num_clusters, + stream, + ) # fmt: skip + + row_ends, alpha, _, workspace = tail + return compile_launch( + launch_rubin if is_sm107() else launch, + fp8_fake((rows, k), (1, 0)), + fp8_fake((experts, n, k), (2, 1, 0)), + scale_fake(experts * n, k), + scale_fake(rows, k), + fake_tensor(cutlass.BFloat16, (rows, n), (1, 0), 16), + fp8_fake((rows, d_cols), (1, 0)), + scale_fake(rows, d_cols), + row_ends, + alpha, + fake_tensor(cutlass.Float32, (1,), (0,), 4), + workspace, + ) + + +def _compile_backward_mxfp8(cfg: DispatchGemmConfig): + """MXFP8 FC2 data gradient with the columnwise forward weight [experts, k, n] read N-major, + SwiGLU backward in the epilogue, and the forward activation recomputed columnwise.""" + kernel, (rows, k, n, experts, num_clusters), tail = _prologue(cfg) + # The SM 10.7 kernel takes the GeGLU linear offset (unused by SwiGLU) ahead of the bias. + rubin = is_sm107() + + @cute.jit + def launch( + a, b, sfb, sfa, c, d, d_col, sfd_row, sfd_col, h, sfh_col, dprob, expert_row_ends, alpha, + prob, norm_const, workspace, stream, + ): # fmt: skip + weight = weight_view(b, n, k, experts, n_major=True) + kernel( + rows_view(a, rows, k), weight, sfb, cutlass.Int32(0), cutlass.Int32(0), + cutlass.Int64(0), OperandMajorMode.K, workspace.iterator, rows_view(c, rows, 2 * n), + rows_view(d, rows, 2 * n), rows_view(d_col, rows, 2 * n), sfa, sfd_row, sfd_col, None, + norm_const, expert_row_ends, alpha, alpha, prob_view(prob, rows), + prob_view(dprob, rows), *((cutlass.Float32(1.0),) if rubin else ()), None, + rows_view(h, rows, n), sfh_col, num_clusters, stream, + ) # fmt: skip + + row_ends, alpha, prob, workspace = tail + return compile_launch( + launch, + fp8_fake((rows, k), (1, 0)), + fp8_fake((experts, k, n), (2, 1, 0)), + scale_fake(experts * n, k), + scale_fake(rows, k), + fake_tensor(cutlass.BFloat16, (rows, 2 * n), (1, 0), 16), + fp8_fake((rows, 2 * n), (1, 0)), + fp8_fake((rows, 2 * n), (1, 0)), + scale_fake(rows, 2 * n), + scale_fake(rows, 2 * n), + fp8_fake((rows, n), (1, 0)), + scale_fake(rows, n), + fake_tensor(cutlass.Float32, (rows,), (0,), 16), + row_ends, + alpha, + prob, + fake_tensor(cutlass.Float32, (1,), (0,), 4), + workspace, + ) + + +_COMPILERS = { + (False, False): _compile_forward, + (True, False): _compile_backward, + (False, True): _compile_forward_mxfp8, + (True, True): _compile_backward_mxfp8, +} + + +def get_ep_dispatch_gemm_workspace_layout( + rows, k, n, num_experts, num_clusters, backward, mxfp8=False, rows_alignment=ROWS_PER_TILE, + device_layout=False, +): # fmt: skip + """[total bytes, counter, signal, state] byte offsets of the workspace of this configuration.""" + cfg = DispatchGemmConfig( + rows, k, n, num_experts, num_clusters, bool(backward), bool(mxfp8), rows_alignment, + bool(device_layout), + ) # fmt: skip + return list(workspace_layout(cfg)) + + +def get_ep_dispatch_gemm_function( + fn_name, rows, k, n, num_experts, num_clusters, backward, mxfp8=False, + rows_alignment=ROWS_PER_TILE, device_layout=False, +) -> bool: # fmt: skip + """Compile and register the GEMM of this configuration as ``fn_name``; return whether it is.""" + cfg = DispatchGemmConfig( + rows, k, n, num_experts, num_clusters, bool(backward), bool(mxfp8), rows_alignment, + bool(device_layout), + ) # fmt: skip + compile_fn = _COMPILERS[cfg.backward, cfg.mxfp8] + return register_function(fn_name, lambda: compile_fn(cfg), "EP dispatch grouped GEMM") diff --git a/transformer_engine/common/CuTeDSL/ep_gemm/requantize.py b/transformer_engine/common/CuTeDSL/ep_gemm/requantize.py new file mode 100644 index 00000000000..2cf7b2b94a6 --- /dev/null +++ b/transformer_engine/common/CuTeDSL/ep_gemm/requantize.py @@ -0,0 +1,331 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""EP weight-gradient requantizer for the received rows. + +When the copy used in the forward is quantized differently from the operand the backward weight +gradient needs, this re-quantizes the received rows into that backward layout. Unlike +``nvte_group_requantize``, it writes the columnwise operand with scales swizzled per expert. Only +MXFP8 (rowwise to columnwise) is supported. + +Rowwise MXFP8 rows ([rows, cols] E4M3 data with compact E8M0 scales, one per 32 columns of a row) +become columnwise MXFP8 ([rows, cols] E4M3 data with one scale per 32 rows of a column, scales +GEMM-swizzled per expert slab), the operand layout of the MXFP8 weight gradients. Each value is +dequantized and quantized again with TE's MXFP8 rule. Operands: ``codes``, ``row_scales``, +``col_codes``, ``col_scales``, ``expert_row_ends`` [experts] int32 (every slab a multiple of 128 +rows), optional ``signal`` [rows / ready_slot_rows] int32, stream. + +A launch is one CTA per (128-row, 256-column) tile of the receive capacity, a programmatic dependent +of the kernel before it that lets the kernel behind it start at once. CTAs past the last expert row +end exit; with ``signal``, a tile starts once the dispatch has set the slots of its rows. The grid +completes after the kernel before it. +""" + +import math +from dataclasses import dataclass +from typing import Optional + +import cuda.bindings.driver as cuda +import cutlass +from cutlass import Float32, Float8E4M3FN, Int32, Int64, Uint8, Uint16, Uint32, cute +from cutlass.experimental.primitives import cvt_packfloat_f32 + +from ._loader import ( + compile_launch, + SF_VEC_SIZE, + fake_tensor, + fp8_fake, + import_gemm_modules, + register_function, +) + +# A thread's rows of a 32-row block, its bytes per row and the warps per CTA. +ROWS_PER_THREAD, VEC, NUM_WARPS = 16, 8, 2 +# Lanes that share a group of VEC columns, each with its own rows of a block. +ROW_LANES = SF_VEC_SIZE // ROWS_PER_THREAD +# Columns per tile. +TILE_COLUMNS = 32 // ROW_LANES * VEC * NUM_WARPS +# Scale blocks along the rows per tile: one 128-row scale atom. +BLOCKS_PER_TILE = 4 +# Rows per tile. +TILE_ROWS = BLOCKS_PER_TILE * SF_VEC_SIZE +# fp32(1 / 448), the reciprocal of the largest E4M3 value. +E4M3_MAX_RCP = 1.0 / 448.0 +# 2^-127, the multiplier of E8M0 0. +E8M0_ZERO_MULTIPLIER = 2.0**-127 +# Order of a thread's four column pairs. Any order gives the same bytes; this one takes 128 +# registers, which fits 8 CTAs per SM. +PAIR_ORDER = (0, 3, 1, 2) + + +@dataclass(frozen=True) +class Mxfp8RequantConfig: + """Static configuration of one compiled MXFP8 requant launch.""" + + rows: int # receive capacity in rows + cols: int # columns + num_experts: int # local experts + ready_slot_rows: int = 0 # wait until signal[] is set for every slot of a tile's rows + + +class _Mxfp8RequantKernel: + """MXFP8 requant of a ``cfg`` operand, one CTA per tile.""" + + def __init__(self, cfg: Mxfp8RequantConfig): + self.cfg = cfg + if cfg.ready_slot_rows: + scheduler = import_gemm_modules("moe_streaming_input_block_tile_scheduler")[0] + self.poll_until_at_least = scheduler._poll_until_at_least + self.wait_slots_ready = scheduler.wait_slots_ready + + @cute.jit + def __call__( + self, + codes: cute.Pointer, + row_scales: cute.Pointer, + col_codes: cute.Pointer, + col_scales: cute.Pointer, + expert_row_ends: cute.Pointer, + signal: Optional[cute.Pointer], + stream: cuda.CUstream, + ): + self.tile_kernel(codes, row_scales, col_codes, col_scales, expert_row_ends, signal).launch( + grid=[self.cfg.rows // TILE_ROWS * (self.cfg.cols // TILE_COLUMNS), 1, 1], + block=[32 * NUM_WARPS, 1, 1], + stream=stream, + use_pdl=True, + ) + + @cute.kernel + def tile_kernel( + self, + codes: cute.Pointer, + row_scales: cute.Pointer, + col_codes: cute.Pointer, + col_scales: cute.Pointer, + expert_row_ends: cute.Pointer, + signal: Optional[cute.Pointer], + ): + """Tile ``blockIdx.x`` if it ends at or before the last expert end; CTA 0 waits for the + kernel before it at exit, so the grid completes after that one.""" + num_experts = self.cfg.num_experts + cute.arch.griddepcontrol_launch_dependents() + ends = cute.make_tensor(expert_row_ends, cute.make_layout((num_experts,))) + num_tiles = ends[num_experts - 1] // TILE_ROWS * (self.cfg.cols // TILE_COLUMNS) + bidx, _, _ = cute.arch.block_idx() + if bidx < num_tiles: + if cutlass.const_expr(signal is not None): + self.wait_rows_ready(signal, Int32(bidx)) + self.tile(codes, row_scales, col_codes, col_scales, expert_row_ends, Int32(bidx)) + if bidx == 0: + cute.arch.griddepcontrol_wait() + + @cute.jit + def wait_rows_ready(self, signal: cute.Pointer, tile: Int32): + """Wait until the dispatch has set the ``signal`` slots of the rows of tile ``tile``: warp 0 + polls them relaxed, then acquires them once.""" + slot_rows = self.cfg.ready_slot_rows + num_slots = max(TILE_ROWS // slot_rows, 1) + tidx, _, _ = cute.arch.thread_idx() + if tidx < 32: + slots = signal + tile // (self.cfg.cols // TILE_COLUMNS) * TILE_ROWS // slot_rows + for j in cutlass.range_constexpr((num_slots + 31) // 32): + self.poll_until_at_least(slots + cutlass.min(tidx + j * 32, num_slots - 1), 0) + self.wait_slots_ready(slots, Int32(num_slots)) + cute.arch.sync_threads() + + @cute.jit + def tile( + self, + codes: cute.Pointer, + row_scales: cute.Pointer, + col_codes: cute.Pointer, + col_scales: cute.Pointer, + expert_row_ends: cute.Pointer, + tile: Int32, + ): + """Requantize tile ``tile`` and write its scale words.""" + # pylint: disable=too-many-locals + R, V, COLS = ROWS_PER_THREAD, VEC, self.cfg.cols + num_experts = self.cfg.num_experts + tidx, _, _ = cute.arch.thread_idx() + # A thread owns VEC columns of the tile and every ROW_LANES-th row of each 32-row block. + lane_row, lane_column = tidx % ROW_LANES, tidx // ROW_LANES + row_group = tile // (COLS // TILE_COLUMNS) + column = (tile % (COLS // TILE_COLUMNS)) * TILE_COLUMNS + lane_column * V + warp_first_column = (tile % (COLS // TILE_COLUMNS)) * TILE_COLUMNS + (tidx // 32) * (R * V) + access = cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), Uint16, num_bits_per_copy=8 * V) + r_codes = cute.make_rmem_tensor((V // 2, R), Uint16) + r_out = cute.make_rmem_tensor((V // 2, R), Uint16) + pair_row, pair_broadcast = cute.make_layout((2,)), cute.make_layout((2,), stride=(0,)) + codes_e4m3 = cute.recast_tensor(r_codes, Float8E4M3FN) + out_e4m3 = cute.recast_tensor(r_out, Float8E4M3FN) + r_values = cute.make_rmem_tensor((2, R), Float32) + r_scales = cute.make_rmem_tensor((R,), Uint8) + row_multiplier_bits = cute.make_rmem_tensor((R,), Int32) + row_multiplier = cute.recast_tensor(row_multiplier_bits, Float32) + multiplier_bits = cute.make_rmem_tensor((2,), Int32) + multiplier = cute.recast_tensor(multiplier_bits, Float32) + scaled_amax = cute.make_rmem_tensor((2,), Float32) + # One word per column pair: this lane's column of the pair, one scale byte per 32-row block. + packed = cute.make_rmem_tensor((V // 2,), Uint32) + packed.fill(0) + for block in cutlass.range(BLOCKS_PER_TILE, unroll=1): + first_row = Int64(row_group) * TILE_ROWS + block * SF_VEC_SIZE + lane_row + rows = cute.make_layout((V // 2, R), stride=(1, ROW_LANES * COLS // 2)) + offset = cute.assume(first_row * COLS + column, divby=V) + cute.copy( + access, + cute.make_tensor(cute.recast_ptr(codes + offset, dtype=Uint16), rows), + r_codes, + ) + # Prefetch the next block's codes and row scales into L2. + if block < BLOCKS_PER_TILE - 1: + next_row = Int64(row_group) * TILE_ROWS + (block + 1) * SF_VEC_SIZE + tidx % 32 + cute.arch.prefetch( + (codes + (next_row * COLS + warp_first_column)).llvm_ptr, cache_level="L2" + ) + cute.arch.prefetch( + ( + row_scales + + (next_row * (COLS // SF_VEC_SIZE) + warp_first_column // SF_VEC_SIZE) + ).llvm_ptr, + cache_level="L2", + ) + cute.autovec_copy( + cute.make_tensor( + row_scales + (first_row * (COLS // SF_VEC_SIZE) + column // SF_VEC_SIZE), + cute.make_layout((R,), stride=(ROW_LANES * (COLS // SF_VEC_SIZE),)), + ), + r_scales, + ) + # 2^(s - 127) from its bits; s = 0 gives 2^-127 and s = 255 NaN. + for i in cutlass.range_constexpr(R): + row_multiplier_bits[i] = Int32(r_scales[i]) << 23 + row_multiplier[i] = cute.arch.fmax(row_multiplier[i], E8M0_ZERO_MULTIPLIER) + row_multiplier_bits[i] = row_multiplier_bits[i] | ( + ((Int32(r_scales[i]) + 1) >> 8) << 22 + ) + for q in PAIR_ORDER: + for i in cutlass.range_constexpr(R): + pair_codes = cute.make_tensor(codes_e4m3.iterator + 2 * q + V * i, pair_row) + pair_row_multiplier = cute.make_tensor( + row_multiplier.iterator + i, pair_broadcast + ) + cute.make_tensor(r_values.iterator + 2 * i, pair_row).store( + pair_codes.load().to(Float32) * pair_row_multiplier.load() + ) + for v in cutlass.range_constexpr(2): + # The amax ignores NaN; the column's other rows sit in the neighboring lanes. + amax = Float32(0.0) + for i in cutlass.range_constexpr(R): + amax = cute.arch.fmax(amax, abs(r_values[v, i])) + for k in cutlass.range_constexpr(int(math.log2(ROW_LANES))): + amax = cute.arch.fmax( + amax, cute.arch.shuffle_sync_bfly(amax, offset=1 << k) + ) + scaled_amax[v] = amax * E4M3_MAX_RCP + # E8M0 of both columns of the pair, rounded up, the first column in the high byte. + pair = cvt_packfloat_f32( + scaled_amax[0], scaled_amax[1], 0, "ue8m0x2", rnd="rp", sat="satfinite" + ) + packed[q] = (packed[q] >> 8) | (Uint32(pair >> (8 * (1 - lane_row))) << 24) + inverse = 0xFEFE - pair + for h in cutlass.range_constexpr(2): + multiplier_bits[h] = cutlass.max( + ((inverse >> (8 * (1 - h))) & 0xFF) << 23, Int32(0x00400000) + ) + for i in cutlass.range_constexpr(R): + pair_values = cute.make_tensor(r_values.iterator + 2 * i, pair_row) + cute.make_tensor(out_e4m3.iterator + 2 * q + V * i, pair_row).store( + (pair_values.load() * multiplier.load()).to(Float8E4M3FN) + ) + cute.copy( + access, + r_out, + cute.make_tensor(cute.recast_ptr(col_codes + offset, dtype=Uint16), rows), + ) + # The tile's expert by a branchless lower bound over the row ends, then its start row. + ends = cute.make_tensor(expert_row_ends, cute.make_layout((num_experts,))) + group_row = row_group * TILE_ROWS + expert, n = Int32(0), num_experts + for _ in cutlass.range_constexpr(int(math.ceil(math.log2(num_experts)))): + half = n // 2 + expert = cutlass.select_(ends[expert + half - 1] <= group_row, expert + half, expert) + n -= half + expert = cutlass.select_(ends[expert] <= group_row, expert + 1, expert) + if expert < num_experts: + start = cutlass.select_(expert > 0, ends[cutlass.max(expert - 1, Int32(0))], Int32(0)) + count = ends[expert] - start + local_group = Int64(row_group - start // TILE_ROWS) + # Int32 word of the expert's swizzled scales: its start, the 128-column atom, the + # 128-row group in it, the column in its 32 and which of the four 32-column groups. + c = column + lane_row + index = ( + Int64(start) * (COLS // 128) + + Int64(c // 128) * count + + local_group * 128 + + (c % 32) * 4 + + (c // 32) % 4 + ) + for j in cutlass.range_constexpr(V // 2): + word = cute.make_tensor( + col_scales + index + j * ROW_LANES * 4, cute.make_layout((1,)) + ) + word[0] = packed[j] + + +def _compile(cfg: Mxfp8RequantConfig): + """Compile the launch wrapper of ``cfg`` with TVM-FFI.""" + assert cfg.rows % TILE_ROWS == 0 and cfg.cols % TILE_COLUMNS == 0, (cfg.rows, cfg.cols) + slot_rows = cfg.ready_slot_rows + # A tile waits for whole slots: one, or the slots it spans; the capacity is whole slots. + assert not slot_rows or ( + (TILE_ROWS % slot_rows == 0 or slot_rows % TILE_ROWS == 0) and cfg.rows % slot_rows == 0 + ), (cfg.rows, slot_rows) + requant = _Mxfp8RequantKernel(cfg) + + @cute.jit + def launch(codes, row_scales, col_codes, col_scales, expert_row_ends, stream): + requant( + cute.recast_ptr(codes.iterator, dtype=Uint8), + cute.recast_ptr(row_scales.iterator, dtype=Uint8), + cute.recast_ptr(col_codes.iterator, dtype=Uint8), + cute.recast_ptr(col_scales.iterator, dtype=Int32), + expert_row_ends.iterator, + None, + stream, + ) + + @cute.jit + def launch_ready(codes, row_scales, col_codes, col_scales, expert_row_ends, signal, stream): + requant( + cute.recast_ptr(codes.iterator, dtype=Uint8), + cute.recast_ptr(row_scales.iterator, dtype=Uint8), + cute.recast_ptr(col_codes.iterator, dtype=Uint8), + cute.recast_ptr(col_scales.iterator, dtype=Int32), + expert_row_ends.iterator, + signal.iterator, + stream, + ) + + rows, cols = cfg.rows, cfg.cols + scales = rows * cols // SF_VEC_SIZE + signal_fake = (fake_tensor(Int32, (rows // slot_rows,), (0,), 4),) if slot_rows else () + return compile_launch( + launch_ready if slot_rows else launch, + fp8_fake((rows, cols), (1, 0)), + fake_tensor(cutlass.Float8E8M0FNU, (scales,), (0,), 16), + fp8_fake((rows, cols), (1, 0)), + fake_tensor(cutlass.Float8E8M0FNU, (scales,), (0,), 16), + fake_tensor(Int32, (cfg.num_experts,), (0,), 16), + *signal_fake, + ) + + +def get_ep_requant_function(fn_name, rows, cols, num_experts, ready_slot_rows=0) -> bool: + """Compile and register the requant of this configuration as ``fn_name``; return whether it + is. ``ready_slot_rows`` > 0: the function takes ``signal`` before the stream.""" + cfg = Mxfp8RequantConfig(rows, cols, num_experts, ready_slot_rows) + return register_function(fn_name, lambda: _compile(cfg), "EP MXFP8 requant") diff --git a/transformer_engine/common/CuTeDSL/ep_gemm/wgrad_gemm.py b/transformer_engine/common/CuTeDSL/ep_gemm/wgrad_gemm.py new file mode 100644 index 00000000000..c36e128d0bb --- /dev/null +++ b/transformer_engine/common/CuTeDSL/ep_gemm/wgrad_gemm.py @@ -0,0 +1,323 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Grouped weight gradient chained into the overlapped backward (SM 10.0, 10.3 and 10.7). + +``weight_grad[e] = grad[rows of e]^T x[rows of e]`` per expert: ``grad`` [rows, m] and ``x`` +[rows, n] BF16, or MXFP8 columnwise data with their GEMM-swizzled columnwise scales, +``weight_grad`` [experts, m, n] BF16 or FP32, written or added to. Operands: ``expert_row_ends`` +[experts] int32, ``workspace`` uint8, stream; ``signal`` [rows / slot rows] int32 with +``ready_slot_rows`` (BF16 only). + +Each launch is a persistent grid that claims output tiles from a counter in ``workspace``, so +launches sharing one workspace split the tiles. It is a programmatic dependent of the kernel +before it and lets the kernel behind it start early. MXFP8 launches of an added or FP32 weight +gradient, and on SM 10.7 of a written BF16 one with long experts, read per-expert descriptors +that a ``descriptors_only`` launch writes into ``workspace`` ahead of them; the others read global +descriptors. +""" + +import functools +from dataclasses import dataclass, field + +import cutlass +from cutlass import cute + +from ._loader import ( + compile_launch, + device_compute_capability, + CLUSTER_MN, + MMA_TILER_MN, + SF_VEC_SIZE, + fake_tensor, + fp8_fake, + import_gemm_modules, + is_sm107, + register_function, + scale_fake, +) + + +@dataclass(frozen=True) +class WgradConfig: + """Static configuration of one compiled weight-gradient launch.""" + + rows: int # receive capacity in rows + m: int # columns of grad, rows of each expert's weight gradient + n: int # columns of x, columns of each expert's weight gradient + num_experts: int # local experts + num_clusters: int # persistent clusters + fp32_out: bool = False # FP32 instead of BF16 weight gradient + accumulate: bool = False # add to the weight gradient instead of writing it + wait_primary_at_start: bool = False # wait for the kernel before it ahead of any load + ready_slot_rows: int = 0 # wait until signal[] is set for every slot of these rows + mxfp8: bool = False # MXFP8 operands + descriptors_only: bool = False # MXFP8: write the per-expert descriptors, no GEMM + rows_alignment: int = 0 # rows per expert slab are a multiple of it + routed_rows: int = 0 # rows a rank receives on average + # Kernels, workspace layouts and compiled functions are specific to the device architecture. + capability: tuple = field(default_factory=device_compute_capability) + + +def _mxfp8_kernels(): + """(per-expert descriptor, reuse, dense or None, moe_utils) MXFP8 kernel classes of the + device. SM 10.7 has no dense one.""" + if is_sm107(): + stock, reuse, moe_utils = import_gemm_modules( + "moe_blockscaled_grouped_gemm_wgrad_rubin", + "moe_blockscaled_grouped_gemm_wgrad_reuse_rubin", + ) + return ( + stock.BlockScaledMoEGroupedGemmWgradRubinKernel, + reuse.BlockScaledMoEGroupedGemmWgradReuseRubinKernel, + None, + moe_utils, + ) + blockscaled, dense, reuse, moe_utils = import_gemm_modules( + "moe_blockscaled_grouped_gemm_wgrad", + "moe_blockscaled_grouped_gemm_wgrad_dense", + "moe_blockscaled_grouped_gemm_wgrad_reuse", + ) + return ( + blockscaled.BlockScaledMoEGroupedGemmWgradKernel, + reuse.BlockScaledMoEGroupedGemmWgradReuseKernel, + dense.BlockScaledMoEGroupedGemmWgradDenseKernel, + moe_utils, + ) + + +def _global_descriptor_kernel(cfg: WgradConfig): + """Class of an MXFP8 weight gradient written in BF16 that reads global descriptors: the one + keeping A across the N tiles of an expert when the experts are short on average, else the + dense one; None for the others.""" + if cfg.accumulate or cfg.fp32_out: + return None + _, reuse, dense, _ = _mxfp8_kernels() + # Each weight gradient picks for its own shape; EPCompOverlap keeps A only when both can, so + # with ffn 256, 1280 or 1792 it runs the first one dense too. + if reuse.can_implement(cfg.m, cfg.n, cfg.rows_alignment, cfg.routed_rows, cfg.num_experts): + return reuse + return dense + + +def reads_descriptors(cfg: WgradConfig) -> bool: + """Whether the launches of ``cfg`` read the per-expert descriptors of a ``descriptors_only`` + launch.""" + return cfg.mxfp8 and _global_descriptor_kernel(cfg) is None + + +def build_kernel(cfg: WgradConfig): + """Kernel for ``cfg``: 256 x 512 output tiles where they divide a BF16 weight gradient that is + written, else 256 x 256 tiles. An MXFP8 weight gradient written in BF16 keeps A across the N + tiles of an expert when the experts are short on average, else reads A, B and the scales + through global descriptors (per-expert ones on SM 10.7).""" + if cfg.mxfp8: + stock, _, _, moe_utils = _mxfp8_kernels() + kernel = _global_descriptor_kernel(cfg) + if kernel is not None: + # The SM 10.7 reuse kernel sizes its A ring to the average padded expert. + padded_rows = cfg.routed_rows + cfg.num_experts * cfg.rows_alignment // 2 + a_stages = ( + {"a_stages": kernel.a_stages_for(padded_rows, cfg.num_experts)} + if is_sm107() + else {} + ) + return kernel( + sf_vec_size=SF_VEC_SIZE, + acc_dtype=cutlass.Float32, + use_2cta_instrs=True, + mma_tiler_mn=MMA_TILER_MN, + cluster_shape_mn=CLUSTER_MN, + expert_cnt=cfg.num_experts, + weight_mode=moe_utils.MoEWeightMode.DENSE, + input_order=moe_utils.WGradInputOrder.Tensor2D, + tile_claim=moe_utils.WgradTileClaim.COUNTER, + use_pdl=True, + wait_primary_at_start=cfg.wait_primary_at_start, + **a_stages, + ) + if is_sm107() and cfg.rows <= 256: + # The SM 10.7 kernel faults when the K extent is a single 256-row tile. + raise ValueError( + "The SM 10.7 MXFP8 weight gradient that is added, FP32 or of long experts needs " + "a receive capacity above 256 rows." + ) + return stock( + sf_vec_size=SF_VEC_SIZE, + acc_dtype=cutlass.Float32, + use_2cta_instrs=True, + mma_tiler_mn=MMA_TILER_MN, + cluster_shape_mn=CLUSTER_MN, + accumulate_on_output=cfg.accumulate, + expert_cnt=cfg.num_experts, + weight_mode=moe_utils.MoEWeightMode.DENSE, + input_order=moe_utils.WGradInputOrder.Tensor2D, + tile_claim=moe_utils.WgradTileClaim.COUNTER, + descriptors_only=cfg.descriptors_only, + descriptors_prebuilt=not cfg.descriptors_only, + use_pdl=not cfg.descriptors_only, + wait_primary_at_start=cfg.wait_primary_at_start, + ) + if is_sm107(): + # Both kernels, sized for the SM 10.7 shared memory. + names = ("moe_grouped_gemm_sm107",) * 2 + else: + names = ("moe_grouped_gemm_wgrad", "moe_grouped_gemm_wgrad_wide") + stock, wide, moe_utils = import_gemm_modules(*names) + options = { + "expert_cnt": cfg.num_experts, + "tile_claim": moe_utils.WgradTileClaim.COUNTER, + } + wide_kernel = wide.MoEGroupedGemmWgradWideBF16Kernel + if not (cfg.accumulate or cfg.fp32_out) and wide_kernel.can_implement(cfg.m, cfg.n): + return wide_kernel( + use_pdl=True, + wait_primary_at_start=cfg.wait_primary_at_start, + ready_slot_rows=cfg.ready_slot_rows, + **options, + ) + return stock.MoEGroupedGemmWgradBF16Kernel( + acc_dtype=cutlass.Float32, + use_2cta_instrs=True, + mma_tiler_mn=MMA_TILER_MN, + cluster_shape_mn=CLUSTER_MN, + accumulate_on_output=cfg.accumulate, + weight_mode=moe_utils.MoEWeightMode.DENSE, + input_order=moe_utils.WGradInputOrder.Tensor2D, + **options, + ) + + +@functools.lru_cache(maxsize=None) +def workspace_layout(cfg: WgradConfig): + """Total bytes and byte offset of the tile counter of ``cfg``'s workspace.""" + kernel = build_kernel(cfg) + return kernel.get_workspace_bytes(), kernel.get_desc_workspace_bytes() + + +def _compile(cfg: WgradConfig): + """Compile the launch wrapper of ``cfg`` with TVM-FFI.""" + kernel = build_kernel(cfg) + rows, m, n, experts, num_clusters = cfg.rows, cfg.m, cfg.n, cfg.num_experts, cfg.num_clusters + + def views(grad, x, weight_grad): + # grad^T as (m, rows), x as (rows, n), weight_grad as (experts, m, n). + return ( + cute.make_tensor(grad.iterator, cute.make_layout((m, rows), stride=(1, m))), + cute.make_tensor(x.iterator, cute.make_layout((rows, n), stride=(n, 1))), + cute.make_tensor( + weight_grad.iterator, cute.make_layout((experts, m, n), stride=(m * n, n, 1)) + ), + ) + + out_fake = fake_tensor( + cutlass.Float32 if cfg.fp32_out else cutlass.BFloat16, (experts, m, n), (2, 1, 0), 16 + ) + tail = ( + fake_tensor(cutlass.Int32, (experts,), (0,), 16), + fake_tensor(cutlass.Uint8, (workspace_layout(cfg)[0],), (0,), 128), + ) + if cfg.mxfp8: + k_scales = rows // SF_VEC_SIZE + + @cute.jit + def launch_mxfp8( + grad, x, grad_scales, x_scales, weight_grad, expert_row_ends, workspace, stream + ): + a, b, out = views(grad, x, weight_grad) + # Columnwise scales as (columns, rows / 32) over the per-expert swizzled atoms. + sfa = cute.make_tensor( + grad_scales.iterator, cute.make_layout((m, k_scales), stride=(k_scales, 1)) + ) + sfb = cute.make_tensor( + x_scales.iterator, cute.make_layout((n, k_scales), stride=(k_scales, 1)) + ) + kernel(a, b, sfa, sfb, out, expert_row_ends, workspace, num_clusters, stream) + + return compile_launch( + launch_mxfp8, + fp8_fake((rows, m), (1, 0)), + fp8_fake((rows, n), (1, 0)), + scale_fake(rows, m), + scale_fake(rows, n), + out_fake, + *tail, + ) + + fakes = ( + fake_tensor(cutlass.BFloat16, (rows, m), (1, 0), 16), + fake_tensor(cutlass.BFloat16, (rows, n), (1, 0), 16), + out_fake, + *tail, + ) + if cfg.ready_slot_rows: + # Only the wide kernel waits on the signal. The stock kernel, chosen for FP32 or + # accumulated gradients and shapes the wide one cannot implement, is launched without PDL, + # so it starts after the signaling kernels complete and has no flags to wait for. + wait_on_signal = getattr(kernel, "ready_slot_rows", 0) > 0 + + @cute.jit + def launch_ready(grad, x, weight_grad, expert_row_ends, workspace, signal, stream): + a, b, out = views(grad, x, weight_grad) + if cutlass.const_expr(wait_on_signal): + kernel( + a, b, out, expert_row_ends, workspace, num_clusters, stream, None, + signal.iterator, + ) # fmt: skip + else: + kernel(a, b, out, expert_row_ends, workspace, num_clusters, stream) + + return compile_launch( + launch_ready, + *fakes, + fake_tensor(cutlass.Int32, (rows // cfg.ready_slot_rows,), (0,), 4), + ) + + @cute.jit + def launch(grad, x, weight_grad, expert_row_ends, workspace, stream): + a, b, out = views(grad, x, weight_grad) + kernel(a, b, out, expert_row_ends, workspace, num_clusters, stream) + + return compile_launch(launch, *fakes) + + +def get_ep_wgrad_workspace_layout( + rows, m, n, num_experts, num_clusters, fp32_out, accumulate, wait_primary_at_start, + ready_slot_rows, mxfp8, descriptors_only, rows_alignment, routed_rows, +): # fmt: skip + """[total bytes, counter offset] of the workspace of this configuration.""" + cfg = WgradConfig( + rows, m, n, num_experts, num_clusters, bool(fp32_out), bool(accumulate), + bool(wait_primary_at_start), ready_slot_rows, bool(mxfp8), bool(descriptors_only), + rows_alignment, routed_rows, + ) # fmt: skip + return list(workspace_layout(cfg)) + + +def get_ep_wgrad_reads_descriptors( + rows, m, n, num_experts, num_clusters, fp32_out, accumulate, wait_primary_at_start, + ready_slot_rows, mxfp8, descriptors_only, rows_alignment, routed_rows, +) -> bool: # fmt: skip + """Whether the launches of this configuration read the per-expert descriptors of a + ``descriptors_only`` launch.""" + cfg = WgradConfig( + rows, m, n, num_experts, num_clusters, bool(fp32_out), bool(accumulate), + bool(wait_primary_at_start), ready_slot_rows, bool(mxfp8), bool(descriptors_only), + rows_alignment, routed_rows, + ) # fmt: skip + return reads_descriptors(cfg) + + +def get_ep_wgrad_function( + fn_name, rows, m, n, num_experts, num_clusters, fp32_out, accumulate, wait_primary_at_start, + ready_slot_rows, mxfp8, descriptors_only, rows_alignment, routed_rows, +) -> bool: # fmt: skip + """Compile and register the weight gradient of this configuration as ``fn_name``; return + whether it is.""" + cfg = WgradConfig( + rows, m, n, num_experts, num_clusters, bool(fp32_out), bool(accumulate), + bool(wait_primary_at_start), ready_slot_rows, bool(mxfp8), bool(descriptors_only), + rows_alignment, routed_rows, + ) # fmt: skip + return register_function(fn_name, lambda: _compile(cfg), "EP weight-gradient grouped GEMM") diff --git a/transformer_engine/common/CuTeDSL/utils.py b/transformer_engine/common/CuTeDSL/utils.py index 3162c8421aa..39cc2d0cb06 100644 --- a/transformer_engine/common/CuTeDSL/utils.py +++ b/transformer_engine/common/CuTeDSL/utils.py @@ -64,6 +64,12 @@ def check(result): return (0, 0) +def tvm_ffi_function(compiled): + """The TVM-FFI function of a kernel compiled with ``--enable-tvm-ffi``, for registration.""" + native = getattr(compiled, "__tvm_ffi_object__", lambda: None)() + return native if native is not None else compiled + + def str_to_cutlass_dtype(dtype_str: str): """Convert a string dtype to a cutlass dtype, or None if unknown.""" return _CUTLASS_DTYPE_FROM_STR.get(dtype_str, None) diff --git a/transformer_engine/common/ep/ep_api.cpp b/transformer_engine/common/ep/ep_api.cpp index 01f79d7445a..fc786799320 100644 --- a/transformer_engine/common/ep/ep_api.cpp +++ b/transformer_engine/common/ep/ep_api.cpp @@ -53,6 +53,10 @@ Cfg normalize_ep_config(const Cfg* user, size_t min_size, const char* name) { } } // namespace +NVTEEpLayerConfig transformer_engine::ep::normalize_layer_config(const NVTEEpLayerConfig* cfg) { + return normalize_ep_config(cfg, kLayerConfigMinSize, "layer_cfg"); +} + void nvte_ep_initialize(void* ep_comm, const NVTEEpGroupConfig* group_config) { NVTE_CHECK(ep_comm != nullptr, "ep_comm must not be null"); NVTEEpGroupConfig cfg = normalize_ep_config(group_config, kGroupConfigMinSize, "group_config"); diff --git a/transformer_engine/common/ep/ep_backend.cpp b/transformer_engine/common/ep/ep_backend.cpp index 73f7ad96dbf..5088b772410 100644 --- a/transformer_engine/common/ep/ep_backend.cpp +++ b/transformer_engine/common/ep/ep_backend.cpp @@ -102,6 +102,51 @@ inline ncclEpTensor_t make_nccl_ep_tensor(const NVTETensor t, NVTEShape& shape_o return desc; } +// NCCL EP first published the overlap completion signal in 0.3.0. An older runtime loaded through +// NCCL_EP_HOME ignores the signal output field and returns success without ever raising the flags, +// leaving the overlapped GEMM waiting forever. Reject a signaled op on such a runtime up front. +void require_signal_capable(const char* call) { + constexpr int kMinSignalVersion = 300; // NCCL EP 0.3.0 + const int version = nccl_ep::runtime_version(); + NVTE_CHECK(version >= kMinSignalVersion, "The loaded NCCL EP runtime ", version / 10000, ".", + (version / 100) % 100, ".", version % 100, + " does not support the overlap completion signal required by the ", call, + "; it needs NCCL EP >= 0.3.0. Use a compatible runtime or disable the overlap."); +} + +// Signal descriptor of a dispatch or combine call; attach() returns the NCCL EP signal to wire into +// the call, or nullptr when no signal is given. +struct SignalDesc { + NVTEShape shape; + ncclEpTensor_t desc = NCCL_EP_TENSOR_INIT; + ncclEpSignal_t sig = NCCL_EP_SIGNAL_INIT; + + ncclEpSignal_t* attach(const EpSignal* signal, ncclEpSignalUnit_t unit, const char* call) { + if (signal == nullptr) { + return nullptr; + } + require_signal_capable(call); + NVTE_CHECK( + nvte_tensor_type(signal->ready) == kNVTEInt32 && nvte_tensor_shape(signal->ready).ndim == 1, + call, " signal must be a 1D int32 tensor"); + NVTE_CHECK(signal->granularity > 0, call, " signal granularity must be positive"); + desc = make_nccl_ep_tensor(signal->ready, shape); + sig.granularity = static_cast(signal->granularity); + sig.unit = unit; + sig.signal = &desc; + return &sig; + } +}; + +// NCCL EP rejects a signaled call it cannot serve with ncclInvalidUsage, e.g. on a group of more +// than one LSA team (NVLink domain). +void check_signaled_call(ncclResult_t status, bool signaled, const char* call) { + NVTE_CHECK(!signaled || status != ncclInvalidUsage, "NCCL EP rejected the signaled ", call, + " (invalid usage): the overlapped EP ops need the EP group within one NVLink domain " + "(one LSA team); NCCL EP prints the cause to stderr"); + NVTE_CHECK_NCCL(status); +} + int current_device() { int device = 0; NVTE_CHECK_CUDA(cudaGetDevice(&device)); @@ -188,6 +233,11 @@ void EPBackend::validate_config(const NVTEEpGroupConfig& config) { ") must be divisible by ep_size (", config.ep_size, ")"); NVTE_CHECK(config.num_comm_sms >= 0, "num_comm_sms must be >= 0 (0 = auto), got ", config.num_comm_sms); + NVTE_CHECK(config.dispatch_fwd_num_sms >= 0 && config.combine_fwd_num_sms >= 0 && + config.dispatch_bwd_num_sms >= 0 && config.combine_bwd_num_sms >= 0, + "dispatch/combine fwd/bwd num_sms must be >= 0 (0 = inherit), got ", + config.dispatch_fwd_num_sms, ", ", config.combine_fwd_num_sms, ", ", + config.dispatch_bwd_num_sms, ", ", config.combine_bwd_num_sms); const int sm = cuda::sm_arch(); NVTE_CHECK(sm >= 90, "NCCL EP requires SM_90+ (Hopper or later), but current device is SM_", sm); @@ -266,6 +316,7 @@ void EPBackend::shutdown_locked() { nccl_ep::group_destroy(ep_group_); ep_group_ = nullptr; } + release_gemm_scratch(); ep_comm_ = nullptr; // borrowed; caller destroys initialized_ = false; } @@ -360,9 +411,43 @@ void EPBackend::init(ncclComm_t ep_comm, NVTEEpGroupConfig group_config) { // Default the dispatch/combine (comm) kernels to 32 SMs, clamped to the device SM count. constexpr int kDefaultCommSms = 32; const int device_sms = cuda::sm_count(); - cfg.max_num_sms = group_config.num_comm_sms > 0 - ? static_cast(group_config.num_comm_sms) - : static_cast(std::min(kDefaultCommSms, device_sms)); + // Resolve the per-call counts here, so that the overlapped GEMMs (group_config()) and NCCL EP + // use the same ones: forward inherits num_comm_sms, backward the forward count of its kind. The + // per-call counts cover every communication kernel, so leave NCCL EP's shared budget on AUTO; + // setting it would also throttle the local permute kernels, which want the whole device. + const auto inherit = [](int num_sms, int fallback) { return num_sms > 0 ? num_sms : fallback; }; + const int comm_sms = group_config.num_comm_sms > 0 ? group_config.num_comm_sms + : std::min(kDefaultCommSms, device_sms); + const int dispatch_fwd = inherit(group_config.dispatch_fwd_num_sms, comm_sms); + const int combine_fwd = inherit(group_config.combine_fwd_num_sms, comm_sms); + const int dispatch_bwd = inherit(group_config.dispatch_bwd_num_sms, dispatch_fwd); + const int combine_bwd = inherit(group_config.combine_bwd_num_sms, combine_fwd); + NVTE_CHECK(comm_sms <= device_sms && dispatch_fwd <= device_sms && combine_fwd <= device_sms && + dispatch_bwd <= device_sms && combine_bwd <= device_sms, + "num_comm_sms and dispatch/combine fwd/bwd num_sms must be <= the device SM count (", + device_sms, "), got num_comm_sms ", comm_sms, ", dispatch fwd/bwd ", dispatch_fwd, "/", + dispatch_bwd, ", combine fwd/bwd ", combine_fwd, "/", combine_bwd); + group_config_.dispatch_fwd_num_sms = dispatch_fwd; + group_config_.combine_fwd_num_sms = combine_fwd; + group_config_.dispatch_bwd_num_sms = dispatch_bwd; + group_config_.combine_bwd_num_sms = combine_bwd; + cfg.dispatch_num_sms = static_cast(dispatch_fwd); + cfg.combine_num_sms = static_cast(combine_fwd); + // NCCL EP's backward counts default to its forward ones; set them only where they differ. +#if NCCL_EP_GROUP_CONFIG_V5_SIZE >= 144u + if (dispatch_bwd != dispatch_fwd) { + cfg.dispatch_bwd_num_sms = static_cast(dispatch_bwd); + } + if (combine_bwd != combine_fwd) { + cfg.combine_bwd_num_sms = static_cast(combine_bwd); + } +#else + NVTE_CHECK(dispatch_bwd == dispatch_fwd && combine_bwd == combine_fwd, + "backward dispatch/combine SM counts other than the forward ones need a Transformer " + "Engine built against an NCCL EP with dispatch_bwd_num_sms and combine_bwd_num_sms; " + "got dispatch fwd/bwd ", + dispatch_fwd, "/", dispatch_bwd, ", combine fwd/bwd ", combine_fwd, "/", combine_bwd); +#endif // 0 = NCCL_EP_AUTO, which enables eager mode (recv buffers sized per routing). cfg.max_recv_tokens_per_rank = static_cast(group_config.max_recv_tokens_per_rank); cfg.zero_copy = group_config.zero_copy ? NCCL_EP_ZERO_COPY_ON : NCCL_EP_ZERO_COPY_OFF; @@ -534,15 +619,14 @@ void EPBackend::prepare(NVTETensor handle_mem, const NVTETensor topk_idx, NVTE_CHECK_NCCL(nccl_ep::update_handle(h, &nccl_topk_idx, &layout_info, stream)); } -void EPBackend::issue_dispatch_locked(ncclEpHandle_t handle, const NVTETensor topk_idx, - const NVTETensor tokens, const NVTECommWindow& tokens_win, - const NVTETensor topk_weights, - const NVTECommWindow& topk_weights_win, - NVTETensor recv_tokens, const NVTECommWindow& recv_tokens_win, - NVTETensor recv_topk_weights, - const NVTECommWindow& recv_topk_weights_win, - NVTETensor recv_tokens_per_expert, - NVTETensor total_recv_tokens_per_rank, cudaStream_t stream) { +void EPBackend::issue_dispatch_locked( + ncclEpHandle_t handle, const NVTETensor topk_idx, const NVTETensor tokens, + const NVTECommWindow& tokens_win, const NVTETensor topk_weights, + const NVTECommWindow& topk_weights_win, NVTETensor recv_tokens, + const NVTECommWindow& recv_tokens_win, NVTETensor recv_topk_weights, + const NVTECommWindow& recv_topk_weights_win, NVTETensor recv_tokens_per_expert, + NVTETensor total_recv_tokens_per_rank, NVTETensor expert_token_offsets, + const EpSignal* dispatch_signal, cudaStream_t stream) { NVTE_CHECK(nvte_tensor_shape(tokens).ndim == 2, "tokens must be 2D [T, hidden_dim]"); NVTE_CHECK(nvte_tensor_shape(recv_tokens).ndim == 2, "recv_tokens must be 2D [recv_T, hidden_dim]"); @@ -617,6 +701,10 @@ void EPBackend::issue_dispatch_locked(ncclEpHandle_t handle, const NVTETensor to out_struct.topk_weights = is_forward ? &nccl_topk_weights_out : nullptr; out_struct.scales = is_scaled ? &nccl_scales_out : nullptr; + SignalDesc dispatch_signal_desc; + out_struct.dispatch_signal = + dispatch_signal_desc.attach(dispatch_signal, NCCL_EP_SIGNAL_UNIT_TILE_ROWS, "dispatch"); + ncclEpDispatchConfig_t dispatch_cfg = NCCL_EP_DISPATCH_CONFIG_INIT; dispatch_cfg.pass_direction = is_forward ? NCCL_EP_FWD_PASS : NCCL_EP_BWD_PASS; // Block-scaled payloads forward the per-token scale-inverse; select the matching recipe. @@ -626,8 +714,8 @@ void EPBackend::issue_dispatch_locked(ncclEpHandle_t handle, const NVTETensor to // fused dispatch writes them (the count scan is fused into the dispatch). NULL leaves // layout_info unset (non-count callers). ncclEpLayoutInfo_t layout_info = NCCL_EP_LAYOUT_INFO_INIT; - NVTEShape recv_counts_shape, total_recv_shape; - ncclEpTensor_t recv_counts_desc, total_recv_desc; + NVTEShape recv_counts_shape, total_recv_shape, recv_offsets_shape; + ncclEpTensor_t recv_counts_desc, total_recv_desc, recv_offsets_desc; const ncclEpLayoutInfo_t* layout_info_ptr = nullptr; if (recv_tokens_per_expert != nullptr) { recv_counts_desc = make_nccl_ep_tensor(recv_tokens_per_expert, recv_counts_shape); @@ -639,9 +727,15 @@ void EPBackend::issue_dispatch_locked(ncclEpHandle_t handle, const NVTETensor to layout_info.recv_total_counter = &total_recv_desc; layout_info_ptr = &layout_info; } + if (expert_token_offsets != nullptr) { + recv_offsets_desc = make_nccl_ep_tensor(expert_token_offsets, recv_offsets_shape); + layout_info.expert_offsets = &recv_offsets_desc; + layout_info_ptr = &layout_info; + } - NVTE_CHECK_NCCL( - nccl_ep::dispatch(handle, &in_struct, &out_struct, layout_info_ptr, &dispatch_cfg, stream)); + check_signaled_call( + nccl_ep::dispatch(handle, &in_struct, &out_struct, layout_info_ptr, &dispatch_cfg, stream), + out_struct.dispatch_signal != nullptr, "dispatch"); } void EPBackend::dispatch(NVTETensor handle_mem, const NVTETensor topk_idx, const NVTETensor tokens, @@ -649,7 +743,8 @@ void EPBackend::dispatch(NVTETensor handle_mem, const NVTETensor topk_idx, const const NVTECommWindow& topk_weights_win, NVTETensor recv_tokens, const NVTECommWindow& recv_tokens_win, NVTETensor recv_topk_weights, const NVTECommWindow& recv_topk_weights_win, - const NVTEEpLayerConfig* layer_cfg, cudaStream_t stream) { + const NVTEEpLayerConfig* layer_cfg, cudaStream_t stream, + const EpSignal* dispatch_signal) { void* hm_ptr = handle_mem_ptr(handle_mem); const int num_tokens = static_cast(nvte_tensor_shape(tokens).data[0]); std::lock_guard lock(mutex_); @@ -661,7 +756,8 @@ void EPBackend::dispatch(NVTETensor handle_mem, const NVTETensor topk_idx, const issue_dispatch_locked(h, topk_idx, tokens, tokens_win, topk_weights, topk_weights_win, recv_tokens, recv_tokens_win, recv_topk_weights, recv_topk_weights_win, /*recv_tokens_per_expert=*/nullptr, - /*total_recv_tokens_per_rank=*/nullptr, stream); + /*total_recv_tokens_per_rank=*/nullptr, + /*expert_token_offsets=*/nullptr, dispatch_signal, stream); } void EPBackend::prepare_and_dispatch( @@ -670,7 +766,8 @@ void EPBackend::prepare_and_dispatch( const NVTECommWindow& topk_weights_win, NVTETensor recv_tokens, const NVTECommWindow& recv_tokens_win, NVTETensor recv_topk_weights, const NVTECommWindow& recv_topk_weights_win, NVTETensor recv_tokens_per_expert, - NVTETensor total_recv_tokens_per_rank, NVTEEpLayerConfig layer_cfg, cudaStream_t stream) { + NVTETensor total_recv_tokens_per_rank, NVTEEpLayerConfig layer_cfg, cudaStream_t stream, + NVTETensor expert_token_offsets, const EpSignal* dispatch_signal) { void* hm_ptr = handle_mem_ptr(handle_mem); NVTE_CHECK(layer_cfg.top_k > 0, "top_k must be > 0, got ", layer_cfg.top_k); NVTE_CHECK(nvte_tensor_shape(topk_idx).ndim == 2, "topk_idx must be 2D [T, top_k]"); @@ -692,19 +789,22 @@ void EPBackend::prepare_and_dispatch( nccl_ep::update_handle(guard.get(), &nccl_topk_idx, /*layout_info=*/nullptr, stream)); issue_dispatch_locked(guard.get(), topk_idx, tokens, tokens_win, topk_weights, topk_weights_win, recv_tokens, recv_tokens_win, recv_topk_weights, recv_topk_weights_win, - recv_tokens_per_expert, total_recv_tokens_per_rank, stream); + recv_tokens_per_expert, total_recv_tokens_per_rank, expert_token_offsets, + dispatch_signal, stream); return; } ncclEpHandle_t h = prepare_handle_locked(hm_ptr, nvte_tensor_size_bytes(handle_mem), layer_cfg); NVTE_CHECK_NCCL(nccl_ep::update_handle(h, &nccl_topk_idx, /*layout_info=*/nullptr, stream)); issue_dispatch_locked(h, topk_idx, tokens, tokens_win, topk_weights, topk_weights_win, recv_tokens, recv_tokens_win, recv_topk_weights, recv_topk_weights_win, - recv_tokens_per_expert, total_recv_tokens_per_rank, stream); + recv_tokens_per_expert, total_recv_tokens_per_rank, expert_token_offsets, + dispatch_signal, stream); } void EPBackend::combine(NVTETensor handle_mem, const NVTETensor expert_out, const NVTECommWindow& expert_out_win, NVTETensor result, - const NVTEEpLayerConfig* layer_cfg, cudaStream_t stream) { + const NVTEEpLayerConfig* layer_cfg, cudaStream_t stream, + const EpSignal* combine_signal) { void* hm_ptr = handle_mem_ptr(handle_mem); NVTE_CHECK(nvte_tensor_shape(expert_out).ndim == 2, "expert_out must be 2D [recv_T, hidden_dim]"); NVTE_CHECK(nvte_tensor_shape(result).ndim == 2, "result must be 2D [T, hidden_dim]"); @@ -717,6 +817,10 @@ void EPBackend::combine(NVTETensor handle_mem, const NVTETensor expert_out, ncclEpCombineInputs_t in_struct = NCCL_EP_COMBINE_INPUTS_INIT; in_struct.tokens = &nccl_expert_in; + SignalDesc combine_signal_desc; + in_struct.combine_signal = + combine_signal_desc.attach(combine_signal, NCCL_EP_SIGNAL_UNIT_CHUNK_ELEMS, "combine"); + ncclEpCombineOutputs_t out_struct = NCCL_EP_COMBINE_OUTPUTS_INIT; out_struct.tokens = &nccl_result_out; @@ -726,14 +830,15 @@ void EPBackend::combine(NVTETensor handle_mem, const NVTETensor expert_out, ncclEpHandle_t h = acquire_step_handle_locked(hm_ptr, nvte_tensor_size_bytes(handle_mem), num_tokens, layer_cfg, &owned); ScopedHandle guard(h, owned); - NVTE_CHECK_NCCL(nccl_ep::combine(h, &in_struct, &out_struct, /*config=*/nullptr, stream)); + check_signaled_call(nccl_ep::combine(h, &in_struct, &out_struct, /*config=*/nullptr, stream), + combine_signal != nullptr, "combine"); } void EPBackend::dispatch_bwd(NVTETensor handle_mem, const NVTETensor grad, const NVTECommWindow& grad_win, const NVTETensor g_recv_topk_weights, const NVTECommWindow& g_recv_topk_weights_win, NVTETensor grad_tokens, NVTETensor grad_topk_weights, const NVTEEpLayerConfig* layer_cfg, - cudaStream_t stream) { + cudaStream_t stream, const EpSignal* combine_signal) { void* hm_ptr = handle_mem_ptr(handle_mem); const int num_tokens = static_cast(nvte_tensor_shape(grad_tokens).data[0]); NVTE_CHECK(nvte_tensor_shape(grad).ndim == 2, "grad must be 2D [recv_capacity, hidden_dim]"); @@ -756,6 +861,10 @@ void EPBackend::dispatch_bwd(NVTETensor handle_mem, const NVTETensor grad, in_struct.tokens = &nccl_tok_in; in_struct.topk_weights = &nccl_w_in; + SignalDesc combine_signal_desc; + in_struct.combine_signal = + combine_signal_desc.attach(combine_signal, NCCL_EP_SIGNAL_UNIT_CHUNK_ELEMS, "combine"); + ncclEpCombineOutputs_t out_struct = NCCL_EP_COMBINE_OUTPUTS_INIT; out_struct.tokens = &nccl_tok_out; out_struct.topk_weights = &nccl_w_out; @@ -769,19 +878,21 @@ void EPBackend::dispatch_bwd(NVTETensor handle_mem, const NVTETensor grad, ncclEpHandle_t h = acquire_step_handle_locked(hm_ptr, nvte_tensor_size_bytes(handle_mem), num_tokens, layer_cfg, &owned); ScopedHandle guard(h, owned); - NVTE_CHECK_NCCL(nccl_ep::combine(h, &in_struct, &out_struct, &cfg, stream)); + check_signaled_call(nccl_ep::combine(h, &in_struct, &out_struct, &cfg, stream), + combine_signal != nullptr, "dispatch backward"); } void EPBackend::combine_bwd(NVTETensor handle_mem, const NVTETensor grad, const NVTECommWindow& grad_win, NVTETensor grad_expert_out, const NVTECommWindow& grad_expert_out_win, - const NVTEEpLayerConfig* layer_cfg, cudaStream_t stream) { + const NVTEEpLayerConfig* layer_cfg, cudaStream_t stream, + const EpSignal* dispatch_signal) { // Backward of combine = reverse-direction dispatch. dispatch(handle_mem, /*topk_idx=*/nullptr, grad, grad_win, /*topk_weights=*/nullptr, /*topk_weights_win=*/NVTECommWindow{}, grad_expert_out, grad_expert_out_win, /*recv_topk_weights=*/nullptr, /*recv_topk_weights_win=*/NVTECommWindow{}, layer_cfg, - stream); + stream, dispatch_signal); } } // namespace ep diff --git a/transformer_engine/common/ep/ep_backend.h b/transformer_engine/common/ep/ep_backend.h index dd9d2a4454e..6ac476e98ab 100644 --- a/transformer_engine/common/ep/ep_backend.h +++ b/transformer_engine/common/ep/ep_backend.h @@ -27,6 +27,18 @@ namespace transformer_engine { namespace ep { +/*! \brief Readiness flags of an overlapped op: 1D int32, one flag per `granularity` units of the + * signaled tensor (rows of recv_tokens for a dispatch, hidden columns for a combine). */ +struct EpSignal { + // int32 readiness signal, one entry per granularity units; reset to -1, set to the slot index i + // when that unit is ready (consumer waits for signal[s] == s). + NVTETensor ready; + size_t granularity; +}; + +/*! \brief Free the current device's grouped GEMM scratch. A no-op without the EP GEMMs. */ +void release_gemm_scratch(); + /*! \brief EP backend per CUDA device; owns the NCCL EP group, borrows the comm. */ class EPBackend { public: @@ -54,16 +66,20 @@ class EPBackend { // Per-step ops below require a prior prepare(). A non-null layer_cfg binds the handle from // handle_mem; null uses the handle cache (NVTE_EP_HANDLE_CACHE_SIZE != 0). + // A non-null dispatch_signal publishes one flag per signal granularity rows of recv_tokens. void dispatch(NVTETensor handle_mem, const NVTETensor topk_idx, const NVTETensor tokens, const NVTECommWindow& tokens_win, const NVTETensor topk_weights, const NVTECommWindow& topk_weights_win, NVTETensor recv_tokens, const NVTECommWindow& recv_tokens_win, NVTETensor recv_topk_weights, const NVTECommWindow& recv_topk_weights_win, const NVTEEpLayerConfig* layer_cfg, - cudaStream_t stream); + cudaStream_t stream, const EpSignal* dispatch_signal = nullptr); // Fused prepare + dispatch: seeds routing then dispatches in one call. Routing writes the // per-expert recv counts to recv_tokens_per_expert and the scalar pre-drop per-rank recv total // to total_recv_tokens_per_rank (nullable); the dispatch then reads the counts from the handle. + // expert_token_offsets (nullable): the padded per-expert token offsets; NCCL EP requires them, + // the counts and the total, when given, to share one integer type. + // dispatch_signal: as for dispatch(). void prepare_and_dispatch(NVTETensor handle_mem, const NVTETensor topk_idx, const NVTETensor tokens, const NVTECommWindow& tokens_win, const NVTETensor topk_weights, const NVTECommWindow& topk_weights_win, @@ -72,22 +88,31 @@ class EPBackend { const NVTECommWindow& recv_topk_weights_win, NVTETensor recv_tokens_per_expert, NVTETensor total_recv_tokens_per_rank, NVTEEpLayerConfig layer_cfg, - cudaStream_t stream); + cudaStream_t stream, NVTETensor expert_token_offsets = nullptr, + const EpSignal* dispatch_signal = nullptr); + // A non-null combine_signal makes the combine start on column chunks as the producer of + // expert_out publishes them. void combine(NVTETensor handle_mem, const NVTETensor expert_out, const NVTECommWindow& expert_out_win, NVTETensor result, - const NVTEEpLayerConfig* layer_cfg, cudaStream_t stream); + const NVTEEpLayerConfig* layer_cfg, cudaStream_t stream, + const EpSignal* combine_signal = nullptr); // g_recv_topk_weights: 1D [recv_capacity] f32; grad_topk_weights: 2D [T, top_k] f32. + // combine_signal: as for combine(). void dispatch_bwd(NVTETensor handle_mem, const NVTETensor grad, const NVTECommWindow& grad_win, const NVTETensor g_recv_topk_weights, const NVTECommWindow& g_recv_topk_weights_win, NVTETensor grad_tokens, NVTETensor grad_topk_weights, const NVTEEpLayerConfig* layer_cfg, - cudaStream_t stream); + cudaStream_t stream, const EpSignal* combine_signal = nullptr); + // dispatch_signal: as for dispatch(), over the rows of grad_expert_out. void combine_bwd(NVTETensor handle_mem, const NVTETensor grad, const NVTECommWindow& grad_win, NVTETensor grad_expert_out, const NVTECommWindow& grad_expert_out_win, - const NVTEEpLayerConfig* layer_cfg, cudaStream_t stream); + const NVTEEpLayerConfig* layer_cfg, cudaStream_t stream, + const EpSignal* dispatch_signal = nullptr); + + const NVTEEpGroupConfig& group_config() const { return group_config_; } ~EPBackend(); @@ -148,7 +173,8 @@ class EPBackend { // Build the dispatch in/out structs and issue ncclEpDispatch on the resolved // handle. When recv_tokens_per_expert != nullptr (count mode), it is wired to - // layout_info.expert_counters so the dispatch writes per-expert recv counts. + // layout_info.expert_counters so the dispatch writes per-expert recv counts; likewise + // expert_token_offsets to layout_info.expert_offsets. // Caller must hold mutex_. void issue_dispatch_locked(ncclEpHandle_t handle, const NVTETensor topk_idx, const NVTETensor tokens, const NVTECommWindow& tokens_win, @@ -157,13 +183,17 @@ class EPBackend { NVTETensor recv_topk_weights, const NVTECommWindow& recv_topk_weights_win, NVTETensor recv_tokens_per_expert, - NVTETensor total_recv_tokens_per_rank, cudaStream_t stream); + NVTETensor total_recv_tokens_per_rank, NVTETensor expert_token_offsets, + const EpSignal* dispatch_signal, cudaStream_t stream); // devices_mutex_ protects the map; each backend's mutex_ protects its state. static std::mutex devices_mutex_; static std::unordered_map> devices_; }; +// A caller's versioned layer config copied into the current layout. +NVTEEpLayerConfig normalize_layer_config(const NVTEEpLayerConfig* cfg); + } // namespace ep } // namespace transformer_engine diff --git a/transformer_engine/common/ep/ep_gemm_api.cpp b/transformer_engine/common/ep/ep_gemm_api.cpp new file mode 100644 index 00000000000..41022b726f7 --- /dev/null +++ b/transformer_engine/common/ep/ep_gemm_api.cpp @@ -0,0 +1,1034 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +#include + +#include "../util/logging.h" + +#if defined(NVTE_WITH_NCCL_EP) && defined(NVTE_WITH_CUTEDSL) + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../common.h" +#include "../util/cuda_runtime.h" +#include "ep_backend.h" +#include "ep_gemm_config.h" +#include "ep_overlap_workspace.h" + +namespace ep = transformer_engine::ep; + +namespace { + +constexpr size_t kWorkspaceAlign = 128; +// Rows per GEMM M tile (cluster tile M). Smaller expert alignments end an expert in a tail tile. +constexpr size_t kRowsPerTile = 256; +// GEMM CTA tile N; combine readiness flags cover a multiple of it. +constexpr size_t kTileN = 256; +constexpr size_t kClusterSize = 2; + +size_t align_up(size_t value) { + return (value + kWorkspaceAlign - 1) / kWorkspaceAlign * kWorkspaceAlign; +} + +// Per-device scratch for the readiness flags and GEMM state, grown geometrically on demand. An +// outgrown block stays allocated until EP shutdown: a captured CUDA graph may still use it. +struct ScratchBlock { + void* ptr = nullptr; + size_t bytes = 0; + std::vector retired; + + void release() { + retired.push_back(ptr); + for (void* p : retired) { + if (p != nullptr) { + cudaFree(p); + } + } + ptr = nullptr; + bytes = 0; + retired.clear(); + } +}; + +std::mutex scratch_mutex; +std::map& scratch_blocks() { + // Leaked: no CUDA call at static teardown. + static auto* blocks = new std::map(); + return *blocks; +} + +uint8_t* get_scratch(size_t bytes) { + std::lock_guard lock(scratch_mutex); + ScratchBlock& block = scratch_blocks()[transformer_engine::cuda::current_device()]; + if (block.bytes < bytes) { + void* ptr = nullptr; + const size_t grown = std::max(bytes, 2 * block.bytes); + NVTE_CHECK_CUDA(cudaMalloc(&ptr, grown)); + block.retired.push_back(block.ptr); + block.ptr = ptr; + block.bytes = grown; + } + return static_cast(block.ptr); +} + +using transformer_engine::tvm_ffi_bridge::DLTensorWrapper; + +// Operands live on the current device. +DLTensorWrapper wrap_data(const NVTEBasicTensor& data, bool flatten_2d) { + return DLTensorWrapper(data, flatten_2d, transformer_engine::cuda::current_device()); +} + +DLTensorWrapper wrap_rowwise(NVTETensor t, bool flatten_2d) { + return wrap_data(NVTEBasicTensor{nvte_tensor_data(t), nvte_tensor_type(t), nvte_tensor_shape(t)}, + flatten_2d); +} + +DLTensorWrapper wrap_raw(void* ptr, NVTEDType dtype, size_t count) { + NVTEShape shape{}; + shape.ndim = 1; + shape.data[0] = count; + return wrap_data(NVTEBasicTensor{ptr, dtype, shape}, false); +} + +bool is_mxfp8(NVTETensor t) { return nvte_tensor_scaling_mode(t) == NVTE_MXFP8_1D_SCALING; } + +// An MXFP8 operand: E4M3 data with compact or GEMM-swizzled scales. +void check_mxfp8(NVTETensor t, bool swizzled, const char* name) { + NVTE_CHECK(t != nullptr && is_mxfp8(t) && nvte_tensor_type(t) == kNVTEFloat8E4M3, name, + " must be MXFP8 E4M3"); + uint8_t with_swizzled = 0; + nvte_get_tensor_param_v2(t, kNVTEWithGEMMSwizzledScales, &with_swizzled, sizeof(with_swizzled), + nullptr); + NVTE_CHECK((with_swizzled != 0) == swizzled, name, + swizzled ? " needs GEMM-swizzled scales" : " needs compact scales"); +} + +// Shapes the MXFP8 GEMMs take: compact scale rows of whole 16-byte TMA boxes, SwiGLU tiles and +// expert slabs of whole 128-row scale tiles. +void check_mxfp8_dims(size_t hidden, size_t ffn, size_t alignment) { + NVTE_CHECK(hidden % 512 == 0 && ffn % 256 == 0, "MXFP8 needs hidden_dim (", hidden, + ") a multiple of 512 and ffn_dim (", ffn, ") a multiple of 256"); + NVTE_CHECK(alignment >= 128, "MXFP8 needs dispatch_output_per_expert_alignment 128 or ", + kRowsPerTile); +} + +// Columnwise data of an MXFP8 operand. +DLTensorWrapper wrap_columnwise(NVTETensor t, bool flatten_2d) { + const NVTEBasicTensor data = nvte_get_tensor_param(t, kNVTEColumnwiseData); + NVTE_CHECK(data.data_ptr != nullptr, "MXFP8 operand without columnwise data"); + return wrap_data(data, flatten_2d); +} + +// E8M0 scales of a [rows, cols] MXFP8 operand, rowwise or columnwise. +DLTensorWrapper wrap_scales(NVTETensor t, bool columnwise, size_t rows, size_t cols) { + const NVTEBasicTensor scales = + nvte_get_tensor_param(t, columnwise ? kNVTEColumnwiseScaleInv : kNVTERowwiseScaleInv); + size_t count = 1; + for (size_t i = 0; i < scales.shape.ndim; ++i) { + count *= scales.shape.data[i]; + } + NVTE_CHECK(scales.data_ptr != nullptr && count == rows * cols / 32, "expected ", rows * cols / 32, + columnwise ? " columnwise" : " rowwise", " MXFP8 scales, got ", count); + return wrap_raw(scales.data_ptr, kNVTEFloat8E8M0, count); +} + +// Value of a positive integer environment variable, 0 when unset. +size_t env_size(const char* name) { + const char* value = std::getenv(name); + if (value == nullptr || value[0] == '\0') { + return 0; + } + char* end = nullptr; + const int64_t parsed = std::strtoll(value, &end, 10); + NVTE_CHECK(*end == '\0' && parsed > 0, name, " must be a positive integer, got '", value, "'"); + return static_cast(parsed); +} + +// Columns per combine flag: the whole row or a multiple of 1024 dividing it, the chunks NCCL EP +// accepts for bfloat16 rows. NVTE_EP_OVERLAP_COMBINE_CHUNK_COLS unset picks 1024 when it divides n, else +// the whole row. +size_t combine_chunk(size_t n) { + NVTE_CHECK(n % kTileN == 0, "output columns (", n, ") must be a multiple of ", kTileN); + const size_t requested = env_size("NVTE_EP_OVERLAP_COMBINE_CHUNK_COLS"); + if (requested == 0) { + return n % 1024 == 0 ? 1024 : n; + } + NVTE_CHECK(requested == n || (requested % 1024 == 0 && n % requested == 0), + "NVTE_EP_OVERLAP_COMBINE_CHUNK_COLS (", requested, ") must be the hidden size (", n, + ") or a multiple of 1024 dividing it"); + return requested; +} + +// The combine (or dispatch backward) starts at the GEMM's entry and waits on its chunk flags. +constexpr bool kCombineAtGemmEntry = true; + +// Group and layer state checked by every entry point. +struct EpGemmContext { + ep::EPBackend& backend; + const NVTEEpGroupConfig& group; + NVTEEpLayerConfig layer_cfg; + int num_clusters; + int all_clusters; // clusters of every SM + // max_recv_tokens_per_rank == 0: the caller sized the recv buffers with nvte_ep_prepare. + bool eager; + + size_t alignment() const { return layer_cfg.dispatch_output_per_expert_alignment; } + // Rows per dispatch readiness flag: the alignment, at most one M tile. + size_t rows_per_slot() const { return std::min(alignment(), kRowsPerTile); } +}; + +// comm_sms: the group's SM count of the NCCL EP call that the entry point overlaps; the GEMM runs +// on the other SMs. +EpGemmContext make_context(const NVTEEpLayerConfig* layer_cfg, int NVTEEpGroupConfig::*comm_sms) { + const NVTEEpLayerConfig cfg = ep::normalize_layer_config(layer_cfg); + const size_t alignment = cfg.dispatch_output_per_expert_alignment; + NVTE_CHECK(alignment == 64 || alignment == 128 || alignment == kRowsPerTile, + "dispatch_output_per_expert_alignment must be 64, 128 or ", kRowsPerTile); + ep::EPBackend& backend = ep::EPBackend::get(); + const NVTEEpGroupConfig& group = backend.group_config(); + NVTE_CHECK(group.num_comm_sms > 0, "the overlapped ops need an explicit num_comm_sms"); + NVTE_CHECK(!group.zero_copy, "the overlapped ops do not support zero-copy"); + // The GEMMs take the SMs left beside the ep_bootstrap counts, which these variables would + // change for the NCCL EP kernels only. + for (const char* name : {"NCCL_EP_COMM_SMS", "NCCL_EP_DISPATCH_SMS", "NCCL_EP_COMBINE_SMS", + "NCCL_EP_DISPATCH_BWD_SMS", "NCCL_EP_COMBINE_BWD_SMS"}) { + NVTE_CHECK(std::getenv(name) == nullptr, + "the overlapped ops split the SMs by the ep_bootstrap " + "counts; unset ", + name); + } + const int sm_count = transformer_engine::cuda::sm_count(); + NVTE_CHECK(sm_count - group.*comm_sms >= static_cast(kClusterSize), "fewer than ", + kClusterSize, " SMs left beside the ", group.*comm_sms, " communication SMs"); + // The GEMM tails take the clusters of every SM. + const int max_clusters = + transformer_engine::cuda::max_active_clusters(static_cast(kClusterSize)); + NVTE_CHECK(max_clusters == sm_count / static_cast(kClusterSize), "the device runs ", + max_clusters, " clusters of ", kClusterSize, " SMs at once, not ", sm_count, " / ", + kClusterSize); + EpGemmContext ctx{backend, + group, + cfg, + (sm_count - group.*comm_sms) / static_cast(kClusterSize), + sm_count / static_cast(kClusterSize), + group.max_recv_tokens_per_rank == 0}; + return ctx; +} + +// A caller-owned [num_experts] int32 count tensor. +void check_counts(NVTETensor counts, size_t num_experts, const char* name) { + NVTE_CHECK(counts != nullptr, name, " must not be null"); + NVTE_CHECK(nvte_tensor_shape(counts).ndim == 1 && nvte_tensor_numel(counts) == num_experts && + nvte_tensor_type(counts) == kNVTEInt32, + name, " must be an int32 tensor of ", num_experts, " elements"); +} + +// A caller-owned float32 [rows] tensor. +void check_row_floats(NVTETensor t, size_t rows, const char* name) { + NVTE_CHECK(t != nullptr, name, " must not be null"); + NVTE_CHECK(nvte_tensor_shape(t).ndim == 1 && nvte_tensor_numel(t) == rows && + nvte_tensor_type(t) == kNVTEFloat32, + name, " must be a float32 tensor of recv_capacity (", rows, ") elements"); +} + +// The GEMM reads the per-expert row ends from expert_token_offsets[1:] and requires them 16-byte +// aligned; callers front-pad the [num_experts + 1] int32 layout by 3 elements to achieve that. +void check_expert_token_offsets(const NVTETensor expert_token_offsets, size_t num_experts) { + NVTE_CHECK(nvte_tensor_numel(expert_token_offsets) == num_experts + 1 && + nvte_tensor_type(expert_token_offsets) == kNVTEInt32, + "expert_token_offsets must be an int32 tensor of num_experts + 1 elements"); + const int32_t* row_ends = static_cast(nvte_tensor_data(expert_token_offsets)) + 1; + NVTE_CHECK(reinterpret_cast(row_ends) % 16 == 0, + "expert_token_offsets[1:] (the per-expert row ends) must start 16-byte aligned; " + "front-pad the num_experts + 1 int32 layout by 3 elements"); +} + +// Rows: the group capacity, or any multiple of the flag rows in eager mode. +void check_rows(const EpGemmContext& ctx, size_t rows, const char* name) { + NVTE_CHECK(rows % ctx.rows_per_slot() == 0, name, " rows (", rows, ") must be a multiple of ", + ctx.rows_per_slot()); + NVTE_CHECK(ctx.eager || rows == static_cast(ctx.group.max_recv_tokens_per_rank), name, + " has ", rows, " rows, max_recv_tokens_per_rank is ", + ctx.group.max_recv_tokens_per_rank); +} + +// Whether the forward dispatch counts the routing itself and writes the caller's expert layout. As +// NCCL EP's dispatch_pull_count_capable for the overlapped group (pull-push flow in one LSA team): +// not in eager mode and at most 65536 / 32 local experts. Without drops only: with them the +// dispatch reports the pre-drop row total. +bool dispatch_writes_layout(const EpGemmContext& ctx, size_t num_local_experts) { + return !ctx.eager && !ctx.group.drop_on_overflow && num_local_experts <= 65536 / 32; +} + +// The forward combine_gemm GEMM of [rows, k] inputs and n output columns. +ep::EpCombineGemmConfig combine_fwd_config(const EpGemmContext& ctx, size_t rows, size_t k, + size_t n, size_t num_experts, bool mxfp8) { + return ep::EpCombineGemmConfig{rows, + k, + n, + num_experts, + ctx.num_clusters, + combine_chunk(n), + kCombineAtGemmEntry, + false, + mxfp8, + ctx.alignment(), + ctx.all_clusters}; +} + +// Scratch of one call: GEMM workspace, row ends (one reserved word before them, which unset_row_ends +// sets), unit alphas, optional unit probs, optional weight-gradient workspace, and the words the +// call's launch resets. +struct GemmScratch { + uint8_t* base; + uint8_t* wgrad = nullptr; + int32_t* row_ends = nullptr; // [num_experts] + float* alpha = nullptr; // [num_experts] + float* ones = nullptr; // [rows] + int32_t* signal = nullptr; // [signal_capacity] readiness flags + int signal_capacity = 0; + int num_experts = 0; + ep::WorkspaceReset reset; +}; + +GemmScratch claim_scratch(size_t gemm_bytes, size_t num_experts, size_t rows, bool unit_probs, + size_t wgrad_bytes) { + const size_t row_ends = align_up(gemm_bytes + sizeof(int32_t)); + const size_t alpha = row_ends + align_up(sizeof(int32_t) * num_experts); + const size_t ones = alpha + align_up(sizeof(float) * num_experts); + const size_t wgrad = ones + (unit_probs ? align_up(sizeof(float) * rows) : 0); + const size_t end = wgrad + align_up(wgrad_bytes); + GemmScratch scratch{get_scratch(end)}; + if (wgrad_bytes > 0) { + scratch.wgrad = scratch.base + wgrad; + } + scratch.row_ends = reinterpret_cast(scratch.base + row_ends); + scratch.alpha = reinterpret_cast(scratch.base + alpha); + scratch.num_experts = static_cast(num_experts); + scratch.reset.fill(scratch.alpha, scratch.num_experts, ep::kFillOne); + if (unit_probs) { + scratch.ones = reinterpret_cast(scratch.base + ones); + scratch.reset.fill(scratch.ones, static_cast(rows), ep::kFillOne); + } + return scratch; +} + +// Words of a GEMM that waits on the dispatch tile flags: its tile counter, flags and scheduler +// state. +void set_signal_words(GemmScratch& scratch, const ep::WorkspaceLayout& layout, size_t slots) { + scratch.signal = reinterpret_cast(scratch.base + layout.signal); + scratch.signal_capacity = static_cast(slots); + scratch.reset.fill(scratch.base + layout.counter, 1, ep::kFillZero); + scratch.reset.fill(scratch.signal, scratch.signal_capacity, ep::kFillMinusOne); + scratch.reset.fill(scratch.base + layout.state, (layout.total_bytes - layout.state) / 4, + ep::kFillZero); +} + +// Words of a GEMM that publishes column chunks: its first zeroed_bytes (scheduler words, zero at +// entry) and chunk flags. +void set_chunk_words(GemmScratch& scratch, const ep::WorkspaceLayout& layout, size_t chunks, + size_t zeroed_bytes) { + scratch.signal = reinterpret_cast(scratch.base + layout.signal); + scratch.signal_capacity = static_cast(chunks); + scratch.reset.fill(scratch.base, zeroed_bytes / 4, ep::kFillZero); + scratch.reset.fill(scratch.signal, scratch.signal_capacity, ep::kFillMinusOne); +} + +// Row ends the dispatch writes: unset until it does, together with the reserved word before them. +void unset_row_ends(GemmScratch& scratch) { + scratch.reset.fill(scratch.row_ends - 1, scratch.num_experts + 1, ep::kFillMinusOne); +} + +// Row ends computed by the reset from the padded int32 counts per local expert; write_layout_base +// also writes the reserved word before them, expert 0's start, as 0. +void set_row_ends(GemmScratch& scratch, NVTETensor counts, bool write_layout_base = false) { + scratch.reset.counts = static_cast(nvte_tensor_data(counts)); + scratch.reset.row_ends = scratch.row_ends; + scratch.reset.num_experts = scratch.num_experts; + if (write_layout_base) { + scratch.reset.fill(scratch.row_ends - 1, 1, ep::kFillZero); + } +} + +// Compiled GEMM and workspace layout of a configuration. +template +tvm::ffi::Function get_gemm(const Config& config, ep::WorkspaceLayout* layout) { + auto kernel = config.get_kernel(); + NVTE_CHECK(kernel.has_value(), + "the EP grouped GEMM is not available: needs NVTE_ENABLE_CUTEDSL_BACKEND=1, " + "NVTE_MOE_OVERLAP and SM 10.0, 10.3 or 10.7"); + const auto offsets = config.workspace_layout(); + NVTE_CHECK(offsets.has_value(), "cannot query the GEMM workspace layout"); + *layout = *offsets; + return *kernel; +} + +// The dispatch GEMM on `clusters` more clusters than the first launch's, which share its workspace. +tvm::ffi::Function get_tail_gemm(ep::EpDispatchGemmConfig config, int clusters, + const ep::WorkspaceLayout& layout, const char* name) { + config.num_clusters = clusters; + ep::WorkspaceLayout tail_layout; + const tvm::ffi::Function tail = get_gemm(config, &tail_layout); + NVTE_CHECK(tail_layout.total_bytes == layout.total_bytes && + tail_layout.counter == layout.counter && tail_layout.signal == layout.signal && + tail_layout.state == layout.state, + name, " has another workspace layout"); + return tail; +} + +// A weight gradient chained behind a backward GEMM: weight_grad[e] = grad^T x over the rows of +// expert e, written or added to. +struct Wgrad { + NVTETensor grad; + NVTETensor x; + NVTETensor weight_grad; + ep::EpWgradConfig config; // each launch sets its clusters and wait options + ep::WorkspaceLayout layout; +}; + +// Checks the weight-gradient operands of a backward entry; nullopt without weight_grad. MXFP8 +// reads both operands columnwise. +std::optional make_wgrad(const EpGemmContext& ctx, NVTETensor grad, NVTETensor x, + NVTETensor weight_grad, bool accumulate, size_t rows, + size_t num_experts, bool mxfp8) { + if (weight_grad == nullptr) { + NVTE_CHECK(x == nullptr, "wgrad_input needs weight_grad"); + return std::nullopt; + } + NVTE_CHECK(x != nullptr, "weight_grad needs wgrad_input"); + NVTE_CHECK(rows > 0, "the chained weight gradient needs received rows"); + NVTE_CHECK(nvte_tensor_shape(grad).ndim == 2 && nvte_tensor_shape(x).ndim == 2, + "wgrad_input and the gradient must be 2D"); + const NVTEShape shape = nvte_tensor_shape(weight_grad); + NVTE_CHECK(shape.ndim == 3 && shape.data[0] == num_experts, + "weight_grad must be [num_local_experts, m, n]"); + NVTE_CHECK(reinterpret_cast(nvte_tensor_data(weight_grad)) % 16 == 0, + "weight_grad must be 16-byte aligned"); + const size_t m = shape.data[1]; + const size_t n = shape.data[2]; + NVTE_CHECK(nvte_tensor_shape(grad).data[1] == m && nvte_tensor_shape(x).data[0] == rows && + nvte_tensor_shape(x).data[1] == n, + "weight_grad must be [num_local_experts, ", nvte_tensor_shape(grad).data[1], + ", n] with wgrad_input [recv_capacity, n]"); + if (mxfp8) { + check_mxfp8(x, true, "wgrad_input"); + // Both operands are read columnwise: [rows, cols] E4M3 data and rows * cols / 32 scales. + const auto check_columnwise = [rows](NVTETensor t, size_t cols, const char* name) { + const NVTEBasicTensor data = nvte_get_tensor_param(t, kNVTEColumnwiseData); + const NVTEBasicTensor scales = nvte_get_tensor_param(t, kNVTEColumnwiseScaleInv); + size_t scale_count = 1; + for (size_t i = 0; i < scales.shape.ndim; ++i) { + scale_count *= scales.shape.data[i]; + } + NVTE_CHECK(data.data_ptr != nullptr && scales.data_ptr != nullptr, + "the MXFP8 weight gradient needs columnwise ", name, " data and scales"); + NVTE_CHECK(data.shape.ndim == 2 && data.shape.data[0] == rows && data.shape.data[1] == cols, + "the columnwise MXFP8 ", name, " must be [", rows, ", ", cols, "]"); + NVTE_CHECK(scale_count == rows * cols / 32, "the columnwise MXFP8 ", name, " needs ", + rows * cols / 32, " scales, got ", scale_count); + }; + check_columnwise(grad, m, "gradient"); + check_columnwise(x, n, "wgrad_input"); + } else { + NVTE_CHECK(nvte_tensor_type(x) == kNVTEBFloat16, "wgrad_input must be bfloat16"); + } + const NVTEDType out_type = nvte_tensor_type(weight_grad); + NVTE_CHECK(out_type == kNVTEBFloat16 || out_type == kNVTEFloat32, + "weight_grad must be bfloat16 or float32"); + const size_t routed_rows = + static_cast(ctx.group.max_tokens_per_rank) * ctx.layer_cfg.top_k; + Wgrad wgrad{grad, + x, + weight_grad, + {rows, m, n, num_experts, 1, out_type == kNVTEFloat32, accumulate, false, 0, mxfp8, + false, ctx.alignment(), routed_rows}, + {}}; + const auto layout = wgrad.config.workspace_layout(); + NVTE_CHECK(layout.has_value(), "cannot query the weight-gradient workspace layout"); + wgrad.layout = *layout; + return wgrad; +} + +// Whether the MXFP8 weight gradient reads per-expert descriptors that a descriptors_only launch +// writes: an added or FP32 one, and on SM 10.7 a written BF16 one with long experts. The others +// read global descriptors. +bool needs_descriptors(const Wgrad& wgrad) { + return wgrad.config.mxfp8 && wgrad.config.reads_descriptors(); +} + +// One launch of a weight gradient on num_clusters clusters (none: no launch), a programmatic +// dependent of the kernel before it. Launches sharing the call's workspace split the tiles. +// wait_primary_at_start: no load before that kernel completes. ready_slot_rows: no load before +// every dispatch flag of the expert rows is set. descriptors_only (MXFP8): a plain launch that +// writes the per-expert descriptors the other launches read; it follows the scratch reset. +void launch_wgrad(const Wgrad& wgrad, const GemmScratch& scratch, int num_clusters, + bool wait_primary_at_start, size_t ready_slot_rows, cudaStream_t stream, + bool descriptors_only) { + if (num_clusters <= 0) { + return; + } + ep::EpWgradConfig config = wgrad.config; + config.num_clusters = num_clusters; + config.wait_primary_at_start = wait_primary_at_start; + config.ready_slot_rows = ready_slot_rows; + config.descriptors_only = descriptors_only; + ep::WorkspaceLayout layout; + const tvm::ffi::Function gemm = get_gemm(config, &layout); + // The call sized the scratch from its base configuration. + NVTE_CHECK( + layout.total_bytes <= wgrad.layout.total_bytes && layout.counter == wgrad.layout.counter, + "weight-gradient workspace layout differs between launches"); + auto out = wrap_rowwise(wgrad.weight_grad, false); + auto row_ends = wrap_raw(scratch.row_ends, kNVTEInt32, config.num_experts); + auto ws = wrap_raw(scratch.wgrad, kNVTEByte, layout.total_bytes); + if (config.mxfp8) { + auto grad = wrap_columnwise(wgrad.grad, true); + auto x = wrap_columnwise(wgrad.x, true); + auto grad_scales = wrap_scales(wgrad.grad, true, config.rows, config.m); + auto x_scales = wrap_scales(wgrad.x, true, config.rows, config.n); + gemm(&grad, &x, &grad_scales, &x_scales, &out, &row_ends, &ws, static_cast(stream)); + return; + } + auto grad = wrap_rowwise(wgrad.grad, true); + auto x = wrap_rowwise(wgrad.x, true); + if (ready_slot_rows > 0) { + auto signal = wrap_raw(scratch.signal, kNVTEInt32, config.rows / ready_slot_rows); + gemm(&grad, &x, &out, &row_ends, &ws, &signal, static_cast(stream)); + return; + } + gemm(&grad, &x, &out, &row_ends, &ws, static_cast(stream)); +} + +// The columnwise MXFP8 copy of t's received rows, requantized from its rowwise data and compact +// scales; columnwise scales GEMM-swizzled per expert slab. A programmatic dependent of the kernel +// before it. ready_slot_rows > 0: a tile starts once the dispatch has set the flags of its rows. +void launch_requant(NVTETensor t, const GemmScratch& scratch, size_t rows, size_t cols, + size_t num_experts, size_t ready_slot_rows, cudaStream_t stream) { + const ep::EpRequantConfig config{rows, cols, num_experts, ready_slot_rows}; + const auto kernel = config.get_kernel(); + NVTE_CHECK(kernel.has_value(), + "the EP MXFP8 requant is not available: needs NVTE_ENABLE_CUTEDSL_BACKEND=1, " + "NVTE_MOE_OVERLAP and SM 10.0, 10.3 or 10.7"); + auto codes = wrap_rowwise(t, true); + auto row_scales = wrap_scales(t, false, rows, cols); + auto col_codes = wrap_columnwise(t, true); + auto col_scales = wrap_scales(t, true, rows, cols); + auto row_ends = wrap_raw(scratch.row_ends, kNVTEInt32, num_experts); + if (ready_slot_rows > 0) { + auto signal = wrap_raw(scratch.signal, kNVTEInt32, rows / ready_slot_rows); + (*kernel)(&codes, &row_scales, &col_codes, &col_scales, &row_ends, &signal, + static_cast(stream)); + return; + } + (*kernel)(&codes, &row_scales, &col_codes, &col_scales, &row_ends, static_cast(stream)); +} + +} // namespace + +namespace transformer_engine { +namespace ep { + +void release_gemm_scratch() { + std::lock_guard lock(scratch_mutex); + scratch_blocks()[cuda::current_device()].release(); +} + +} // namespace ep +} // namespace transformer_engine + +extern "C" { + +void nvte_ep_dispatch_gemm_fwd(NVTETensor handle_mem, const NVTETensor topk_idx, + const NVTETensor tokens, NVTECommWindow tokens_win, + const NVTETensor topk_weights, const NVTETensor weight, + NVTETensor recv_tokens, NVTETensor recv_topk_weights, + NVTETensor gemm_out, NVTETensor act_out, + NVTETensor recv_tokens_per_expert, NVTETensor expert_token_offsets, + NVTETensor total_recv_tokens, const NVTEEpLayerConfig* layer_cfg, + cudaStream_t stream) { + using namespace transformer_engine; + const EpGemmContext ctx = make_context(layer_cfg, &NVTEEpGroupConfig::dispatch_fwd_num_sms); + checkCuDriverContext(stream); + + const size_t rows = nvte_tensor_shape(recv_tokens).data[0]; + const size_t hidden = nvte_tensor_shape(tokens).data[1]; + const size_t num_experts = nvte_tensor_shape(weight).data[0]; + const size_t n = nvte_tensor_shape(weight).data[1]; + check_rows(ctx, rows, "recv_tokens"); + NVTE_CHECK(num_experts * ctx.group.ep_size == static_cast(ctx.group.num_experts), + "weight has ", num_experts, " experts, expected num_experts / ep_size = ", + ctx.group.num_experts / ctx.group.ep_size); + NVTE_CHECK(nvte_tensor_shape(act_out).data[1] * 2 == n, "weight rows (", n, + ") must be 2 * the act_out columns: SwiGLU"); + check_counts(recv_tokens_per_expert, num_experts, "recv_tokens_per_expert"); + check_row_floats(recv_topk_weights, rows, "recv_topk_weights"); + NVTE_CHECK(total_recv_tokens == nullptr || (nvte_tensor_numel(total_recv_tokens) == 1 && + nvte_tensor_type(total_recv_tokens) == kNVTEInt32), + "total_recv_tokens must be an int32 tensor of 1 element"); + const bool mxfp8 = is_mxfp8(tokens); + if (mxfp8) { + check_mxfp8(tokens, false, "tokens"); + check_mxfp8(recv_tokens, false, "recv_tokens"); + check_mxfp8(weight, true, "weight"); + check_mxfp8(act_out, true, "act_out"); + NVTE_CHECK(nvte_tensor_type(gemm_out) == kNVTEBFloat16, "gemm_out must be bfloat16"); + check_mxfp8_dims(hidden, n / 2, ctx.alignment()); + } else { + NVTE_CHECK( + nvte_tensor_type(tokens) == kNVTEBFloat16 && nvte_tensor_type(weight) == kNVTEBFloat16, + "tokens and weight must be bfloat16 or MXFP8"); + } + + // No count exchange before the dispatch when the dispatch writes the GEMM's expert layout. + const bool device_layout = dispatch_writes_layout(ctx, num_experts); + + // The GEMM runs beside the dispatch, and its tail on the SMs the dispatch frees. Both launches + // claim tiles from one workspace. + ep::WorkspaceLayout layout; + const ep::EpDispatchGemmConfig config{ + rows, hidden, n, num_experts, ctx.num_clusters, false, mxfp8, ctx.alignment(), device_layout}; + std::vector gemms{get_gemm(config, &layout)}; + const int tail_clusters = ctx.all_clusters - ctx.num_clusters; + if (tail_clusters > 0) { + gemms.push_back(get_tail_gemm(config, tail_clusters, layout, "the GEMM tail")); + } + GemmScratch scratch = claim_scratch(layout.total_bytes, num_experts, rows, false, 0); + set_signal_words(scratch, layout, rows / ctx.rows_per_slot()); + // The GEMM reads the expert row ends from the caller's [num_experts + 1] int32 + // expert_token_offsets (expert 0's start, then the per-expert ends); the dispatch writes them in + // capacity mode. + check_expert_token_offsets(expert_token_offsets, num_experts); + scratch.row_ends = static_cast(nvte_tensor_data(expert_token_offsets)) + 1; + + // The dispatch publishes one flag per slot of rows; the GEMM behind it consumes ready tiles. + TensorWrapper signal(scratch.signal, + std::vector{static_cast(scratch.signal_capacity)}, + DType::kInt32); + const ep::EpSignal dispatch_signal{signal.data(), ctx.rows_per_slot()}; + NVTECommWindow no_window{}; + // Expert offsets, then the row total: the expert row ends sit one word in. + int32_t* const expert_layout = scratch.row_ends - 1; + if (device_layout) { + // Flags, scheduler state and unset layout words, before the dispatch that sets them. The fused + // dispatch writes the GEMM's expert layout and the padded counts in one scan. + unset_row_ends(scratch); + ep::reset_workspace(scratch.reset, stream); + TensorWrapper offsets(expert_layout, std::vector{num_experts}, DType::kInt32); + TensorWrapper total(expert_layout + num_experts, std::vector{1}, DType::kInt32); + ctx.backend.prepare_and_dispatch(handle_mem, topk_idx, tokens, tokens_win, topk_weights, + no_window, recv_tokens, no_window, recv_topk_weights, + no_window, recv_tokens_per_expert, total.data(), ctx.layer_cfg, + stream, offsets.data(), &dispatch_signal); + } else { + // Capacity mode runs the routing prepare; eager mode ran nvte_ep_prepare already. + if (!ctx.eager) { + ctx.backend.prepare(handle_mem, topk_idx, recv_tokens_per_expert, total_recv_tokens, + ctx.layer_cfg, stream); + } + // Flags, scheduler state and expert row ends, before the dispatch that sets the flags. The + // reset also writes the caller offsets' leading 0 (expert 0's start), which the dispatch + // layout scan does not touch here. + set_row_ends(scratch, recv_tokens_per_expert, /*write_layout_base=*/true); + ep::reset_workspace(scratch.reset, stream); + ctx.backend.dispatch(handle_mem, topk_idx, tokens, tokens_win, topk_weights, no_window, + recv_tokens, no_window, recv_topk_weights, no_window, &ctx.layer_cfg, + stream, &dispatch_signal); + } + + // The GEMM is a programmatic dependent of the dispatch: tiles start as their flags are set. The + // tail is the GEMM's programmatic dependent. + auto a = wrap_rowwise(recv_tokens, true); + auto b = wrap_rowwise(weight, false); + auto c = wrap_rowwise(gemm_out, true); + auto d = wrap_rowwise(act_out, true); + auto row_ends = wrap_raw(scratch.row_ends, kNVTEInt32, num_experts); + auto alpha = wrap_raw(scratch.alpha, kNVTEFloat32, num_experts); + auto ws = wrap_raw(scratch.base, kNVTEByte, layout.total_bytes); + if (mxfp8) { + auto sfb = wrap_scales(weight, false, num_experts * n, hidden); + auto sfa = wrap_scales(recv_tokens, false, rows, hidden); + auto sfd = wrap_scales(act_out, false, rows, n / 2); + auto norm_const = wrap_raw(scratch.alpha, kNVTEFloat32, 1); + for (const tvm::ffi::Function& gemm : gemms) { + gemm(&a, &b, &sfb, &sfa, &c, &d, &sfd, &row_ends, &alpha, &norm_const, &ws, + static_cast(stream)); + } + } else { + auto prob = wrap_rowwise(recv_topk_weights, false); + for (const tvm::ffi::Function& gemm : gemms) { + gemm(&a, &b, &c, &d, &row_ends, &alpha, &prob, &ws, static_cast(stream)); + } + } +} + +void nvte_ep_combine_gemm_fwd(NVTETensor handle_mem, const NVTETensor gemm_in, + const NVTETensor recv_topk_weights, + const NVTETensor expert_token_offsets, const NVTETensor weight, + NVTETensor expert_out, NVTETensor result, + const NVTEEpLayerConfig* layer_cfg, cudaStream_t stream) { + using namespace transformer_engine; + const EpGemmContext ctx = make_context(layer_cfg, &NVTEEpGroupConfig::combine_fwd_num_sms); + checkCuDriverContext(stream); + + const size_t rows = nvte_tensor_shape(gemm_in).data[0]; + const size_t k = nvte_tensor_shape(gemm_in).data[1]; + const size_t num_experts = nvte_tensor_shape(weight).data[0]; + const size_t n = nvte_tensor_shape(weight).data[1]; + check_rows(ctx, rows, "gemm_in"); + NVTE_CHECK(nvte_tensor_shape(weight).data[2] == k, "weight columns must equal gemm_in columns"); + const bool mxfp8 = is_mxfp8(gemm_in); + if (mxfp8) { + check_mxfp8(gemm_in, true, "gemm_in"); + check_mxfp8(weight, true, "weight"); + check_mxfp8_dims(n, k, ctx.alignment()); + } else { + NVTE_CHECK( + nvte_tensor_type(gemm_in) == kNVTEBFloat16 && nvte_tensor_type(weight) == kNVTEBFloat16, + "gemm_in and weight must be bfloat16 or MXFP8"); + } + check_expert_token_offsets(expert_token_offsets, num_experts); + const bool weighted = recv_topk_weights != nullptr; + NVTE_CHECK(!weighted || (nvte_tensor_type(recv_topk_weights) == kNVTEFloat32 && + nvte_tensor_numel(recv_topk_weights) == rows), + "recv_topk_weights must be float32 [recv_capacity]"); + + // The GEMM launches on every SM; the clusters past those beside the combine stop after the + // first chunk and leave their SMs to the combine. + ep::WorkspaceLayout layout; + const ep::EpCombineGemmConfig config = combine_fwd_config(ctx, rows, k, n, num_experts, mxfp8); + const size_t n_chunk = config.n_chunk; + const tvm::ffi::Function gemm = get_gemm(config, &layout); + // Own workspace for the signal, scheduler and alpha words; the expert layout is the caller's + // expert_token_offsets the dispatch wrote. + GemmScratch scratch = claim_scratch(layout.total_bytes, num_experts, rows, !weighted, 0); + set_chunk_words(scratch, layout, n / n_chunk, layout.signal); + ep::reset_workspace(scratch.reset, stream); + + // The GEMM fills the flags; the combine is its programmatic dependent. + auto a = wrap_rowwise(gemm_in, true); + auto b = wrap_rowwise(weight, false); + auto d = wrap_rowwise(expert_out, true); + auto row_ends = wrap_raw(static_cast(nvte_tensor_data(expert_token_offsets)) + 1, + kNVTEInt32, num_experts); + auto alpha = wrap_raw(scratch.alpha, kNVTEFloat32, num_experts); + auto prob = weighted ? wrap_rowwise(recv_topk_weights, false) + : wrap_raw(scratch.ones, kNVTEFloat32, rows); + auto ws = wrap_raw(scratch.base, kNVTEByte, layout.total_bytes); + if (mxfp8) { + auto sfb = wrap_scales(weight, false, num_experts * n, k); + auto sfa = wrap_scales(gemm_in, false, rows, k); + gemm(&a, &b, &sfb, &sfa, &d, &row_ends, &alpha, &prob, &ws, static_cast(stream)); + } else { + gemm(&a, &b, &d, &row_ends, &alpha, &prob, &ws, static_cast(stream)); + } + + NVTECommWindow no_window{}; + TensorWrapper signal(scratch.base + layout.signal, std::vector{n / n_chunk}, + DType::kInt32); + const ep::EpSignal combine_signal{signal.data(), n_chunk}; + ctx.backend.combine(handle_mem, expert_out, no_window, result, &ctx.layer_cfg, stream, + &combine_signal); +} + +void nvte_ep_combine_gemm_bwd(NVTETensor handle_mem, const NVTETensor grad_result, + NVTECommWindow grad_result_win, const NVTETensor gemm_out, + const NVTETensor recv_topk_weights, + const NVTETensor recv_tokens_per_expert, const NVTETensor weight, + const NVTETensor wgrad_input, NVTETensor grad_expert_out, + NVTETensor grad_gemm_out, NVTETensor grad_recv_topk_weights, + NVTETensor act_out, NVTETensor weight_grad, + int accumulate_weight_grad, const NVTEEpLayerConfig* layer_cfg, + cudaStream_t stream) { + using namespace transformer_engine; + const EpGemmContext ctx = make_context(layer_cfg, &NVTEEpGroupConfig::dispatch_bwd_num_sms); + checkCuDriverContext(stream); + + const size_t rows = nvte_tensor_shape(grad_expert_out).data[0]; + const size_t hidden = nvte_tensor_shape(grad_expert_out).data[1]; + const size_t num_experts = nvte_tensor_shape(weight).data[0]; + const size_t ffn = nvte_tensor_shape(weight).data[2]; + check_rows(ctx, rows, "grad_expert_out"); + NVTE_CHECK(nvte_tensor_shape(weight).data[1] == hidden && + nvte_tensor_shape(gemm_out).data[1] == 2 * ffn && + nvte_tensor_shape(grad_gemm_out).data[1] == 2 * ffn, + "weight must be [experts, hidden, ffn] with 2 * ffn the gemm_out columns: SwiGLU"); + NVTE_CHECK(nvte_tensor_type(gemm_out) == kNVTEBFloat16, "gemm_out must be bfloat16"); + const bool mxfp8 = is_mxfp8(grad_result); + if (mxfp8) { + check_mxfp8(grad_result, false, "grad_result"); + check_mxfp8(grad_expert_out, false, "grad_expert_out"); + check_mxfp8(weight, true, "weight"); + check_mxfp8(grad_gemm_out, true, "grad_gemm_out"); + check_mxfp8(act_out, true, "act_out"); + NVTE_CHECK(nvte_tensor_shape(act_out).data[1] == ffn, "act_out must have ffn columns"); + check_mxfp8_dims(hidden, ffn, ctx.alignment()); + } else { + NVTE_CHECK( + nvte_tensor_type(grad_result) == kNVTEBFloat16 && nvte_tensor_type(weight) == kNVTEBFloat16, + "grad_result and weight must be bfloat16 or MXFP8"); + NVTE_CHECK(act_out == nullptr, "act_out is written for MXFP8 only"); + } + check_counts(recv_tokens_per_expert, num_experts, "recv_tokens_per_expert"); + check_row_floats(recv_topk_weights, rows, "recv_topk_weights"); + check_row_floats(grad_recv_topk_weights, rows, "grad_recv_topk_weights"); + + // The MXFP8 weight gradient reads the columnwise grad_expert_out that this call writes. + const bool requant = + mxfp8 && nvte_get_tensor_param(grad_expert_out, kNVTEColumnwiseData).data_ptr != nullptr; + NVTE_CHECK(!mxfp8 || weight_grad == nullptr || requant, + "the MXFP8 weight gradient needs the columnwise grad_expert_out"); + const std::optional wgrad = make_wgrad(ctx, grad_expert_out, wgrad_input, weight_grad, + accumulate_weight_grad, rows, num_experts, mxfp8); + + ep::WorkspaceLayout layout; + const ep::EpDispatchGemmConfig config{ + rows, hidden, ffn, num_experts, ctx.num_clusters, true, mxfp8, ctx.alignment()}; + const tvm::ffi::Function gemm = get_gemm(config, &layout); + GemmScratch scratch = claim_scratch(layout.total_bytes, num_experts, rows, false, + wgrad ? wgrad->layout.total_bytes : 0); + set_signal_words(scratch, layout, rows / ctx.rows_per_slot()); + if (wgrad) { + scratch.reset.fill(scratch.wgrad + wgrad->layout.counter, 1, ep::kFillZero); + } + // The GEMM adds each row's routing-weight gradient into grad_recv_topk_weights. + scratch.reset.fill(nvte_tensor_data(grad_recv_topk_weights), static_cast(rows), + ep::kFillZero); + set_row_ends(scratch, recv_tokens_per_expert); + ep::reset_workspace(scratch.reset, stream); + if (wgrad && needs_descriptors(*wgrad)) { + launch_wgrad(*wgrad, scratch, 1, false, 0, stream, true); + } + + // The reverse dispatch publishes one flag per slot; the GEMM consumes ready tiles. + TensorWrapper signal(scratch.signal, + std::vector{static_cast(scratch.signal_capacity)}, + DType::kInt32); + const ep::EpSignal dispatch_signal{signal.data(), ctx.rows_per_slot()}; + NVTECommWindow no_window{}; + ctx.backend.combine_bwd(handle_mem, grad_result, grad_result_win, grad_expert_out, no_window, + &ctx.layer_cfg, stream, &dispatch_signal); + + // FC2 data gradient with the SwiGLU backward in its epilogue. + auto a = wrap_rowwise(grad_expert_out, true); + auto c = wrap_rowwise(gemm_out, true); + auto d = wrap_rowwise(grad_gemm_out, true); + auto dprob = wrap_rowwise(grad_recv_topk_weights, false); + auto row_ends = wrap_raw(scratch.row_ends, kNVTEInt32, num_experts); + auto alpha = wrap_raw(scratch.alpha, kNVTEFloat32, num_experts); + auto prob = wrap_rowwise(recv_topk_weights, false); + auto ws = wrap_raw(scratch.base, kNVTEByte, layout.total_bytes); + if (mxfp8) { + // The columnwise forward weight; the epilogue also recomputes act_out columnwise. + auto b = wrap_columnwise(weight, false); + auto sfb = wrap_scales(weight, true, num_experts * hidden, ffn); + auto sfa = wrap_scales(grad_expert_out, false, rows, hidden); + auto d_col = wrap_columnwise(grad_gemm_out, true); + auto sfd_row = wrap_scales(grad_gemm_out, false, rows, 2 * ffn); + auto sfd_col = wrap_scales(grad_gemm_out, true, rows, 2 * ffn); + auto h = wrap_columnwise(act_out, true); + auto sfh_col = wrap_scales(act_out, true, rows, ffn); + auto norm_const = wrap_raw(scratch.alpha, kNVTEFloat32, 1); + gemm(&a, &b, &sfb, &sfa, &c, &d, &d_col, &sfd_row, &sfd_col, &h, &sfh_col, &dprob, &row_ends, + &alpha, &prob, &norm_const, &ws, static_cast(stream)); + // The columnwise grad_expert_out, a dependent of the GEMM, which starts it at entry: each + // tile waits for the reverse dispatch's flags of its rows. + if (requant) { + launch_requant(grad_expert_out, scratch, rows, hidden, num_experts, ctx.rows_per_slot(), + stream); + } + // Its tail on the SMs the reverse dispatch frees takes the tiles the GEMM has not claimed. + const int tail_clusters = ctx.all_clusters - ctx.num_clusters; + if (tail_clusters > 0) { + const tvm::ffi::Function tail = get_tail_gemm(config, tail_clusters, layout, "the dGLU tail"); + tail(&a, &b, &sfb, &sfa, &c, &d, &d_col, &sfd_row, &sfd_col, &h, &sfh_col, &dprob, &row_ends, + &alpha, &prob, &norm_const, &ws, static_cast(stream)); + } + // The weight gradient once the GEMM and the requant complete: its base, which then lets its + // tail in. + if (wgrad) { + launch_wgrad(*wgrad, scratch, ctx.num_clusters, true, 0, stream, false); + launch_wgrad(*wgrad, scratch, ctx.all_clusters - ctx.num_clusters, false, 0, stream, false); + } + return; + } + auto b = wrap_rowwise(weight, false); + gemm(&a, &b, &c, &d, &dprob, &row_ends, &alpha, &prob, &ws, static_cast(stream)); + // The weight gradient once every row is received: its tail on the SMs the reverse dispatch + // frees, its base on those the GEMM frees. + if (wgrad) { + launch_wgrad(*wgrad, scratch, ctx.all_clusters - ctx.num_clusters, false, ctx.rows_per_slot(), + stream, false); + launch_wgrad(*wgrad, scratch, ctx.num_clusters, false, ctx.rows_per_slot(), stream, false); + } +} + +void nvte_ep_dispatch_gemm_bwd(NVTETensor handle_mem, const NVTETensor grad_gemm_out, + const NVTETensor grad_recv_topk_weights, + const NVTETensor recv_tokens_per_expert, const NVTETensor weight, + const NVTETensor wgrad_input, NVTETensor grad_recv_tokens, + NVTETensor grad_tokens, NVTETensor grad_topk_weights, + NVTETensor weight_grad, int accumulate_weight_grad, + const NVTEEpLayerConfig* layer_cfg, cudaStream_t stream) { + using namespace transformer_engine; + const EpGemmContext ctx = make_context(layer_cfg, &NVTEEpGroupConfig::combine_bwd_num_sms); + checkCuDriverContext(stream); + + const size_t rows = nvte_tensor_shape(grad_gemm_out).data[0]; + const size_t k = nvte_tensor_shape(grad_gemm_out).data[1]; + const size_t hidden = nvte_tensor_shape(grad_recv_tokens).data[1]; + const size_t num_experts = nvte_tensor_shape(weight).data[0]; + check_rows(ctx, rows, "grad_gemm_out"); + NVTE_CHECK(nvte_tensor_shape(weight).data[1] == k && nvte_tensor_shape(weight).data[2] == hidden, + "weight must be [experts, 2 * ffn, hidden]"); + check_counts(recv_tokens_per_expert, num_experts, "recv_tokens_per_expert"); + check_row_floats(grad_recv_topk_weights, rows, "grad_recv_topk_weights"); + const bool mxfp8 = is_mxfp8(grad_gemm_out); + if (mxfp8) { + check_mxfp8(grad_gemm_out, true, "grad_gemm_out"); + check_mxfp8(weight, true, "weight"); + NVTE_CHECK(nvte_tensor_type(grad_recv_tokens) == kNVTEBFloat16, + "grad_recv_tokens must be bfloat16"); + check_mxfp8_dims(hidden, k / 2, ctx.alignment()); + } else { + NVTE_CHECK(nvte_tensor_type(grad_gemm_out) == kNVTEBFloat16 && + nvte_tensor_type(weight) == kNVTEBFloat16, + "grad_gemm_out and weight must be bfloat16 or MXFP8"); + } + const std::optional wgrad = make_wgrad(ctx, grad_gemm_out, wgrad_input, weight_grad, + accumulate_weight_grad, rows, num_experts, mxfp8); + // MXFP8: the call writes the columnwise wgrad_input from its rowwise data when that is set. + const bool requant = + wgrad && mxfp8 && nvte_get_tensor_param(wgrad_input, kNVTERowwiseData).data_ptr != nullptr; + // With the weight gradient the dispatch backward starts behind the weight gradient's base, which + // waits for the GEMM to complete: the GEMM publishes nothing and runs in the stock tile order, and + // the one flag of the whole row is reset to 0, published. + const bool plain = wgrad && ctx.num_clusters > 0; + const size_t n_chunk = plain ? 0 : combine_chunk(hidden); + const size_t chunk = plain ? hidden : n_chunk; + + // With the weight gradient the GEMM runs alone on every SM, and the weight gradient beside the + // dispatch backward. + ep::WorkspaceLayout layout; + const ep::EpCombineGemmConfig config{rows, + k, + hidden, + num_experts, + wgrad ? ctx.all_clusters : ctx.num_clusters, + n_chunk, + !plain && kCombineAtGemmEntry, + true, + mxfp8, + ctx.alignment()}; + const tvm::ffi::Function gemm = get_gemm(config, &layout); + GemmScratch scratch = claim_scratch(layout.total_bytes, num_experts, rows, true, + wgrad ? wgrad->layout.total_bytes : 0); + set_chunk_words(scratch, layout, plain ? 0 : hidden / chunk, + plain ? layout.total_bytes : layout.signal); + if (wgrad) { + scratch.reset.fill(scratch.wgrad + wgrad->layout.counter, 1, ep::kFillZero); + } + set_row_ends(scratch, recv_tokens_per_expert); + ep::reset_workspace(scratch.reset, stream); + if (wgrad && needs_descriptors(*wgrad)) { + launch_wgrad(*wgrad, scratch, 1, false, 0, stream, true); + } + // The columnwise wgrad_input, before the GEMM. + if (requant) { + launch_requant(wgrad_input, scratch, rows, hidden, num_experts, 0, stream); + } + + // The GEMM fills the flags; the dispatch backward is its programmatic dependent. + auto a = wrap_rowwise(grad_gemm_out, true); + auto d = wrap_rowwise(grad_recv_tokens, true); + auto row_ends = wrap_raw(scratch.row_ends, kNVTEInt32, num_experts); + auto alpha = wrap_raw(scratch.alpha, kNVTEFloat32, num_experts); + auto prob = wrap_raw(scratch.ones, kNVTEFloat32, rows); + auto ws = wrap_raw(scratch.base, kNVTEByte, layout.total_bytes); + if (mxfp8) { + // The columnwise forward weight. + auto b = wrap_columnwise(weight, false); + auto sfb = wrap_scales(weight, true, num_experts * k, hidden); + auto sfa = wrap_scales(grad_gemm_out, false, rows, k); + gemm(&a, &b, &sfb, &sfa, &d, &row_ends, &alpha, &prob, &ws, static_cast(stream)); + } else { + auto b = wrap_rowwise(weight, false); + gemm(&a, &b, &d, &row_ends, &alpha, &prob, &ws, static_cast(stream)); + } + // The weight gradient's base starts once the GEMM completes and lets the dispatch backward in. + if (wgrad) { + launch_wgrad(*wgrad, scratch, ctx.num_clusters, true, 0, stream, false); + } + + NVTECommWindow no_window{}; + TensorWrapper signal(scratch.base + layout.signal, std::vector{hidden / chunk}, + DType::kInt32); + const ep::EpSignal combine_signal{signal.data(), chunk}; + ctx.backend.dispatch_bwd(handle_mem, grad_recv_tokens, no_window, grad_recv_topk_weights, + no_window, grad_tokens, grad_topk_weights, &ctx.layer_cfg, stream, + &combine_signal); + // Its tail takes the tiles left on the communication SMs. + if (wgrad) { + launch_wgrad(*wgrad, scratch, ctx.all_clusters - ctx.num_clusters, false, 0, stream, false); + } +} + +} // extern "C" + +#else // !(NVTE_WITH_NCCL_EP && NVTE_WITH_CUTEDSL): throwing stubs. + +namespace { +[[noreturn]] void ep_gemm_not_built() { + NVTE_ERROR( + "The EP grouped GEMMs need TransformerEngine built with NVTE_WITH_NCCL_EP=1 and " + "NVTE_WITH_CUTEDSL=1."); +} +} // namespace + +namespace transformer_engine { +namespace ep { +void release_gemm_scratch() {} +} // namespace ep +} // namespace transformer_engine + +extern "C" { + +void nvte_ep_dispatch_gemm_fwd(NVTETensor, NVTETensor, NVTETensor, NVTECommWindow, NVTETensor, + NVTETensor, NVTETensor, NVTETensor, NVTETensor, NVTETensor, + NVTETensor, NVTETensor, NVTETensor, const NVTEEpLayerConfig*, + cudaStream_t) { + ep_gemm_not_built(); +} + +void nvte_ep_combine_gemm_fwd(NVTETensor, NVTETensor, NVTETensor, NVTETensor, NVTETensor, + NVTETensor, NVTETensor, const NVTEEpLayerConfig*, cudaStream_t) { + ep_gemm_not_built(); +} + +void nvte_ep_combine_gemm_bwd(NVTETensor, NVTETensor, NVTECommWindow, NVTETensor, NVTETensor, + NVTETensor, NVTETensor, NVTETensor, NVTETensor, NVTETensor, + NVTETensor, NVTETensor, NVTETensor, int, const NVTEEpLayerConfig*, + cudaStream_t) { + ep_gemm_not_built(); +} + +void nvte_ep_dispatch_gemm_bwd(NVTETensor, NVTETensor, NVTETensor, NVTETensor, NVTETensor, + NVTETensor, NVTETensor, NVTETensor, NVTETensor, NVTETensor, int, + const NVTEEpLayerConfig*, cudaStream_t) { + ep_gemm_not_built(); +} + +} // extern "C" + +#endif diff --git a/transformer_engine/common/ep/ep_gemm_config.h b/transformer_engine/common/ep/ep_gemm_config.h new file mode 100644 index 00000000000..989003fe475 --- /dev/null +++ b/transformer_engine/common/ep/ep_gemm_config.h @@ -0,0 +1,307 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +#ifndef TRANSFORMER_ENGINE_COMMON_EP_EP_GEMM_CONFIG_H_ +#define TRANSFORMER_ENGINE_COMMON_EP_EP_GEMM_CONFIG_H_ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "../tvm_ffi_bridge.h" +#include "../util/cuda_runtime.h" + +namespace transformer_engine { +namespace ep { + +// Signal-aware grouped GEMMs (CuTeDSL via TVM-FFI), keyed by configuration. The first +// get_kernel of a configuration compiles: call it at warmup, outside graph capture. + +// Byte offsets inside a GEMM workspace: total size, then the words the entry points reset. +struct WorkspaceLayout { + size_t total_bytes; + size_t counter; // dispatch GEMM, weight gradient: claim counter + size_t signal; // dispatch GEMM: tile flags; combine GEMM: column-chunk flags + size_t state; // dispatch GEMM: scheduler state +}; + +// Compiled functions and workspace layouts depend on the architecture of the current device. +inline std::string arch_key() { return "_sm" + std::to_string(cuda::sm_arch()); } + +// Cache keyed by a configuration string, read under a shared lock. Configurations carry sizes that +// do not fit the 32-bit id of tvm_ffi_bridge::TVMFFIConfigCache. +template +class KeyedCache { + public: + // Process lifetime, as TVMFFIConfigCache: nothing is destroyed at static teardown. + static KeyedCache &create() { return *new KeyedCache(); } + + // The value of key, from make() on a miss. A missing value is cached only with cache_missing. + template + std::optional get(const std::string &key, Make &&make, bool cache_missing = false) { + { + std::shared_lock lock(mutex_); + auto it = map_.find(key); + if (it != map_.end()) { + return it->second; + } + } + std::unique_lock lock(mutex_); + auto it = map_.find(key); + if (it != map_.end()) { + return it->second; + } + std::optional value = make(); + if (value.has_value() || cache_missing) { + map_.emplace(key, value); + } + return value; + } + + private: + KeyedCache() = default; + + std::shared_mutex mutex_; + std::unordered_map> map_; +}; + +// Cache key: the prefix, every argument of the configuration, and the device architecture. +template +std::string config_key(const char *prefix, const Args &args) { + std::string key = prefix; + std::apply([&](const auto &...value) { ((key += "_" + std::to_string(value)), ...); }, args); + return key + arch_key(); +} + +template +std::optional get_cached_kernel(const Config &config) { + tvm_ffi_bridge::TVMFFICentral ¢ral = tvm_ffi_bridge::TVMFFICentral::getInstance(); + if (!central.get_cutedsl_backend_enabled()) { + return central.load_tvm_ffi_function(config); + } + static auto &cache = KeyedCache::create(); + return cache.get( + config.to_key(), [&] { return central.load_tvm_ffi_function(config); }, + /*cache_missing=*/true); +} + +// Calls the global function `name` with the configuration's arguments, prefixed by `prefix`. +template +std::optional call_entry(const char *name, const Config &config, + const Prefix &...prefix) { + auto fn = tvm::ffi::Function::GetGlobal(name); + if (!fn.has_value()) { + return std::nullopt; + } + return std::apply([&](const auto &...args) { return (*fn)(prefix..., args...); }, config.args()); +} + +template +bool retrieve_from_python(const Config &config, const std::string &fn_name) { + auto result = call_entry(Config::kEntrypointName, config, tvm::ffi::String(fn_name)); + return result.has_value() && result->template try_cast().value_or(false); +} + +// The layout entry returns [total, counter, signal, state], [total, signal, entries] or +// [total, counter]. +template +std::optional query_layout_from_python(const Config &config) { + auto result = call_entry(Config::kLayoutName, config); + if (!result.has_value()) { + return std::nullopt; + } + auto offsets = result->template try_cast>(); + if (!offsets.has_value()) { + return std::nullopt; + } + const auto at = [&](size_t i) { return static_cast((*offsets)[i]); }; + if (offsets->size() == 4) { + return WorkspaceLayout{at(0), at(1), at(2), at(3)}; + } + if (offsets->size() == 3) { + return WorkspaceLayout{at(0), 0, at(1), 0}; + } + if (offsets->size() == 2) { + return WorkspaceLayout{at(0), at(1), 0, 0}; + } + return std::nullopt; +} + +// query_layout_from_python, kept per configuration once found. +template +std::optional query_layout(const Config &config) { + static auto &cache = KeyedCache::create(); + return cache.get(config.to_key(), [&] { return query_layout_from_python(config); }); +} + +// GEMM behind the dispatch: waits on the dispatch tile flags. +struct EpDispatchGemmConfig { + static constexpr const char *kEntrypointName = "get_ep_dispatch_gemm_function"; + static constexpr const char *kLayoutName = "get_ep_dispatch_gemm_workspace_layout"; + + size_t rows; // receive capacity + size_t k; // reduction size + size_t n; // GEMM output columns + size_t num_experts; + int num_clusters; // persistent clusters beside its NCCL EP call's SMs + bool backward; // the second GEMM's data gradient and SwiGLU backward instead of the first GEMM + bool mxfp8; // MXFP8 operands instead of BF16 + size_t rows_alignment; // rows per expert slab are a multiple of it + // Forward: the dispatch writes the expert offsets and the row total, the word before the row + // ends and the row ends, after the caller set them to -1. + bool device_layout = false; + + // In the parameter order of the get_ep_* function of CuTeDSL/ep_gemm/dispatch_gemm.py. + auto args() const { + return std::make_tuple(static_cast(rows), static_cast(k), + static_cast(n), static_cast(num_experts), num_clusters, + backward, mxfp8, static_cast(rows_alignment), device_layout); + } + + std::string to_key() const { return config_key("ep_dispatch_gemm", args()); } + + bool retrieve_func_from_python(const std::string &fn_name) const { + return retrieve_from_python(*this, fn_name); + } + + std::optional get_kernel() const { return get_cached_kernel(*this); } + + std::optional workspace_layout() const { return query_layout(*this); } +}; + +// GEMM in front of the combine: publishes finished column chunks of its output. +struct EpCombineGemmConfig { + static constexpr const char *kEntrypointName = "get_ep_combine_gemm_function"; + static constexpr const char *kLayoutName = "get_ep_combine_gemm_workspace_layout"; + + size_t rows; // receive capacity + size_t k; // reduction size + size_t n; // output columns + size_t num_experts; + int num_clusters; // persistent clusters beside the combine's SMs + size_t n_chunk; // columns per readiness flag + bool dependents_at_entry; + bool backward; // the data gradient of the first GEMM instead of the second GEMM + bool mxfp8; // MXFP8 operands instead of BF16 + size_t rows_alignment; // rows per expert slab are a multiple of it + // Clusters launched, 0: num_clusters. Those past num_clusters stop after the first chunk. + int grid_clusters = 0; + + // In the parameter order of the get_ep_* function of CuTeDSL/ep_gemm/combine_gemm.py. + auto args() const { + return std::make_tuple(static_cast(rows), static_cast(k), + static_cast(n), static_cast(num_experts), num_clusters, + static_cast(n_chunk), dependents_at_entry, backward, mxfp8, + static_cast(rows_alignment), grid_clusters); + } + + std::string to_key() const { return config_key("ep_combine_gemm", args()); } + + bool retrieve_func_from_python(const std::string &fn_name) const { + return retrieve_from_python(*this, fn_name); + } + + std::optional get_kernel() const { return get_cached_kernel(*this); } + + // total_bytes and the byte offset of signal[]; the other words are unused. + std::optional workspace_layout() const { return query_layout(*this); } +}; + +// Weight gradient chained behind a backward GEMM: a persistent grid that claims output tiles +// from the workspace counter, so launches sharing one workspace split the tiles. +struct EpWgradConfig { + static constexpr const char *kEntrypointName = "get_ep_wgrad_function"; + static constexpr const char *kLayoutName = "get_ep_wgrad_workspace_layout"; + static constexpr const char *kDescriptorsName = "get_ep_wgrad_reads_descriptors"; + + size_t rows; // receive capacity + size_t m; // rows of each expert's weight gradient + size_t n; // columns of each expert's weight gradient + size_t num_experts; + int num_clusters; // persistent clusters + bool fp32_out; // FP32 instead of BF16 weight gradient + bool accumulate; // add to the weight gradient instead of writing it + bool wait_primary_at_start; // wait for the kernel before it ahead of any load + size_t ready_slot_rows; // wait for every dispatch flag of this many rows, 0: none + // MXFP8 operands: the GEMM launches of some weight gradients (reads_descriptors) read the + // per-expert descriptors that one launch with descriptors_only writes into the workspace first. + bool mxfp8; + bool descriptors_only; + size_t rows_alignment; // rows per expert slab are a multiple of it + size_t routed_rows; // rows a rank receives at capacity under uniform routing, unpadded + + // In the parameter order of the get_ep_* function of CuTeDSL/ep_gemm/wgrad_gemm.py. + auto args() const { + return std::make_tuple(static_cast(rows), static_cast(m), + static_cast(n), static_cast(num_experts), num_clusters, + fp32_out, accumulate, wait_primary_at_start, + static_cast(ready_slot_rows), mxfp8, descriptors_only, + static_cast(rows_alignment), static_cast(routed_rows)); + } + + std::string to_key() const { return config_key("ep_wgrad", args()); } + + bool retrieve_func_from_python(const std::string &fn_name) const { + return retrieve_from_python(*this, fn_name); + } + + std::optional get_kernel() const { return get_cached_kernel(*this); } + + // total_bytes and the byte offset of the claim counter. + std::optional workspace_layout() const { return query_layout(*this); } + + // Whether the GEMM launches read the per-expert descriptors of a descriptors_only launch, kept + // per configuration once found. Without an answer, the added and FP32 ones read them. + bool reads_descriptors() const { + static auto &cache = KeyedCache::create(); + return cache + .get(to_key(), + [&]() -> std::optional { + auto result = call_entry(kDescriptorsName, *this); + return result.has_value() ? result->template try_cast() : std::nullopt; + }) + .value_or(mxfp8 && (accumulate || fp32_out)); + } +}; + +// Rowwise to columnwise MXFP8 requant of received rows for a weight gradient: one CTA per tile of +// the receive capacity. +struct EpRequantConfig { + static constexpr const char *kEntrypointName = "get_ep_requant_function"; + + size_t rows; // receive capacity + size_t cols; + size_t num_experts; + size_t ready_slot_rows; // wait for the dispatch flags of a tile's rows, this many each; 0: none + + // In the parameter order of the get_ep_* function of CuTeDSL/ep_gemm/requantize.py. + auto args() const { + return std::make_tuple(static_cast(rows), static_cast(cols), + static_cast(num_experts), + static_cast(ready_slot_rows)); + } + + std::string to_key() const { return config_key("ep_requant", args()); } + + bool retrieve_func_from_python(const std::string &fn_name) const { + return retrieve_from_python(*this, fn_name); + } + + std::optional get_kernel() const { return get_cached_kernel(*this); } +}; + +} // namespace ep +} // namespace transformer_engine + +#endif // TRANSFORMER_ENGINE_COMMON_EP_EP_GEMM_CONFIG_H_ diff --git a/transformer_engine/common/ep/ep_overlap_workspace.cu b/transformer_engine/common/ep/ep_overlap_workspace.cu new file mode 100644 index 00000000000..02f38f3f249 --- /dev/null +++ b/transformer_engine/common/ep/ep_overlap_workspace.cu @@ -0,0 +1,75 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +#include +#include + +#include "../util/logging.h" +#include "ep_overlap_workspace.h" + +namespace transformer_engine { +namespace ep { + +namespace { + +constexpr int kThreads = 1024; + +__global__ void reset_workspace_kernel(WorkspaceReset reset) { + const int tid = threadIdx.x; + const int index = blockIdx.x * kThreads + tid; + const int stride = gridDim.x * kThreads; +#pragma unroll + for (int f = 0; f < WorkspaceReset::kMaxFills; ++f) { + if (f < reset.num_fills) { + for (int i = index; i < reset.fills[f].count; i += stride) { + reset.fills[f].words[i] = reset.fills[f].value; + } + } + } + if (blockIdx.x != 0 || reset.counts == nullptr) { + return; + } + // Inclusive prefix sum, kThreads experts per step. + using BlockScan = cub::BlockScan; + __shared__ typename BlockScan::TempStorage scan_storage; + int32_t running = 0; + for (int first = 0; first < reset.num_experts; first += kThreads) { + const int e = first + tid; + const int32_t count = e < reset.num_experts ? reset.counts[e] : 0; + int32_t row_end = 0; + int32_t step_total = 0; + BlockScan(scan_storage).InclusiveSum(count, row_end, step_total); + if (e < reset.num_experts) { + reset.row_ends[e] = running + row_end; + } + running += step_total; + __syncthreads(); + } +} + +} // namespace + +void WorkspaceReset::fill(void* words, int count, int32_t value) { + if (count == 0) { + return; + } + NVTE_CHECK(num_fills < kMaxFills, "too many workspace fills"); + fills[num_fills++] = {static_cast(words), count, value}; +} + +void reset_workspace(const WorkspaceReset& reset, cudaStream_t stream) { + // One block per kThreads words of the largest fill. + int most_words = 1; + for (int f = 0; f < reset.num_fills; ++f) { + most_words = std::max(most_words, reset.fills[f].count); + } + const int blocks = (most_words + kThreads - 1) / kThreads; + reset_workspace_kernel<<>>(reset); + NVTE_CHECK_CUDA(cudaGetLastError()); +} + +} // namespace ep +} // namespace transformer_engine diff --git a/transformer_engine/common/ep/ep_overlap_workspace.h b/transformer_engine/common/ep/ep_overlap_workspace.h new file mode 100644 index 00000000000..820cb35ef3e --- /dev/null +++ b/transformer_engine/common/ep/ep_overlap_workspace.h @@ -0,0 +1,47 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +#ifndef TRANSFORMER_ENGINE_COMMON_EP_EP_OVERLAP_WORKSPACE_H_ +#define TRANSFORMER_ENGINE_COMMON_EP_EP_OVERLAP_WORKSPACE_H_ + +#include +#include +#include + +namespace transformer_engine { +namespace ep { + +// Bit patterns of the words a reset fills with: int32 0 and -1, float 0.0f and 1.0f. +constexpr int32_t kFillZero = 0; +constexpr int32_t kFillMinusOne = -1; +constexpr int32_t kFillOne = 0x3F800000; + +// The words one launch writes before a call: constant fills and, optionally, the expert row ends +// as the inclusive prefix sums of the padded int32 counts. +struct WorkspaceReset { + static constexpr int kMaxFills = 8; + struct Fill { + int32_t* words; + int count; + int32_t value; + }; + Fill fills[kMaxFills]; + int num_fills = 0; + const int32_t* counts = nullptr; // [num_experts] padded counts; null: no row ends + int32_t* row_ends = nullptr; // [num_experts] row ends + int num_experts = 0; + + // Fill count words from words with value (a kFill* pattern); a zero count adds nothing. + void fill(void* words, int count, int32_t value); +}; + +// One launch performing every fill and the row-ends scan of reset. +void reset_workspace(const WorkspaceReset& reset, cudaStream_t stream); + +} // namespace ep +} // namespace transformer_engine + +#endif // TRANSFORMER_ENGINE_COMMON_EP_EP_OVERLAP_WORKSPACE_H_ diff --git a/transformer_engine/common/ep/nccl_ep_provider.cpp b/transformer_engine/common/ep/nccl_ep_provider.cpp index 4e4aa42599c..fa8d069f9a3 100644 --- a/transformer_engine/common/ep/nccl_ep_provider.cpp +++ b/transformer_engine/common/ep/nccl_ep_provider.cpp @@ -216,6 +216,12 @@ void* open_nccl_ep_library() { return nullptr; // Unreachable. } +// Set once in library_handle to the loaded runtime's NCCL_EP_VERSION_CODE. +int& runtime_version_storage() { + static int version = 0; + return version; +} + void* library_handle() { // Deliberately keep libnccl_ep loaded until process exit. EPBackend owns // objects whose implementation lives in this library. @@ -230,6 +236,7 @@ void* library_handle() { NVTE_CHECK(runtime_version / 10000 == NCCL_EP_MAJOR, "Incompatible NCCL EP library major version ", runtime_version / 10000, "; expected ", NCCL_EP_MAJOR); + runtime_version_storage() = runtime_version; const std::filesystem::path library_path = shared_library_path(reinterpret_cast(get_version)); NVTE_CHECK(!library_path.empty(), "Could not determine the NCCL EP library path"); @@ -249,6 +256,11 @@ void initialize() { (void)library_handle(); } void* get_symbol(const char* symbol) { return load_symbol(library_handle(), symbol); } +int runtime_version() { + (void)library_handle(); + return runtime_version_storage(); +} + namespace { template diff --git a/transformer_engine/common/ep/nccl_ep_provider.h b/transformer_engine/common/ep/nccl_ep_provider.h index 1d64d2e573e..44e0e4186b6 100644 --- a/transformer_engine/common/ep/nccl_ep_provider.h +++ b/transformer_engine/common/ep/nccl_ep_provider.h @@ -20,6 +20,9 @@ void initialize(); /*! \brief Get a function pointer from the runtime NCCL EP library. */ void* get_symbol(const char* symbol); +/*! \brief NCCL_EP_VERSION_CODE of the loaded runtime library (loads it if needed). */ +int runtime_version(); + ncclResult_t create_group(ncclEpGroup_t* ep_group, ncclComm_t comm, const ncclEpGroupConfig_t* config); ncclResult_t group_destroy(ncclEpGroup_t ep_group); diff --git a/transformer_engine/common/include/transformer_engine/ep.h b/transformer_engine/common/include/transformer_engine/ep.h index 1cd432a3a0c..9b996ad58b7 100644 --- a/transformer_engine/common/include/transformer_engine/ep.h +++ b/transformer_engine/common/include/transformer_engine/ep.h @@ -56,7 +56,8 @@ typedef struct { int max_recv_tokens_per_rank; /*! Token hidden dimension. */ int hidden_dim; - /*! Max SMs for NCCL EP dispatch/combine kernels. 0 = default (32, clamped to device SM count). */ + /*! SMs for NCCL EP dispatch/combine kernels where no per-direction count below is set. + * 0 = default (32, clamped to device SM count). */ int num_comm_sms; /*! Widest token dtype the group will dispatch; sizes staging buffers. * Required (no default): must be set to a real token dtype. Per-dispatch @@ -72,6 +73,21 @@ typedef struct { /*! Recv overflow policy. Nonzero drops tokens past max_recv_tokens_per_rank * and continues; 0 (default) traps. Not supported in eager mode. */ int drop_on_overflow; + /* SMs of the NCCL EP dispatch and combine kernels per pass direction, for every EP op of the + * group. 0 (default) inherits: a forward count from num_comm_sms, a backward count from the + * forward count of the same kind. Each ep_gemm.h call runs its GEMM on the SMs its NCCL EP call + * leaves. */ + /*! Forward dispatch: nvte_ep_dispatch(_v2), nvte_ep_prepare_and_dispatch, + * nvte_ep_dispatch_gemm_fwd. */ + int dispatch_fwd_num_sms; + /*! Forward combine: nvte_ep_combine(_v2), nvte_ep_combine_gemm_fwd. */ + int combine_fwd_num_sms; + /*! Backward dispatch, which carries the combine's gradient: nvte_ep_combine_bwd(_v2), + * nvte_ep_combine_gemm_bwd. */ + int dispatch_bwd_num_sms; + /*! Backward combine, which carries the dispatch's gradient: nvte_ep_dispatch_bwd(_v2), + * nvte_ep_dispatch_gemm_bwd. */ + int combine_bwd_num_sms; } NVTEEpGroupConfig; /*! \brief Per-layer configuration consumed by nvte_ep_handle_mem_size, diff --git a/transformer_engine/common/include/transformer_engine/ep_gemm.h b/transformer_engine/common/include/transformer_engine/ep_gemm.h new file mode 100644 index 00000000000..927b36a91a4 --- /dev/null +++ b/transformer_engine/common/include/transformer_engine/ep_gemm.h @@ -0,0 +1,223 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +/*! \file ep_gemm.h + * \brief Expert-parallel grouped GEMMs overlapped with the NCCL EP dispatch and combine. + * + * Each call overlaps the grouped GEMMs of one expert-MLP stage with an NCCL EP call: the EP call + * signals its progress in chunks and the GEMM follows chunk by chunk. The backward calls also + * run the stage's weight-gradient GEMM, weight_grad[e] = grad^T wgrad_input over the rows of + * expert e, beside the EP call when weight_grad is given (null skips it). + * + * entry GEMMs EP call SMs of the EP call + * nvte_ep_dispatch_gemm_fwd FC1 + SwiGLU dispatch dispatch_fwd_num_sms + * [dispatch on its SMs] || [FC1 + SwiGLU on the rest, tile by tile as rows arrive] + * nvte_ep_combine_gemm_fwd FC2 combine combine_fwd_num_sms + * [FC2 on the rest, chunk by chunk] || [combine on its SMs] + * nvte_ep_combine_gemm_bwd FC2 dgrad + SwiGLU bwd, wgrad dispatch bwd dispatch_bwd_num_sms + * [FC2 dgrad + SwiGLU bwd (+ wgrad) on the rest] || [dispatch bwd on its SMs] + * nvte_ep_dispatch_gemm_bwd FC1 dgrad, wgrad combine bwd combine_bwd_num_sms + * [FC1 dgrad (+ wgrad) on the rest] || [combine bwd on its SMs] + * + * The SM counts are NVTEEpGroupConfig fields. A GEMM runs on the SMs its EP call leaves and, in + * its tail, on the SMs the EP call frees as it ends. A combine readiness flag spans + * NVTE_EP_OVERLAP_COMBINE_CHUNK_COLS hidden columns (hidden_dim or a multiple of 1024 dividing it; + * default 1024 when it divides hidden_dim, else the whole row); smaller chunks let the combine + * start earlier but cost more flags. + * + * Requirements: SM 10.0, 10.3 or 10.7, EP within one NVLink domain, num_comm_sms > 0, zero_copy + * off, dispatch_output_per_expert_alignment 256, 128 (not MXFP8 on SM 10.7) or (bfloat16 only) + * 64, and NVTE_ENABLE_CUTEDSL_BACKEND=1 with NVTE_MOE_OVERLAP. MXFP8 needs hidden_dim a multiple + * of 512 and ffn_dim a multiple of 256. The first call per shape compiles the GEMM and allocates + * scratch, freed by nvte_ep_shutdown: make it at warmup, outside CUDA graph capture. Issue the calls of one EP group on one + * stream, in the same order on every rank. A call throws when a requirement does not hold. + * + * An input with a symm-mem window (tokens_win, grad_result_win) is read in place by the peers: + * every rank passes a window, at the same data and scale offsets, or none, and the input is not + * written before the call's dispatch completes, then only by a launch without programmatic + * dependency on the stream. + * + * Row-sized tensors have recv_capacity rows, a multiple of the alignment. Under MXFP8, tokens and + * received rows travel with compact rowwise scales, weights and GEMM inputs carry GEMM-swizzled + * scales and the backward reads the weights and wgrad operands columnwise. + */ + +#ifndef TRANSFORMER_ENGINE_EP_GEMM_H_ +#define TRANSFORMER_ENGINE_EP_GEMM_H_ + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/*! \brief Dispatch overlapped with the first grouped GEMM and SwiGLU. + * + * Per expert slab e: gemm_out = recv_tokens @ weight[e]^T and + * act_out = silu(gate) * up * recv_topk_weights, gate and up being alternating 32-column blocks of + * gemm_out. MXFP8 tokens: tokens, recv_tokens, weight and act_out are MXFP8 and act_out is + * silu(gate) * up, the routing weights left to nvte_ep_combine_gemm_fwd. + * + * \param[in,out] handle_mem uint8 routing-state buffer. + * \param[in] topk_idx [T, top_k] int32 or int64 routing indices. + * \param[in] tokens [T, hidden_dim] input tokens. + * \param[in] tokens_win Optional symm-mem window for tokens (MXFP8: and, in + * scale_window, for its scales); NVTECommWindow{} stages + * them. + * \param[in] topk_weights [T, top_k] float32 routing weights. + * \param[in] weight [num_local_experts, 2 * ffn_dim, hidden_dim] weights. + * \param[out] recv_tokens [recv_capacity, hidden_dim] received tokens. + * \param[out] recv_topk_weights [recv_capacity] float32 per-row routing weights. + * \param[out] gemm_out [recv_capacity, 2 * ffn_dim] GEMM output. + * \param[out] act_out [recv_capacity, ffn_dim] SwiGLU output. + * \param[in,out] recv_tokens_per_expert [num_local_experts] int32 padded counts. Eager mode + * provides them to build the expert layout. Capacity mode + * writes them: the dispatch without drops and with at most + * 2048 local experts, otherwise the routing prepare. + * \param[out] expert_token_offsets [num_local_experts + 1] int32 expert layout in tokens + * (rows): expert 0's start (0), then each expert's padded + * row end, the GEMM operand. expert_token_offsets[1:] must + * start 16-byte aligned, so front-pad the allocation by 3 + * int32 elements. Capacity mode writes it; eager mode builds + * it from recv_tokens_per_expert. The last element is the + * recv total; the per-expert counts are its diffs. + * \param[out] total_recv_tokens [1] int32 recv total before any overflow drop. The routing + * prepare writes it; null when the dispatch writes + * expert_token_offsets, where expert_token_offsets[-1] is + * the padded total of the kept rows. + * \param[in] layer_cfg Per-layer EP configuration. + * \param[in] stream CUDA stream. + */ +void nvte_ep_dispatch_gemm_fwd(NVTETensor handle_mem, const NVTETensor topk_idx, + const NVTETensor tokens, NVTECommWindow tokens_win, + const NVTETensor topk_weights, const NVTETensor weight, + NVTETensor recv_tokens, NVTETensor recv_topk_weights, + NVTETensor gemm_out, NVTETensor act_out, + NVTETensor recv_tokens_per_expert, NVTETensor expert_token_offsets, + NVTETensor total_recv_tokens, const NVTEEpLayerConfig* layer_cfg, + cudaStream_t stream); + +/*! \brief Second grouped GEMM overlapped with the combine. + * + * Per expert slab e: expert_out = (gemm_in @ weight[e]^T) * recv_topk_weights; result is the + * unweighted combine of expert_out. BF16 applies the routing weights in the dispatch GEMM's + * activation, so recv_topk_weights is null here; MXFP8 leaves them off the activation and applies + * them to this FC2 output. + * + * \param[in,out] handle_mem uint8 routing-state buffer. + * \param[in] gemm_in [recv_capacity, ffn_dim] act_out of + * nvte_ep_dispatch_gemm_fwd, bfloat16 or MXFP8. + * \param[in] recv_topk_weights [recv_capacity] float32 per-row routing weights; null for + * unit weights (bfloat16 act_out carries them). + * \param[in] expert_token_offsets [num_local_experts + 1] int32 expert layout from + * nvte_ep_dispatch_gemm_fwd (the GEMM reads its row ends); + * expert_token_offsets[1:] must start 16-byte aligned. + * \param[in] weight [num_local_experts, hidden_dim, ffn_dim] weights. + * \param[out] expert_out [recv_capacity, hidden_dim] GEMM output. + * \param[out] result [T, hidden_dim] combined output. + * \param[in] layer_cfg Per-layer EP configuration. + * \param[in] stream CUDA stream. + */ +void nvte_ep_combine_gemm_fwd(NVTETensor handle_mem, const NVTETensor gemm_in, + const NVTETensor recv_topk_weights, + const NVTETensor expert_token_offsets, const NVTETensor weight, + NVTETensor expert_out, NVTETensor result, + const NVTEEpLayerConfig* layer_cfg, cudaStream_t stream); + +/*! \brief Backward of nvte_ep_combine_gemm_fwd and of the SwiGLU: the combine backward + * overlapped with the second GEMM's data gradient, SwiGLU backward in its epilogue. + * + * The combine backward scatters grad_result to the expert rows as grad_expert_out (dy). SwiGLU + * backward over gemm_out = [gate | up] (gate and up in alternating 32-column blocks) gives + * grad_gemm_out, scaled by recv_topk_weights (MXFP8) or carrying it from the activation (BF16). + * Over expert e's rows, with W2 its forward weight and wgrad_input the route-weighted activation: + * + * grad_recv_topk_weights = sum((dy @ W2) * (silu(gate) * up), over ffn) one value per row + * weight_grad[e] = dy^T @ wgrad_input [hidden_dim, ffn_dim] + * + * grad_recv_topk_weights uses the unweighted activation silu(gate) * up. weight_grad is written + * only when non-null; accumulate_weight_grad adds to it instead of overwriting it. + * + * \param[in,out] handle_mem uint8 routing-state buffer. + * \param[in] grad_result [T, hidden_dim] gradient of result. + * \param[in] grad_result_win Optional symm-mem window for grad_result (MXFP8: and, + * in scale_window, for its scales); NVTECommWindow{} + * stages them. + * \param[in] gemm_out [recv_capacity, 2 * ffn_dim] gemm_out of + * nvte_ep_dispatch_gemm_fwd. + * \param[in] recv_topk_weights [recv_capacity] float32 per-row routing weights. + * \param[in] recv_tokens_per_expert [num_local_experts] int32 padded counts. + * \param[in] weight [num_local_experts, hidden_dim, ffn_dim] forward weights. + * \param[in] wgrad_input [recv_capacity, ffn_dim] bfloat16 act_out of + * nvte_ep_dispatch_gemm_fwd, or MXFP8 act_out of this + * call; with weight_grad only. + * \param[out] grad_expert_out [recv_capacity, hidden_dim] received gradient of + * expert_out; MXFP8: rowwise with compact scales and, when + * its columnwise data is set, also columnwise, its scales + * GEMM-swizzled per expert slab, for the weight gradient. + * \param[out] grad_gemm_out [recv_capacity, 2 * ffn_dim] gradient of gemm_out; MXFP8: + * rowwise and columnwise. + * \param[out] grad_recv_topk_weights [recv_capacity] float32 gradient of recv_topk_weights. + * \param[out] act_out [recv_capacity, ffn_dim] columnwise MXFP8 + * recv_topk_weights * silu(gate) * up, the input of the + * second GEMM's weight gradient; MXFP8 only, else null. + * \param[in,out] weight_grad [num_local_experts, hidden_dim, ffn_dim] bfloat16 or + * float32 gradient of weight, or null. + * \param[in] accumulate_weight_grad Add to weight_grad instead of writing it. + * \param[in] layer_cfg Per-layer EP configuration. + * \param[in] stream CUDA stream. + */ +void nvte_ep_combine_gemm_bwd(NVTETensor handle_mem, const NVTETensor grad_result, + NVTECommWindow grad_result_win, const NVTETensor gemm_out, + const NVTETensor recv_topk_weights, + const NVTETensor recv_tokens_per_expert, const NVTETensor weight, + const NVTETensor wgrad_input, NVTETensor grad_expert_out, + NVTETensor grad_gemm_out, NVTETensor grad_recv_topk_weights, + NVTETensor act_out, NVTETensor weight_grad, + int accumulate_weight_grad, const NVTEEpLayerConfig* layer_cfg, + cudaStream_t stream); + +/*! \brief Backward of nvte_ep_dispatch_gemm_fwd: the first GEMM's data gradient overlapped with + * the dispatch backward. + * + * \param[in,out] handle_mem uint8 routing-state buffer. + * \param[in] grad_gemm_out [recv_capacity, 2 * ffn_dim] from + * nvte_ep_combine_gemm_bwd; MXFP8 with weight_grad: also + * columnwise. + * \param[in] grad_recv_topk_weights [recv_capacity] float32 from nvte_ep_combine_gemm_bwd. + * \param[in] recv_tokens_per_expert [num_local_experts] int32 padded counts. + * \param[in] weight [num_local_experts, 2 * ffn_dim, hidden_dim] forward + * weights. + * \param[in,out] wgrad_input [recv_capacity, hidden_dim] recv_tokens of + * nvte_ep_dispatch_gemm_fwd, bfloat16 or columnwise + * MXFP8; with weight_grad only. MXFP8 with its rowwise + * data and compact scales also set: the call writes the + * columnwise data and scales from them. + * \param[out] grad_recv_tokens [recv_capacity, hidden_dim] gradient of recv_tokens. + * \param[out] grad_tokens [T, hidden_dim] gradient of tokens. + * \param[out] grad_topk_weights [T, top_k] float32 gradient of topk_weights. + * \param[in,out] weight_grad [num_local_experts, 2 * ffn_dim, hidden_dim] bfloat16 or + * float32 gradient of weight, or null. + * \param[in] accumulate_weight_grad Add to weight_grad instead of writing it. + * \param[in] layer_cfg Per-layer EP configuration. + * \param[in] stream CUDA stream. + */ +void nvte_ep_dispatch_gemm_bwd(NVTETensor handle_mem, const NVTETensor grad_gemm_out, + const NVTETensor grad_recv_topk_weights, + const NVTETensor recv_tokens_per_expert, const NVTETensor weight, + const NVTETensor wgrad_input, NVTETensor grad_recv_tokens, + NVTETensor grad_tokens, NVTETensor grad_topk_weights, + NVTETensor weight_grad, int accumulate_weight_grad, + const NVTEEpLayerConfig* layer_cfg, cudaStream_t stream); + +#ifdef __cplusplus +} +#endif + +#endif // TRANSFORMER_ENGINE_EP_GEMM_H_ diff --git a/transformer_engine/common/tvm_ffi_bridge.h b/transformer_engine/common/tvm_ffi_bridge.h index 1c2e7fffe6a..7470b2b5890 100644 --- a/transformer_engine/common/tvm_ffi_bridge.h +++ b/transformer_engine/common/tvm_ffi_bridge.h @@ -507,4 +507,10 @@ extern "C" inline void TVMFFIErrorMoveFromRaised(TVMFFIObjectHandle *result) { transformer_engine::tvm_ffi_bridge::TVMFFICentral::getInstance().move_error(result); } +// TVM-FFI error construction (e.g. in try_cast) references this symbol; backtraces stay empty. +extern "C" inline const TVMFFIByteArray *TVMFFIBacktrace(const char *, int, const char *, int) { + static const TVMFFIByteArray empty{"", 0}; + return ∅ +} + #endif // TRANSFORMER_ENGINE_COMMON_TVM_FFI_BRIDGE_H_ diff --git a/transformer_engine/common/util/cuda_runtime.h b/transformer_engine/common/util/cuda_runtime.h index 02f95c08f1e..9ef9e334d03 100644 --- a/transformer_engine/common/util/cuda_runtime.h +++ b/transformer_engine/common/util/cuda_runtime.h @@ -38,6 +38,9 @@ int sm_arch(int device_id = -1); */ int sm_count(int device_id = -1); +/* \brief Clusters of cluster_size CTAs, one CTA per SM, the current device runs at once */ +int max_active_clusters(int cluster_size); + /* \brief Maximum shared memory per block available with opt-in * * This is the upper bound for the static plus dynamic shared memory diff --git a/transformer_engine/common/util/utils.cu b/transformer_engine/common/util/utils.cu index 39d82624632..3f39423592e 100644 --- a/transformer_engine/common/util/utils.cu +++ b/transformer_engine/common/util/utils.cu @@ -9,8 +9,11 @@ #include #include +#include +#include #include "../common.h" +#include "../util/cuda_runtime.h" #include "../util/logging.h" namespace transformer_engine { @@ -80,3 +83,51 @@ void nvte_convert_pointers_to_tensor(const uint64_t *host_ptrs, NVTETensor outpu nvte_copy_host_to_device_via_kernel(host_ptrs, out_tensor->data.dptr, static_cast(count) * sizeof(uint64_t), stream); } + +namespace transformer_engine { + +namespace { + +// Occupancy probe for cuda::max_active_clusters: empty on purpose, only its launch bounds and +// cluster dim matter. +__global__ void cluster_probe_kernel() {} + +} // namespace + +namespace cuda { + +int max_active_clusters(int cluster_size) { + static std::mutex mutex; + // Occupancy depends on the device and the cluster size, so the cache keys on both. + static std::unordered_map clusters_by_device_and_size; + const int device = current_device(); + const int64_t key = (static_cast(device) << 32) | static_cast(cluster_size); + std::lock_guard lock(mutex); + auto it = clusters_by_device_and_size.find(key); + if (it == clusters_by_device_and_size.end()) { + // All the shared memory a CTA may take: one CTA per SM. + int smem = 0; + NVTE_CHECK_CUDA(cudaDeviceGetAttribute(&smem, cudaDevAttrMaxSharedMemoryPerBlockOptin, device)); + NVTE_CHECK_CUDA(cudaFuncSetAttribute(cluster_probe_kernel, + cudaFuncAttributeMaxDynamicSharedMemorySize, smem)); + cudaLaunchAttribute attr{}; + attr.id = cudaLaunchAttributeClusterDimension; + attr.val.clusterDim.x = cluster_size; + attr.val.clusterDim.y = 1; + attr.val.clusterDim.z = 1; + cudaLaunchConfig_t config{}; + config.gridDim = dim3(cluster_size, 1, 1); + config.blockDim = dim3(128, 1, 1); + config.dynamicSmemBytes = smem; + config.attrs = &attr; + config.numAttrs = 1; + int clusters = 0; + NVTE_CHECK_CUDA(cudaOccupancyMaxActiveClusters(&clusters, cluster_probe_kernel, &config)); + it = clusters_by_device_and_size.emplace(key, clusters).first; + } + return it->second; +} + +} // namespace cuda + +} // namespace transformer_engine diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index 02bf6773442..56594101603 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -770,7 +770,9 @@ int64_t get_nccl_comm_ptr(c10d::ProcessGroup *process_group); void ep_initialize(uintptr_t comm_ptr, const std::string &group_name, int64_t num_experts, int64_t max_tokens_per_rank, int64_t max_recv_tokens_per_rank, int64_t hidden_dim, int64_t max_num_sms, pybind11::object max_token_dtype, - bool zero_copy, int64_t num_topk, bool drop_on_overflow); + bool zero_copy, int64_t num_topk, bool drop_on_overflow, + int64_t dispatch_fwd_num_sms, int64_t combine_fwd_num_sms, + int64_t dispatch_bwd_num_sms, int64_t combine_bwd_num_sms); void ep_finalize(); diff --git a/transformer_engine/pytorch/csrc/extensions/ep.cpp b/transformer_engine/pytorch/csrc/extensions/ep.cpp index 27bc7e1d418..4a3b6317456 100644 --- a/transformer_engine/pytorch/csrc/extensions/ep.cpp +++ b/transformer_engine/pytorch/csrc/extensions/ep.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include "transformer_engine/comm_window.h" +#include "transformer_engine/ep_gemm.h" // torch's NCCL symm-mem headers (zero-copy path) exist only since torch 2.11; // without them NCCL_HAS_SYMMEM_SUPPORT stays undefined and zero-copy is compiled out. @@ -74,13 +76,10 @@ size_t symm_mem_window_offset(T* sm, ...) { } #endif -// Resolve ``t`` to an NCCL symm-mem window for the zero-copy one-sided path. -// Returns ``kNoWindow`` when symm-mem support isn't compiled in, zero-copy is -// disabled, no group is set, or ``t`` isn't symm-mem-backed; callers pass the -// resulting window unconditionally to the backend. -NVTECommWindow maybe_make_window(const at::Tensor& t) { +// Resolve ``t`` to its NCCL symm-mem window. Returns ``kNoWindow`` when symm-mem +// support isn't compiled in, no group is set, or ``t`` isn't symm-mem-backed. +NVTECommWindow make_window(const at::Tensor& t) { #ifdef NCCL_HAS_SYMMEM_SUPPORT - if (!g_zero_copy_enabled.load(std::memory_order_relaxed)) return kNoWindow; if (g_ep_group_name.empty()) return kNoWindow; c10::intrusive_ptr sm; try { @@ -106,6 +105,29 @@ NVTECommWindow maybe_make_window(const at::Tensor& t) { #endif } +// Resolve ``t`` to an NCCL symm-mem window for the zero-copy one-sided path. +// Returns ``kNoWindow`` when zero-copy is disabled or make_window finds none; +// callers pass the resulting window unconditionally to the backend. +NVTECommWindow maybe_make_window(const at::Tensor& t) { + if (!g_zero_copy_enabled.load(std::memory_order_relaxed)) return kNoWindow; + return make_window(t); +} + +// Window of a symm-mem input and, for MXFP8, of its scales, whatever the +// zero-copy setting. Throws when either is not symm-mem-backed on the EP group. +NVTECommWindow make_input_window(const at::Tensor& t, const std::optional& scale_inv) { + NVTECommWindow win = make_window(t); + NVTE_CHECK(win.window != nullptr, "symm_input needs a symm-mem input on the EP group"); + if (scale_inv.has_value()) { + const NVTECommWindow scale_win = make_window(*scale_inv); + NVTE_CHECK(scale_win.window != nullptr, + "symm_input needs symm-mem input scales on the EP group"); + win.scale_window = scale_win.window; + win.scale_offset = scale_win.offset; + } + return win; +} + // When zero-copy is enabled, the named tensor must be symm-mem-backed on the // EP group. Throws a clear error otherwise. No-op when zero-copy is off or // symm-mem support isn't compiled in. Mirrors maybe_make_window's resolution @@ -172,6 +194,89 @@ size_t check_mxfp8_scale_pair(const at::Tensor& send_scale, const at::Tensor& re return sc_cols; } +// MXFP8 TE tensor of E4M3 rowwise and/or columnwise data (undefined: absent) and E8M0 scales. +TensorWrapper make_mxfp8_tensor(const at::Tensor& rowwise, const at::Tensor& rowwise_scale_inv, + const at::Tensor& columnwise, + const at::Tensor& columnwise_scale_inv, bool swizzled) { + TensorWrapper t(NVTE_MXFP8_1D_SCALING); + for (const bool col : {false, true}) { + const at::Tensor& data = col ? columnwise : rowwise; + const at::Tensor& scale_inv = col ? columnwise_scale_inv : rowwise_scale_inv; + if (!data.defined()) { + continue; + } + NVTE_CHECK(data.scalar_type() == at::kFloat8_e4m3fn && data.is_contiguous() && + scale_inv.defined() && scale_inv.is_contiguous(), + "MXFP8 operands need contiguous E4M3 data and contiguous scales"); + if (col) { + t.set_columnwise_data(data.data_ptr(), DType::kFloat8E4M3, getTensorShape(data)); + t.set_columnwise_scale_inv(scale_inv.data_ptr(), DType::kFloat8E8M0, + getTensorShape(scale_inv)); + } else { + t.set_rowwise_data(data.data_ptr(), DType::kFloat8E4M3, getTensorShape(data)); + t.set_rowwise_scale_inv(scale_inv.data_ptr(), DType::kFloat8E8M0, getTensorShape(scale_inv)); + } + } + t.set_with_gemm_swizzled_scales(swizzled); + return t; +} + +// Columnwise-only MXFP8 TE tensor with GEMM-swizzled scales. +TensorWrapper make_columnwise_mxfp8_tensor(const at::Tensor& data, const at::Tensor& scale_inv) { + return make_mxfp8_tensor(at::Tensor(), at::Tensor(), data, scale_inv, true); +} + +// Rowwise MXFP8 TE tensor when scale_inv is given, else the plain tensor. +TensorWrapper make_maybe_mxfp8_tensor(const at::Tensor& data, + const std::optional& scale_inv, bool swizzled) { + if (!scale_inv.has_value()) { + return makeTransformerEngineTensor(data); + } + return make_mxfp8_tensor(data, *scale_inv, at::Tensor(), at::Tensor(), swizzled); +} + +// Optional weight gradient of an overlapped backward GEMM and the activations it reads: plain, or +// columnwise MXFP8 data with its GEMM-swizzled scales. +class WeightGradTensors { + public: + // input_rowwise: the rowwise MXFP8 input the call requantizes into the columnwise input. + WeightGradTensors(const std::optional& input, + const std::optional& weight_grad, + const std::optional& input_scale_inv = std::nullopt, + const std::optional& input_rowwise = std::nullopt, + const std::optional& input_rowwise_scale_inv = std::nullopt) + : enabled_(weight_grad.has_value()) { + NVTE_CHECK(input.has_value() == enabled_, "wgrad_input and weight_grad must be given together"); + NVTE_CHECK(enabled_ || !input_scale_inv.has_value(), "wgrad_input_scale_inv needs weight_grad"); + NVTE_CHECK(input_rowwise.has_value() == input_rowwise_scale_inv.has_value() && + (input_scale_inv.has_value() || !input_rowwise.has_value()), + "the rowwise wgrad_input needs its scales and a columnwise MXFP8 wgrad_input"); + if (enabled_) { + NVTE_CHECK(input->is_contiguous() && weight_grad->is_contiguous(), + "wgrad_input and weight_grad must be contiguous"); + input_ = input_scale_inv.has_value() + ? make_mxfp8_tensor(input_rowwise.value_or(at::Tensor()), + input_rowwise_scale_inv.value_or(at::Tensor()), *input, + *input_scale_inv, true) + : makeTransformerEngineTensor(*input); + output_ = makeTransformerEngineTensor(*weight_grad); + } + } + NVTETensor input() { return enabled_ ? input_.data() : nullptr; } + NVTETensor output() { return enabled_ ? output_.data() : nullptr; } + + private: + bool enabled_; + TensorWrapper input_; + TensorWrapper output_; +}; + +// A tensor the native call reads or writes through a raw pointer: dense, on the current device. +void check_dense_cuda(const at::Tensor& t, const char* name) { + NVTE_CHECK(t.is_cuda() && t.is_contiguous() && t.device().index() == c10::cuda::current_device(), + name, " must be a contiguous tensor on the current CUDA device"); +} + } // namespace bool ep_get_zero_copy() { return g_zero_copy_enabled.load(std::memory_order_relaxed); } @@ -191,7 +296,9 @@ bool ep_zero_copy_supported() { void ep_initialize(uintptr_t comm_ptr, const std::string& group_name, int64_t num_experts, int64_t max_tokens_per_rank, int64_t max_recv_tokens_per_rank, int64_t hidden_dim, int64_t max_num_sms, pybind11::object max_token_dtype, - bool zero_copy, int64_t num_topk, bool drop_on_overflow) { + bool zero_copy, int64_t num_topk, bool drop_on_overflow, + int64_t dispatch_fwd_num_sms, int64_t combine_fwd_num_sms, + int64_t dispatch_bwd_num_sms, int64_t combine_bwd_num_sms) { NVTE_CHECK(!group_name.empty(), "group_name must be non-empty (used for symm-mem lookup)"); NVTE_CHECK(comm_ptr != 0, "comm_ptr must be non-null (torch NCCL host comm pointer)"); NVTE_CHECK(!g_ep_initialized, "ep_initialize called twice without ep_finalize"); @@ -212,6 +319,10 @@ void ep_initialize(uintptr_t comm_ptr, const std::string& group_name, int64_t nu .zero_copy = zero_copy ? 1 : 0, .num_topk = static_cast(num_topk), .drop_on_overflow = drop_on_overflow ? 1 : 0, + .dispatch_fwd_num_sms = static_cast(dispatch_fwd_num_sms), + .combine_fwd_num_sms = static_cast(combine_fwd_num_sms), + .dispatch_bwd_num_sms = static_cast(dispatch_bwd_num_sms), + .combine_bwd_num_sms = static_cast(combine_bwd_num_sms), }; // Release the GIL only around the native init. It must stay held while pybind11 casts // the ``max_token_dtype`` object above and destroys the by-value ``pybind11::object`` @@ -262,9 +373,14 @@ void ep_prepare(at::Tensor handle_mem, at::Tensor topk_idx, at::Tensor tokens_pe auto stream = at::cuda::getCurrentCUDAStream().stream(); NVTE_CHECK(topk_idx.dim() >= 2, "topk_idx must be at least 2D [..., top_k]"); auto idx_dtype = get_contiguous_topk_idx_te_dtype(topk_idx); - // NCCL EP requires all prepare int output counters to share a dtype. - NVTE_CHECK(tokens_per_expert.scalar_type() == at::kLong, "tokens_per_expert must be int64"); - NVTE_CHECK(total_recv_tokens.scalar_type() == at::kLong, "total_recv_tokens must be int64"); + // NCCL EP requires all prepare int output counters to share a dtype: int64, or int32 for the + // overlapped GEMMs. + NVTE_CHECK( + tokens_per_expert.scalar_type() == at::kLong || tokens_per_expert.scalar_type() == at::kInt, + "tokens_per_expert must be int64 or int32"); + NVTE_CHECK(total_recv_tokens.scalar_type() == tokens_per_expert.scalar_type(), + "total_recv_tokens must have the dtype of tokens_per_expert"); + const DType count_dtype = GetTransformerEngineDType(tokens_per_expert.scalar_type()); const size_t T_flat = topk_idx.numel() / topk_idx.size(-1); const size_t topk_n = static_cast(topk_idx.size(-1)); @@ -272,14 +388,14 @@ void ep_prepare(at::Tensor handle_mem, at::Tensor topk_idx, at::Tensor tokens_pe makeTransformerEngineTensor(topk_idx.data_ptr(), Shape{T_flat, topk_n}, idx_dtype); auto tokens_per_expert_te = makeTransformerEngineTensor( tokens_per_expert.data_ptr(), Shape{static_cast(tokens_per_expert.numel())}, - DType::kInt64); + count_dtype); auto handle_mem_te = makeTransformerEngineTensor( handle_mem.data_ptr(), Shape{static_cast(handle_mem.numel())}, DType::kByte); - // [1] int64 scalar recv-slot total; lets the caller size dispatch outputs + // [1] scalar recv-slot total; lets the caller size dispatch outputs // (eager) or detect overflow past recv_capacity_per_rank (graph mode). auto total_recv_tokens_te = makeTransformerEngineTensor( total_recv_tokens.data_ptr(), Shape{static_cast(total_recv_tokens.numel())}, - DType::kInt64); + count_dtype); auto layer_cfg = make_layer_cfg(top_k, dispatch_output_per_expert_alignment, idx_dtype); nvte_ep_prepare(handle_mem_te.data(), topk_idx_te.data(), tokens_per_expert_te.data(), @@ -712,6 +828,257 @@ void ep_combine_bwd(at::Tensor handle_mem, at::Tensor grad, at::Tensor grad_expe grad_expert_out_win, &layer_cfg, stream); } +std::vector ep_dispatch_gemm_fwd( + at::Tensor handle_mem, at::Tensor topk_idx, at::Tensor tokens, at::Tensor topk_weights, + at::Tensor weight, std::optional recv_tokens_opt, + std::optional recv_topk_weights_opt, at::Tensor tokens_per_expert, + std::optional gemm_out_opt, std::optional act_out_opt, int64_t top_k, + int64_t dispatch_output_per_expert_alignment, std::optional total_recv_tokens, + int64_t eager_row_bucket, std::optional tokens_scale_inv, + std::optional recv_scale_inv, std::optional weight_scale_inv, + std::optional act_out_scale_inv, bool symm_input) { + auto stream = at::cuda::getCurrentCUDAStream().stream(); + const bool eager = !recv_tokens_opt.has_value(); + NVTE_CHECK(recv_topk_weights_opt.has_value() != eager && gemm_out_opt.has_value() != eager && + act_out_opt.has_value() != eager, + "pass all of the recv and GEMM outputs, or none (eager)"); + const bool mxfp8 = tokens_scale_inv.has_value(); + NVTE_CHECK(weight_scale_inv.has_value() == mxfp8 && + recv_scale_inv.has_value() == (mxfp8 && !eager) && + act_out_scale_inv.has_value() == (mxfp8 && !eager), + "pass all of the MXFP8 scales, or none; eager allocates the output scales"); + NVTE_CHECK(!total_recv_tokens.has_value() || total_recv_tokens->scalar_type() == at::kInt, + "total_recv_tokens must be int32"); + at::Tensor recv_tokens, recv_topk_weights, gemm_out, act_out; + if (eager) { + // Prepare, read the recv total on the host and size the outputs to whole row buckets, all + // in this call; the fused entry then skips the prepare. + NVTE_CHECK(total_recv_tokens.has_value() && total_recv_tokens->is_pinned(), + "eager needs total_recv_tokens in pinned host memory"); + NVTE_CHECK(eager_row_bucket > 0, "eager needs eager_row_bucket > 0"); + ep_prepare(handle_mem, topk_idx, tokens_per_expert, top_k, dispatch_output_per_expert_alignment, + *total_recv_tokens); + NVTE_CHECK_CUDA(cudaStreamSynchronize(stream)); + const int64_t total = *total_recv_tokens->data_ptr(); + const int64_t buckets = std::max((total + eager_row_bucket - 1) / eager_row_bucket, 1); + const int64_t rows = buckets * eager_row_bucket; + const int64_t hidden = tokens.size(1); + const int64_t ffn = weight.size(1) / 2; + recv_tokens = at::empty({rows, hidden}, tokens.options()); + recv_topk_weights = at::empty({rows}, topk_weights.options()); + gemm_out = at::empty({rows, 2 * ffn}, tokens.options().dtype(at::kBFloat16)); + act_out = at::empty({rows, ffn}, tokens.options()); + if (mxfp8) { + const int64_t scale_cols = tokens_scale_inv->size(-1); + NVTE_CHECK(scale_cols > 0 && hidden % scale_cols == 0, "eager MXFP8 tokens_scale_inv cols (", + scale_cols, ") must be a non-zero divisor of hidden (", hidden, ")"); + const int64_t block = hidden / scale_cols; + recv_scale_inv = at::empty({rows, scale_cols}, tokens_scale_inv->options()); + act_out_scale_inv = at::empty({rows * ffn / block}, tokens_scale_inv->options()); + } + } else { + recv_tokens = *recv_tokens_opt; + recv_topk_weights = *recv_topk_weights_opt; + gemm_out = *gemm_out_opt; + act_out = *act_out_opt; + } + NVTE_CHECK(tokens.is_contiguous() && recv_tokens.is_contiguous() && weight.is_contiguous() && + gemm_out.is_contiguous() && act_out.is_contiguous(), + "operands must be contiguous"); + check_dense_cuda(tokens_per_expert, "tokens_per_expert"); + check_dense_cuda(recv_topk_weights, "recv_topk_weights"); + // The expert layout the GEMMs read: expert 0's start (0), then each expert's padded row end. + // Front-pad by 3 so the row ends (expert_token_offsets[1:]) start 16-byte aligned, as the GEMM + // needs. + auto expert_token_offsets = + at::empty({weight.size(0) + 4}, tokens.options().dtype(at::kInt)).slice(0, 3); + auto handle_mem_te = makeTransformerEngineTensor(handle_mem); + auto topk_idx_te = makeTransformerEngineTensor(topk_idx); + auto tokens_te = make_maybe_mxfp8_tensor(tokens, tokens_scale_inv, false); + auto topk_weights_te = makeTransformerEngineTensor(topk_weights); + auto weight_te = make_maybe_mxfp8_tensor(weight, weight_scale_inv, true); + auto recv_tokens_te = make_maybe_mxfp8_tensor(recv_tokens, recv_scale_inv, false); + auto recv_topk_weights_te = makeTransformerEngineTensor(recv_topk_weights); + auto tokens_per_expert_te = makeTransformerEngineTensor(tokens_per_expert); + auto gemm_out_te = makeTransformerEngineTensor(gemm_out); + auto act_out_te = make_maybe_mxfp8_tensor(act_out, act_out_scale_inv, true); + auto expert_token_offsets_te = makeTransformerEngineTensor(expert_token_offsets); + // In eager mode the prepare above wrote the total. + const bool report_total = total_recv_tokens.has_value() && !eager; + TensorWrapper total_recv_tokens_te; + if (report_total) { + total_recv_tokens_te = makeTransformerEngineTensor(*total_recv_tokens); + } + const NVTECommWindow tokens_win = + symm_input ? make_input_window(tokens, tokens_scale_inv) : kNoWindow; + auto layer_cfg = make_layer_cfg(top_k, dispatch_output_per_expert_alignment, + get_contiguous_topk_idx_te_dtype(topk_idx)); + nvte_ep_dispatch_gemm_fwd(handle_mem_te.data(), topk_idx_te.data(), tokens_te.data(), tokens_win, + topk_weights_te.data(), weight_te.data(), recv_tokens_te.data(), + recv_topk_weights_te.data(), gemm_out_te.data(), act_out_te.data(), + tokens_per_expert_te.data(), expert_token_offsets_te.data(), + report_total ? total_recv_tokens_te.data() : nullptr, &layer_cfg, + stream); + if (mxfp8) { + return {recv_tokens, recv_topk_weights, gemm_out, act_out, + expert_token_offsets, *recv_scale_inv, *act_out_scale_inv}; + } + return {recv_tokens, recv_topk_weights, gemm_out, act_out, expert_token_offsets}; +} + +void ep_combine_gemm_fwd(at::Tensor handle_mem, at::Tensor gemm_in, at::Tensor expert_token_offsets, + at::Tensor weight, at::Tensor expert_out, at::Tensor result, int64_t top_k, + int64_t dispatch_output_per_expert_alignment, + at::ScalarType topk_idx_dtype, std::optional recv_topk_weights, + std::optional gemm_in_scale_inv, + std::optional weight_scale_inv) { + auto stream = at::cuda::getCurrentCUDAStream().stream(); + NVTE_CHECK(gemm_in.is_contiguous() && weight.is_contiguous() && expert_out.is_contiguous() && + result.is_contiguous(), + "operands must be contiguous"); + check_dense_cuda(expert_token_offsets, "expert_token_offsets"); + NVTE_CHECK(gemm_in_scale_inv.has_value() == weight_scale_inv.has_value(), + "pass both MXFP8 scales, or none"); + auto handle_mem_te = makeTransformerEngineTensor(handle_mem); + auto gemm_in_te = make_maybe_mxfp8_tensor(gemm_in, gemm_in_scale_inv, true); + TensorWrapper recv_topk_weights_te; + if (recv_topk_weights.has_value()) { + recv_topk_weights_te = makeTransformerEngineTensor(*recv_topk_weights); + } + auto expert_token_offsets_te = makeTransformerEngineTensor(expert_token_offsets); + auto weight_te = make_maybe_mxfp8_tensor(weight, weight_scale_inv, true); + auto expert_out_te = makeTransformerEngineTensor(expert_out); + auto result_te = makeTransformerEngineTensor(result); + auto layer_cfg = make_layer_cfg(top_k, dispatch_output_per_expert_alignment, + get_topk_idx_te_dtype(topk_idx_dtype)); + nvte_ep_combine_gemm_fwd(handle_mem_te.data(), gemm_in_te.data(), + recv_topk_weights.has_value() ? recv_topk_weights_te.data() : nullptr, + expert_token_offsets_te.data(), weight_te.data(), expert_out_te.data(), + result_te.data(), &layer_cfg, stream); +} + +void ep_combine_gemm_bwd( + at::Tensor handle_mem, at::Tensor grad_result, at::Tensor gemm_out, + at::Tensor recv_topk_weights, at::Tensor tokens_per_expert, at::Tensor weight, + at::Tensor grad_expert_out, at::Tensor grad_gemm_out, at::Tensor grad_recv_topk_weights, + int64_t top_k, int64_t dispatch_output_per_expert_alignment, at::ScalarType topk_idx_dtype, + std::optional grad_result_scale_inv, + std::optional grad_expert_out_scale_inv, std::optional weight_scale_inv, + std::optional grad_gemm_out_scale_inv, + std::optional grad_gemm_out_columnwise, + std::optional grad_gemm_out_columnwise_scale_inv, std::optional act_out, + std::optional act_out_scale_inv, std::optional wgrad_input, + std::optional weight_grad, bool accumulate_weight_grad, + std::optional grad_expert_out_columnwise, + std::optional grad_expert_out_columnwise_scale_inv, + std::optional wgrad_input_scale_inv, bool symm_input) { + auto stream = at::cuda::getCurrentCUDAStream().stream(); + NVTE_CHECK(grad_result.is_contiguous() && gemm_out.is_contiguous() && weight.is_contiguous() && + grad_expert_out.is_contiguous() && grad_gemm_out.is_contiguous(), + "operands must be contiguous"); + check_dense_cuda(tokens_per_expert, "tokens_per_expert"); + check_dense_cuda(grad_recv_topk_weights, "grad_recv_topk_weights"); + const bool mxfp8 = grad_result_scale_inv.has_value(); + NVTE_CHECK(grad_expert_out_scale_inv.has_value() == mxfp8 && + weight_scale_inv.has_value() == mxfp8 && + grad_gemm_out_scale_inv.has_value() == mxfp8 && + grad_gemm_out_columnwise.has_value() == mxfp8 && + grad_gemm_out_columnwise_scale_inv.has_value() == mxfp8 && + act_out.has_value() == mxfp8 && act_out_scale_inv.has_value() == mxfp8, + "pass all of the MXFP8 operands, or none"); + // The columnwise grad_expert_out of the MXFP8 weight gradient, written by the call. + const bool grad_expert_out_columns = grad_expert_out_columnwise.has_value(); + NVTE_CHECK(grad_expert_out_columnwise_scale_inv.has_value() == grad_expert_out_columns && + (mxfp8 || !grad_expert_out_columns), + "pass the MXFP8 columnwise grad_expert_out with its scales, or neither"); + auto handle_mem_te = makeTransformerEngineTensor(handle_mem); + auto grad_result_te = make_maybe_mxfp8_tensor(grad_result, grad_result_scale_inv, false); + auto gemm_out_te = makeTransformerEngineTensor(gemm_out); + auto recv_topk_weights_te = makeTransformerEngineTensor(recv_topk_weights); + auto tokens_per_expert_te = makeTransformerEngineTensor(tokens_per_expert); + // MXFP8 reads the forward weight columnwise. + auto weight_te = mxfp8 ? make_columnwise_mxfp8_tensor(weight, *weight_scale_inv) + : makeTransformerEngineTensor(weight); + auto grad_expert_out_te = + grad_expert_out_columns + ? make_mxfp8_tensor(grad_expert_out, *grad_expert_out_scale_inv, + *grad_expert_out_columnwise, *grad_expert_out_columnwise_scale_inv, + false) + : make_maybe_mxfp8_tensor(grad_expert_out, grad_expert_out_scale_inv, false); + auto grad_gemm_out_te = + mxfp8 ? make_mxfp8_tensor(grad_gemm_out, *grad_gemm_out_scale_inv, *grad_gemm_out_columnwise, + *grad_gemm_out_columnwise_scale_inv, true) + : makeTransformerEngineTensor(grad_gemm_out); + auto grad_recv_topk_weights_te = makeTransformerEngineTensor(grad_recv_topk_weights); + TensorWrapper act_out_te; + if (mxfp8) { + act_out_te = make_columnwise_mxfp8_tensor(*act_out, *act_out_scale_inv); + } + NVTE_CHECK(mxfp8 || !wgrad_input_scale_inv.has_value(), + "wgrad_input_scale_inv is for MXFP8 only"); + WeightGradTensors wgrad(wgrad_input, weight_grad, wgrad_input_scale_inv); + const NVTECommWindow grad_result_win = + symm_input ? make_input_window(grad_result, grad_result_scale_inv) : kNoWindow; + auto layer_cfg = make_layer_cfg(top_k, dispatch_output_per_expert_alignment, + get_topk_idx_te_dtype(topk_idx_dtype)); + nvte_ep_combine_gemm_bwd(handle_mem_te.data(), grad_result_te.data(), grad_result_win, + gemm_out_te.data(), recv_topk_weights_te.data(), + tokens_per_expert_te.data(), weight_te.data(), wgrad.input(), + grad_expert_out_te.data(), grad_gemm_out_te.data(), + grad_recv_topk_weights_te.data(), mxfp8 ? act_out_te.data() : nullptr, + wgrad.output(), accumulate_weight_grad, &layer_cfg, stream); +} + +void ep_dispatch_gemm_bwd( + at::Tensor handle_mem, at::Tensor grad_gemm_out, at::Tensor grad_recv_topk_weights, + at::Tensor tokens_per_expert, at::Tensor weight, at::Tensor grad_recv_tokens, + at::Tensor grad_tokens, at::Tensor grad_topk_weights, int64_t top_k, + int64_t dispatch_output_per_expert_alignment, at::ScalarType topk_idx_dtype, + std::optional grad_gemm_out_scale_inv, std::optional weight_scale_inv, + std::optional grad_gemm_out_columnwise, + std::optional grad_gemm_out_columnwise_scale_inv, + std::optional wgrad_input, std::optional wgrad_input_scale_inv, + std::optional weight_grad, bool accumulate_weight_grad, + std::optional wgrad_input_rowwise, + std::optional wgrad_input_rowwise_scale_inv) { + auto stream = at::cuda::getCurrentCUDAStream().stream(); + NVTE_CHECK( + grad_gemm_out.is_contiguous() && weight.is_contiguous() && grad_recv_tokens.is_contiguous(), + "operands must be contiguous"); + check_dense_cuda(tokens_per_expert, "tokens_per_expert"); + check_dense_cuda(grad_recv_topk_weights, "grad_recv_topk_weights"); + const bool mxfp8 = grad_gemm_out_scale_inv.has_value(); + NVTE_CHECK(weight_scale_inv.has_value() == mxfp8, "pass both MXFP8 scales, or none"); + // The MXFP8 weight gradient reads grad_gemm_out and wgrad_input columnwise. + const bool columnwise = grad_gemm_out_columnwise.has_value(); + NVTE_CHECK(grad_gemm_out_columnwise_scale_inv.has_value() == columnwise && + wgrad_input_scale_inv.has_value() == columnwise && (mxfp8 || !columnwise), + "pass the MXFP8 columnwise operands of the weight gradient together, or none"); + auto handle_mem_te = makeTransformerEngineTensor(handle_mem); + auto grad_gemm_out_te = + columnwise + ? make_mxfp8_tensor(grad_gemm_out, *grad_gemm_out_scale_inv, *grad_gemm_out_columnwise, + *grad_gemm_out_columnwise_scale_inv, true) + : make_maybe_mxfp8_tensor(grad_gemm_out, grad_gemm_out_scale_inv, true); + auto grad_recv_topk_weights_te = makeTransformerEngineTensor(grad_recv_topk_weights); + auto tokens_per_expert_te = makeTransformerEngineTensor(tokens_per_expert); + // MXFP8 reads the forward weight columnwise. + auto weight_te = mxfp8 ? make_columnwise_mxfp8_tensor(weight, *weight_scale_inv) + : makeTransformerEngineTensor(weight); + auto grad_recv_tokens_te = makeTransformerEngineTensor(grad_recv_tokens); + auto grad_tokens_te = makeTransformerEngineTensor(grad_tokens); + auto grad_topk_weights_te = makeTransformerEngineTensor(grad_topk_weights); + WeightGradTensors wgrad(wgrad_input, weight_grad, wgrad_input_scale_inv, wgrad_input_rowwise, + wgrad_input_rowwise_scale_inv); + auto layer_cfg = make_layer_cfg(top_k, dispatch_output_per_expert_alignment, + get_topk_idx_te_dtype(topk_idx_dtype)); + nvte_ep_dispatch_gemm_bwd(handle_mem_te.data(), grad_gemm_out_te.data(), + grad_recv_topk_weights_te.data(), tokens_per_expert_te.data(), + weight_te.data(), wgrad.input(), grad_recv_tokens_te.data(), + grad_tokens_te.data(), grad_topk_weights_te.data(), wgrad.output(), + accumulate_weight_grad, &layer_cfg, stream); +} + void register_ep_bindings(pybind11::module_& m) { namespace py = pybind11; m.def("ep_initialize", &ep_initialize, @@ -719,7 +1086,9 @@ void register_ep_bindings(pybind11::module_& m) { py::arg("comm_ptr"), py::arg("group_name"), py::arg("num_experts"), py::arg("max_tokens_per_rank"), py::arg("max_recv_tokens_per_rank"), py::arg("hidden_dim"), py::arg("max_num_sms") = 0, py::arg("max_token_dtype"), py::arg("zero_copy") = false, - py::arg("num_topk") = 0, py::arg("drop_on_overflow") = false); + py::arg("num_topk") = 0, py::arg("drop_on_overflow") = false, + py::arg("dispatch_fwd_num_sms") = 0, py::arg("combine_fwd_num_sms") = 0, + py::arg("dispatch_bwd_num_sms") = 0, py::arg("combine_bwd_num_sms") = 0); m.def("ep_finalize", &ep_finalize, "Tear down the EP backend. Idempotent.", py::call_guard()); m.def("ep_get_zero_copy", &ep_get_zero_copy, "Return the current EP zero-copy toggle state."); @@ -746,6 +1115,57 @@ void register_ep_bindings(pybind11::module_& m) { m.def("ep_combine", &ep_combine, "EP combine", py::arg("handle_mem"), py::arg("expert_out"), py::arg("result"), py::arg("top_k"), py::arg("dispatch_output_per_expert_alignment"), py::arg("topk_idx_dtype"), py::call_guard()); + m.def("ep_dispatch_gemm_fwd", &ep_dispatch_gemm_fwd, + "EP dispatch overlapped with the first GEMM. Returns recv_tokens, recv_topk_weights, " + "gemm_out, act_out and expert_token_offsets, then the recv and act_out scales when " + "MXFP8; in eager mode omit them and they are sized and allocated here", + py::arg("handle_mem"), py::arg("topk_idx"), py::arg("tokens"), py::arg("topk_weights"), + py::arg("weight"), py::arg("recv_tokens"), py::arg("recv_topk_weights"), + py::arg("tokens_per_expert"), py::arg("gemm_out"), py::arg("act_out"), py::arg("top_k"), + py::arg("dispatch_output_per_expert_alignment"), + py::arg("total_recv_tokens") = std::nullopt, py::arg("eager_row_bucket") = 0, + py::arg("tokens_scale_inv") = std::nullopt, py::arg("recv_scale_inv") = std::nullopt, + py::arg("weight_scale_inv") = std::nullopt, py::arg("act_out_scale_inv") = std::nullopt, + py::arg("symm_input") = false, py::call_guard()); + m.def("ep_combine_gemm_fwd", &ep_combine_gemm_fwd, "EP second GEMM overlapped with the combine", + py::arg("handle_mem"), py::arg("gemm_in"), py::arg("expert_token_offsets"), + py::arg("weight"), py::arg("expert_out"), py::arg("result"), py::arg("top_k"), + py::arg("dispatch_output_per_expert_alignment"), py::arg("topk_idx_dtype"), + py::arg("recv_topk_weights") = std::nullopt, py::arg("gemm_in_scale_inv") = std::nullopt, + py::arg("weight_scale_inv") = std::nullopt, py::call_guard()); + m.def("ep_combine_gemm_bwd", &ep_combine_gemm_bwd, + "Backward of the second GEMM and the combine, with the activation backward", + py::arg("handle_mem"), py::arg("grad_result"), py::arg("gemm_out"), + py::arg("recv_topk_weights"), py::arg("tokens_per_expert"), py::arg("weight"), + py::arg("grad_expert_out"), py::arg("grad_gemm_out"), py::arg("grad_recv_topk_weights"), + py::arg("top_k"), py::arg("dispatch_output_per_expert_alignment"), + py::arg("topk_idx_dtype"), py::arg("grad_result_scale_inv") = std::nullopt, + py::arg("grad_expert_out_scale_inv") = std::nullopt, + py::arg("weight_scale_inv") = std::nullopt, + py::arg("grad_gemm_out_scale_inv") = std::nullopt, + py::arg("grad_gemm_out_columnwise") = std::nullopt, + py::arg("grad_gemm_out_columnwise_scale_inv") = std::nullopt, + py::arg("act_out") = std::nullopt, py::arg("act_out_scale_inv") = std::nullopt, + py::arg("wgrad_input") = std::nullopt, py::arg("weight_grad") = std::nullopt, + py::arg("accumulate_weight_grad") = false, + py::arg("grad_expert_out_columnwise") = std::nullopt, + py::arg("grad_expert_out_columnwise_scale_inv") = std::nullopt, + py::arg("wgrad_input_scale_inv") = std::nullopt, py::arg("symm_input") = false, + py::call_guard()); + m.def( + "ep_dispatch_gemm_bwd", &ep_dispatch_gemm_bwd, "Backward of the first GEMM and the dispatch", + py::arg("handle_mem"), py::arg("grad_gemm_out"), py::arg("grad_recv_topk_weights"), + py::arg("tokens_per_expert"), py::arg("weight"), py::arg("grad_recv_tokens"), + py::arg("grad_tokens"), py::arg("grad_topk_weights"), py::arg("top_k"), + py::arg("dispatch_output_per_expert_alignment"), py::arg("topk_idx_dtype"), + py::arg("grad_gemm_out_scale_inv") = std::nullopt, py::arg("weight_scale_inv") = std::nullopt, + py::arg("grad_gemm_out_columnwise") = std::nullopt, + py::arg("grad_gemm_out_columnwise_scale_inv") = std::nullopt, + py::arg("wgrad_input") = std::nullopt, py::arg("wgrad_input_scale_inv") = std::nullopt, + py::arg("weight_grad") = std::nullopt, py::arg("accumulate_weight_grad") = false, + py::arg("wgrad_input_rowwise") = std::nullopt, + py::arg("wgrad_input_rowwise_scale_inv") = std::nullopt, + py::call_guard()); m.def("ep_dispatch_bwd", &ep_dispatch_bwd, "EP dispatch backward", py::arg("handle_mem"), py::arg("grad"), py::arg("g_recv_topk_weights"), py::arg("grad_tokens"), py::arg("grad_topk_weights"), py::arg("top_k"), diff --git a/transformer_engine/pytorch/ep.py b/transformer_engine/pytorch/ep.py index cf4c7b1e51b..18e9a7a7b45 100644 --- a/transformer_engine/pytorch/ep.py +++ b/transformer_engine/pytorch/ep.py @@ -6,6 +6,9 @@ from __future__ import annotations import atexit +import functools +import math +import os import warnings from dataclasses import dataclass from typing import Optional, TYPE_CHECKING @@ -15,11 +18,13 @@ import transformer_engine_torch as tex +from ..common import _get_shared_object_file from .constants import DType from .cpu_offload import mark_not_offload from .distributed import symm_mem_alloc, release_symm_mem_pool from .quantized_tensor import QuantizedTensor, QuantizedTensorStorage from .tensor.storage.mxfp8_tensor_storage import MXFP8TensorStorage +from .utils import round_up_to_nearest_multiple # Type-hint-only import; keeps the ``Recipe`` annotation without a runtime import of # common.recipe (the concrete recipe classes are imported lazily where used). @@ -84,11 +89,15 @@ def _check_nccl_runtime_version() -> None: _ATEXIT_REGISTERED = False # EP group captured at bootstrap; used by the zero-copy symm-mem pool allocator. _EP_GROUP: Optional[dist.ProcessGroup] = None +# Communication SMs of the overlapped MoE ops when ep_bootstrap picks them. +_OVERLAP_COMM_SMS = 24 # Eager-mode toggle captured at bootstrap (set when recv_capacity_per_rank is # omitted); ep_dispatch reads it to size the recv outputs from the per-step # recv-token total instead of a fixed recv_capacity_per_rank. _EAGER = False _BOOTSTRAP_SETTINGS: Optional[dict[str, object]] = None +# Symm-mem data and scale buffers the overlapped MoE ops quantize their MXFP8 dispatch inputs into. +_DISPATCH_WINDOW: Optional[tuple[torch.Tensor, torch.Tensor]] = None @dataclass(frozen=True, slots=True) @@ -125,9 +134,10 @@ class EpConfig: def _atexit_finalize() -> None: """Best-effort teardown at interpreter shutdown; swallows errors.""" - global _BOOTSTRAPPED, _EP_GROUP, _EAGER, _BOOTSTRAP_SETTINGS + global _BOOTSTRAPPED, _EP_GROUP, _EAGER, _BOOTSTRAP_SETTINGS, _DISPATCH_WINDOW if _BOOTSTRAPPED: try: + _DISPATCH_WINDOW = None tex.ep_finalize() except Exception: # pylint: disable=broad-exception-caught import traceback @@ -151,6 +161,10 @@ def ep_bootstrap( zero_copy: bool = False, drop_on_overflow: bool = False, max_token_dtype: torch.dtype = torch.bfloat16, + dispatch_fwd_num_sms: int = 0, + combine_fwd_num_sms: int = 0, + dispatch_bwd_num_sms: int = 0, + combine_bwd_num_sms: int = 0, ) -> None: """Initialize EP by borrowing ep_group's NCCL comm. Call once per process. @@ -172,9 +186,22 @@ def ep_bootstrap( ``drop_on_overflow`` drops tokens exceeding ``recv_capacity_per_rank`` instead of trapping. Requires ``recv_capacity_per_rank``. + + ``max_num_sms`` defaults to 24 when the ``te.ops`` MoE overlap is available + (``NVTE_ENABLE_CUTEDSL_BACKEND=1``, ``NVTE_MOE_OVERLAP`` set) and ``zero_copy`` is off. + + ``dispatch_fwd_num_sms``, ``combine_fwd_num_sms``, ``dispatch_bwd_num_sms`` and + ``combine_bwd_num_sms`` set the SMs of the NCCL EP kernels of the dispatch and combine of the + layer forward and backward, overlapped or not. The backward of MoeCombine is a dispatch and the + backward of MoeDispatch a combine. 0 inherits: a forward budget takes ``max_num_sms``, a + backward budget the forward one. ``FusedMoE`` runs the GEMMs on the remaining SMs. A backward + budget other than the forward one needs NCCL EP with backward SM budgets. ``NCCL_EP_COMM_SMS`` + and NCCL EP's per-call SM variables override these counts, and ``FusedMoE`` rejects them. """ global _BOOTSTRAPPED, _ATEXIT_REGISTERED, _EP_GROUP, _EAGER, _BOOTSTRAP_SETTINGS eager = recv_capacity_per_rank is None + if max_num_sms == 0 and _moe_overlap_available() and not zero_copy: + max_num_sms = _OVERLAP_COMM_SMS if _BOOTSTRAPPED: raise RuntimeError("ep_bootstrap was already called in this process") if ep_group.size() < 2: @@ -216,6 +243,10 @@ def ep_bootstrap( bool(zero_copy), int(num_topk), bool(drop_on_overflow), + int(dispatch_fwd_num_sms), + int(combine_fwd_num_sms), + int(dispatch_bwd_num_sms), + int(combine_bwd_num_sms), ) _BOOTSTRAPPED = True _EP_GROUP = ep_group @@ -254,6 +285,30 @@ def get_ep_drop_on_overflow() -> Optional[bool]: return bool(_BOOTSTRAP_SETTINGS["drop_on_overflow"]) +@functools.lru_cache(maxsize=None) +def _cutedsl_backend_built() -> bool: + """Whether the core library was built with the CuTeDSL backend.""" + import ctypes + + core = ctypes.CDLL(str(_get_shared_object_file("core"))) + return hasattr(core, "nvte_is_cutedsl_backend_built") and bool( + core.nvte_is_cutedsl_backend_built() + ) + + +def _moe_overlap_available() -> bool: + """Whether the overlapped MoE GEMMs can run: SM 10.0, 10.3 or 10.7 (tcgen05 kernels), the + CuTeDSL backend built and on, and the GEMM path set.""" + if not torch.cuda.is_available(): + return False + return ( + torch.cuda.get_device_capability() in ((10, 0), (10, 3), (10, 7)) + and os.environ.get("NVTE_ENABLE_CUTEDSL_BACKEND", "0") != "0" + and bool(os.environ.get("NVTE_MOE_OVERLAP")) + and _cutedsl_backend_built() + ) + + def _ep_is_eager() -> bool: """Return whether the bootstrapped EP group uses variable-size eager buffers.""" return _EAGER @@ -269,10 +324,11 @@ def ep_finalize() -> None: a caller that used ``symm_mem_alloc(use_pool=True)`` does not need a separate ``release_symm_mem_pool()`` before destroying the PG. """ - global _BOOTSTRAPPED, _EP_GROUP, _EAGER, _BOOTSTRAP_SETTINGS + global _BOOTSTRAPPED, _EP_GROUP, _EAGER, _BOOTSTRAP_SETTINGS, _DISPATCH_WINDOW if not _BOOTSTRAPPED: return try: + _DISPATCH_WINDOW = None # Deregister pooled symm-mem windows while the group's comm is still valid. release_symm_mem_pool() tex.ep_finalize() @@ -330,6 +386,8 @@ class EpBuffer: "zero_copy", "eager", "total_recv_tokens", + "overlap_tokens_per_expert", + "overlap_total_recv_tokens", "dispatch_fwd_quant_recipe", "combine_bwd_quant_recipe", ) @@ -394,6 +452,15 @@ def __init__( else: self.total_recv_tokens = torch.empty(1, dtype=torch.int64, device=device) mark_not_offload(self.total_recv_tokens) + # The int32 counts and recv total of the fused MoE overlap, placed as above. + self.overlap_tokens_per_expert = torch.empty( + self.num_local_experts, dtype=torch.int32, device=device + ) + if self.eager: + self.overlap_total_recv_tokens = torch.empty(1, dtype=torch.int32, pin_memory=True) + else: + self.overlap_total_recv_tokens = torch.empty(1, dtype=torch.int32, device=device) + mark_not_offload(self.overlap_total_recv_tokens) # torch.library custom ops (so they don't graph-break under torch.compile) @@ -1084,17 +1151,68 @@ def _alloc_io(shape, dtype: torch.dtype, device, zero_copy: bool) -> torch.Tenso return torch.empty(*shape, dtype=dtype, device=device) +def _dispatch_window(device) -> Optional[tuple[torch.Tensor, torch.Tensor]]: + """Symm-mem data and scale buffers of one MXFP8 dispatch input, or None without symm-mem + support. Allocated collectively on first use and shared by every layer and direction, so the + EP calls of the group must share one stream.""" + global _DISPATCH_WINDOW + if _DISPATCH_WINDOW is None: + if not tex.ep_zero_copy_supported(): + return None + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError( + "The EP dispatch window is allocated on first use; run a warmup step before " + "CUDA graph capture." + ) + from .tensor.mxfp8_tensor import MXFP8Quantizer + + shape = (_BOOTSTRAP_SETTINGS["max_tokens_per_rank"], _BOOTSTRAP_SETTINGS["hidden_dim"]) + data_bytes = round_up_to_nearest_multiple(math.prod(shape), 256) + scale_shape = MXFP8Quantizer(DType.kFloat8E4M3).get_scale_shape(shape, False) + window = symm_mem_alloc( + (data_bytes + math.prod(scale_shape),), torch.uint8, _EP_GROUP, device=device + ) + mark_not_offload(window) + _DISPATCH_WINDOW = (window[:data_bytes], window[data_bytes:]) + return _DISPATCH_WINDOW + + +def _mxfp8_storage(out, shape, quantizer: "MXFP8Quantizer", fake_dtype=None) -> MXFP8TensorStorage: + """Rowwise MXFP8 storage of ``shape`` at the start of the uint8 data and scale buffers ``out``.""" + data, scales = out + scale_shape = quantizer.get_scale_shape(shape, False) + if data.numel() < math.prod(shape) or scales.numel() < math.prod(scale_shape): + raise ValueError(f"MXFP8 output buffers too small for {tuple(shape)}.") + return MXFP8TensorStorage( + data[: math.prod(shape)].view(shape), + scales[: math.prod(scale_shape)].view(scale_shape), + None, + None, + quantizer.dtype, + quantizer, + False, + fake_dtype=fake_dtype, + ) + + def quantize_for_ep( input_: torch.Tensor | QuantizedTensorStorage, quantizer: Optional["Quantizer"], + out: Optional[tuple[torch.Tensor, torch.Tensor]] = None, ) -> tuple[MXFP8TensorStorage, torch.Tensor]: - """Quantize an EP input with a supported quantizer.""" + """Quantize an EP input with a supported quantizer. + + ``out`` is a pair of contiguous uint8 tensors on the input's device, the data and the scales. + The data holds ``input_.numel()`` bytes and the scales the rowwise scale shape of the quantizer + (``get_scale_shape(shape, False)``, rows padded to 128). When given, the returned storage and + the compact scales it returns are views of them, so they alias ``out``. + """ from .tensor.mxfp8_tensor import MXFP8Quantizer if quantizer is None: raise ValueError("EP quantization requires a quantizer.") if isinstance(quantizer, MXFP8Quantizer): - return _quantize_mxfp8(input_, quantizer) + return _quantize_mxfp8(input_, quantizer, out) raise NotImplementedError( "EP dispatch/combine quantization currently supports only MXFP8; " f"got {type(quantizer).__name__}." @@ -1104,6 +1222,7 @@ def quantize_for_ep( def _quantize_mxfp8( input_: torch.Tensor | QuantizedTensorStorage, quantizer: "MXFP8Quantizer", + out: Optional[tuple[torch.Tensor, torch.Tensor]] = None, ) -> tuple[MXFP8TensorStorage, torch.Tensor]: """Return E4M3 MXFP8 storage and compact rowwise scales for EP.""" from .constants import MXFP8_BLOCK_SCALING_SIZE @@ -1118,7 +1237,11 @@ def _quantize_mxfp8( if not quantizer.internal: quantizer = quantizer.copy() quantizer.internal = True - quantized = quantizer(input_) + if out is None: + quantized = quantizer(input_) + else: + quantized = _mxfp8_storage(out, input_.shape, quantizer, input_.dtype) + tex.quantize(input_, quantizer, quantized, None) if quantized._fp8_dtype != DType.kFloat8E4M3: raise NotImplementedError("EP supports only E4M3 MXFP8 data.") @@ -1145,6 +1268,12 @@ def _quantize_mxfp8( scale_inv = scale_inv[:t_flat, :scale_cols] if not scale_inv.is_contiguous(): raise ValueError("EP requires compact contiguous MXFP8 scales.") + if out is not None and quantized is input_: + # An MXFP8 input is copied into out, so the result is there whatever the input. + quantized = _mxfp8_storage(out, input_.shape, quantizer) + quantized._rowwise_data.copy_(data) + quantized._rowwise_scale_inv[:t_flat, :scale_cols].copy_(scale_inv) + scale_inv = quantized._rowwise_scale_inv[:t_flat, :scale_cols] return quantized, scale_inv diff --git a/transformer_engine/pytorch/ops/__init__.py b/transformer_engine/pytorch/ops/__init__.py index 5e21bb39e8d..4fbbe2c5e8e 100644 --- a/transformer_engine/pytorch/ops/__init__.py +++ b/transformer_engine/pytorch/ops/__init__.py @@ -18,3 +18,4 @@ from .op import BasicOperation, FusedOperation, FusibleOperation from .sequential import Sequential from . import fused +from .fused import moe_overlap_supported, moe_overlap_unsupported_reason diff --git a/transformer_engine/pytorch/ops/basic/combine.py b/transformer_engine/pytorch/ops/basic/combine.py index 9f9825d8b28..a0c02eb0f00 100644 --- a/transformer_engine/pytorch/ops/basic/combine.py +++ b/transformer_engine/pytorch/ops/basic/combine.py @@ -70,6 +70,8 @@ class MoeCombine(BasicOperation): The quantization format of the communication is an EP backend detail, so it is configured by ``EpConfig`` rather than this operation's quantizers. + + ``buffer`` is the same ``EpBuffer`` the ``MoeDispatch`` was given. """ num_extra_inputs: int = 1 diff --git a/transformer_engine/pytorch/ops/basic/dispatch.py b/transformer_engine/pytorch/ops/basic/dispatch.py index 4613be2c689..9d2dd378e8c 100644 --- a/transformer_engine/pytorch/ops/basic/dispatch.py +++ b/transformer_engine/pytorch/ops/basic/dispatch.py @@ -70,13 +70,25 @@ class MoeDispatch(BasicOperation): The quantization format of the communication is an EP backend detail, so it is configured by ``EpConfig`` rather than this operation's quantizers. + + ``buffer`` is the ``EpBuffer`` of every call. For several concurrent in-flight calls + (e.g. pipeline microbatches), give each its own ``EpBuffer`` and op instance. + + ``recv_tokens_peak``, an optional int64 tensor of shape ``[1]`` on the EP device, is raised + on every forward with a fixed receive capacity to the call's receive total before any + overflow drop. A peak above the capacity means tokens were dropped. """ num_extra_inputs: int = 2 # tokens-per-expert and received routing weights consumed by the expert MLP. num_extra_outputs: int = 2 - def __init__(self, config: EpConfig, buffer: Optional[EpBuffer] = None) -> None: + def __init__( + self, + config: EpConfig, + buffer: Optional[EpBuffer] = None, + recv_tokens_peak: Optional[torch.Tensor] = None, + ) -> None: # EpBuffer is specific to NCCL EP. Fused implementations using another communication # backend, such as NVSHMEM, do not need it. super().__init__() @@ -86,6 +98,17 @@ def __init__(self, config: EpConfig, buffer: Optional[EpBuffer] = None) -> None: raise NotImplementedError("MoeDispatch does not support zero-copy EP.") self.config = config self.buffer = buffer + self.recv_tokens_peak = recv_tokens_peak + + def update_recv_tokens_peak( + self, buffer: EpBuffer, total_recv_tokens: Optional[torch.Tensor] = None + ) -> None: + """Raise ``recv_tokens_peak`` to the receive total of the last prepare on ``buffer``, or to + ``total_recv_tokens`` when given.""" + if self.recv_tokens_peak is not None and not buffer.eager: + if total_recv_tokens is None: + total_recv_tokens = buffer.total_recv_tokens + torch.maximum(self.recv_tokens_peak, total_recv_tokens, out=self.recv_tokens_peak) def op_forward(self, *args: Any, **kwargs: Any) -> None: raise RuntimeError("MoeDispatch uses fuser_forward") @@ -122,6 +145,7 @@ def fuser_forward( None, None, ) + self.update_recv_tokens_peak(buffer) tokens_per_expert = buffer.tokens_per_expert if ctx.requires_grad: ctx.dispatch_state = dispatch_state diff --git a/transformer_engine/pytorch/ops/fused/__init__.py b/transformer_engine/pytorch/ops/fused/__init__.py index ff1075e6fe2..d1b0b1b715f 100644 --- a/transformer_engine/pytorch/ops/fused/__init__.py +++ b/transformer_engine/pytorch/ops/fused/__init__.py @@ -36,3 +36,8 @@ GroupedMLP_CuTeGEMMUnary, GroupedMLP_CuTeGEMMGLUBf16, ) +from .moe_overlap import ( # pylint: disable=wrong-import-position + FusedMoE, + moe_overlap_supported, + moe_overlap_unsupported_reason, +) diff --git a/transformer_engine/pytorch/ops/fused/moe_overlap.py b/transformer_engine/pytorch/ops/fused/moe_overlap.py new file mode 100644 index 00000000000..1428cdfb8da --- /dev/null +++ b/transformer_engine/pytorch/ops/fused/moe_overlap.py @@ -0,0 +1,1141 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Fused MoE layer with EP dispatch and combine overlapped with the grouped GEMMs.""" + +from __future__ import annotations + +import os +import sys +import warnings +from collections.abc import Callable, Sequence +from typing import Any, NamedTuple, Optional + +import torch + +import transformer_engine_torch as tex +from ....common.recipe import Format, MXFP8BlockScaling +from ...constants import MXFP8_BLOCK_SCALING_SIZE +from ...cpu_offload import ( + is_cpu_offload_enabled, + mark_activation_offload, + mark_not_offload, + start_offload, +) +from ...ep import ( + EpConfig, + _dispatch_window, + _make_grouped_mxfp8, + _moe_overlap_available, + get_ep_drop_on_overflow, + quantize_for_ep, +) +from ...distributed_weight import is_distributed_weight +from ...quantization import Recipe +from ...tensor.grouped_tensor import GroupedTensor +from ...tensor.mxfp8_tensor import MXFP8Quantizer +from ...utils import clear_tensor_data, mark_grouped_tensor +from ..basic import ( + GroupedLinear, + MoeCombine, + MoeDispatch, + ScaledClampedQGeGLU, + ScaledSiTUGLU, + ScaledSwiGLU, +) +from ..basic.dispatch import _validate_dispatch_input, _validate_routing_inputs +from ..fuser import register_forward_backward_fusion +from ..op import FusedOperation, FusibleOperation, OperationContext +from .._common import maybe_dequantize, validate_ep_buffer +from .grouped_mlp import ( + GroupedMLP_CuTeGEMMGLU, + GroupedMLP_CuTeGEMMGLUBf16, + _bf16_packed_weight, + _bf16_wgrad_fn_factory, + _compute_grad_params, + _deterministic_algorithms_required, + validate_grouped_mlp_dims, +) + +# Recv rows are multiples of this tile, as are hidden_dim and the MXFP8 ffn_hidden_size. +_TILE = 256 +# Each distinct recv-row count compiles a new GEMM, so eager rounds the recv rows up to a multiple +# of this to bound how many are compiled. +_EAGER_ROW_BUCKET = 1024 +_SF_VEC = MXFP8_BLOCK_SCALING_SIZE +# Width of the alternating FC1 gate and up blocks the GEMMs take. +_GLU_BLOCK = 32 + + +class FusedMoE(FusedOperation): + """MoeDispatch, GroupedLinear, ScaledSwiGLU, GroupedLinear and MoeCombine as one op. + + Pipeline: dispatch -> FC1 -> SwiGLU -> FC2 -> combine, with the dispatch overlapped with FC1 + and the combine overlapped with FC2, forward and backward. + + Requirements: + - Packed weights (``single_grouped_weight=True``) without bias: BF16 parameters, or MXFP8 + primary weights (``quantized_model_init``) under ``MXFP8BlockScaling``. + - Contiguous forward inputs. + - EP within one NVLink domain. + - ``EpConfig`` ``alignment`` of 256, 128 or (without a recipe) 64, with the receive capacity + a multiple of 256. Below 256 the unfused ops run where FusedMoE cannot, also skipping the + grouped MLP fusion, and run slower. + - Recipe of none or ``MXFP8BlockScaling``. + + Notes: + - The GEMMs keep process-wide scratch for their flags and state until EP shutdown and + share it between layers, so every FusedMoE must run on one CUDA stream. + - With a fixed receive capacity the op can be captured in a CUDA graph after a warmup call; + with eager receive sizing (no ``recv_capacity_per_rank``) the host is synchronized and it + cannot be captured. + - Without a recipe the routing weights are applied in the activation; under + ``MXFP8BlockScaling`` the GEMMs and the EP transport run in MXFP8 (the ``EpConfig`` sets + MXFP8 ``dispatch_fwd_quant_recipe`` and ``combine_bwd_quant_recipe``) and the routing + weights are applied to the second GEMM's output. Either way their gradient goes to the + dispatch's ``topk_weights``. + - The weights are prepared for the GEMMs (quantized to MXFP8, or their FC1 rows reordered) at + each forward, unless the GroupedLinear's ``op_kwargs`` set ``is_first_microbatch``: True + keeps the prepared weights, False reuses them for the next microbatches of the same + weights. + - The GEMMs take the FC1 gate and up rows in alternating blocks of 32 + (``glu_interleave_size=32``). With ``glu_interleave_size=None`` the FC1 weight rows are + reordered when the weights are prepared and the FC1 weight gradient is returned in the + parameter's own layout. + """ + + @classmethod + def is_supported(cls) -> bool: + """Whether this fused operation can run on the current system.""" + if not _moe_overlap_available() or _gemm_interface_reason() is not None: + return False + if torch.cuda.get_device_capability() != (10, 7): + return GroupedMLP_CuTeGEMMGLUBf16.is_supported() + # SM 10.7 has no BF16 cuDNN GLU kernels; of those kernels this op uses only the wgrad. + try: + kernel = GroupedMLP_CuTeGEMMGLUBf16.grouped_gemm_wgrad_kernel() + except ImportError: + return False + return getattr(kernel, "supports_bf16_canonical_layouts", False) + + def __init__( + self, + *, + dispatch: MoeDispatch, + fc1: GroupedLinear, + activation: ScaledSwiGLU, + fc2: GroupedLinear, + combine: MoeCombine, + recipe: Optional[Recipe] = None, + _validated: bool = False, + ) -> None: + super().__init__((dispatch, fc1, activation, fc2, combine)) + if not _validated: + reason = _unsupported_reason(dispatch, fc1, activation, fc2, combine, recipe=recipe) + if reason is not None: + raise RuntimeError(f"FusedMoE is not supported: {reason}.") + self._mxfp8 = recipe is not None + # The GEMMs take FC1 gate and up rows in alternating blocks of _GLU_BLOCK. + self._reorder_glu = activation.glu_interleave_size is None + # MXFP8 weight operands kept across microbatches, per GroupedLinear index. + self._weight_cache: dict[int, _PreparedWeight] = {} + # The dispatch and combine share one EpBuffer (see _unsupported_reason); FusedMoE reads it + # from self.basic_ops[0].buffer at each forward. + + def fuser_forward( + self, + basic_op_ctxs: list[OperationContext], + input_: torch.Tensor, + *, + basic_op_extra_inputs: list[tuple[torch.Tensor, ...]], + prev_op_grad_output_quantizer, + next_op_input_quantizer, + basic_op_kwargs: list[dict[str, Any]], + ) -> tuple[torch.Tensor, Sequence[Sequence[torch.Tensor]]]: + del prev_op_grad_output_quantizer, next_op_input_quantizer + _, fc1_op, _, fc2_op, _ = self.basic_ops + topk_idx, topk_weights = basic_op_extra_inputs[0] + # Splits and scales come from the dispatch inside this op. + for idx in (1, 2, 3): + if any(t is not None for t in basic_op_extra_inputs[idx]): + raise ValueError( + "FusedMoE takes the splits and routing scales from its dispatch; " + "do not pass them as extra inputs." + ) + + buffer = validate_ep_buffer("FusedMoE", self.basic_ops[0].config, self.basic_ops[0].buffer) + input_shape = _validate_dispatch_input(input_, buffer) + _validate_routing_inputs(topk_idx, topk_weights, device=buffer.device) + if topk_idx.dtype not in (torch.int32, torch.int64): + raise TypeError(f"topk_idx must be int32 or int64, got {topk_idx.dtype}.") + if torch.is_autocast_enabled() and torch.get_autocast_dtype("cuda") != torch.bfloat16: + raise RuntimeError("FusedMoE computes in BF16; keep the autocast dtype BF16.") + if not buffer.eager and buffer.recv_capacity_per_rank % _TILE != 0: + raise ValueError( + f"FusedMoE needs the recv rows ({buffer.recv_capacity_per_rank}) to be a" + f" multiple of {_TILE}." + ) + buffer.num_local_tokens = input_shape[0] + buffer.topk_idx_dtype = topk_idx.dtype + + if self._mxfp8: + return self._forward_mxfp8( + basic_op_ctxs, buffer, input_, topk_idx, topk_weights, basic_op_kwargs + ) + device = buffer.device + num_groups = fc1_op.num_groups + hidden = buffer.hidden_dim + ffn = fc2_op.in_features + w1 = self._weight_operands(1, False, basic_op_kwargs[1].get("is_first_microbatch")) + w2 = _bf16_weight(fc2_op, num_groups) + tokens = _require_contiguous(input_, "input") + topk_idx = _require_contiguous(topk_idx, "topk_idx") + topk_weights = _require_contiguous(topk_weights, "topk_weights") + + tokens_per_expert = buffer.overlap_tokens_per_expert + if buffer.eager: + # The binding runs the prepare, reads the recv total on the host and allocates the + # recv rows in whole buckets, so the number of compiled GEMMs stays bounded. + recv_tokens, recv_weights, gemm_out, act_out, expert_token_offsets = ( + tex.ep_dispatch_gemm_fwd( + buffer.handle_mem, topk_idx, tokens, topk_weights, w1, None, None, + tokens_per_expert, None, None, buffer.top_k, buffer.alignment, + buffer.overlap_total_recv_tokens, _EAGER_ROW_BUCKET, + ) + ) # fmt: skip + rows = recv_tokens.shape[0] + else: + rows = buffer.recv_capacity_per_rank + recv_tokens = torch.empty(rows, hidden, dtype=torch.bfloat16, device=device) + recv_weights = torch.empty(rows, dtype=torch.float32, device=device) + gemm_out = torch.empty(rows, 2 * ffn, dtype=torch.bfloat16, device=device) + act_out = torch.empty(rows, ffn, dtype=torch.bfloat16, device=device) + # The dispatch writes the expert layout the GEMMs read. + expert_token_offsets = tex.ep_dispatch_gemm_fwd( + buffer.handle_mem, topk_idx, tokens, topk_weights, w1, recv_tokens, recv_weights, + tokens_per_expert, gemm_out, act_out, buffer.top_k, buffer.alignment, + buffer.overlap_total_recv_tokens, + )[4] # fmt: skip + expert_out = torch.empty(rows, hidden, dtype=torch.bfloat16, device=device) + result = torch.empty(buffer.num_local_tokens, hidden, dtype=torch.bfloat16, device=device) + tex.ep_combine_gemm_fwd( + buffer.handle_mem, act_out, expert_token_offsets, w2, expert_out, result, + buffer.top_k, buffer.alignment, buffer.topk_idx_dtype, + ) # fmt: skip + recv_total = _recv_total_from_offsets(buffer, expert_token_offsets) + self.basic_ops[0].update_recv_tokens_peak(buffer, recv_total) + + # The BF16 SwiGLU backward GEMM cannot write the SwiGLU output, so the FC2 wgrad reads it. + self._save_for_backward( + basic_op_ctxs, buffer, recv_total, (recv_tokens, gemm_out, act_out, recv_weights), + (w1,) if self._reorder_glu else (), topk_weights, + ) # fmt: skip + return result, [(tokens_per_expert, recv_weights), (), (), (), ()] + + def _save_for_backward( + self, basic_op_ctxs, buffer, recv_total, row_tensors, other_tensors, topk_weights + ): + """Save the tensors the backward reads, when any output needs a gradient.""" + _, fc1_op, _, fc2_op, _ = self.basic_ops + fc1_ctx, fc2_ctx = basic_op_ctxs[1], basic_op_ctxs[3] + if any(ctx.requires_grad for ctx in basic_op_ctxs): + fc1_ctx.weight_requires_grad = fc1_op.weight.requires_grad + fc2_ctx.weight_requires_grad = fc2_op.weight.requires_grad + fc1_ctx.dtype = fc2_ctx.dtype = torch.bfloat16 + # Paged stashing keeps only the received rows of each row-sized tensor. The recv total + # counts them, except in eager mode (a host total) and when rows past the capacity drop. + mark_grouped_tensor(*row_tensors) + if buffer.eager or get_ep_drop_on_overflow(): + num_tokens = buffer.overlap_tokens_per_expert.sum( + 0, keepdim=True, dtype=torch.int32 + ) + else: + num_tokens = recv_total + for tensor in row_tensors: + tensor.grouped_tensor_num_tokens = num_tokens + # CPU offload takes the activations, as the unfused ops do, not the routing weights. + # The weight copies stay on the GPU, as in TE modules, since they may be kept for + # the next microbatches. + if is_cpu_offload_enabled(): + start_offload(*row_tensors[:3]) + mark_activation_offload(*row_tensors[:3]) + mark_not_offload(*row_tensors[3:], *other_tensors) + fc1_ctx.save_for_backward(*row_tensors, *other_tensors) + fc1_ctx.buffer = buffer + fc1_ctx.topk_weights_shape = topk_weights.shape + + def _forward_mxfp8( + self, basic_op_ctxs, buffer, input_, topk_idx, topk_weights, basic_op_kwargs + ): + """Forward with MXFP8 GEMMs and EP transport.""" + fc2_op = self.basic_ops[3] + device = buffer.device + hidden = buffer.hidden_dim + ffn = fc2_op.in_features + backward = any(ctx.requires_grad for ctx in basic_op_ctxs) + w1, w2 = ( + self._weight_operands(idx, backward, basic_op_kwargs[idx].get("is_first_microbatch")) + for idx in (1, 3) + ) + tokens, tokens_scale, symm_input = _quantize_for_dispatch( + _require_contiguous(input_, "input"), device + ) + topk_idx = _require_contiguous(topk_idx, "topk_idx") + topk_weights = _require_contiguous(topk_weights, "topk_weights") + tokens_per_expert = buffer.overlap_tokens_per_expert + if buffer.eager: + ( + recv_tokens, recv_weights, gemm_out, act_out, expert_token_offsets, recv_scale, + act_scale, + ) = ( + tex.ep_dispatch_gemm_fwd( + buffer.handle_mem, topk_idx, tokens, topk_weights, w1.data, None, None, + tokens_per_expert, None, None, buffer.top_k, buffer.alignment, + buffer.overlap_total_recv_tokens, _EAGER_ROW_BUCKET, + tokens_scale_inv=tokens_scale, weight_scale_inv=w1.scale_inv, symm_input=symm_input, + ) + ) # fmt: skip + rows = recv_tokens.shape[0] + else: + rows = buffer.recv_capacity_per_rank + fp8 = torch.float8_e4m3fn + recv_tokens = torch.empty(rows, hidden, dtype=fp8, device=device) + recv_scale = torch.empty(rows, hidden // _SF_VEC, dtype=torch.uint8, device=device) + recv_weights = torch.empty(rows, dtype=torch.float32, device=device) + gemm_out = torch.empty(rows, 2 * ffn, dtype=torch.bfloat16, device=device) + act_out = torch.empty(rows, ffn, dtype=fp8, device=device) + act_scale = torch.empty(rows * ffn // _SF_VEC, dtype=torch.uint8, device=device) + expert_token_offsets = tex.ep_dispatch_gemm_fwd( + buffer.handle_mem, topk_idx, tokens, topk_weights, w1.data, recv_tokens, + recv_weights, tokens_per_expert, gemm_out, act_out, buffer.top_k, + buffer.alignment, buffer.overlap_total_recv_tokens, tokens_scale_inv=tokens_scale, + recv_scale_inv=recv_scale, weight_scale_inv=w1.scale_inv, act_out_scale_inv=act_scale, + symm_input=symm_input, + )[4] # fmt: skip + expert_out = torch.empty(rows, hidden, dtype=torch.bfloat16, device=device) + result = torch.empty(buffer.num_local_tokens, hidden, dtype=torch.bfloat16, device=device) + # The routing weights scale the second GEMM's output rows. + tex.ep_combine_gemm_fwd( + buffer.handle_mem, act_out, expert_token_offsets, w2.data, expert_out, result, + buffer.top_k, buffer.alignment, buffer.topk_idx_dtype, recv_topk_weights=recv_weights, + gemm_in_scale_inv=act_scale, weight_scale_inv=w2.scale_inv, + ) # fmt: skip + recv_total = _recv_total_from_offsets(buffer, expert_token_offsets) + self.basic_ops[0].update_recv_tokens_peak(buffer, recv_total) + # The FC2 dgrad recomputes the SwiGLU output for the FC2 wgrad, so it is not saved. + self._save_for_backward( + basic_op_ctxs, buffer, recv_total, (recv_tokens, recv_scale, gemm_out, recv_weights), + (w1.columnwise_data, w1.columnwise_scale_inv, w2.columnwise_data, w2.columnwise_scale_inv) + if backward else (), topk_weights, + ) # fmt: skip + return result, [(tokens_per_expert, recv_weights), (), (), (), ()] + + def _weight_operands(self, idx, columnwise, is_first_microbatch): + """GEMM operands of the weight of the GroupedLinear at ``idx``: its ``_MxFp8Weight``, or + the BF16 weight. As in TE modules, ``is_first_microbatch`` True prepares and keeps them, + False reuses the kept ones unless the weight changed in place, and None prepares them + without keeping them.""" + fc_op = self.basic_ops[idx] + reorder = self._reorder_glu and idx == 1 + if not (self._mxfp8 or reorder): + return _bf16_weight(fc_op, fc_op.num_groups) + key = None if is_first_microbatch is None else _weight_key(fc_op.weight) + kept = self._weight_cache.get(idx) + if is_first_microbatch is False and key is not None and kept is not None: + operands = kept.operands + has_columnwise = not isinstance(operands, _MxFp8Weight) or ( + operands.columnwise_data is not None + ) + if kept.key == key and (has_columnwise or not columnwise): + return operands + if self._mxfp8: + operands = _mxfp8_weight(fc_op, fc_op.num_groups, columnwise, reorder_glu=reorder) + else: + operands = _interleave_glu_rows(_bf16_weight(fc_op, fc_op.num_groups)) + if key is None: + self._weight_cache.pop(idx, None) + else: + self._weight_cache[idx] = _PreparedWeight(key, operands) + return operands + + def fuser_backward( + self, + basic_op_ctxs: list[OperationContext], + grad_output: torch.Tensor, + *, + basic_op_grad_extra_outputs: list[tuple[Optional[torch.Tensor], ...]], + **unused, # pylint: disable=unused-argument + ): + # A later op that also reads the dispatch's routing weights delivers their gradient here; + # fold it into the routing-weight gradient the dispatch backward scatters to topk_weights. + grad_recv_weights_extra = basic_op_grad_extra_outputs[0][1] + if self._mxfp8: + return self._backward_mxfp8(basic_op_ctxs, grad_output, grad_recv_weights_extra) + _, fc1_op, _, fc2_op, _ = self.basic_ops + fc1_ctx, fc2_ctx = basic_op_ctxs[1], basic_op_ctxs[3] + recv_tokens, gemm_out, act_out, recv_weights, *w1 = fc1_ctx.saved_tensors + buffer = fc1_ctx.buffer + device = buffer.device + num_groups = fc1_op.num_groups + hidden = buffer.hidden_dim + ffn = fc2_op.in_features + rows = gemm_out.shape[0] + tokens_per_expert = buffer.overlap_tokens_per_expert + w1 = w1[0] if w1 else _bf16_weight(fc1_op, num_groups) + w2 = _bf16_weight(fc2_op, num_groups) + grad_result = _bf16_grad_output(grad_output) + num_tokens = grad_result.shape[0] + + wgrad_kernel_fn = GroupedMLP_CuTeGEMMGLUBf16.grouped_gemm_wgrad_kernel() + fc2_shape = (fc2_op.out_features, fc2_op.in_features) + fc1_shape = (fc1_op.out_features, fc1_op.in_features) + + def grad_params(fc_op, ctx, shape, grouped_x, grouped_dy, label, wgrad_args=None): + # With wgrad_args, the backward call computes the weight gradient and reads no offsets. + offsets = tokens_per_expert.cumsum(0, dtype=torch.int32) if wgrad_args is None else None + factory = ( + _bf16_wgrad_fn_factory( + weight_shape=shape, offsets=offsets, wgrad_kernel_fn=wgrad_kernel_fn, + single_grouped_weight=True, num_groups=num_groups, + ) + if wgrad_args is None + else wgrad_args.gemm_fn_factory(num_groups, shape) + ) # fmt: skip + grads = _compute_grad_params( + fc_op=fc_op, ctx=ctx, num_groups=num_groups, weight_shape=shape, + grouped_x=grouped_x, grouped_dy=grouped_dy, dtype=torch.bfloat16, device=device, + bias_grads=None, bias_grad_packed=None, label=label, + cudnn_wgrad_kernel_fn=wgrad_kernel_fn, use_nvfp4=False, + data_dtype=torch.bfloat16, scale_view_dtype=None, sf_vec_size=None, + offsets=offsets, use_dense_single_group=False, wgrad_gemm_fn_factory=factory, + ) # fmt: skip + if wgrad_args is not None and ctx.weight_requires_grad: + wgrad_args.require_recorded() + return grads + + grad_expert_out = torch.empty(rows, hidden, dtype=torch.bfloat16, device=device) + grad_gemm = torch.empty(rows, 2 * ffn, dtype=torch.bfloat16, device=device) + grad_recv_weights = torch.empty(rows, dtype=torch.float32, device=device) + fc2_grads = None + fc2_wgrad = _CallWgrad() + if fc2_ctx.weight_requires_grad: + fc2_grads = grad_params( + fc2_op, fc2_ctx, fc2_shape, act_out, grad_expert_out, "FC2", fc2_wgrad + ) + tex.ep_combine_gemm_bwd( + buffer.handle_mem, grad_result, gemm_out, recv_weights, tokens_per_expert, w2, + grad_expert_out, grad_gemm, grad_recv_weights, buffer.top_k, buffer.alignment, + buffer.topk_idx_dtype, **fc2_wgrad, + ) # fmt: skip + if fc2_grads is None: + fc2_grads = grad_params(fc2_op, fc2_ctx, fc2_shape, act_out, grad_expert_out, "FC2") + + # Neither the input nor the routing weights need a gradient: no dispatch backward. + grad_tokens = grad_topk = fc1_grads = None + if basic_op_ctxs[0].requires_grad: + if grad_recv_weights_extra is not None: + grad_recv_weights.add_(grad_recv_weights_extra.to(torch.float32)) + fc1_wgrad = _CallWgrad() + # The weight gradient of reordered FC1 rows is computed after the reorder back. + if not self._reorder_glu and fc1_ctx.weight_requires_grad: + fc1_grads = grad_params( + fc1_op, fc1_ctx, fc1_shape, recv_tokens, grad_gemm, "FC1", fc1_wgrad + ) + grad_recv = torch.empty(rows, hidden, dtype=torch.bfloat16, device=device) + grad_tokens = torch.empty(num_tokens, hidden, dtype=torch.bfloat16, device=device) + grad_topk = torch.empty(fc1_ctx.topk_weights_shape, dtype=torch.float32, device=device) + tex.ep_dispatch_gemm_bwd( + buffer.handle_mem, grad_gemm, grad_recv_weights, tokens_per_expert, w1, + grad_recv, grad_tokens, grad_topk, buffer.top_k, buffer.alignment, + buffer.topk_idx_dtype, **fc1_wgrad, + ) # fmt: skip + + if fc1_grads is None: + if self._reorder_glu: + grad_gemm = _deinterleave_glu_columns(grad_gemm) + fc1_grads = grad_params(fc1_op, fc1_ctx, fc1_shape, recv_tokens, grad_gemm, "FC1") + + clear_tensor_data(recv_tokens, act_out) + return self._finish_backward(grad_tokens, grad_topk, fc1_grads, fc2_grads) + + def _finish_backward(self, grad_tokens, grad_topk, fc1_grads, fc2_grads): + """Return the gradients; the routing-weight gradient is part of the dispatch's + ``topk_weights`` gradient.""" + return ( + grad_tokens, + [(), fc1_grads, (), fc2_grads, ()], + [(None, grad_topk), (None,), (None,), (None,), (None,)], + ) + + def _backward_mxfp8(self, basic_op_ctxs, grad_output, grad_recv_weights_extra=None): + """Backward with MXFP8 GEMMs and EP transport.""" + _, fc1_op, _, fc2_op, _ = self.basic_ops + fc1_ctx, fc2_ctx = basic_op_ctxs[1], basic_op_ctxs[3] + ( + recv_tokens, + recv_scale, + gemm_out, + recv_weights, + w1_col, + w1_col_scale, + w2_col, + w2_col_scale, + ) = fc1_ctx.saved_tensors + buffer = fc1_ctx.buffer + device = buffer.device + num_groups = fc1_op.num_groups + hidden = buffer.hidden_dim + ffn = fc2_op.in_features + rows = gemm_out.shape[0] + tokens_per_expert = buffer.overlap_tokens_per_expert + # GroupedTensor group sizes are int64. + first_dims = tokens_per_expert.long() + grad_result = _bf16_grad_output(grad_output) + num_tokens = grad_result.shape[0] + grad_result, grad_result_scale, symm_input = _quantize_for_dispatch(grad_result, device) + + def fp8_rows(cols): + data = torch.empty(rows, cols, dtype=torch.float8_e4m3fn, device=device) + return data, torch.empty(rows * cols // _SF_VEC, dtype=torch.uint8, device=device) + + grad_expert_out, grad_expert_out_scale = fp8_rows(hidden) + grad_expert_out_scale = grad_expert_out_scale.view(rows, hidden // _SF_VEC) + # The call also writes the received rows columnwise for the FC2 weight gradient. + grad_expert_out_col, grad_expert_out_col_scale = fp8_rows(hidden) + grad_gemm, grad_gemm_scale = fp8_rows(2 * ffn) + grad_gemm_col, grad_gemm_col_scale = fp8_rows(2 * ffn) + act_col, act_col_scale = fp8_rows(ffn) + grad_recv_weights = torch.empty(rows, dtype=torch.float32, device=device) + + wgrad_kernel_fn = GroupedMLP_CuTeGEMMGLU.grouped_gemm_wgrad_kernel() + + def grouped(cols, data, scale, col_data, col_scale, in_call): + # Expert slabs of a row-sized MXFP8 tensor with GEMM-swizzled scales. A weight + # gradient in a backward call reads no tensor offsets. + return GroupedTensor( + shape=(rows, cols), dtype=torch.bfloat16, num_tensors=num_groups, + quantizer=_ep_quantizer(), data=data, columnwise_data=col_data.view(-1), + scale_inv=scale, columnwise_scale_inv=col_scale, first_dims=first_dims, + tensor_offsets=None if in_call else tex.splits_to_offsets(first_dims, cols), + with_gemm_swizzled_scales=True, + ) # fmt: skip + + def grad_params(fc_op, ctx, grouped_x, grouped_dy, label, wgrad_args=None): + # With wgrad_args, the backward call computes the weight gradient. + shape = (fc_op.out_features, fc_op.in_features) + grads = _compute_grad_params( + fc_op=fc_op, ctx=ctx, num_groups=num_groups, weight_shape=shape, + grouped_x=grouped_x, grouped_dy=grouped_dy, dtype=torch.bfloat16, device=device, + bias_grads=None, bias_grad_packed=None, label=label, + cudnn_wgrad_kernel_fn=wgrad_kernel_fn, use_nvfp4=False, + data_dtype=torch.float8_e4m3fn, scale_view_dtype=torch.float8_e8m0fnu, + sf_vec_size=_SF_VEC, use_dense_single_group=False, + offsets=( + tokens_per_expert.cumsum(0, dtype=torch.int32) if wgrad_args is None else None + ), + wgrad_gemm_fn_factory=( + None + if wgrad_args is None + else wgrad_args.gemm_fn_factory(num_groups, shape) + ), + ) # fmt: skip + if wgrad_args is not None and ctx.weight_requires_grad: + wgrad_args.require_recorded() + return grads + + def fc1_grad_params(wgrad_args=None): + in_call = wgrad_args is not None + if in_call: + # The call writes the columnwise received rows from their rowwise copy. + x_col, x_col_scale = fp8_rows(hidden) + grouped_x = grouped(hidden, None, None, x_col, x_col_scale, True) + wgrad_args.update( + wgrad_input_rowwise=recv_tokens, wgrad_input_rowwise_scale_inv=recv_scale + ) + else: + grouped_x = _columnwise_rows(recv_tokens, recv_scale, first_dims) + grad_gemm_cols = ( + grouped(2 * ffn, None, None, _deinterleave_glu_columns(grad_gemm_col), + _deinterleave_glu_col_scales(grad_gemm_col_scale, tokens_per_expert, + 2 * ffn), in_call) + if self._reorder_glu + else grouped(2 * ffn, grad_gemm.view(-1), grad_gemm_scale, grad_gemm_col, + grad_gemm_col_scale, in_call) + ) # fmt: skip + return grad_params( + fc1_op, fc1_ctx, grouped_x, grad_gemm_cols, "FC1", wgrad_args + ) # fmt: skip + + # The call computes the FC2 weight gradient too. + fc2_wgrad = _CallWgrad() + fc2_grads = grad_params( + fc2_op, fc2_ctx, grouped(ffn, None, None, act_col, act_col_scale, True), + grouped(hidden, None, None, grad_expert_out_col, grad_expert_out_col_scale, True), + "FC2", fc2_wgrad, + ) # fmt: skip + tex.ep_combine_gemm_bwd( + buffer.handle_mem, grad_result, gemm_out, + recv_weights, tokens_per_expert, w2_col, grad_expert_out, grad_gemm, + grad_recv_weights, buffer.top_k, buffer.alignment, buffer.topk_idx_dtype, + grad_result_scale_inv=grad_result_scale, + grad_expert_out_scale_inv=grad_expert_out_scale, weight_scale_inv=w2_col_scale, + grad_gemm_out_scale_inv=grad_gemm_scale, grad_gemm_out_columnwise=grad_gemm_col, + grad_gemm_out_columnwise_scale_inv=grad_gemm_col_scale, act_out=act_col, + act_out_scale_inv=act_col_scale, grad_expert_out_columnwise=grad_expert_out_col, + grad_expert_out_columnwise_scale_inv=grad_expert_out_col_scale, **fc2_wgrad, + symm_input=symm_input, + ) # fmt: skip + + grad_tokens = grad_topk = fc1_grads = None + if basic_op_ctxs[0].requires_grad: + if grad_recv_weights_extra is not None: + grad_recv_weights.add_(grad_recv_weights_extra.to(torch.float32)) + fc1_wgrad = _CallWgrad() + # The weight gradient of reordered FC1 rows is computed after the reorder back. + if not self._reorder_glu and fc1_ctx.weight_requires_grad: + fc1_wgrad.update( + grad_gemm_out_columnwise=grad_gemm_col, + grad_gemm_out_columnwise_scale_inv=grad_gemm_col_scale, + ) + fc1_grads = fc1_grad_params(fc1_wgrad) + grad_recv = torch.empty(rows, hidden, dtype=torch.bfloat16, device=device) + grad_tokens = torch.empty(num_tokens, hidden, dtype=torch.bfloat16, device=device) + grad_topk = torch.empty(fc1_ctx.topk_weights_shape, dtype=torch.float32, device=device) + tex.ep_dispatch_gemm_bwd( + buffer.handle_mem, grad_gemm, grad_recv_weights, tokens_per_expert, w1_col, + grad_recv, grad_tokens, grad_topk, buffer.top_k, buffer.alignment, + buffer.topk_idx_dtype, grad_gemm_out_scale_inv=grad_gemm_scale, + weight_scale_inv=w1_col_scale, **fc1_wgrad, + ) # fmt: skip + + if fc1_grads is None: + fc1_grads = fc1_grad_params() + + clear_tensor_data(recv_tokens, recv_scale) + return self._finish_backward(grad_tokens, grad_topk, fc1_grads, fc2_grads) + + +def _recv_total_from_offsets(buffer, expert_token_offsets) -> torch.Tensor: + """The int32 [1] recv total: in capacity mode the last element of the expert token offsets. + Eager and overflow-drop modes get it from the routing prepare instead.""" + if buffer.eager or get_ep_drop_on_overflow(): + return buffer.overlap_total_recv_tokens + return expert_token_offsets[-1:] + + +def _require_contiguous(tensor: torch.Tensor, name: str) -> torch.Tensor: + """Forward inputs must already be contiguous; making them contiguous here would hide a copy.""" + if not tensor.is_contiguous(): + raise ValueError(f"FusedMoE requires a contiguous {name}, got strides {tensor.stride()}.") + return tensor + + +def _bf16_grad_output(grad_output) -> torch.Tensor: + """Contiguous BF16 gradient of the combine output.""" + if isinstance(grad_output, GroupedTensor): + raise RuntimeError("FusedMoE does not accept a quantized grad output.") + return maybe_dequantize(grad_output, torch.bfloat16).contiguous() + + +class _CallWgrad(dict): + """Keyword arguments of a weight gradient computed inside a backward call, passed as + ``**wgrad``. Its wgrad factory only records the operands; the call runs the kernel later, so + distributed weights, which FusedMoE rejects, do not fit.""" + + def gemm_fn_factory(self, num_groups: int, weight_shape) -> Callable: + """Wgrad factory that records the operands of a weight of shape ``weight_shape``.""" + + def factory(accumulate: bool) -> Callable: + def gemm_fn(grouped_x, grouped_dy, wgrad_output): + del grouped_dy + if isinstance(grouped_x, GroupedTensor): + self.update( + wgrad_input=grouped_x.columnwise_data.view(torch.float8_e4m3fn).view( + -1, weight_shape[1] + ), + wgrad_input_scale_inv=grouped_x.columnwise_scale_inv, + ) + else: + self.update(wgrad_input=grouped_x) + self.update( + weight_grad=wgrad_output.rowwise_data.view(num_groups, *weight_shape), + accumulate_weight_grad=accumulate, + ) + + return gemm_fn + + return factory + + def require_recorded(self) -> None: + """Raise unless the GEMM function ran: the call would otherwise skip the gradient.""" + if "weight_grad" not in self: + raise RuntimeError("FusedMoE: the weight-gradient GEMM function was not called.") + + +class _MxFp8Weight(NamedTuple): + """MXFP8 operands of a weight: data and scales, rowwise and, when prepared, columnwise.""" + + data: torch.Tensor + scale_inv: torch.Tensor + columnwise_data: Optional[torch.Tensor] = None + columnwise_scale_inv: Optional[torch.Tensor] = None + + +class _PreparedWeight(NamedTuple): + """Operands kept for ``is_first_microbatch=False`` and the ``_weight_key`` they were made at.""" + + key: tuple + operands: torch.Tensor | _MxFp8Weight + + +def _gemm_interface_reason(mxfp8: bool = False) -> Optional[str]: + """Why the EP grouped GEMM kernel tree does not match Transformer Engine, or ``None``.""" + try: + from ....common.CuTeDSL.ep_gemm._loader import ( # pylint: disable=import-outside-toplevel + gemm_interface_reason, + ) + except ImportError as err: + return f"the CuTeDSL EP grouped GEMMs cannot be imported: {err}" + return gemm_interface_reason(mxfp8) + + +def _weight_key(weight) -> Optional[tuple]: + """Identity and in-place update counters of the tensors that store ``weight``, or ``None`` + when one has no counter (inference tensors).""" + try: + key = [weight._version] + for name in ("rowwise_data", "columnwise_data", "scale_inv", "columnwise_scale_inv"): + tensor = getattr(weight, name, None) + if tensor is not None: + key += [tensor.data_ptr(), tensor._version] + except RuntimeError: + return None + return tuple(key) + + +def _ep_quantizer() -> MXFP8Quantizer: + """Rowwise E4M3 quantizer of the MXFP8 EP transport.""" + quantizer = MXFP8Quantizer(tex.DType.kFloat8E4M3, rowwise=True, columnwise=False) + quantizer.internal = True + return quantizer + + +def _quantize_for_dispatch(input_, device): + """MXFP8 data and compact scales of a dispatch input, and whether the dispatch reads them in + place: they are in the EP dispatch window whenever it exists, alike on every rank.""" + window = _dispatch_window(device) + quantized, scale = quantize_for_ep(input_, _ep_quantizer(), window) + return quantized._rowwise_data.view(torch.float8_e4m3fn), scale, window is not None + + +def _columnwise_rows(data, scale, tokens_per_expert): + """Expert slabs of received MXFP8 rows with compact scales, given columnwise data for the + weight gradient.""" + grouped = _make_grouped_mxfp8( + data, scale, tokens_per_expert, tex.DType.kFloat8E4M3, torch.bfloat16 + ) + quantizer = _ep_quantizer() + quantizer.set_usage(rowwise=True, columnwise=True) + quantizer.optimize_for_gemm = True + tex.group_requantize_inplace( + grouped, quantizer, tokens_per_expert.numel(), tokens_per_expert, tex.DType.kBFloat16, + tensor_offsets=grouped.tensor_offsets, + ) # fmt: skip + return grouped + + +def _mxfp8_weight( + fc_op: GroupedLinear, num_groups: int, columnwise: bool, reorder_glu: bool = False +) -> _MxFp8Weight: + """GEMM operands of the weight quantized to MXFP8, with the columnwise ones when asked; data + is [num_groups, out_features, in_features]. + BF16 weights are quantized with GEMM-swizzled scales; the scales of MXFP8 weight parameters + are swizzled into a copy. Each direction is swizzled along its own scaling axis. + ``reorder_glu`` reorders FC1 rows as ``_interleave_glu_rows`` does, data and scales alike.""" + weight = fc_op.weight + shape = (num_groups, fc_op.out_features, fc_op.in_features) + if weight.quantizer is not None: + # Only add usages, never drop: later weight updates keep what this op needs. + weight.quantizer.set_usage(rowwise=True, columnwise=True if columnwise else None) + if columnwise and weight.columnwise_data is None: + raise RuntimeError("FusedMoE needs columnwise data in the MXFP8 weight parameters.") + weight = weight.copy() + if reorder_glu: + # Rowwise scales follow the rows; a columnwise scale row covers one block of rows. + rows, cols = shape[1:] + weight.rowwise_data = _interleave_glu_rows(weight.rowwise_data.view(shape)).view(-1) + weight.scale_inv = _interleave_glu_rows( + weight.scale_inv.view(num_groups, rows, -1) + ).view(-1) + if weight.columnwise_data is not None: + weight.columnwise_data = _interleave_glu_rows( + weight.columnwise_data.view(shape) + ).view(-1) + weight.columnwise_scale_inv = _interleave_glu_rows( + weight.columnwise_scale_inv.view(num_groups, rows // _SF_VEC, cols), + block=_GLU_BLOCK // _SF_VEC, + ).view(-1) + tex.grouped_swizzle_for_gemm(weight, rowwise=True, columnwise=columnwise) + else: + quantizer = fc_op.get_quantizer("forward", 1).copy() + quantizer.set_usage(rowwise=True, columnwise=columnwise) + quantizer.optimize_for_gemm = True + data = weight.rowwise_data.view(shape) + if reorder_glu: + data = _interleave_glu_rows(data) + weight = tex.group_quantize(data.view(weight.logical_shape), quantizer, num_groups, None) + fp8 = torch.float8_e4m3fn + operands = _MxFp8Weight(weight.rowwise_data.view(fp8).view(shape), weight.scale_inv.reshape(-1)) + if columnwise: + operands = operands._replace( + columnwise_data=weight.columnwise_data.view(fp8).view(shape), + columnwise_scale_inv=weight.columnwise_scale_inv.reshape(-1), + ) + return operands + + +def _bf16_weight(fc_op: GroupedLinear, num_groups: int) -> torch.Tensor: + """Expert-major BF16 weight of a packed GroupedLinear weight, cast when its data has another + dtype, as after ``Module.to(torch.bfloat16)`` on FP32 weights.""" + return _bf16_packed_weight(fc_op, num_groups).to(torch.bfloat16) + + +# TODO(phuonguyen): the cuDNN GEMMs require the interleaved GLU layout, so FC1 weights are reordered +# here. Drop this reorder and pass the non-interleaved layout through once cuDNN supports it. +def _interleave_glu_rows(t: torch.Tensor, block: int = _GLU_BLOCK) -> torch.Tensor: + """Copy of per-expert rows [experts, 2 * n, ...], gate rows then up rows, with gate and up + alternating in blocks of ``block`` rows, the GLU layout of ``glu_interleave_size=32``.""" + experts, rows = t.shape[:2] + blocks = t.reshape(experts, 2, rows // (2 * block), block, *t.shape[2:]) + return blocks.transpose(1, 2).reshape(t.shape) + + +def _deinterleave_glu_columns(t: torch.Tensor) -> torch.Tensor: + """Copy of [rows, 2 * n] with gate and up in alternating column blocks, gate columns first.""" + rows, cols = t.shape + # Moved in 16-byte words (complex128); a column block spans whole words. + words = t.view(torch.complex128) + blocks = words.reshape(rows, cols // (2 * _GLU_BLOCK), 2, _GLU_BLOCK * t.element_size() // 16) + return blocks.transpose(1, 2).reshape(rows, -1).view(t.dtype) + + +def _deinterleave_glu_col_scales(scale, tokens_per_expert, cols): + """Columnwise MXFP8 scales, swizzled per expert, of a tensor reordered by + ``_deinterleave_glu_columns``.""" + tiles = cols // 128 + blocks = scale.view(torch.int32).view(-1, 128) + units = blocks.shape[0] // tiles # 128-row units + device = scale.device + # First unit and unit count of each unit's expert; the rows after the experts form a last group. + ends = tokens_per_expert.cumsum(0) // 128 + starts = torch.nn.functional.pad(ends, (1, 0)) + stops = torch.nn.functional.pad(ends, (0, 1), value=units) + unit = torch.arange(units, device=device) + group = torch.searchsorted(stops, unit, right=True) + first = starts[group] + count = stops[group] - first + # Block of each (tile, unit) in the slabs. + tile = torch.arange(tiles, device=device)[:, None] + index = (torch.add(unit, first, alpha=tiles - 1) + tile * count).view(-1) + # Tile 2 * i + h holds column blocks 8 * i + 4 * h + 2 * j + g, of gate (g = 0) or up (g = 1), + # which go to tile g * tiles / 2 + i at slot 2 * h + j. + src = blocks[index].view(tiles // 2, 2, units, 32, 2, 2) + out = torch.empty_like(blocks) + out[index] = src.permute(5, 0, 2, 3, 1, 4).reshape(-1, 128) + return out.view(torch.uint8).view(scale.shape) + + +def _routing_extras_internal(dispatch, fc1, activation, fc2) -> bool: + """Whether tokens_per_expert and the routing weights flow from the dispatch to the MLP.""" + channels = dispatch._extra_output_channels + return ( + not any(dispatch._extra_output_to_caller) + and channels[0] is not None + and channels[1] is not None + and fc1._extra_input_channels[0] == channels[0] + and fc2._extra_input_channels[0] == channels[0] + and activation._extra_input_channels[0] == channels[1] + ) + + +def moe_overlap_unsupported_reason( + config: EpConfig, + ffn_hidden_size: int, + *, + params_dtype: torch.dtype, + activation: str = "swiglu", + glu_interleave_size: Optional[int] = None, + bias: bool = False, + single_grouped_weight: bool = False, + recipe: Optional[Recipe] = None, + primary_weights_in_fp8: bool = False, +) -> Optional[str]: + """Why a ``Sequential`` of ``MoeDispatch``, ``GroupedLinear``, ``ScaledSwiGLU``, + ``GroupedLinear`` and ``MoeCombine`` built with these settings does not run as one + ``FusedMoE``, or ``None`` when it does. The reason names an unsupported setting or the + grouped GEMM backend failing to load on this device. See ``moe_overlap_supported`` for what + the query covers. + + Parameters + ---------- + config : EpConfig + EP configuration shared by the dispatch and the combine. + ffn_hidden_size : int + Output features of the activation, input features of the second GroupedLinear. + params_dtype : torch.dtype + Dtype of the GroupedLinear weights. + activation : str, default = "swiglu" + Activation between the GroupedLinear ops. + glu_interleave_size : int, optional + ``glu_interleave_size`` of the ``ScaledSwiGLU``, ``None`` or 32. + bias : bool, default = ``False`` + Whether the GroupedLinear ops have biases. + single_grouped_weight : bool, default = ``False`` + ``single_grouped_weight`` of the GroupedLinear ops. + recipe : Recipe, optional + Quantization recipe the ops run under, ``None`` for BF16. ``MXFP8BlockScaling`` needs + MXFP8 EP transport in ``config``. + primary_weights_in_fp8 : bool, default = ``False`` + Whether the GroupedLinear weights are quantized parameters (``quantized_model_init``). + They must be MXFP8 and need the ``MXFP8BlockScaling`` recipe. + """ + if not FusedMoE.is_supported(): + reason = ( + "needs SM 10.0, 10.3 or 10.7, NVTE_ENABLE_CUTEDSL_BACKEND=1, NVTE_MOE_OVERLAP and" + " the BF16 cuDNN grouped GEMM kernels" + ) + mismatch = _gemm_interface_reason() if _moe_overlap_available() else None + return f"{reason}: {mismatch}" if mismatch else reason + if _deterministic_algorithms_required(): + return ( + "deterministic execution was requested, and the routing-weight gradient is reduced" + " with atomics" + ) + if torch.is_autocast_enabled() and torch.get_autocast_dtype("cuda") != torch.bfloat16: + return "needs a BF16 autocast dtype" + if config.payload_dtype is not torch.bfloat16: + return "needs a BF16 payload" + alignments = (64, 128, _TILE) + if isinstance(recipe, MXFP8BlockScaling): + # The SM 10.7 MXFP8 GEMMs have no tail tiles. + alignments = (_TILE,) if torch.cuda.get_device_capability() == (10, 7) else (128, _TILE) + if config.alignment not in alignments: + return f"the EP alignment must be one of {alignments}" + if config.zero_copy: + return "zero-copy EP is not supported" + if recipe is not None: + if not isinstance(recipe, MXFP8BlockScaling): + return "quantized GEMMs other than MXFP8BlockScaling are not supported" + if not GroupedMLP_CuTeGEMMGLU.is_supported(): + return "MXFP8 needs the MXFP8 cuDNN grouped GEMM kernels" + mismatch = _gemm_interface_reason(mxfp8=True) + if mismatch: + return mismatch + if recipe.fp8_format != Format.E4M3 or recipe.backward_override is not None: + return "MXFP8 needs fp8_format E4M3 and no backward_override" + if not all( + isinstance(r, MXFP8BlockScaling) + for r in (config.dispatch_fwd_quant_recipe, config.combine_bwd_quant_recipe) + ): + return ( + "MXFP8 needs MXFP8 EP transport (dispatch_fwd_quant_recipe and" + " combine_bwd_quant_recipe)" + ) + if config.hidden_dim % 512 != 0 or ffn_hidden_size % _TILE != 0: + return f"MXFP8 needs hidden_dim a multiple of 512 and ffn_hidden_size of {_TILE}" + elif ( + config.dispatch_fwd_quant_recipe is not None or config.combine_bwd_quant_recipe is not None + ): + return "quantized EP transport is not supported" + if config.hidden_dim % _TILE != 0: + return f"hidden_dim must be a multiple of {_TILE}" + capacity = config.recv_capacity_per_rank + if capacity is not None and capacity % _TILE != 0: + return f"recv_capacity_per_rank must be a multiple of {_TILE}" + if ffn_hidden_size % 64 != 0: + return "ffn_hidden_size must be a multiple of 64" + if activation != "swiglu": + return "needs the SwiGLU activation" + if glu_interleave_size not in (None, _GLU_BLOCK): + return f"needs glu_interleave_size None or {_GLU_BLOCK}" + if not single_grouped_weight: + return "needs single_grouped_weight=True" + if bias: + return "biases are not supported" + if params_dtype != torch.bfloat16: + return "needs BF16 weights" + if primary_weights_in_fp8 and recipe is None: + return "quantized weight parameters need the MXFP8BlockScaling recipe" + return None + + +def moe_overlap_supported( + config: EpConfig, + ffn_hidden_size: int, + *, + params_dtype: torch.dtype, + activation: str = "swiglu", + glu_interleave_size: Optional[int] = None, + bias: bool = False, + single_grouped_weight: bool = False, + recipe: Optional[Recipe] = None, + primary_weights_in_fp8: bool = False, +) -> bool: + """Whether a ``Sequential`` of ``MoeDispatch``, ``GroupedLinear``, ``ScaledSwiGLU``, + ``GroupedLinear`` and ``MoeCombine`` built with these settings runs as one ``FusedMoE``; + ``moe_overlap_unsupported_reason`` gives the reason when it does not and takes the same + parameters. + + Call it after ``ep_bootstrap``. It only queries the device. Deterministic execution + (``NVTE_ALLOW_NONDETERMINISTIC_ALGO=0`` or ``torch.use_deterministic_algorithms``) and an + autocast dtype other than BF16 run the unfused ops; the answer follows the settings in effect + at the call. GroupedLinear settings it does not take, such as ``delay_wgrad_compute`` or + ``accumulate_into_main_grad``, can still make the ops run unfused. + """ + return ( + moe_overlap_unsupported_reason( + config, + ffn_hidden_size, + params_dtype=params_dtype, + activation=activation, + glu_interleave_size=glu_interleave_size, + bias=bias, + single_grouped_weight=single_grouped_weight, + recipe=recipe, + primary_weights_in_fp8=primary_weights_in_fp8, + ) + is None + ) + + +def _unsupported_reason(dispatch, fc1, activation, fc2, combine, recipe=None) -> Optional[str]: + """Why a window cannot use the overlapped ops, or ``None`` when it can.""" + config = dispatch.config + dtypes = {w.dtype for fc in (fc1, fc2) for w in fc._get_weight_tensors()} + reason = moe_overlap_unsupported_reason( + config, + fc2.in_features, + params_dtype=dtypes.pop() if len(dtypes) == 1 else None, + activation="swiglu" if isinstance(activation, ScaledSwiGLU) else type(activation).__name__, + glu_interleave_size=activation.glu_interleave_size, + bias=fc1.has_bias or fc2.has_bias, + single_grouped_weight=fc1.single_grouped_weight and fc2.single_grouped_weight, + recipe=recipe, + primary_weights_in_fp8=any( + getattr(w, "quantizer", None) is not None + for fc in (fc1, fc2) + for w in fc._get_weight_tensors() + ), + ) + if reason is not None: + return reason + if any( + not isinstance(getattr(fc.weight, "quantizer", None), (type(None), MXFP8Quantizer)) + for fc in (fc1, fc2) + ): + return "quantized weight parameters must be MXFP8" + if combine.config != config: + return "dispatch and combine must use the same EpConfig" + if dispatch.buffer is not combine.buffer: + return "dispatch and combine must share one EpBuffer" + if not _routing_extras_internal(dispatch, fc1, activation, fc2): + return "the dispatch extra outputs must feed the grouped MLP and not the caller" + if any(is_distributed_weight(fc.weight) for fc in (fc1, fc2)): + return "distributed (sharded) weights are not supported" + if any( + fc.wgrad_store is not None and fc.wgrad_store.delay_wgrad_compute() for fc in (fc1, fc2) + ): + return "delayed weight gradient computation (delay_wgrad_compute) is not supported" + try: + # The GLU layout is checked above. + validate_grouped_mlp_dims(fc1, ScaledSwiGLU(glu_interleave_size=_GLU_BLOCK), fc2) + except (TypeError, ValueError) as err: + return str(err) + if fc1.in_features != config.hidden_dim or fc2.out_features != config.hidden_dim: + return "the GroupedLinear sizes do not match the EP hidden_dim" + if ( + recipe is not None + and torch.cuda.get_device_capability() == (10, 7) + and config.recv_capacity_per_rank == _TILE + ): + # Some SM 10.7 MXFP8 weight gradients need more than one receive tile; one into + # main_grad counts as added. + from ....common.CuTeDSL.ep_gemm import ( # pylint: disable=import-outside-toplevel + wgrad_gemm, + ) + + for fc in (fc1, fc2): + cfg = wgrad_gemm.WgradConfig( + _TILE, fc.out_features, fc.in_features, config.num_local_experts, 1, + accumulate=fc._accumulate_into_main_grad, mxfp8=True, + rows_alignment=config.alignment, + routed_rows=config.max_tokens_per_rank * config.top_k, + ) # fmt: skip + try: + wgrad_gemm.build_kernel(cfg) + except ValueError as err: + return str(err) + return None + + +# The GLU activations of the grouped MLP; the unsupported ones run unfused with a warning. +_WINDOW = ( + MoeDispatch, + GroupedLinear, + (ScaledSwiGLU, ScaledClampedQGeGLU, ScaledSiTUGLU), + GroupedLinear, + MoeCombine, +) + + +_INTERNAL_DIRS = tuple( + os.path.abspath(path) + os.sep + for path in ( + os.path.join(os.path.dirname(__file__), *[os.pardir] * 3), + os.path.dirname(torch.__file__), + ) +) + + +def _stacklevel_outside_package() -> int: + """``warnings.warn`` stack level, from the caller of this function, of the first frame outside + Transformer Engine and PyTorch: the fuser runs inside the user's forward call.""" + level, frame = 1, sys._getframe(1) # pylint: disable=protected-access + while frame is not None and frame.f_code.co_filename.startswith(_INTERNAL_DIRS): + level, frame = level + 1, frame.f_back + return level if frame is not None else 1 + + +def fuse_moe_overlap_ops( + ops: list[FusibleOperation], + *, + recipe=None, + **unused, # pylint: disable=unused-argument +) -> list[FusibleOperation]: + """Fuse dispatch, grouped MLP and combine; other windows keep the unfused ops.""" + out = [] + idx = 0 + while idx < len(ops): + window = ops[idx : idx + 5] + if len(window) == len(_WINDOW) and all(map(isinstance, window, _WINDOW)): + reason = _unsupported_reason(*window, recipe=recipe) + if reason is None: + out.append(FusedMoE( + dispatch=window[0], fc1=window[1], activation=window[2], + fc2=window[3], combine=window[4], recipe=recipe, _validated=True, + )) # fmt: skip + idx += 5 + continue + # Without the overlap enabled the unfused ops are the expected ones. + if _moe_overlap_available(): + warnings.warn( + f"FusedMoE fusion skipped, running the unfused ops: {reason}.", + stacklevel=_stacklevel_outside_package(), + ) + out.append(ops[idx]) + idx += 1 + return out + + +register_forward_backward_fusion(fuse_moe_overlap_ops, prepend=True) diff --git a/transformer_engine/pytorch/ops/fuser.py b/transformer_engine/pytorch/ops/fuser.py index 6d13ba43942..f6a2f93e4ea 100644 --- a/transformer_engine/pytorch/ops/fuser.py +++ b/transformer_engine/pytorch/ops/fuser.py @@ -8,6 +8,7 @@ from collections.abc import Callable, Iterable, Sequence import copy import itertools +import os from typing import Any, Optional, TypeAlias import torch @@ -51,7 +52,7 @@ def _is_graph_capturing() -> bool: "Callable[tuple[list[FusibleOperation], ...], list[FusibleOperation]]" ) _FusedOpList: TypeAlias = list[tuple[FusibleOperation, list[int]]] -_FusionParams: TypeAlias = tuple[str, int] +_FusionParams: TypeAlias = tuple[str, int, tuple] class _OperationFuserAutogradFunction(torch.autograd.Function): @@ -701,9 +702,19 @@ def maybe_fuse_ops( # backward boundary in the key, but pay construction cost only once for # each configuration. Full recompute therefore builds at most one # no-grad plan and one grad-enabled plan for a stable recipe. + # Runtime settings that fusions may fall back on are part of the key. + runtime_policy = ( + torch.are_deterministic_algorithms_enabled(), + os.getenv("NVTE_ALLOW_NONDETERMINISTIC_ALGO", "1"), + torch.get_autocast_dtype("cuda") if torch.is_autocast_enabled() else None, + tuple( + op.need_backward_dw() for op in self._basic_ops if hasattr(op, "need_backward_dw") + ), + ) fusion_params: _FusionParams = ( str(recipe) if recipe is not None else "", first_op_requiring_backward, + runtime_policy, ) cached_ops = self._fused_ops_cache.get(fusion_params) if cached_ops is not None: