diff --git a/docs/api/pytorch.rst b/docs/api/pytorch.rst index 2d8cc87827e..4a7ea9377cb 100644 --- a/docs/api/pytorch.rst +++ b/docs/api/pytorch.rst @@ -6,6 +6,9 @@ PyTorch ======= +General-purpose layers +---------------------- + .. autoapiclass:: transformer_engine.pytorch.Linear(in_features, out_features, **kwargs) :members: forward, set_tensor_parallel_group @@ -43,6 +46,24 @@ PyTorch .. autoapiclass:: transformer_engine.pytorch.TransformerLayer(hidden_size, ffn_hidden_size, num_attention_heads, **kwargs) :members: forward, set_context_parallel_group, set_tensor_parallel_group +Model layers +------------ + +DeepSeek-V3 +~~~~~~~~~~~ + +.. autoapiclass:: transformer_engine.pytorch.models.DeepSeekV3Layer(hidden_size, num_attention_heads, **kwargs) + :members: forward + +.. autoapiclass:: transformer_engine.pytorch.models.DeepSeekV3MoE(hidden_size, moe_ffn_hidden_size, num_experts, **kwargs) + :members: forward, update_expert_bias + +.. autoapiclass:: transformer_engine.pytorch.models.MultiLatentAttention(hidden_size, num_attention_heads, **kwargs) + :members: forward + +Other +----- + .. autoapiclass:: transformer_engine.pytorch.dot_product_attention.inference.InferenceParams(max_batch_size, max_sequence_length) :members: reset, allocate_memory, pre_step, get_seqlens_pre_step, convert_paged_to_nonpaged, step @@ -71,6 +92,15 @@ PyTorch .. autoapifunction:: transformer_engine.pytorch.deinterleave_glu_tensor +MLA rotary position embeddings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoapifunction:: transformer_engine.pytorch.attention.mla_rope.build_rope_tables + +.. autoapifunction:: transformer_engine.pytorch.attention.mla_rope.apply_mla_rope_q + +.. autoapifunction:: transformer_engine.pytorch.attention.mla_rope.apply_mla_rope_kv + Data types ---------- diff --git a/docs/examples/jax/test_attention.py b/docs/examples/jax/test_attention.py index c86631dbc2a..f4caa35a950 100644 --- a/docs/examples/jax/test_attention.py +++ b/docs/examples/jax/test_attention.py @@ -59,9 +59,20 @@ def test_bshd_gqa_swa_matches_baseline(): @requires_hopper_or_newer -def test_single_gpu_benchmark(): +def test_single_gpu_benchmark(monkeypatch): import attention + # The benchmark synchronizes on the native baseline, which has the same memory problem + # as test_bshd_gqa_swa_matches_baseline. Benchmark the same smaller prefix; the tutorial script itself + # still runs at the full sequence length. + seq = 1024 + monkeypatch.setattr(attention, "qkv", tuple(t[:, :seq] for t in attention.qkv)) + monkeypatch.setattr(attention, "dout", attention.dout[:, :seq]) + monkeypatch.setattr( + attention, + "sequence_descriptor", + SequenceDescriptor.from_seqlens(jnp.full((attention.batch,), seq, dtype=jnp.int32)), + ) attention.run_single_gpu_bench() diff --git a/examples/pytorch/deepseek_v3/README.md b/examples/pytorch/deepseek_v3/README.md new file mode 100644 index 00000000000..c52fe579bee --- /dev/null +++ b/examples/pytorch/deepseek_v3/README.md @@ -0,0 +1,124 @@ +# DeepSeekV3Layer with expert parallelism + +A forward + backward benchmark of one `DeepSeekV3Layer`: RMSNorm, multi-latent attention, +and MoE with a shared expert. Routed experts are sharded across GPUs using NCCL EP. + +## Quick start + +Requires SM90+ GPUs with NVLink and an NCCL EP-enabled TE build; +see [requirements](#requirements). From this directory: + +```bash +bash run_deepseek_v3_layer_ep.sh --dsv3 +``` + +For MXFP8 with the fused grouped MLP (SM100-class GPUs): + +```bash +NVTE_CUTEDSL_FUSED_GROUPED_MLP=1 bash run_deepseek_v3_layer_ep.sh --dsv3 --recipe mxfp8 +``` + +## Requirements + +The following NCCL EP requirements apply to `--impl te`. The naive variants use +ordinary NCCL collectives and can also run on older GPUs without NVLink. + +- SM90 or newer GPUs connected with NVLink. +- NCCL >= 2.30.4, PyTorch >= 2.11 (symmetric memory), Transformer Engine built with NCCL EP + enabled (`NVTE_WITH_NCCL_EP`, on by default when an SM90+ architecture is targeted; + `NVTE_WITH_NCCL_EP=0` disables it). +- `NVTE_CUTEDSL_FUSED_GROUPED_MLP=1` additionally needs SM100-class GPUs and the CuTe DSL + (`nvidia-cutlass-dsl`) for the fused MXFP8 grouped MLP. + +## Running + +Run these commands from this directory. Single node, all local GPUs: + +```bash +bash run_deepseek_v3_layer_ep.sh # small dims, bf16 +bash run_deepseek_v3_layer_ep.sh --dsv3 # DeepSeek-V3 layer dims, bf16 +bash run_deepseek_v3_layer_ep.sh --dsv3 --recipe mxfp8 # MXFP8 experts (unfused grouped GEMM) +NVTE_CUTEDSL_FUSED_GROUPED_MLP=1 bash run_deepseek_v3_layer_ep.sh --dsv3 --recipe mxfp8 +bash run_deepseek_v3_layer_ep.sh --dsv3 --impl naive # all_to_all + loop over experts +bash run_deepseek_v3_layer_ep.sh --dsv3 --impl naive_grouped # all_to_all + TE grouped GEMM +``` + +`--impl` selects the MoE block inside the same layer (attention and norms are identical): + +- `te` (default): `DeepSeekV3MoE`, NCCL EP dispatch/combine, experts as one grouped GEMM. +- `naive`: MoE written with plain PyTorch, no TE MoE code: sigmoid top-k router with expert + bias, `all_to_all_single` dispatch and combine (two host syncs per layer for the split sizes), + a Python loop over the local experts with dense `F.linear` SwiGLU MLPs, `index_copy` / + `index_add` to place results, and a shared expert. This is what an EP MoE looks like before + any fused kernels. +- `naive_grouped`: the same all_to_all dispatch and combine, but the received rows are sorted by + local expert and run through one `te.ops.GroupedLinear` / `ScaledSwiGLU` / `GroupedLinear` + stack. Isolates the cost of the Python loop from the cost of the communication path. + +Multi-node: launch `torchrun` yourself, EP spans every rank: + +```bash +torchrun --nnodes=2 --nproc-per-node=4 --rdzv-backend=c10d --rdzv-endpoint=
:29500 \ + deepseek_v3_layer_ep.py --dsv3 --recipe mxfp8 +``` + +Every rank owns `--num-local-experts` experts (default 8), so the expert count is +`8 * world_size`. Other knobs: `--tokens-per-rank`, `--topk`, `--hidden`, `--num-heads`, +`--moe-ffn`, the MLA dims (`--q-lora-rank`, `--kv-lora-rank`, `--qk-nope-head-dim`, +`--qk-rope-head-dim`, `--v-head-dim`), `--warmup`, `--iters`. +`--warmup 0` is supported; `--iters` must be positive and `--tokens-per-rank` must be a +positive multiple of four. MXFP8 requires `--impl te`. + +## Results + +GB200 and GB300 GPUs, 4096 tokens per rank, top-k 8, 8 local experts per GPU. +Times cover one layer's forward + backward; throughput is global, in millions of tokens/s. +Every number is the median of three independent runs (spread within 0.3 ms). +See [benchmark setup](#benchmark-setup) below for the full configuration. + +### TE vs. plain PyTorch (`--dsv3`) + +Same layer and dimensions, BF16 unless noted. + +All variants run the router backward and clear parameter gradients before each step. + +| MoE implementation | 4 GB200 (32 experts) | 4 GB300 (32 experts) | 8 GB300 (64 experts) | +|---|---:|---:|---:| +| `naive`: all_to_all + loop over experts | 28.61 ms | 27.21 ms | 27.70 ms | +| `naive_grouped`: all_to_all + TE grouped GEMM | 17.79 ms | 16.72 ms | 17.13 ms | +| `te`: NCCL EP + grouped GEMM | 13.68 ms | 12.37 ms | 14.09 ms | +| `te`, mxfp8 (unfused grouped GEMM) | 10.99 ms | 10.32 ms | 12.11 ms | +| `te`, mxfp8 fused | 10.11 ms | 9.49 ms | 10.45 ms | + +### TE throughput (`--dsv3`) + +| Precision | 4 GB200 · Mtok/s | 4 GB300 · Mtok/s | 8 GB300 · Mtok/s | +|---|---:|---:|---:| +| BF16 | 1.20 | 1.33 | 2.32 | +| MXFP8 | 1.49 | 1.59 | 2.71 | +| MXFP8 fused | 1.62 | 1.73 | 3.14 | + +4 GPUs = 1 node / 32 experts; 8 GPUs = 2 nodes / 64 experts. +Both nodes share one NVLink domain (MNNVL). + +“Fused” enables `NVTE_CUTEDSL_FUSED_GROUPED_MLP=1`. + +### Benchmark setup + +| Setting | Value | +|---|---| +| hardware | GB200 (SM100), one node; GB300 (SM103), one or two nodes; 4 GPUs per node, both GB300 nodes in one NVLink domain (MNNVL) | +| software | CUDA 13.3, NCCL 2.30.7, PyTorch 2.13.0a0+9186a08b2c.nv26.07, cuDNN 9.26, cuDNN Frontend 1.29.0, CUTLASS DSL 4.6.2 | +| NVIDIA driver | GB200: 580.173.02; GB300: 580.173.10 | +| `--dsv3` dims | hidden 7168, 128 heads, MLA q_lora 1536 / kv_lora 512 / nope 128 / rope 64 / v 128, expert ffn 2048, shared expert ffn 2048 | +| MoE | 8 local experts per rank (32 on 4 GPUs, 64 on 8), top-k 8, 4096 tokens per rank | +| precision | bf16 params and activations; `--recipe mxfp8` = MXFP8 block scaling for the expert GEMMs and dense projections | +| timing | fwd + bwd, 5 warmup, 10 timed iterations per run, median of 3 runs; wall-clock timing with CUDA synchronization, no CUDA graphs or profiler | +| environment | `OMP_NUM_THREADS=8`, `NVTE_GROUPED_LINEAR_SINGLE_PARAM=0`, `NVTE_ALLOW_NONDETERMINISTIC_ALGO=1`, `NVTE_FLASH_ATTN_V2=1`, `NVTE_FLASH_ATTN_V3=0`, `NVTE_FLASH_ATTN_V4=0`; EP buffer reused | + +Each run reuses a normally distributed random input and backpropagates an all-ones +output gradient. Parameter and input gradients are cleared before each iteration. + +`ep_bootstrap` is given the recv capacity returned by +`DeepSeekV3MoE.ep_recv_capacity(ep_size, tokens_per_rank, topk, num_local_experts)`, with +per-expert zones aligned to 256 rows. diff --git a/examples/pytorch/deepseek_v3/deepseek_v3_layer_ep.py b/examples/pytorch/deepseek_v3/deepseek_v3_layer_ep.py new file mode 100644 index 00000000000..e95a7ecba6d --- /dev/null +++ b/examples/pytorch/deepseek_v3/deepseek_v3_layer_ep.py @@ -0,0 +1,283 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. +"""DeepSeekV3Layer with expert parallelism over all ranks: forward + backward timing. + +One process per GPU, launched via run_deepseek_v3_layer_ep.sh (torchrun). Every rank +holds ``--num-local-experts`` routed experts. ``--impl te`` exchanges tokens with NCCL EP +and runs the experts as one grouped GEMM (DeepSeekV3MoE); ``--impl naive`` is a plain +PyTorch MoE (all_to_all_single + a Python loop over experts) dropped into the same layer; +``--impl naive_grouped`` keeps the all_to_all but runs the experts as one TE grouped GEMM. +Timed iterations run inside a ``torch.cuda.profiler`` window, so +``nsys profile -c cudaProfilerApi --capture-range-end=stop torchrun ...`` records +only them. +""" + +import argparse +import os +import sys +import time +from contextlib import nullcontext + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch.distributed.nn.functional import all_to_all_single + +import transformer_engine.pytorch as te +from transformer_engine.common import recipe as te_recipe +from transformer_engine.pytorch.ep import ep_bootstrap, ep_finalize, release_symm_mem_pool +from transformer_engine.pytorch.models import DeepSeekV3Layer, DeepSeekV3MoE + + +def _parse_args(argv=None): + p = argparse.ArgumentParser(description="DeepSeekV3Layer EP example (fwd + bwd)") + p.add_argument("--tokens-per-rank", type=int, default=4096) + p.add_argument("--hidden", type=int, default=2048) + p.add_argument("--num-heads", type=int, default=16) + p.add_argument("--moe-ffn", type=int, default=1024) + p.add_argument("--num-local-experts", type=int, default=8) + p.add_argument("--topk", type=int, default=8) + p.add_argument("--q-lora-rank", type=int, default=512) + p.add_argument("--kv-lora-rank", type=int, default=256) + p.add_argument("--qk-nope-head-dim", type=int, default=64) + p.add_argument("--qk-rope-head-dim", type=int, default=32) + p.add_argument("--v-head-dim", type=int, default=64) + p.add_argument( + "--dsv3", + action="store_true", + help=( + "DeepSeek-V3 layer dims (hidden 7168, 128 heads, MLA 1536/512/128/64/128, expert ffn" + " 2048)." + ), + ) + p.add_argument("--impl", choices=["te", "naive", "naive_grouped"], default="te") + p.add_argument("--recipe", choices=["none", "mxfp8"], default="none") + p.add_argument( + "--ep-buffer-per-call", + action="store_true", + help="Let the layer create a new EpBuffer per forward instead of reusing one.", + ) + p.add_argument("--warmup", type=int, default=5) + p.add_argument("--iters", type=int, default=10) + args = p.parse_args(argv) + if args.warmup < 0: + p.error("--warmup must be non-negative") + if args.iters <= 0: + p.error("--iters must be positive") + if args.tokens_per_rank <= 0 or args.tokens_per_rank % 4: + p.error("--tokens-per-rank must be a positive multiple of 4") + if args.impl != "te" and args.recipe != "none": + p.error("--recipe mxfp8 is only supported with --impl te") + if args.dsv3: + args.hidden, args.num_heads, args.moe_ffn = 7168, 128, 2048 + args.q_lora_rank, args.kv_lora_rank = 1536, 512 + args.qk_nope_head_dim, args.qk_rope_head_dim, args.v_head_dim = 128, 64, 128 + return args + + +def _autocast(name): + if name == "none": + return nullcontext() + return te.autocast(enabled=True, recipe=te_recipe.MXFP8BlockScaling()) + + +class NaiveMoE(torch.nn.Module): + """DeepSeek-style MoE with torch all_to_all dispatch/combine: sigmoid top-k router with + expert bias, a shared expert, and experts either as a Python loop of dense SwiGLU MLPs or + (``grouped=True``) as one TE grouped GEMM stack.""" + + def __init__(self, hidden, ffn, num_experts, topk, ep_group, shared_ffn, dtype, grouped=False): + super().__init__() + self.grouped = grouped + self.hidden, self.topk, self.group = hidden, topk, ep_group + self.ws, self.rank = dist.get_world_size(ep_group), dist.get_rank(ep_group) + self.num_experts, self.local = num_experts, num_experts // self.ws + self.gate = torch.nn.Linear(hidden, num_experts, bias=False, dtype=dtype, device="cuda") + self.register_buffer("expert_bias", torch.zeros(num_experts, device="cuda")) + std = hidden**-0.5 + if grouped: + self.experts = te.ops.Sequential( + te.ops.GroupedLinear(self.local, hidden, 2 * ffn, bias=False, dtype=dtype), + te.ops.ScaledSwiGLU(glu_interleave_size=32), + te.ops.GroupedLinear(self.local, ffn, hidden, bias=False, dtype=dtype), + ) + else: + self.w1 = torch.nn.Parameter( + torch.randn(self.local, 2 * ffn, hidden, dtype=dtype, device="cuda") * std + ) + self.w2 = torch.nn.Parameter( + torch.randn(self.local, hidden, ffn, dtype=dtype, device="cuda") * ffn**-0.5 + ) + self.shared_w1 = torch.nn.Linear( + hidden, 2 * shared_ffn, bias=False, dtype=dtype, device="cuda" + ) + self.shared_w2 = torch.nn.Linear(shared_ffn, hidden, bias=False, dtype=dtype, device="cuda") + + @staticmethod + def _swiglu(h): + a, g = h.chunk(2, dim=-1) + return F.silu(a) * g + + def forward(self, hidden_states): + x = hidden_states.reshape(-1, self.hidden) + scores = torch.sigmoid(self.gate(x).float()) + _, idx = torch.topk(scores + self.expert_bias, self.topk, dim=-1) + probs = scores.gather(1, idx) + probs = probs / probs.sum(-1, keepdim=True) * 2.5 + # Dispatch: sort (token, expert) pairs by destination rank, exchange counts, all_to_all. + flat_e, flat_p = idx.reshape(-1), probs.reshape(-1) + tok = torch.arange(x.shape[0], device=x.device).repeat_interleave(self.topk) + order = torch.argsort(flat_e // self.local, stable=True) + flat_e, flat_p, tok = flat_e[order], flat_p[order], tok[order] + send = torch.bincount(flat_e // self.local, minlength=self.ws) + recv = torch.empty_like(send) + dist.all_to_all_single(recv, send, group=self.group) + send, recv = send.tolist(), recv.tolist() + n_recv = sum(recv) + x_recv = all_to_all_single( + torch.empty(n_recv, self.hidden, dtype=x.dtype, device=x.device), + x[tok], + recv, + send, + group=self.group, + ) + e_recv = torch.empty(n_recv, dtype=flat_e.dtype, device=x.device) + p_recv = torch.empty(n_recv, dtype=flat_p.dtype, device=x.device) + dist.all_to_all_single(e_recv, flat_e.contiguous(), recv, send, group=self.group) + p_recv = all_to_all_single(p_recv, flat_p.contiguous(), recv, send, group=self.group) + local_e = e_recv - self.rank * self.local + if self.grouped: + # Experts: sort received rows by local expert, one grouped GEMM stack. + by_expert = torch.argsort(local_e, stable=True) + counts = torch.bincount(local_e, minlength=self.local) + y_sorted = self.experts( + x_recv[by_expert], counts, p_recv[by_expert].to(x.dtype), counts + ) + y_recv = torch.empty_like(x_recv).index_copy(0, by_expert, y_sorted) + else: + # Experts: one dense SwiGLU MLP per local expert. + y_recv = torch.zeros_like(x_recv) + for e in range(self.local): + sel = (local_e == e).nonzero().squeeze(1) + if sel.numel() == 0: + continue + h = self._swiglu(F.linear(x_recv[sel], self.w1[e])) * p_recv[sel, None].to(x.dtype) + y_recv = y_recv.index_copy(0, sel, F.linear(h, self.w2[e])) + # Combine: reverse all_to_all, sum the top-k contributions per token. + y = all_to_all_single(torch.empty_like(x[tok]), y_recv, send, recv, group=self.group) + out = torch.zeros_like(x).index_add(0, tok, y) + out = out + self.shared_w2(self._swiglu(self.shared_w1(x))) + return out.view_as(hidden_states) + + +def main(): + """Build the layer, run warmup + timed fwd/bwd iterations, print throughput on rank 0.""" + args = _parse_args() + local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local_rank) + dist.init_process_group("nccl", device_id=torch.device("cuda", local_rank)) + rank, world_size = dist.get_rank(), dist.get_world_size() + + major, minor = torch.cuda.get_device_capability() + if args.impl == "te" and major * 10 + minor < 90: + if rank == 0: + print(f"SKIPPED: NCCL EP requires SM>=90 (got SM{major}{minor})") + dist.destroy_process_group() + return 0 + + ep_group = dist.new_group(ranks=list(range(world_size)), backend="nccl") + dist.all_reduce(torch.zeros(1, device="cuda"), group=ep_group) + num_experts = args.num_local_experts * world_size + if args.impl == "te": + ep_bootstrap( + ep_group, + num_experts=num_experts, + max_tokens_per_rank=args.tokens_per_rank, + hidden_dim=args.hidden, + num_topk=args.topk, + recv_capacity_per_rank=DeepSeekV3MoE.ep_recv_capacity( + world_size, args.tokens_per_rank, args.topk, args.num_local_experts + ), + ) + + torch.manual_seed(0) + mlp_kwargs = dict( + num_experts=num_experts, + moe_ffn_hidden_size=args.moe_ffn, + shared_expert_ffn_hidden_size=args.moe_ffn, + topk=args.topk, + ep_group=ep_group, + ep_max_tokens_per_rank=args.tokens_per_rank, + ) + if args.impl != "te": + # Build the TE MoE without EP (replaced below); keeps the pre-MLP RMSNorm. + mlp_kwargs.pop("ep_group"), mlp_kwargs.pop("ep_max_tokens_per_rank") + layer = DeepSeekV3Layer( + args.hidden, + args.num_heads, + params_dtype=torch.bfloat16, + **mlp_kwargs, + q_lora_rank=args.q_lora_rank, + kv_lora_rank=args.kv_lora_rank, + qk_nope_head_dim=args.qk_nope_head_dim, + qk_rope_head_dim=args.qk_rope_head_dim, + v_head_dim=args.v_head_dim, + ) + if args.impl != "te": + layer.mlp = NaiveMoE( + args.hidden, + args.moe_ffn, + num_experts, + args.topk, + ep_group, + args.moe_ffn, + torch.bfloat16, + grouped=args.impl == "naive_grouped", + ) + seq = args.tokens_per_rank // 4 + x = torch.randn(seq, 4, args.hidden, dtype=torch.bfloat16, device="cuda", requires_grad=True) + + ep_buffer = None + if args.impl == "te" and not args.ep_buffer_per_call: + ep_buffer = layer.mlp.make_ep_buffer() + + def step(): + layer.zero_grad(set_to_none=True) + x.grad = None + with _autocast(args.recipe): + out = layer(x, ep_buffer=ep_buffer) + out.backward(torch.ones_like(out)) + + for _ in range(args.warmup): + step() + torch.cuda.synchronize() + dist.barrier() + + torch.cuda.profiler.start() + start = time.perf_counter() + for i in range(args.iters): + with torch.cuda.nvtx.range(f"iter{i}"): + step() + torch.cuda.synchronize() + ms = (time.perf_counter() - start) / args.iters * 1e3 + torch.cuda.profiler.stop() + dist.barrier() + + if rank == 0: + tok_s = args.tokens_per_rank * world_size / (ms / 1e3) + print( + f"DeepSeekV3Layer impl={args.impl}:" + f" ranks={world_size} experts={num_experts} topk={args.topk} tokens/rank={args.tokens_per_rank} hidden={args.hidden} recipe={args.recipe} fused_mlp={os.environ.get('NVTE_CUTEDSL_FUSED_GROUPED_MLP', '0')} ep_buffer={'per_call' if ep_buffer is None else 'reused'} fwd+bwd" + f" {ms:.3f} ms/iter ({tok_s / 1e6:.2f} Mtok/s)", + flush=True, + ) + if args.impl == "te": + ep_finalize() + release_symm_mem_pool() + dist.destroy_process_group() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/pytorch/deepseek_v3/run_deepseek_v3_layer_ep.sh b/examples/pytorch/deepseek_v3/run_deepseek_v3_layer_ep.sh new file mode 100755 index 00000000000..9341b7f14a6 --- /dev/null +++ b/examples/pytorch/deepseek_v3/run_deepseek_v3_layer_ep.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. +# +# Launcher for deepseek_v3_layer_ep.py on all local GPUs. Extra args go to the script: +# bash run_deepseek_v3_layer_ep.sh # bf16, small dims +# bash run_deepseek_v3_layer_ep.sh --dsv3 --recipe mxfp8 # DeepSeek-V3 dims, MXFP8 experts +# NVTE_CUTEDSL_FUSED_GROUPED_MLP=1 bash run_deepseek_v3_layer_ep.sh --dsv3 --recipe mxfp8 +# NSYS=1 bash run_deepseek_v3_layer_ep.sh --dsv3 # nsys report in results/ +# Multi-node: run torchrun yourself with --nnodes/--rdzv-endpoint; EP spans all ranks. + +set -uo pipefail + +DETECTED_GPUS=$(nvidia-smi -L 2>/dev/null | wc -l) +NUM_GPUS="${NUM_GPUS:-${DETECTED_GPUS}}" +if [ "${NUM_GPUS}" -lt 2 ]; then + echo "EP requires >= 2 GPUs (found ${NUM_GPUS}); SKIPPING." + exit 0 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +: ${NCCL_EP_JIT_CACHE_DIR:="${TMPDIR:-/tmp}/nccl_ep_jit_cache_$(id -u)"} +export NCCL_EP_JIT_CACHE_DIR +mkdir -p "$NCCL_EP_JIT_CACHE_DIR" + +PREFIX=() +if [ "${NSYS:-0}" = "1" ]; then + mkdir -p "${SCRIPT_DIR}/results" + PREFIX=(nsys profile -t cuda,nvtx,nccl -c cudaProfilerApi --capture-range-end=stop + --cuda-graph-trace=node -o "${SCRIPT_DIR}/results/deepseek_v3_layer_ep_%h") +fi + +"${PREFIX[@]}" torchrun --standalone --nnodes=1 --nproc-per-node="${NUM_GPUS}" \ + "${SCRIPT_DIR}/deepseek_v3_layer_ep.py" "$@" diff --git a/qa/L0_pytorch_unittest/test.sh b/qa/L0_pytorch_unittest/test.sh index 2aa02340018..f215114d679 100644 --- a/qa/L0_pytorch_unittest/test.sh +++ b/qa/L0_pytorch_unittest/test.sh @@ -42,6 +42,7 @@ PYTORCH_JIT=0 NVTE_TORCH_COMPILE=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 NVTE_FUSED PYTORCH_JIT=0 NVTE_TORCH_COMPILE=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 NVTE_FUSED_ATTN=0 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_cuda_graphs.xml $TE_PATH/tests/pytorch/test_cuda_graphs.py || test_fail "test_cuda_graphs.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_jit.xml $TE_PATH/tests/pytorch/test_jit.py || test_fail "test_jit.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_fused_rope.xml $TE_PATH/tests/pytorch/test_fused_rope.py || test_fail "test_fused_rope.py" +python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_mla_rope.xml $TE_PATH/tests/pytorch/attention/test_mla_rope.py || test_fail "test_mla_rope.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_nvfp4.xml $TE_PATH/tests/pytorch/nvfp4 || test_fail "test_nvfp4" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_mxfp8.xml $TE_PATH/tests/pytorch/mxfp8 || test_fail "test_mxfp8" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_strided_batched_gemm.xml $TE_PATH/tests/pytorch/test_strided_batched_gemm.py || test_fail "test_strided_batched_gemm.py" @@ -65,6 +66,7 @@ python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_backward_overrid python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_permutation.xml $TE_PATH/tests/pytorch/test_permutation.py || test_fail "test_permutation.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_cross_entropy.xml $TE_PATH/tests/pytorch/test_cross_entropy.py || test_fail "test_cross_entropy.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_cpu_offloading.xml $TE_PATH/tests/pytorch/test_cpu_offloading.py || test_fail "test_cpu_offloading.py" +python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_models.xml $TE_PATH/tests/pytorch/test_models.py || test_fail "test_models.py" NVTE_FLASH_ATTN=0 NVTE_CPU_OFFLOAD_V1=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_cpu_offloading_v1.xml $TE_PATH/tests/pytorch/test_cpu_offloading_v1.py || test_fail "test_cpu_offloading_v1.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_hybrid_quantization.xml $TE_PATH/tests/pytorch/test_hybrid_quantization.py || test_fail "test_hybrid_quantization.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_identity_quantizer.xml $TE_PATH/tests/pytorch/test_identity_quantizer.py || test_fail "test_identity_quantizer.py" diff --git a/qa/L1_pytorch_distributed_unittest/test.sh b/qa/L1_pytorch_distributed_unittest/test.sh index 41ab6aeb444..9b17315fabf 100644 --- a/qa/L1_pytorch_distributed_unittest/test.sh +++ b/qa/L1_pytorch_distributed_unittest/test.sh @@ -57,6 +57,7 @@ python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_cu_seqlens_cache.xml python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_cast_master_weights_to_fp8.xml $TE_PATH/tests/pytorch/distributed/test_cast_master_weights_to_fp8.py || test_fail "test_cast_master_weights_to_fp8.py" python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_newton_schulz.xml $TE_PATH/tests/pytorch/distributed/test_newton_schulz.py || test_fail "test_newton_schulz.py" python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_ep.xml $TE_PATH/tests/pytorch/distributed/test_ep.py || test_fail "test_ep.py" +python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_models.xml $TE_PATH/tests/pytorch/distributed/test_models.py || test_fail "distributed/test_models.py" # debug tests diff --git a/tests/pytorch/attention/test_linear_mxfp8_attention.py b/tests/pytorch/attention/test_linear_mxfp8_attention.py index e5cb1c8a655..f1926814f3a 100644 --- a/tests/pytorch/attention/test_linear_mxfp8_attention.py +++ b/tests/pytorch/attention/test_linear_mxfp8_attention.py @@ -35,7 +35,11 @@ _current_file = pathlib.Path(__file__).resolve() sys.path = [str(_current_file.parent.parent)] + sys.path from utils import ModelConfig, compare_and_assert, get_available_attention_backends -from mla_rope_utils import apply_mla_rope, build_rope_tables +from transformer_engine.pytorch.attention.mla_rope import ( + apply_mla_rope_kv, + apply_mla_rope_q, + build_rope_tables, +) try: @@ -179,6 +183,13 @@ def _run_projections( return q_flat, kv_flat, q, kv, k_pos_emb +def _apply_rope(q, kv, k_pos_emb, rope_tables): + cos, sin = rope_tables + q = apply_mla_rope_q(q, cos, sin, HEAD_DIM_NOPE, HEAD_DIM_ROPE) + k, v = apply_mla_rope_kv(kv, k_pos_emb, cos, sin, HEAD_DIM_NOPE, HEAD_DIM_ROPE, HEAD_DIM_V) + return q, k, v + + def _run_forward_bf16( modules: tuple, x: torch.Tensor, @@ -186,7 +197,7 @@ def _run_forward_bf16( ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: q_proj, kv_proj, dpa, out_linear = modules _, _, q, kv, k_pos_emb = _run_projections(q_proj, kv_proj, x) - q, k, v = apply_mla_rope(q, kv, k_pos_emb, cos_table=rope_tables[0], sin_table=rope_tables[1]) + q, k, v = _apply_rope(q, kv, k_pos_emb, rope_tables) attn_out = dpa(q, k, v, qkv_format="sbhd") return q, k, v, out_linear(attn_out.view(x.shape[0], x.shape[1], HIDDEN_SIZE)) @@ -208,13 +219,7 @@ def _run_forward_mxfp8( x, is_first_microbatch, ) - q, k, v = apply_mla_rope( - q, - kv, - k_pos_emb, - cos_table=rope_tables[0], - sin_table=rope_tables[1], - ) + q, k, v = _apply_rope(q, kv, k_pos_emb, rope_tables) attn_out = dpa(q, k, v, qkv_format="sbhd") out = out_linear( attn_out.view(x.shape[0], x.shape[1], HIDDEN_SIZE), @@ -288,7 +293,7 @@ def test_accuracy(self, batch_size: int, seq_len: int) -> None: _set_seed() baseline_modules, mxfp8_modules = _build_modules() x = torch.randn(seq_len, batch_size, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda") - rope_tables = build_rope_tables(seq_len, device=x.device) + rope_tables = build_rope_tables(seq_len, HEAD_DIM_ROPE, device=x.device) q_bf16, k_bf16, v_bf16, out_bf16 = _run_forward_bf16(baseline_modules, x, rope_tables) q_mxfp8, k_mxfp8, v_mxfp8, out_mxfp8 = _run_forward_mxfp8( @@ -374,7 +379,7 @@ def test_backward(self, batch_size: int, seq_len: int) -> None: device="cuda", requires_grad=True, ) - rope_tables = build_rope_tables(seq_len, device=x.device) + rope_tables = build_rope_tables(seq_len, HEAD_DIM_ROPE, device=x.device) *_, out_mxfp8 = _run_forward_mxfp8(mxfp8_modules, x, fp8_recipe, rope_tables) out_mxfp8.sum().backward() @@ -408,7 +413,7 @@ def test_performance(self, batch_size: int, seq_len: int) -> None: device="cuda", requires_grad=True, ) - rope_tables = build_rope_tables(seq_len, device=x.device) + rope_tables = build_rope_tables(seq_len, HEAD_DIM_ROPE, device=x.device) mxfp8_fprop_ms, mxfp8_bprop_ms = _benchmark_training_step( _run_forward_mxfp8, mxfp8_modules, x, fp8_recipe, rope_tables diff --git a/tests/pytorch/attention/test_mla_rope.py b/tests/pytorch/attention/test_mla_rope.py new file mode 100644 index 00000000000..2b35cec5cef --- /dev/null +++ b/tests/pytorch/attention/test_mla_rope.py @@ -0,0 +1,184 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +import math + +import pytest +import torch + +from transformer_engine.pytorch.attention import mla_rope +from transformer_engine.pytorch.module import LayerNormLinear + + +@pytest.mark.parametrize("nope,rope,vdim", [(64, 32, 64), (48, 32, 64), (64, 48, 64), (64, 32, 48)]) +def test_mla_rope_matches_pytorch(nope, rope, vdim): + if not mla_rope.HAVE_TRITON: + pytest.skip("Triton unavailable") + s, b, h = 64, 2, 4 + cos, sin = mla_rope.build_rope_tables(s, rope, device="cuda") + + torch.manual_seed(0) + q_leaf = torch.randn(s, b, h, nope + rope, device="cuda", requires_grad=True) + kv_leaf = torch.randn(s, b, h, nope + vdim, device="cuda", requires_grad=True) + pos_leaf = torch.randn(s, b, 1, rope, device="cuda", requires_grad=True) + grad_q = torch.randn(s, b, h, nope + rope, device="cuda") + grad_k = torch.randn(s, b, h, nope + rope, device="cuda") + grad_v = torch.randn(s, b, h, vdim, device="cuda") + + def run(fmt): + q, kv, pos = q_leaf * 1.0, kv_leaf * 1.0, pos_leaf * 1.0 + q_out = mla_rope.apply_mla_rope_q(q, cos, sin, nope, rope, fmt) + k_out, v_out = mla_rope.apply_mla_rope_kv(kv, pos, cos, sin, nope, rope, vdim, fmt) + torch.autograd.backward( + [q_out, k_out, v_out], [grad_q.clone(), grad_k.clone(), grad_v.clone()] + ) + grads = (q_leaf.grad.clone(), kv_leaf.grad.clone(), pos_leaf.grad.clone()) + q_leaf.grad = kv_leaf.grad = pos_leaf.grad = None + return (q_out.clone(), k_out, v_out), grads + + (q_t, k_t, v_t), grads_t = run("sbhd") + + seq_dim = 0 + q_ref = torch.cat( + ( + (q_leaf * 1.0)[..., :nope], + mla_rope._rotate_interleaved_to_neox((q_leaf * 1.0)[..., nope:], cos, sin, seq_dim), + ), + dim=-1, + ) + k_ref = torch.cat( + ( + (kv_leaf * 1.0)[..., :nope], + mla_rope._rotate_interleaved_to_neox(pos_leaf * 1.0, cos, sin, seq_dim).expand( + s, b, h, rope + ), + ), + dim=-1, + ) + v_ref = (kv_leaf * 1.0)[..., nope:] + torch.autograd.backward([q_ref, k_ref, v_ref], [grad_q.clone(), grad_k.clone(), grad_v.clone()]) + + torch.testing.assert_close(q_t, q_ref, rtol=1e-5, atol=1e-5) + torch.testing.assert_close(k_t, k_ref, rtol=1e-5, atol=1e-5) + torch.testing.assert_close(v_t, v_ref, rtol=1e-5, atol=1e-5) + torch.testing.assert_close(grads_t[0], q_leaf.grad, rtol=1e-5, atol=1e-5) + torch.testing.assert_close(grads_t[1], kv_leaf.grad, rtol=1e-5, atol=1e-5) + torch.testing.assert_close(grads_t[2], pos_leaf.grad, rtol=1e-5, atol=1e-5) + + +@pytest.mark.parametrize("compiled", [False, True]) +def test_mla_rope_q_preserves_input_and_gradient(compiled): + if not mla_rope.HAVE_TRITON: + pytest.skip("Triton unavailable") + s, b, h, nope, rope = 8, 2, 4, 64, 32 + cos, sin = mla_rope.build_rope_tables(s, rope, device="cuda") + q = torch.randn(s, b, h, nope + rope, device="cuda", requires_grad=True) + q_before = q.detach().clone() + incoming_grad = torch.randn_like(q) + grad_before = incoming_grad.clone() + apply_rope = lambda x: mla_rope.apply_mla_rope_q(x, cos, sin, nope, rope) + if compiled: + apply_rope = torch.compile(apply_rope, fullgraph=True) + + aux = (q * q).sum() + rotated = apply_rope(q) + assert rotated.data_ptr() != q.data_ptr() + torch.autograd.backward((rotated, aux), (incoming_grad, torch.ones_like(aux))) + + q_ref = q_before.requires_grad_() + rotated_ref = torch.cat( + (q_ref[..., :nope], mla_rope._rotate_interleaved_to_neox(q_ref[..., nope:], cos, sin, 0)), + dim=-1, + ) + torch.autograd.backward( + (rotated_ref, (q_ref * q_ref).sum()), (grad_before, torch.ones_like(aux)) + ) + torch.testing.assert_close(q, q_before) + torch.testing.assert_close(incoming_grad, grad_before) + torch.testing.assert_close(rotated, rotated_ref) + torch.testing.assert_close(q.grad, q_ref.grad) + + +def test_mla_rope_q_in_place_eager(): + if not mla_rope.HAVE_TRITON: + pytest.skip("Triton unavailable") + s, b, h, nope, rope = 8, 2, 4, 64, 32 + cos, sin = mla_rope.build_rope_tables(s, rope, device="cuda") + leaf = torch.randn(s, b, h, nope + rope, device="cuda", requires_grad=True) + q = (leaf * 1).view(s, b, h, nope + rope) + q_before = q.detach().clone() + incoming_grad = torch.randn_like(q) + grad_before = incoming_grad.clone() + rotated = mla_rope.apply_mla_rope_q(q, cos, sin, nope, rope, in_place=True) + assert rotated.data_ptr() == q.data_ptr() + torch.autograd.backward(rotated, incoming_grad) + + q_ref = q_before.requires_grad_() + rotated_ref = torch.cat( + (q_ref[..., :nope], mla_rope._rotate_interleaved_to_neox(q_ref[..., nope:], cos, sin, 0)), + dim=-1, + ) + torch.autograd.backward(rotated_ref, grad_before) + torch.testing.assert_close(rotated, rotated_ref) + torch.testing.assert_close(incoming_grad, grad_before) + torch.testing.assert_close(leaf.grad, q_ref.grad) + + +def test_mla_rope_q_in_place_rejects_compile(): + if not mla_rope.HAVE_TRITON: + pytest.skip("Triton unavailable") + s, b, h, nope, rope = 8, 2, 4, 64, 32 + cos, sin = mla_rope.build_rope_tables(s, rope, device="cuda") + q = torch.randn(s, b, h, nope + rope, device="cuda") + compiled = torch.compile( + lambda x: mla_rope.apply_mla_rope_q(x, cos, sin, nope, rope, in_place=True), + fullgraph=True, + ) + with pytest.raises(RuntimeError, match="in_place=True is not supported under torch.compile"): + compiled(q) + + +def test_mla_rope_q_layernormlinear_view(): + if not mla_rope.HAVE_TRITON: + pytest.skip("Triton unavailable") + s, b, h, nope, rope = 8, 2, 4, 64, 32 + cos, sin = mla_rope.build_rope_tables(s, rope, device="cuda") + projection = LayerNormLinear( + 64, h * (nope + rope), normalization="RMSNorm", params_dtype=torch.float32, device="cuda" + ) + x = torch.randn(s, b, 64, device="cuda", requires_grad=True) + q = projection(x).view(s, b, h, nope + rope) + q_before = q.detach().clone() + rotated = mla_rope.apply_mla_rope_q(q, cos, sin, nope, rope) + rotated.sum().backward() + + rotated_ref = torch.cat( + ( + q_before[..., :nope], + mla_rope._rotate_interleaved_to_neox(q_before[..., nope:], cos, sin, 0), + ), + dim=-1, + ) + torch.testing.assert_close(q, q_before) + torch.testing.assert_close(rotated, rotated_ref) + assert x.grad is not None + + +def test_rope_tables_yarn(): + s, rope = 8192, 64 + cos, sin = mla_rope.build_rope_tables(s, rope, device="cuda") + cos_none, sin_none = mla_rope.build_rope_tables(s, rope, device="cuda", scaling_factor=None) + assert torch.equal(cos, cos_none) and torch.equal(sin, sin_none) + + yarn = dict(scaling_factor=40.0, original_max_position_embeddings=4096) + cos_y, sin_y = mla_rope.build_rope_tables(s, rope, device="cuda", **yarn) + factor = mla_rope.yarn_concentration_factor(40.0, 1.0, 0.0) + assert factor == pytest.approx(0.1 * math.log(40.0) + 1.0) + # amplitude scaled by the concentration factor + torch.testing.assert_close(cos_y**2 + sin_y**2, torch.full_like(cos_y, factor**2)) + # high-frequency dims untouched, low-frequency dims interpolated by 1/scaling_factor + torch.testing.assert_close(cos_y[:, 0] / factor, cos[:, 0]) + angle_y = torch.atan2(sin_y[:, rope // 2 - 1], cos_y[:, rope // 2 - 1]) + angle = torch.atan2(sin[:, rope // 2 - 1], cos[:, rope // 2 - 1]) + torch.testing.assert_close(angle_y[:64], angle[:64] / 40.0, atol=1e-4, rtol=0) diff --git a/tests/pytorch/distributed/run_models.py b/tests/pytorch/distributed/run_models.py new file mode 100644 index 00000000000..e38a45f2672 --- /dev/null +++ b/tests/pytorch/distributed/run_models.py @@ -0,0 +1,241 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. +"""Multi-process tests for model-specific layers (te.models), launched via torchrun.""" + +import os +import sys + +import torch +import torch.distributed as dist + +from transformer_engine.pytorch.ep import ep_bootstrap, ep_finalize, release_symm_mem_pool +from transformer_engine.pytorch.models import DeepSeekV3Layer, DeepSeekV3MoE, MultiLatentAttention + +HIDDEN = 256 +MOE_FFN = 128 +SHARED_FFN = 128 +NUM_LOCAL_EXPERTS = 2 +TOP_K = 2 +TOKENS_PER_RANK = 64 +HEADS = 4 +DTYPE = torch.bfloat16 + +MLA_KWARGS = dict( + q_lora_rank=96, + kv_lora_rank=64, + qk_nope_head_dim=64, + qk_rope_head_dim=32, + v_head_dim=64, +) + + +def _device_sm() -> int: + major, minor = torch.cuda.get_device_capability() + return major * 10 + minor + + +def _recv_capacity(ep_size: int) -> int: + return DeepSeekV3MoE.ep_recv_capacity(ep_size, TOKENS_PER_RANK, TOP_K, NUM_LOCAL_EXPERTS) + + +def _broadcast_params(module: torch.nn.Module) -> None: + for t in list(module.parameters()) + list(module.buffers()): + dist.broadcast(t.detach(), src=0) + + +def _make_layer(ep_group, ep_size: int, num_experts: int) -> DeepSeekV3Layer: + ep = ep_group is not None + return DeepSeekV3Layer( + HIDDEN, + HEADS, + num_experts=num_experts, + moe_ffn_hidden_size=MOE_FFN, + topk=TOP_K, + shared_expert_ffn_hidden_size=SHARED_FFN, + params_dtype=DTYPE, + ep_group=ep_group, + ep_max_tokens_per_rank=TOKENS_PER_RANK if ep else None, + **MLA_KWARGS, + ) + + +def _copy_weights(ep_layer: DeepSeekV3Layer, ref: DeepSeekV3Layer, rank: int) -> None: + ref_params = dict(ref.named_parameters()) + ref_bufs = dict(ref.named_buffers()) + with torch.no_grad(): + for name, p in ep_layer.named_parameters(): + if not name.startswith("mlp.experts."): + p.copy_(ref_params[name]) + for name, b in ep_layer.named_buffers(): + if name in ref_bufs and b.shape == ref_bufs[name].shape: + b.copy_(ref_bufs[name]) + ep_fc1, _, ep_fc2 = ep_layer.mlp.experts[1:4] + ref_fc1, _, ref_fc2 = ref.mlp.experts + for local_e in range(NUM_LOCAL_EXPERTS): + global_e = rank * NUM_LOCAL_EXPERTS + local_e + getattr(ep_fc1, f"weight{local_e}").copy_(getattr(ref_fc1, f"weight{global_e}")) + getattr(ep_fc2, f"weight{local_e}").copy_(getattr(ref_fc2, f"weight{global_e}")) + + +def test_layer_ep_matches_local( + rank: int, ep_size: int, ep_group, num_microbatches: int = 1 +) -> None: + """Full DeepSeekV3Layer with EP must match the all-experts-local layer numerically.""" + num_experts = NUM_LOCAL_EXPERTS * ep_size + torch.manual_seed(0) + ref = _make_layer(None, ep_size, num_experts) + _broadcast_params(ref) + ep_layer = _make_layer(ep_group, ep_size, num_experts) + _copy_weights(ep_layer, ref, rank) + + torch.manual_seed(1234 + rank) + microbatches = [] + for mb in range(num_microbatches): + seq_len = TOKENS_PER_RANK // 2 - 8 * mb + x = torch.randn(seq_len, 2, HIDDEN, dtype=DTYPE, device="cuda") + x_ep = x.clone().requires_grad_(True) + x_ref = x.clone().requires_grad_(True) + out_ep = ep_layer(x_ep) + out_ref = ref(x_ref) + assert out_ep.shape == x.shape + torch.testing.assert_close(out_ep, out_ref, rtol=0.05, atol=0.05) + microbatches.append((x_ep, x_ref, out_ep, out_ref)) + + # All microbatches must retain their routing until their own backward. + for x_ep, x_ref, out_ep, out_ref in microbatches: + grad_out = torch.randn_like(out_ep) + out_ep.backward(grad_out.clone()) + out_ref.backward(grad_out.clone()) + torch.testing.assert_close(x_ep.grad, x_ref.grad, rtol=0.05, atol=0.05) + + ref_params = dict(ref.named_parameters()) + for name, p in ep_layer.named_parameters(): + if name.startswith("mlp.experts.") or p.grad is None: + continue + torch.testing.assert_close(p.grad, ref_params[name].grad, rtol=0.1, atol=0.1, msg=name) + + # A local expert's wgrad on its owner rank equals the sum of the + # reference wgrads over all ranks. all_reduce is collective, so every + # rank must reduce every expert's grad (in the same order). + ep_fc1, _, ep_fc2 = ep_layer.mlp.experts[1:4] + ref_fc1, _, ref_fc2 = ref.mlp.experts + for ep_fc, ref_fc in ((ep_fc1, ref_fc1), (ep_fc2, ref_fc2)): + ref_grads = [getattr(ref_fc, f"weight{e}").grad.float().clone() for e in range(num_experts)] + for g in ref_grads: + dist.all_reduce(g) + for local_e in range(NUM_LOCAL_EXPERTS): + global_e = rank * NUM_LOCAL_EXPERTS + local_e + ep_grad = getattr(ep_fc, f"weight{local_e}").grad.float() + torch.testing.assert_close(ep_grad, ref_grads[global_e], rtol=0.1, atol=0.1) + + counts = ep_layer.mlp._last_tokens_per_expert.clone() + dist.all_reduce(counts, group=ep_group) + last_num_tokens = microbatches[-1][0].numel() // HIDDEN + assert counts.sum().item() == ep_size * last_num_tokens * TOP_K + + expected_bias = ep_layer.mlp.expert_bias.clone() + expected_bias += ep_layer.mlp.expert_bias_update_rate * torch.sign( + counts.float().mean() - counts.float() + ) + ep_layer.mlp._last_tokens_per_expert = counts + ep_layer.mlp.update_expert_bias() + torch.testing.assert_close(ep_layer.mlp.expert_bias, expected_bias, rtol=0, atol=0) + biases = [torch.empty_like(expected_bias) for _ in range(ep_size)] + dist.all_gather(biases, ep_layer.mlp.expert_bias, group=ep_group) + for bias in biases: + torch.testing.assert_close(bias, expected_bias, rtol=0, atol=0) + + +def test_mla_tp_matches_local(rank: int, tp_size: int, tp_group) -> None: + """Compare sharded MLA outputs and gradients with an unsharded reference.""" + torch.backends.cuda.matmul.allow_tf32 = False + for fmt in ("sbhd", "bshd"): + for explicit_size in (False, True): + torch.manual_seed(0) + common = dict(params_dtype=torch.float32, qkv_format=fmt, **MLA_KWARGS) + ref = MultiLatentAttention(HIDDEN, 2 * tp_size, **common) + _broadcast_params(ref) + tp = MultiLatentAttention( + HIDDEN, + 2 * tp_size, + tp_group=tp_group, + **({"tp_size": tp_size} if explicit_size else {}), + **common, + ) + ref_params = dict(ref.named_parameters()) + + def shard(name, tensor): + if name in ("q_up_proj.weight", "kv_up_proj.weight"): + return tensor.chunk(tp_size, dim=0)[rank] + if name == "out_proj.weight": + return tensor.chunk(tp_size, dim=1)[rank] + return tensor + + with torch.no_grad(): + for name, param in tp.named_parameters(): + param.copy_(shard(name, ref_params[name])) + + torch.manual_seed(1234) + shape = (16, 2, HIDDEN) if fmt == "sbhd" else (2, 16, HIDDEN) + x = torch.randn(shape, device="cuda", requires_grad=True) + x_ref = x.detach().clone().requires_grad_() + out = tp(x) + out_ref = ref(x_ref) + torch.testing.assert_close(out, out_ref, rtol=1e-3, atol=1e-3) + grad = torch.randn_like(out) + out.backward(grad.clone()) + out_ref.backward(grad.clone()) + torch.testing.assert_close(x.grad, x_ref.grad, rtol=1e-3, atol=1e-3) + for name, param in tp.named_parameters(): + torch.testing.assert_close( + param.grad, + shard(name, ref_params[name].grad), + rtol=1e-3, + atol=1e-3, + msg=name, + ) + + +def main() -> int: + dist.init_process_group(backend="nccl") + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + if "--tp" in sys.argv: + test_mla_tp_matches_local(dist.get_rank(), dist.get_world_size(), dist.group.WORLD) + print(f"[rank {dist.get_rank()}] TP PASSED") + dist.destroy_process_group() + return 0 + from torch.distributed import _symmetric_memory as _symm_mem + + _symm_mem.set_backend("NCCL") + + rank = dist.get_rank() + ep_size = dist.get_world_size() + if _device_sm() < 90: + if rank == 0: + print(f"NCCL EP requires SM>=90 (got SM{_device_sm()}); skipping.") + dist.destroy_process_group() + return 0 + + ep_group = dist.new_group(ranks=list(range(ep_size)), backend="nccl") + ep_bootstrap( + ep_group, + num_experts=NUM_LOCAL_EXPERTS * ep_size, + max_tokens_per_rank=TOKENS_PER_RANK, + hidden_dim=HIDDEN, + num_topk=TOP_K, + recv_capacity_per_rank=None if "--eager" in sys.argv else _recv_capacity(ep_size), + ) + for num_microbatches in (1, 3): + test_layer_ep_matches_local(rank, ep_size, ep_group, num_microbatches) + print(f"[rank {rank}] PASSED") + + dist.barrier() + ep_finalize() + release_symm_mem_pool() + dist.destroy_process_group() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/pytorch/distributed/test_models.py b/tests/pytorch/distributed/test_models.py new file mode 100644 index 00000000000..d748b6bac76 --- /dev/null +++ b/tests/pytorch/distributed/test_models.py @@ -0,0 +1,46 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +import os +import subprocess +from pathlib import Path + +import pytest +import torch + +TEST_ROOT = Path(__file__).parent.resolve() +NUM_PROCS = min(8, torch.cuda.device_count()) +LAUNCH_CMD = ["torchrun", f"--nproc_per_node={NUM_PROCS}"] + + +def _has_nvlink() -> bool: + # NCCL EP falls back to the network transport and deadlocks on PCIe-only nodes. + out = subprocess.run( + ["nvidia-smi", "nvlink", "--status"], capture_output=True, text=True, check=False + ).stdout + return "GB/s" in out + + +@pytest.mark.skipif(NUM_PROCS < 2, reason="EP requires >= 2 GPUs") +@pytest.mark.skipif(not _has_nvlink(), reason="NCCL EP requires NVLink") +@pytest.mark.parametrize("eager", [False, True], ids=["fixed", "eager"]) +def test_deepseek_layer_ep(eager): + result = subprocess.run( + LAUNCH_CMD + [str(TEST_ROOT / "run_models.py")] + (["--eager"] if eager else []), + env=os.environ, + check=False, + timeout=300, + ) + assert result.returncode == 0 + + +@pytest.mark.skipif(NUM_PROCS < 2, reason="TP requires >= 2 GPUs") +def test_mla_tp(): + result = subprocess.run( + LAUNCH_CMD + [str(TEST_ROOT / "run_models.py"), "--tp"], + env=os.environ, + check=False, + timeout=300, + ) + assert result.returncode == 0 diff --git a/tests/pytorch/mxfp8/test_mxfp8_cutedsl_backend.py b/tests/pytorch/mxfp8/test_mxfp8_cutedsl_backend.py index 5d7feeb6ce6..57436eeb720 100644 --- a/tests/pytorch/mxfp8/test_mxfp8_cutedsl_backend.py +++ b/tests/pytorch/mxfp8/test_mxfp8_cutedsl_backend.py @@ -196,9 +196,14 @@ def fill(): return fill(), fill() -def run_quantize(method, act, x, ain, rowwise, columnwise, fp8_dtype, swizzled): +def run_quantize(method, act, x, ain, rowwise, columnwise, fp8_dtype, swizzled, use_2d_quant): """Quantize via the public dispatch; returns (mxfp8_tensor, dbias_or_None).""" - q = MXFP8Quantizer(fp8_dtype=fp8_dtype, rowwise=rowwise, columnwise=columnwise) + q = MXFP8Quantizer( + fp8_dtype=fp8_dtype, + rowwise=rowwise, + columnwise=columnwise, + with_2d_quantization=use_2d_quant, + ) # Emit scales in the GEMM-swizzled layout (MXFP8QuantConfig::swizzled). q.optimize_for_gemm = swizzled if method == "CAST_ONLY": @@ -216,7 +221,7 @@ def run_quantize(method, act, x, ain, rowwise, columnwise, fp8_dtype, swizzled): raise ValueError(f"unknown method {method!r}") -def get_cfg_key(method, act, in_dtype, fp8_dtype, rowwise, colwise, swizzled): +def get_cfg_key(method, act, in_dtype, fp8_dtype, rowwise, colwise, swizzled, use_2d_quant): """Mirror of MXFP8QuantConfig::to_key (quantize_mxfp8_cutedsl.cuh): the name the CuTeDSL backend registers its compiled kernel under for this config. Used to check if the CuTeDSL implementation is registered """ @@ -229,8 +234,7 @@ def get_cfg_key(method, act, in_dtype, fp8_dtype, rowwise, colwise, swizzled): elif with_dact: desc = f"d{act['desc']}" # with_amax is hardcoded to False for now because currently MXFP8 quantizers never allocate an amax - # The last False is for use_2d_quantization; these cases never request 2D block scaling - flags = (rowwise, colwise, swizzled, False, with_dbias, with_dact, with_act, False) + flags = (rowwise, colwise, swizzled, False, with_dbias, with_dact, with_act, use_2d_quant) major, minor = device_compute_capability() return ( f"cutedsl_mxfp8_sm{major * 10 + minor}_" @@ -271,7 +275,15 @@ def extract_quantized_output(out, rowwise, columnwise, swizzled): def run_test_case( - method, act, shape, block_size, in_dtype, fp8_dtype, swizzled=False, input_offset=0 + method, + act, + shape, + block_size, + in_dtype, + fp8_dtype, + swizzled=False, + input_offset=0, + use_2d_quant=False, ): """Assert the CuTeDSL and CUDA backends produce bit-identical outputs for the same input and config. input_offset (in elements) shifts x's data pointer past the @@ -287,14 +299,14 @@ def run_test_case( set_cutedsl_backend(False) out_cuda, dbias_cuda = run_quantize( - method, act, x, act_input, rowwise, columnwise, fp8_dtype, swizzled + method, act, x, act_input, rowwise, columnwise, fp8_dtype, swizzled, use_2d_quant ) cuda_output = extract_quantized_output(out_cuda, rowwise, columnwise, swizzled) set_cutedsl_backend(True) try: out_cutedsl, dbias_cutedsl = run_quantize( - method, act, x, act_input, rowwise, columnwise, fp8_dtype, swizzled + method, act, x, act_input, rowwise, columnwise, fp8_dtype, swizzled, use_2d_quant ) cutedsl_output = extract_quantized_output(out_cutedsl, rowwise, columnwise, swizzled) finally: @@ -304,7 +316,7 @@ def run_test_case( # CuTeDSL backend supports, so its kernel must have been registered under the # config key. If not, the backend rejected or missed the config and the # comparison above was CUDA vs CUDA. - key = get_cfg_key(method, act, in_dtype, fp8_dtype, rowwise, columnwise, swizzled) + key = get_cfg_key(method, act, in_dtype, fp8_dtype, rowwise, columnwise, swizzled, use_2d_quant) assert tvm_ffi.get_global_func(key, allow_missing=True) is not None, ( f"CuTeDSL kernel not registered for {key}; the CuTeDSL backend fell back " "to CUDA and this case compared CUDA against itself" @@ -314,6 +326,8 @@ def run_test_case( tag = ( f"{method}/{act['name']}/{M}x{N}/{DTYPE_TO_STR[in_dtype]}/{FP8_TO_STR[fp8_dtype]}/{layout}" ) + if use_2d_quant: + tag += "/2d" for name, cuda_bytes in cuda_output.items(): assert torch.equal( cutedsl_output[name], cuda_bytes @@ -334,6 +348,18 @@ def test_cast_only(swizzled, fp8_dtype, in_dtype, block_size, shape): run_test_case("CAST_ONLY", IDENTITY, shape, block_size, in_dtype, fp8_dtype, swizzled) +# 2D block scaling (one scale per 32x32 block) is only reachable as a plain cast. +@pytest.mark.parametrize("shape", MATRIX_SIZES, ids=get_shape_id) +@pytest.mark.parametrize("block_size", BLOCK_SIZES, ids=get_block_id) +@pytest.mark.parametrize("in_dtype", IN_DTYPES, ids=get_dtype_id) +@pytest.mark.parametrize("fp8_dtype", FP8_DTYPES, ids=get_fp8_id) +@pytest.mark.parametrize("swizzled", SWIZZLE_MODES, ids=get_swizzle_id) +def test_cast_only_2d(swizzled, fp8_dtype, in_dtype, block_size, shape): + run_test_case( + "CAST_ONLY", IDENTITY, shape, block_size, in_dtype, fp8_dtype, swizzled, use_2d_quant=True + ) + + def test_cast_only_rowwise_specialized_grid_y_overflow(): """Fall back when the specialized rowwise launch would exceed CUDA's grid.y limit.""" # The specialized kernel covers four rows per CTA, so this shape would require @@ -431,3 +457,43 @@ def test_specialized_kernels(shape, block_size, in_dtype, swizzled, input_offset swizzled, input_offset, ) + + +OUT_DTYPES = [torch.float32, torch.bfloat16, torch.float16] +DEQUANT_SHAPES = MATRIX_SIZES + [(100, 48)] + + +# Test cases for dequantization (mirrors C++ test's OperatorTest_DequantizeMXFP8). +# With both data copies present, dequantize reads the rowwise one. +@pytest.mark.parametrize("shape", DEQUANT_SHAPES, ids=get_shape_id) +@pytest.mark.parametrize("block_size", BLOCK_SIZES, ids=get_block_id) +@pytest.mark.parametrize("out_dtype", OUT_DTYPES, ids=get_dtype_id) +@pytest.mark.parametrize("fp8_dtype", FP8_DTYPES, ids=get_fp8_id) +@pytest.mark.parametrize("swizzled", SWIZZLE_MODES, ids=get_swizzle_id) +def test_dequantize(swizzled, fp8_dtype, out_dtype, block_size, shape): + M, N = shape + rowwise = block_size[1] != 1 + columnwise = block_size[0] != 1 + x, _ = generate_inputs(M, N, torch.bfloat16) + set_cutedsl_backend(False) + q = MXFP8Quantizer(fp8_dtype=fp8_dtype, rowwise=rowwise, columnwise=columnwise) + q.optimize_for_gemm = swizzled + qx = q(x) + out_cuda = qx.dequantize(dtype=out_dtype) + + set_cutedsl_backend(True) + try: + out_cutedsl = qx.dequantize(dtype=out_dtype) + finally: + set_cutedsl_backend(False) + + major, minor = device_compute_capability() + key = ( + f"cutedsl_mxfp8_dequant_sm{major * 10 + minor}_{FP8_TO_KEY[fp8_dtype]}_" + f"{DTYPE_TO_STR[out_dtype]}_{int(rowwise)}_{int(columnwise)}_{int(swizzled)}" + ) + assert tvm_ffi.get_global_func(key, allow_missing=True) is not None, ( + f"CuTeDSL kernel not registered for {key}; the CuTeDSL backend fell back " + "to CUDA and this case compared CUDA against itself" + ) + assert torch.equal(out_cutedsl, out_cuda) diff --git a/tests/pytorch/mxfp8/test_mxfp8_group_quantize_graph_safe.py b/tests/pytorch/mxfp8/test_mxfp8_group_quantize_graph_safe.py index 7a1edc071e9..26fb7c5c65b 100644 --- a/tests/pytorch/mxfp8/test_mxfp8_group_quantize_graph_safe.py +++ b/tests/pytorch/mxfp8/test_mxfp8_group_quantize_graph_safe.py @@ -804,6 +804,23 @@ def _requantize_setup(M: int = 1024, N: int = 256): return x, torch.tensor(split_sections, dtype=torch.int64, device="cuda"), len(split_sections) +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +@pytest.mark.parametrize("columnwise", [False, True]) +def test_prequantized_requantize_empty_gemm_metadata(columnwise): + """Empty dispatch results must still advertise GEMM-ready scale layout.""" + inputs, splits, num_groups = _requantize_setup(M=0, N=512) + wire = make_prequantized_wire_tensor(inputs, splits) + tex.group_requantize_inplace( + wire, make_op_quantizer(columnwise), num_groups, splits, te.DType.kBFloat16 + ) + assert wire._with_gemm_swizzled_scales + assert wire.rowwise_data.numel() == 0 + assert wire.scale_inv.numel() == 0 + if columnwise: + assert wire.columnwise_data is not None + assert wire.columnwise_data.numel() == 0 + + @pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) def test_prequantized_requantize_passes_through_gemm_ready_input(): """A tensor already GEMM-ready in both directions is left untouched.""" diff --git a/tests/pytorch/nvfp4/test_nvfp4_global_scale.py b/tests/pytorch/nvfp4/test_nvfp4_global_scale.py new file mode 100644 index 00000000000..945e6ff7bc1 --- /dev/null +++ b/tests/pytorch/nvfp4/test_nvfp4_global_scale.py @@ -0,0 +1,107 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. +"""Tests for the fused NVFP4 global-scale / GEMM-alpha Triton kernel. + +The kernel replaces the eager ``amax.float() / (6 * scale_max)`` divisions and the +product / square root of the two operand scales used by the grouped MLP. It must be +bit-identical to the eager ops, including with a broadcast (single-group) amax, and must +be safe to capture in a CUDA graph.""" + +import pytest +import torch + +from transformer_engine.pytorch.triton.nvfp4_global_scale import nvfp4_global_scale_pair + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") + +_DENOMS = [6.0 * 448.0, 6.0 * 256.0, 6.0 * 114688.0] + + +def _random_amax(n, device): + # Wide dynamic range plus exact zeros (empty experts) and a few special magnitudes. + amax = torch.exp2(torch.empty(n, device=device).uniform_(-40.0, 40.0)) + amax *= torch.rand(n, device=device) + 0.5 + if n >= 4: + amax[0] = 0.0 + amax[1] = 1.0 + amax[2] = torch.finfo(torch.float32).tiny + amax[3] = 448.0 * 6.0 + return amax + + +def _reference(amax_a, amax_b, denom_a, denom_b, combine): + scale_a = amax_a.to(torch.float32) / denom_a + scale_b = amax_b.to(torch.float32) / denom_b + if combine is None: + return scale_a, scale_b + if combine == "sqrt_product": + return torch.sqrt(scale_a * scale_b) + return (scale_a * scale_b).to(torch.float32) + + +def _assert_bitwise_equal(out, ref): + if isinstance(ref, tuple): + assert isinstance(out, tuple) and len(out) == len(ref) + for o, r in zip(out, ref): + _assert_bitwise_equal(o, r) + return + assert out.dtype == torch.float32 + assert out.shape == ref.shape + assert torch.equal(out.view(torch.int32), ref.contiguous().view(torch.int32)) + + +@pytest.mark.parametrize("n", [1, 3, 8, 32, 257, 4096]) +@pytest.mark.parametrize("combine", [None, "product", "sqrt_product"]) +@pytest.mark.parametrize("denoms", [(_DENOMS[0], _DENOMS[0]), (_DENOMS[1], _DENOMS[2])]) +@pytest.mark.parametrize("broadcast", ["none", "a", "b"]) +def test_matches_eager(n, combine, denoms, broadcast): + torch.manual_seed(n) + device = torch.device("cuda") + amax_a = _random_amax(1 if broadcast == "a" else n, device) + amax_b = _random_amax(1 if broadcast == "b" else n, device) + out = nvfp4_global_scale_pair(amax_a, amax_b, denoms[0], denoms[1], combine=combine) + if combine is None and amax_a.numel() != amax_b.numel(): + # Separate scales keep their own shapes, so the caller falls back to eager. + assert out is None + return + assert out is not None + _assert_bitwise_equal(out, _reference(amax_a, amax_b, denoms[0], denoms[1], combine)) + + +def test_falls_back_on_unsupported_inputs(): + device = torch.device("cuda") + amax = torch.ones(8, device=device) + assert nvfp4_global_scale_pair(amax.half(), amax, 1.0, 1.0) is None + assert nvfp4_global_scale_pair(amax, amax[:3], 1.0, 1.0) is None + assert nvfp4_global_scale_pair(amax.cpu(), amax.cpu(), 1.0, 1.0) is None + big = torch.ones(4097, device=device) + assert nvfp4_global_scale_pair(big, big, 1.0, 1.0) is None + + +@pytest.mark.parametrize("combine", [None, "product", "sqrt_product"]) +def test_cuda_graph_replay(combine): + device = torch.device("cuda") + n = 32 + denom = _DENOMS[0] + amax_a = _random_amax(n, device) + amax_b = _random_amax(n, device) + + # Warm up (compiles the kernel) on a side stream before capture. + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + nvfp4_global_scale_pair(amax_a, amax_b, denom, denom, combine=combine) + torch.cuda.current_stream().wait_stream(stream) + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = nvfp4_global_scale_pair(amax_a, amax_b, denom, denom, combine=combine) + + for seed in range(3): + torch.manual_seed(seed) + amax_a.copy_(_random_amax(n, device)) + amax_b.copy_(_random_amax(n, device)) + graph.replay() + torch.cuda.synchronize() + _assert_bitwise_equal(out, _reference(amax_a, amax_b, denom, denom, combine)) diff --git a/tests/pytorch/nvfp4/test_nvfp4_quantize_exact.py b/tests/pytorch/nvfp4/test_nvfp4_quantize_exact.py index 51b5b46c158..0947e4e744c 100644 --- a/tests/pytorch/nvfp4/test_nvfp4_quantize_exact.py +++ b/tests/pytorch/nvfp4/test_nvfp4_quantize_exact.py @@ -123,6 +123,7 @@ def check_quantization_nvfp4_versus_reference( nvfp4_4over6_err_mode: str = "MAE", nvfp4_4over6_err_use_fast_math: bool = False, scale_dtype: te.DType = te.DType.kFloat8E4M3, + check_amax_reset: bool = False, ) -> None: scale_dtype = te.DType.cast(scale_dtype) if nvfp4_e4m3_max not in NVFP4_SUPPORTED_SCALE_MAX_VALUES: @@ -252,6 +253,19 @@ def check_quantization_nvfp4_versus_reference( torch.testing.assert_close(qx_amax, ref_amax, atol=0.0, rtol=0.0) + if check_amax_reset: + assert not row_scaled_nvfp4 + # Reusing an output whose maximum decreases verifies that every amax buffer is reset. + for magnitude in (16.0, 0.5, 0.0, 2.0): + x = torch.full((M, N), -magnitude, dtype=x_dtype, device=device) + nvfp4_quantizer.update_quantized(x, x_nvfp4_sut) + expected_amax = x.float().abs().amax().reshape(1) + torch.testing.assert_close(x_nvfp4_sut._amax_rowwise, expected_amax, atol=0.0, rtol=0.0) + if return_transpose: + torch.testing.assert_close( + x_nvfp4_sut._amax_columnwise, expected_amax, atol=0.0, rtol=0.0 + ) + @pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) @pytest.mark.parametrize("return_transpose", [False, True], ids=["rowwise", "with_columnwise"]) @@ -359,6 +373,7 @@ def test_quantization_block_tiling_versus_reference( row_scaled_nvfp4: bool, nvfp4_4over6_config: NVFP44Over6TestConfig, ) -> None: + check_amax_reset = not row_scaled_nvfp4 and not nvfp4_4over6_config.use_4over6 check_quantization_nvfp4_versus_reference( x_dtype=x_dtype, M=M, @@ -372,6 +387,7 @@ def test_quantization_block_tiling_versus_reference( nvfp4_e4m3_max=nvfp4_4over6_config.e4m3_max, nvfp4_4over6_err_mode=nvfp4_4over6_config.err_mode, nvfp4_4over6_err_use_fast_math=nvfp4_4over6_config.err_use_fast_math, + check_amax_reset=check_amax_reset, ) @@ -807,29 +823,42 @@ def test_nvfp4_quantization_noncontiguous_inputs( @pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) @pytest.mark.parametrize( - "M, N", + "with_2d_quantization, M, N", [ - # Aligned tiles - (128, 128), - (256, 256), - (512, 512), - (2048, 2048), - # Padded tiles (non-multiple of kTileDim=128) - (256, 272), - (304, 304), - (320, 256), + # Existing 2D coverage: aligned and padded tiles. + (True, 128, 128), + (True, 256, 256), + (True, 512, 512), + (True, 2048, 2048), + (True, 256, 272), + (True, 304, 304), + (True, 320, 256), + # Focused 1D coverage for aligned and padded output reuse. + (False, 128, 128), + (False, 256, 272), + ], + ids=[ + "2d-128x128", + "2d-256x256", + "2d-512x512", + "2d-2048x2048", + "2d-256x272", + "2d-304x304", + "2d-320x256", + "1d-128x128", + "1d-256x272", ], ) @pytest.mark.parametrize("x_dtype", [torch.float32, torch.bfloat16], ids=str) -def test_nvfp4_2d_columnwise_only_matches_both_directions( +def test_nvfp4_columnwise_only_matches_both_directions( x_dtype: torch.dtype, + with_2d_quantization: bool, M: int, N: int, ): - """Bitwise check: 2D NVFP4 with columnwise-only must produce the same - columnwise data/scales as the columnwise half of (rowwise + columnwise) 2D. + """Columnwise-only NVFP4 must match the columnwise half of both-directions. - Covers both kernels depending on the (dtype, shape) routing: + The 2D cases cover both kernels depending on the (dtype, shape) routing: - bf16 with rows % 32 == 0 and cols % 32 == 0 routes to the optimized ``quantize_transpose_nvfp4_2D_kernel`` (instantiated with RETURN_ROWWISE=false), validating that gating the rowwise pass/store leaves the shared @@ -853,7 +882,7 @@ def _make_quantizer(*, rowwise: bool, columnwise: bool) -> NVFP4Quantizer: amax_reduction_group=None, with_rht=False, with_post_rht_amax=False, - with_2d_quantization=True, + with_2d_quantization=with_2d_quantization, row_scaled_nvfp4=False, ) @@ -861,7 +890,7 @@ def _make_quantizer(*, rowwise: bool, columnwise: bool) -> NVFP4Quantizer: q_both = _make_quantizer(rowwise=True, columnwise=True) out_both = q_both(x) - # SUT: produce columnwise only (the path that hits the new amax-only pass). + # SUT: produce columnwise only. q_col_only = _make_quantizer(rowwise=False, columnwise=True) out_col_only = q_col_only(x) @@ -901,5 +930,15 @@ def _make_quantizer(*, rowwise: bool, columnwise: bool) -> NVFP4Quantizer: out_col_only._amax_columnwise, out_both._amax_columnwise, atol=0, rtol=0 ) + if (M, N) in ((128, 128), (256, 272)): + # Reusing a columnwise-only output exercises the path where its amax is primary. + for magnitude in (16.0, 0.5, 0.0, 2.0): + x = torch.full((M, N), -magnitude, dtype=x_dtype, device=device) + q_col_only.update_quantized(x, out_col_only) + expected_amax = x.float().abs().amax().reshape(1) + torch.testing.assert_close( + out_col_only._amax_columnwise, expected_amax, atol=0.0, rtol=0.0 + ) + # Sanity: column-only path must not allocate a rowwise output. assert out_col_only._rowwise_data is None diff --git a/tests/pytorch/test_models.py b/tests/pytorch/test_models.py new file mode 100644 index 00000000000..9d677be2fe2 --- /dev/null +++ b/tests/pytorch/test_models.py @@ -0,0 +1,194 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +import math + +import pytest +import torch + +import transformer_engine.pytorch as te +from transformer_engine.pytorch.ops.fused.grouped_mlp import ( + GroupedMLP_CuTeGEMMGLU, + fuse_glu_ops, +) +from transformer_engine.pytorch.ops.fuser import OperationFuser +from transformer_engine.pytorch.utils import deinterleave_glu_tensor +from transformer_engine.pytorch.models import DeepSeekV3MoE, MultiLatentAttention +from utils import make_recipe, quantization_tols + +SEQ_LEN = 128 +BATCH = 2 +HIDDEN = 256 +HEADS = 4 +DTYPE = torch.bfloat16 + +MLA_KWARGS = dict( + q_lora_rank=96, + kv_lora_rank=64, + qk_nope_head_dim=64, + qk_rope_head_dim=32, + v_head_dim=64, +) + + +def _input(requires_grad=True): + torch.manual_seed(1234) + return torch.randn( + SEQ_LEN, BATCH, HIDDEN, dtype=DTYPE, device="cuda", requires_grad=requires_grad + ) + + +@pytest.mark.parametrize("mscale_all_dim", [0.0, 1.0]) +def test_mla_yarn_softmax_scale(mscale_all_dim): + mla = MultiLatentAttention( + HIDDEN, + HEADS, + params_dtype=DTYPE, + rope_scaling_factor=40.0, + original_max_position_embeddings=64, + mscale_all_dim=mscale_all_dim, + **MLA_KWARGS, + ) + m = 0.1 * mscale_all_dim * math.log(40.0) + 1.0 + qk_head_dim = MLA_KWARGS["qk_nope_head_dim"] + MLA_KWARGS["qk_rope_head_dim"] + assert mla.softmax_scale == pytest.approx(m * m / math.sqrt(qk_head_dim)) + + +@pytest.mark.parametrize("shared", [False, True], ids=["no_shared", "shared"]) +@pytest.mark.parametrize("grouped", [False, True], ids=["ungrouped", "grouped"]) +@pytest.mark.parametrize("topk", [2, 4]) +@pytest.mark.parametrize("quantization", [None, "fp8_block_scaling"]) +def test_moe_matches_dense_reference(shared, grouped, topk, quantization): + """Routed output must equal the prob-weighted sum of the selected expert MLPs.""" + if quantization is not None: + available, reason = te.is_fp8_block_scaling_available(return_reason=True) + if not available: + pytest.skip(reason) + torch.manual_seed(0) + num_experts = 4 + moe = DeepSeekV3MoE( + HIDDEN, + moe_ffn_hidden_size=128, + num_experts=num_experts, + topk=topk, + num_groups=2 if grouped else None, + group_topk=topk // 2 if grouped else None, + shared_expert_ffn_hidden_size=128 if shared else None, + params_dtype=DTYPE, + ) + x = _input() + with te.autocast(enabled=quantization is not None, recipe=make_recipe(quantization)): + out = moe(x) + assert out.shape == x.shape + assert torch.isfinite(out).all() + out.sum().backward() + assert torch.isfinite(x.grad).all() + for name, parameter in moe.named_parameters(): + if parameter.requires_grad: + assert parameter.grad is not None, name + assert torch.isfinite(parameter.grad).all(), name + + tokens = x.detach().reshape(-1, HIDDEN) + probs, _ = moe._route(moe.gate(tokens).float()) + assert (probs > 0).sum(dim=1).eq(topk).all() + assert moe._last_tokens_per_expert.sum().item() == tokens.shape[0] * topk + + fc1, _, fc2 = moe.experts + quantizers = [ + te.Float8BlockQuantizer( + fp8_dtype=te.DType.kFloat8E4M3, + rowwise=True, + columnwise=False, + block_scaling_dim=dim, + ) + for dim in (1, 2) + ] + + def qdq(tensor, block_dim): + if quantization is None: + return tensor + return quantizers[block_dim - 1](tensor).dequantize(dtype=DTYPE) + + ref_tokens = qdq(tokens, 1) + ref = torch.zeros_like(tokens) + for e in range(num_experts): + w1 = deinterleave_glu_tensor(qdq(getattr(fc1, f"weight{e}"), 2), 32) + w2 = qdq(getattr(fc2, f"weight{e}"), 2) + gate_part, lin_part = (ref_tokens @ w1.t()).chunk(2, dim=-1) + act = torch.nn.functional.silu(gate_part.float()) * lin_part.float() + act = qdq(act.to(DTYPE) * probs[:, e : e + 1].to(DTYPE), 1) + ref += act @ w2.t() + if shared: + with te.autocast(enabled=quantization is not None, recipe=make_recipe(quantization)): + ref += moe.shared_expert(tokens) + torch.testing.assert_close(out.reshape(-1, HIDDEN), ref, rtol=0.05, atol=0.05) + + bias_before = moe.expert_bias.clone() + moe.update_expert_bias() + assert torch.isfinite(moe.expert_bias).all() + if topk < num_experts: + assert not torch.equal(bias_before, moe.expert_bias) + + +@pytest.mark.parametrize("quantization", ["mxfp8", "nvfp4_rht"]) +def test_moe_fused_quantized_uneven_expert_rows(monkeypatch, quantization): + available_fn = te.is_mxfp8_available if quantization == "mxfp8" else te.is_nvfp4_available + available, reason = available_fn(return_reason=True) + if not available: + pytest.skip(reason) + + recipe = make_recipe(quantization) + torch.manual_seed(0) + moe = DeepSeekV3MoE(HIDDEN, 128, num_experts=2, topk=1, params_dtype=DTYPE) + fused_ops = fuse_glu_ops(list(moe.experts), recipe=recipe) + if ( + fuse_glu_ops not in OperationFuser.forward_backward_fusion_functions + or len(fused_ops) != 1 + or not isinstance(fused_ops[0], GroupedMLP_CuTeGEMMGLU) + ): + pytest.skip("requires the fused grouped MLP") + reference = DeepSeekV3MoE(HIDDEN, 128, num_experts=2, topk=1, params_dtype=DTYPE) + reference.load_state_dict(moe.state_dict()) + with torch.no_grad(): + for module in (moe, reference): + module.gate.weight.zero_() + module.gate.weight[0, 0] = 1 + module.gate.weight[1, 0] = -1 + + tokens = torch.empty(512, HIDDEN, device="cuda", dtype=DTYPE).uniform_(-0.25, 0.25) + tokens[:128, 0] = 2 + tokens[128:, 0] = -2 + x = tokens.detach().requires_grad_() + x_ref = tokens.detach().clone().requires_grad_() + grad = torch.empty_like(tokens).uniform_(-0.25, 0.25) + splits = [] + moe.experts.register_forward_pre_hook(lambda _module, args: splits.append(args[1].clone())) + + with te.autocast(enabled=True, recipe=recipe): + out = moe(x) + out.backward(grad.clone()) + assert torch.equal(splits[0], torch.tensor([256, 512], device="cuda")) + fused_op = moe.experts._module_groups[0]._forward_ops[0][0] + assert isinstance(fused_op, GroupedMLP_CuTeGEMMGLU) + + monkeypatch.setattr( + OperationFuser, + "forward_backward_fusion_functions", + [fn for fn in OperationFuser.forward_backward_fusion_functions if fn is not fuse_glu_ops], + ) + ref_splits = [] + reference.experts.register_forward_pre_hook( + lambda _module, args: ref_splits.append(args[1].clone()) + ) + with te.autocast(enabled=True, recipe=make_recipe(quantization)): + ref_out = reference(x_ref) + ref_out.backward(grad.clone()) + assert torch.equal(ref_splits[0], splits[0]) + + tols = quantization_tols(quantization) + torch.testing.assert_close(out, ref_out, **tols) + torch.testing.assert_close(x.grad, x_ref.grad, **tols) + ref_params = dict(reference.named_parameters()) + for name, param in moe.named_parameters(): + torch.testing.assert_close(param.grad, ref_params[name].grad, **tols) diff --git a/tests/pytorch/test_recipe.py b/tests/pytorch/test_recipe.py index 9558003eda6..e1447999db2 100644 --- a/tests/pytorch/test_recipe.py +++ b/tests/pytorch/test_recipe.py @@ -12,6 +12,7 @@ import transformer_engine.common.recipe import transformer_engine.pytorch as te +import transformer_engine.pytorch.quantization as quantization from transformer_engine.pytorch import ( Float8BlockQuantizer, MXFP8Quantizer, @@ -57,6 +58,37 @@ fp4_available, reason_for_no_fp4 = te.is_nvfp4_available(return_reason=True) +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +def test_autocast_reuse_after_failed_entry(monkeypatch): + monkeypatch.setattr(FP8GlobalStateManager, "quantization_state", quantization.FP8GlobalState()) + original_check = quantization.check_recipe_support + fail_next_check = True + + def fail_once(recipe): + nonlocal fail_next_check + if fail_next_check: + fail_next_check = False + raise RuntimeError("recipe validation failed") + original_check(recipe) + + monkeypatch.setattr(quantization, "check_recipe_support", fail_once) + context = te.autocast(recipe=Float8CurrentScaling()) + with te.autocast(enabled=False, calibrating=True, recipe=Float8CurrentScaling()): + saved_state = FP8GlobalStateManager.get_autocast_state() + saved_depth = FP8GlobalStateManager.quantization_state.autocast_depth + with pytest.raises(RuntimeError, match="recipe validation failed"): + with context: + pytest.fail("Autocast body ran after failed validation") + assert FP8GlobalStateManager.get_autocast_state() == saved_state + assert FP8GlobalStateManager.quantization_state.autocast_depth == saved_depth + + with context: + assert FP8GlobalStateManager.is_fp8_enabled() + assert FP8GlobalStateManager.quantization_state.autocast_depth == saved_depth + 1 + assert FP8GlobalStateManager.get_autocast_state() == saved_state + assert FP8GlobalStateManager.quantization_state.autocast_depth == saved_depth + + # FP8 per tensor delayed scaling @pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) class TestFP8Recipe: diff --git a/tests/pytorch/test_sanity.py b/tests/pytorch/test_sanity.py index 2627de023ac..0fe2bacc6cf 100644 --- a/tests/pytorch/test_sanity.py +++ b/tests/pytorch/test_sanity.py @@ -35,6 +35,7 @@ is_bf16_available, ) from transformer_engine.common import recipe +from transformer_engine.pytorch.models import DeepSeekV3Layer from transformer_engine.pytorch.cpp_extensions import general_gemm from transformer_engine.pytorch.tensor.utils import replace_raw_data from transformer_engine.pytorch.module import ( @@ -285,7 +286,7 @@ def _test_sanity_e2e_gradient_accumulation_fusion(block, dtype, config, fp8_reci ), f"grad_added_to_main_grad not set to True for {failed_grad_added_flags}." -def _test_sanity_e2e(block, dtype, config, fp8_recipe, skip_wgrad): +def _test_sanity_e2e(block, dtype, config, fp8_recipe, skip_wgrad, *, check_finite=False): te_inp_hidden_states = torch.randn( (config.max_seqlen_q, config.batch_size, config.hidden_size), dtype=dtype, @@ -303,6 +304,14 @@ def _test_sanity_e2e(block, dtype, config, fp8_recipe, skip_wgrad): loss.backward() torch.cuda.synchronize() + if check_finite: + assert torch.isfinite(te_out).all(), "Non-finite output" + assert torch.isfinite(te_inp_hidden_states.grad).all(), "Non-finite input gradient" + for name, parameter in block.named_parameters(): + if parameter.requires_grad: + assert parameter.grad is not None, name + assert torch.isfinite(parameter.grad).all(), name + def _test_sanity_e2e_bert(block, dtype, config, fp8_recipe, skip_wgrad): te_inp_hidden_states = torch.randn( @@ -858,6 +867,41 @@ def checked_norm(*args, _norm=norm_, _suffix=suffix, **kwargs): assert set(seen_norm_stages) == {"fwd", "bwd"} +@pytest.mark.parametrize("dtype", param_types) +@pytest.mark.parametrize("fp8_recipe", fp8_recipes, ids=recipe_id) +@pytest.mark.parametrize("moe", all_boolean, ids=["moe", "dense"]) +def test_sanity_deepseek_v3_layer(dtype, fp8_recipe, moe): + config = model_configs["small"] + + if fp8_recipe is not None: + if not is_fp8_supported(config): + pytest.skip("Model config does not support FP8") + if fp8_recipe.nvfp4() and dtype == torch.float16: + pytest.skip("FP16 output for NVFP4 not supported") + if fp8_recipe.nvfp4() and moe and dtype != torch.bfloat16: + pytest.skip("NVFP4 GroupedLinear requires BF16") + + mlp_kwargs = ( + dict(num_experts=4, topk=2, moe_ffn_hidden_size=64, shared_expert_ffn_hidden_size=64) + if moe + else dict(ffn_hidden_size=4 * config.hidden_size) + ) + block = DeepSeekV3Layer( + config.hidden_size, + config.num_heads, + q_lora_rank=64, + kv_lora_rank=64, + qk_nope_head_dim=32, + qk_rope_head_dim=32, + v_head_dim=32, + params_dtype=dtype, + device="cuda", + **mlp_kwargs, + ) + + _test_sanity_e2e(block, dtype, config, fp8_recipe, skip_wgrad=False, check_finite=True) + + @pytest.mark.parametrize("dtype", param_types) @pytest.mark.parametrize("fp8_recipe", fp8_recipes, ids=recipe_id) @pytest.mark.parametrize("model", ["small"]) diff --git a/transformer_engine/common/CuTeDSL/__init__.py b/transformer_engine/common/CuTeDSL/__init__.py index 15aae1aba2d..24f4b3202bc 100644 --- a/transformer_engine/common/CuTeDSL/__init__.py +++ b/transformer_engine/common/CuTeDSL/__init__.py @@ -10,6 +10,9 @@ import tvm_ffi +from transformer_engine.common.CuTeDSL.cast.mxfp8.dequantize_mxfp8 import ( + get_mxfp8_dequantization_function, +) from transformer_engine.common.CuTeDSL.cast.mxfp8.quantize_mxfp8_common import ( get_mxfp8_quantization_function, ) @@ -25,6 +28,9 @@ def register_cutedsl_backends(): tvm_ffi.register_global_func( "get_mxfp8_quantization_function", get_mxfp8_quantization_function, override=True ) + tvm_ffi.register_global_func( + "get_mxfp8_dequantization_function", get_mxfp8_dequantization_function, override=True + ) tvm_ffi.register_global_func( "get_mxfp8_requantization_function", get_mxfp8_requantization_function, override=True ) diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/__init__.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/__init__.py index 444d85234ec..95f3b89882e 100644 --- a/transformer_engine/common/CuTeDSL/cast/mxfp8/__init__.py +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/__init__.py @@ -2,6 +2,7 @@ # # See LICENSE for license information. -"""CuTeDSL MXFP8 quantization kernels.""" +"""CuTeDSL MXFP8 quantization and dequantization kernels.""" +from . import dequantize_mxfp8 from . import quantize_mxfp8_common diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/dequantize_mxfp8.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/dequantize_mxfp8.py new file mode 100644 index 00000000000..cdf5b435ef8 --- /dev/null +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/dequantize_mxfp8.py @@ -0,0 +1,455 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""CuTeDSL MXFP8 dequantization kernel, with its compilation and TVM-FFI registration.""" + +# Local @cute.struct classes are shared-memory storage descriptors. +# pylint: disable=missing-class-docstring + +import logging +import cutlass +from cutlass import cute +from cutlass import pipeline +from cutlass import Float8E8M0FNU, Float32, Int32 +from cuda.bindings.driver import CUstream # pylint: disable=no-name-in-module +import tvm_ffi + +from transformer_engine.common.CuTeDSL.utils import ( + device_compute_capability, + exp2f, + str_to_cutlass_dtype, +) +from .quantize_mxfp8_common import ( + CUTEDSL_DEBUG_LOGGING, + MXFP8_BLOCK_SCALING_SIZE, + SYM_N_DIVISIBILITY, + THREADS_PER_WARP, + derive_swizzled_scale_layout, +) + +logger = logging.getLogger("transformer_engine.cutedsl.mxfp8") + + +@cute.jit +def dequantize_mxfp8_smem_tile( + sQ: cute.Tensor, # Input quantized SMEM tile + sO: cute.Tensor, # Output dequantized SMEM tile + gS: cute.Tensor, # This tile's E8M0 scales in GMEM + TILE_ROWS: cutlass.Constexpr[int], # How many rows in the tile + TILE_COLS: cutlass.Constexpr[int], # How many columns in the tile + ELEMS_PER_THREAD: cutlass.Constexpr[int], # How many values each thread dequantizes + DEQUANTIZE_FROM_ROWWISE: cutlass.Constexpr[bool], # Which direction to dequantize from +): + """Dequantize a smem tile sQ into sO using the scales in gS. + + Requires CTA threads multiplied by ELEMS_PER_THREAD to cover the tile. + + sQ, sO: (TILE_ROWS, TILE_COLS) + gS: + - rowwise: (TILE_ROWS, max(1, TILE_COLS // MXFP8_BLOCK_SCALING_SIZE)) + - colwise: (max(1, TILE_ROWS // MXFP8_BLOCK_SCALING_SIZE), TILE_COLS) + """ + + tidx, _, _ = cute.arch.thread_idx() + + # MXFP8_BLOCK_SCALING_SIZE must be divisible by ELEMS_PER_THREAD + assert MXFP8_BLOCK_SCALING_SIZE % ELEMS_PER_THREAD == 0 + FRAGS_SHARING_SAME_SCALE = MXFP8_BLOCK_SCALING_SIZE // ELEMS_PER_THREAD + if cutlass.const_expr(DEQUANTIZE_FROM_ROWWISE): + assert TILE_COLS % ELEMS_PER_THREAD == 0 + THR_ROWS = TILE_ROWS + THR_COLS = TILE_COLS // ELEMS_PER_THREAD + # Row-major threads: a warp reads contiguous 16 B chunks along a row (no bank conflicts) + thr_layout = cute.make_layout((THR_ROWS, THR_COLS), stride=(THR_COLS, 1)) + val_layout = cute.make_layout((1, ELEMS_PER_THREAD), stride=(0, 1)) + scale_y = tidx // THR_COLS + scale_x = tidx % THR_COLS + # Every `FRAGS_SHARING_SAME_SCALE` threads along x direction share the same scale + scale_e8m0 = gS[scale_y, scale_x // FRAGS_SHARING_SAME_SCALE] + else: + assert TILE_ROWS % ELEMS_PER_THREAD == 0 and MXFP8_BLOCK_SCALING_SIZE % TILE_ROWS == 0 + THR_ROWS = TILE_ROWS // ELEMS_PER_THREAD + THR_COLS = TILE_COLS + thr_layout = cute.make_layout((THR_ROWS, THR_COLS)) + val_layout = cute.make_layout((ELEMS_PER_THREAD, 1), stride=(1, 0)) + scale_y = tidx % THR_ROWS + scale_x = tidx // THR_ROWS + # Every `FRAGS_SHARING_SAME_SCALE` threads along y direction share the same scale + scale_e8m0 = gS[scale_y // FRAGS_SHARING_SAME_SCALE, scale_x] + + _, tv_layout = cute.make_layout_tv( + thr_layout=thr_layout, + val_layout=val_layout, + ) + + sQ_frag = cute.composition(sQ, tv_layout)[tidx, None] + sO_frag = cute.composition(sO, tv_layout)[tidx, None] + + rQ_frag = cute.make_rmem_tensor(cute.make_layout(ELEMS_PER_THREAD), sQ.element_type) + rO_frag = cute.make_rmem_tensor(cute.make_layout(ELEMS_PER_THREAD), sO.element_type) + + cute.autovec_copy(sQ_frag, rQ_frag) + + scale_f32 = exp2f(scale_e8m0) + rO_frag.store((rQ_frag.load().to(Float32) * scale_f32).to(sO.element_type)) + + cute.autovec_copy(rO_frag, sO_frag) + + +class MXFP8DequantizeConfig: + """Compile-time configuration of the CuTeDSL MXFP8 dequantize kernel.""" + + def __init__( + self, + fp8_dtype: str, + out_dtype: str, + rowwise: bool, + colwise: bool, + with_gemm_swizzled_scales: bool, + ): + if not (rowwise or colwise): + raise ValueError("at least one of rowwise and columnwise data must be present") + if fp8_dtype not in ("Float8E4M3", "Float8E5M2"): + raise ValueError( + f"unknown FP8 dtype {fp8_dtype!r}; expected 'Float8E4M3' or 'Float8E5M2'" + ) + if out_dtype not in ("Float32", "Float16", "BFloat16"): + raise ValueError( + f"unknown output dtype {out_dtype!r}; expected Float32|Float16|BFloat16" + ) + self.FP8_DTYPE = str_to_cutlass_dtype(fp8_dtype) + self.FP8_DTYPE_STR = fp8_dtype + self.OUT_DTYPE = str_to_cutlass_dtype(out_dtype) + self.OUT_DTYPE_STR = out_dtype + # Which data copies (each with its scales) the caller passes. Both are (M, N) row-major. + self.ROWWISE = rowwise + self.COLWISE = colwise + self.WITH_GEMM_SWIZZLED_SCALES = with_gemm_swizzled_scales + + def __str__(self): + return ( + f"MXFP8DequantizeConfig(fp8_dtype={self.FP8_DTYPE_STR}, out_dtype={self.OUT_DTYPE_STR}," + f" rowwise={self.ROWWISE}, colwise={self.COLWISE}," + f" swizzled={self.WITH_GEMM_SWIZZLED_SCALES})" + ) + + __repr__ = __str__ + + +class MXFP8DequantizeKernel: + """MXFP8 dequantize with staged TMA I/O, a 1:1 port of dequantize_mxfp8_kernel in + cast/mxfp8/dequantize_mxfp8.cuh (same tiling, thread mapping, barriers and pipelining). + + Each CTA processes _NUM_TILES (_TILE_ROWS, _TILE_COLS) tiles stacked vertically through a + _NUM_STAGES-deep smem ring. The next tile's TMA load is issued before the current one is + consumed, and the output is written back with a TMA store. + """ + + _NUM_STAGES = 2 + _NUM_TILES = 8 # Each CTA handles 8 tiles stacked vertically + + _THREADS_PER_CTA = 128 + _NUM_WARPS = _THREADS_PER_CTA // THREADS_PER_WARP + # How many rows and columns are processed by a CTA at a time + # Rowwise: each thread owns _ELEMS_PER_THREAD elements in a row + # Colwise: each thread owns _ELEMS_PER_THREAD elements in a column + _TILE_ROWS = 16 + _TILE_COLS = _THREADS_PER_CTA + _ELEMS_PER_THREAD = _TILE_ROWS * _TILE_COLS // _THREADS_PER_CTA # 16 + + # How many rows and columns are processed by a CTA in total + CTA_BLOCK_ROWS = _TILE_ROWS * _NUM_TILES + CTA_BLOCK_COLS = _TILE_COLS + + def __init__(self, cfg: MXFP8DequantizeConfig): + self.cfg = cfg + # Prefer using rowwise quantization result to dequantize + self.DEQUANTIZE_FROM_ROWWISE = cfg.ROWWISE + + @cute.jit + def __call__( + self, + mX_row: cute.Tensor, + mS_row: cute.Tensor, + mX_col: cute.Tensor, + mS_col: cute.Tensor, + mO: cute.Tensor, + stream: CUstream, + ): + if cutlass.const_expr(CUTEDSL_DEBUG_LOGGING): + cute.printf(f"[CuTeDSL] MXFP8DequantizeKernel.__call__() with config: {self.cfg}\n") + + mX = mX_row if cutlass.const_expr(self.DEQUANTIZE_FROM_ROWWISE) else mX_col + mS = mS_row if cutlass.const_expr(self.DEQUANTIZE_FROM_ROWWISE) else mS_col + M, N = mX.shape + if cutlass.const_expr(self.cfg.WITH_GEMM_SWIZZLED_SCALES): + if cutlass.const_expr(self.DEQUANTIZE_FROM_ROWWISE): + mS, _ = derive_swizzled_scale_layout(M, N, True, False, mS, None) + else: + _, mS = derive_swizzled_scale_layout(M, N, False, True, None, mS) + + # Unswizzled boxes, like create_2D_tensor_map's default. + tiler = (self._TILE_ROWS, self._TILE_COLS) + smem_tile_layout = cute.make_ordered_layout(tiler, order=(1, 0)) + tma_atom_in, tma_in = cute.nvgpu.cpasync.make_tiled_tma_atom( + cute.nvgpu.cpasync.CopyBulkTensorTileG2SOp(), mX, smem_tile_layout, tiler + ) + tma_atom_out, tma_out = cute.nvgpu.cpasync.make_tiled_tma_atom( + cute.nvgpu.cpasync.CopyBulkTensorTileS2GOp(), mO, smem_tile_layout, tiler + ) + + grid = [ + cute.ceil_div(Int32(N), self.CTA_BLOCK_COLS), + cute.ceil_div(M, self.CTA_BLOCK_ROWS), + 1, + ] + self.kernel(mS, tma_atom_in, tma_in, tma_atom_out, tma_out).launch( + grid=grid, block=[self._THREADS_PER_CTA, 1, 1], stream=stream + ) + + @cute.kernel + def kernel( + self, + mS: cute.Tensor, + tma_atom_in: cute.CopyAtom, + tma_in: cute.Tensor, + tma_atom_out: cute.CopyAtom, + tma_out: cute.Tensor, + ): + """Device entry: out = exp2(scale - 127) * x, computed in FP32.""" + + TILE_SIZE = self._TILE_ROWS * self._TILE_COLS + + bidx, bidy, _ = cute.arch.block_idx() + warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + + # This CTA's block, split into _NUM_TILES tiles: ((_TILE_ROWS, _TILE_COLS), (_NUM_TILES, 1)). + cta_block_tiler = (self.CTA_BLOCK_ROWS, self.CTA_BLOCK_COLS) + tiler = (self._TILE_ROWS, self._TILE_COLS) + + # Select the block of tiles that belongs to this CTA + tma_in = cute.local_tile(tma_in, cta_block_tiler, (bidy, bidx)) + tma_out = cute.local_tile(tma_out, cta_block_tiler, (bidy, bidx)) + + gQ = cute.zipped_divide(tma_in, tiler) + gO = cute.zipped_divide(tma_out, tiler) + + if cutlass.const_expr(self.DEQUANTIZE_FROM_ROWWISE): + scale_cta_block_tiler = ( + self.CTA_BLOCK_ROWS, + self.CTA_BLOCK_COLS // MXFP8_BLOCK_SCALING_SIZE, + ) + # self._TILE_COLS // MXFP8_BLOCK_SCALING_SIZE might be fractional meaning a tile only owns a + # partial MXFP8 block. In which case this tile still own at least 1 scale on that direction + scale_tiler = ( + self._TILE_ROWS, + max(1, self._TILE_COLS // MXFP8_BLOCK_SCALING_SIZE), + ) + # Each data row has its own scales, so tiles never share them + tiles_sharing_same_scale = 1 + else: + scale_cta_block_tiler = ( + self.CTA_BLOCK_ROWS // MXFP8_BLOCK_SCALING_SIZE, + self.CTA_BLOCK_COLS, + ) + # self._TILE_ROWS // MXFP8_BLOCK_SCALING_SIZE might be fractional meaning a tile only owns a + # partial MXFP8 block. In which case this tile still own at least 1 scale on that direction + scale_tiler = ( + max(1, self._TILE_ROWS // MXFP8_BLOCK_SCALING_SIZE), + self._TILE_COLS, + ) + # A tile shorter than an MXFP8 block shares its scale row with the other tiles in it + tiles_sharing_same_scale = max(1, MXFP8_BLOCK_SCALING_SIZE // self._TILE_ROWS) + # If tiles_sharing_same_scale > 1, then this tile owns only a fraction of the MXFP8 block and + # shares its scale with the other tiles, so we need to index with tile_idx // tiles_sharing_same_scale + # to obtain the scale slice that belongs to this tile + mS = cute.local_tile(mS, scale_cta_block_tiler, (bidy, bidx)) + + @cute.struct + class SharedStorage: + mbar: cute.struct.MemRange[cute.Int64, 2 * self._NUM_STAGES] + sQ: cute.struct.Align[ + cute.struct.MemRange[self.cfg.FP8_DTYPE, self._NUM_STAGES * TILE_SIZE], 128 + ] + sO: cute.struct.Align[ + cute.struct.MemRange[self.cfg.OUT_DTYPE, self._NUM_STAGES * TILE_SIZE], 128 + ] + + storage = cutlass.memory.SmemAllocator().allocate(SharedStorage) + ring_layout = cute.zipped_product( + cute.make_ordered_layout(tiler, order=(1, 0)), cute.make_layout(self._NUM_STAGES) + ) + sQ = storage.sQ.get_tensor(ring_layout) + sO = storage.sO.get_tensor(ring_layout) + + tXs_in, tXg_in = cute.nvgpu.cpasync.tma_partition( + tma_atom_in, 0, cute.make_layout(1), sQ, gQ + ) + tXs_out, tXg_out = cute.nvgpu.cpasync.tma_partition( + tma_atom_out, 0, cute.make_layout(1), sO, gO + ) + + # Only warp 0 is the producer (issues TMA); every warp is a consumer. + mainloop_pipeline = pipeline.PipelineTmaAsync.create( + barrier_storage=storage.mbar.data_ptr(), + num_stages=self._NUM_STAGES, + producer_group=pipeline.CooperativeGroup(pipeline.Agent.Thread, 1), + consumer_group=pipeline.CooperativeGroup(pipeline.Agent.Thread, self._NUM_WARPS), + tx_count=TILE_SIZE * self.cfg.FP8_DTYPE.width // 8, + cta_layout_vmnk=None, + ) + prod_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Producer, self._NUM_STAGES + ) + cons_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self._NUM_STAGES + ) + + # Prologue: fill the pipeline. + if warp_idx == 0: + for it in cutlass.range_constexpr(self._NUM_STAGES): + mainloop_pipeline.producer_acquire(prod_state) + cute.copy( + tma_atom_in, + tXg_in[(None, it)], + tXs_in[(None, prod_state.index)], + tma_bar_ptr=mainloop_pipeline.producer_get_barrier(prod_state), + ) + mainloop_pipeline.producer_commit(prod_state) + prod_state.advance() + + for tile_idx in cutlass.range_constexpr(self._NUM_TILES): + stage = cons_state.index + # The first _NUM_STAGES tiles don't have to wait for previous TMA read because they are not used yet + if cutlass.const_expr(tile_idx >= self._NUM_STAGES): + if warp_idx == 0: + # Wait for previous TMA to finish their read so we can write to this output tile + cute.arch.cp_async_bulk_wait_group(self._NUM_STAGES - 1, read=True) + cute.arch.sync_threads() + + mainloop_pipeline.consumer_wait(cons_state) + + gS = cute.local_tile(mS, scale_tiler, (tile_idx // tiles_sharing_same_scale, 0)) + + dequantize_mxfp8_smem_tile( + sQ[(None, None), stage], + sO[(None, None), stage], + gS, + self._TILE_ROWS, + self._TILE_COLS, + self._ELEMS_PER_THREAD, + self.DEQUANTIZE_FROM_ROWWISE, + ) + + # Make the smem writes visible to the TMA engine before the store. + cute.arch.fence_proxy("async.shared", space="cta") + cute.arch.sync_threads() + mainloop_pipeline.consumer_release(cons_state) + cons_state.advance() + + if warp_idx == 0: + cute.copy( + tma_atom_out, + tXs_out[(None, stage)], + tXg_out[(None, tile_idx)], + ) + cute.arch.cp_async_bulk_commit_group() + if cutlass.const_expr(tile_idx + self._NUM_STAGES < self._NUM_TILES): + mainloop_pipeline.producer_acquire(prod_state) + cute.copy( + tma_atom_in, + tXg_in[(None, tile_idx + self._NUM_STAGES)], + tXs_in[(None, prod_state.index)], + tma_bar_ptr=mainloop_pipeline.producer_get_barrier(prod_state), + ) + mainloop_pipeline.producer_commit(prod_state) + prod_state.advance() + + if warp_idx == 0: + cute.arch.cp_async_bulk_wait_group(0, read=False) + cute.arch.sync_threads() + + +def compile_dequantize_function_from_cfg(cfg: MXFP8DequantizeConfig): + """Compile the dequantize kernel for cfg with symbolic shapes and a TVM-FFI entrypoint.""" + sym_M = cute.sym_int32() + sym_N = cute.sym_int32(divisibility=SYM_N_DIVISIBILITY) + # Scales are allocated at the padded shapes described in compile_cutedsl_function_from_cfg. + scale_row_shape = (cute.sym_int32(divisibility=128), cute.sym_int32(divisibility=4)) + scale_col_shape = (cute.sym_int32(divisibility=4), cute.sym_int32(divisibility=128)) + + def fake(dtype, shape, align, present=True): + if not present: + return None + return cute.runtime.make_fake_compact_tensor( + dtype, shape, stride_order=(1, 0), memspace=cute.AddressSpace.gmem, assumed_align=align + ) + + return cute.compile( + MXFP8DequantizeKernel(cfg), + fake(cfg.FP8_DTYPE, (sym_M, sym_N), 16, cfg.ROWWISE), # mX_row + fake(Float8E8M0FNU, scale_row_shape, 4, cfg.ROWWISE), # mS_row + fake(cfg.FP8_DTYPE, (sym_M, sym_N), 16, cfg.COLWISE), # mX_col + fake(Float8E8M0FNU, scale_col_shape, 4, cfg.COLWISE), # mS_col + fake(cfg.OUT_DTYPE, (sym_M, sym_N), 16), # mO + cute.runtime.make_fake_stream(), + options="--enable-tvm-ffi", + ) + + +def get_mxfp8_dequantization_function( + fn_name: str, + fp8_dtype: str, + out_dtype: str, + rowwise: bool, + colwise: bool, + with_gemm_swizzled_scales: bool, +) -> bool: + """Compile the MXFP8 dequantize kernel for this config and register it in the TVM-FFI global + registry under `fn_name`. rowwise/colwise say which data copies the C++ caller passes; the + direction actually dequantized is chosen in MXFP8DequantizeConfig. Returns False if the config is unsupported, so the C++ dispatcher + falls back to the CUDA kernel. Mirrors get_mxfp8_quantization_function. + """ + try: + if tvm_ffi.get_global_func(fn_name, allow_missing=True) is not None: + return True + + major, minor = device_compute_capability() + if major < 10: + logger.warning( + "CuTeDSL MXFP8 backend requires compute capability >= 10.0 (Blackwell), " + "but detected %d.%d; falling back to the CUDA C++ kernel.", + major, + minor, + ) + return False + + try: + cfg = MXFP8DequantizeConfig( + fp8_dtype, out_dtype, rowwise, colwise, with_gemm_swizzled_scales + ) + except ValueError as e: + logger.warning( + "CuTeDSL MXFP8 backend does not support this dequantize config, " + "falling back to the CUDA C++ kernel: %s", + e, + ) + return False + + logger.debug("Compiling CuTeDSL MXFP8 dequantization kernel for %s", cfg) + compiled = compile_dequantize_function_from_cfg(cfg) + # Register the native function to skip the Python argument-parsing wrapper + # (see get_mxfp8_quantization_function). + native = getattr(compiled, "__tvm_ffi_object__", lambda: None)() + tvm_ffi.register_global_func( + fn_name, native if native is not None else compiled, override=True + ) + return True + except Exception as e: # pylint: disable=broad-exception-caught + logger.error( + "CuTeDSL MXFP8 dequantize kernel compilation & registration failed, falling back to" + " the CUDA C++ kernel: %s", + e, + ) + return False diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_2d.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_2d.py new file mode 100644 index 00000000000..fdca0825d44 --- /dev/null +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_2d.py @@ -0,0 +1,431 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""MXFP8 2D block-scaling quantization kernel implemented in CuTeDSL. + +Ports the kIs2DBlockScaling path of quantize_mxfp8_kernel (quantize_mxfp8.cuh): every 32x32 +block of the input shares one E8M0 scale, which the rowwise scales repeat down the block's 32 +rows and the colwise scales repeat across its 32 columns. + +The TMA pipeline is the general G2S kernel's; only the per-tile math differs, so this subclasses +it and swaps in its own tile function. A tile is 32 rows by TILE_COLS, i.e. TILE_COLS // 32 +blocks side by side, and the colwise pass maps warp w onto columns [32w, 32w + 32), which is +exactly block w, so a warp reduction of the per-column amaxes yields the block amax. The rowwise +pass reads it back from SMEM, or, when there is no colwise pass, reduces its per-row amaxes over +the block's 32 rows itself. + +Only a plain cast is implemented: the C APIs that fuse dbias or an activation pass no +quantization config, so only plain casts (nvte_quantize_v2, nvte_multi_tensor_quantize) can +request 2D scaling. +""" + +from typing import Optional, Type + +import cutlass +from cutlass import cute +from cutlass import Float32, Int32, Int64, Uint16, Uint32, Uint8 + +from transformer_engine.common.CuTeDSL.utils import ( + device_compute_capability, + is_packed16, + abs_max_x2_bf16, + abs_max_x2_f16, + abs_max_scalar_bf16, + abs_max_scalar_f16, + to_f32_bf16, + to_f32_f16, + x2_lo_to_f32_bf16, + x2_lo_to_f32_f16, + x2_hi_to_f32_bf16, + x2_hi_to_f32_f16, + fabs_f32, + exp2f_rcp, + pack_f32x2, + unpack_i64_to_i32x2, +) +from transformer_engine.common.CuTeDSL.utils_fp8 import ( + get_cvt_f32x2_to_fp8x2_func, + cvt_f32_to_fp8e8m0fnu, + mul_f32x2_cvt_f32x4_to_fp8x4, + mul_f32x2_cvt_packed16x4_to_fp8x4, +) +from .quantize_mxfp8_common import ( + MXFP8QuantizeConfig, + MXFP8_BLOCK_SCALING_SIZE, + THREADS_PER_WARP, +) +from .quantize_mxfp8_g2s_general import MXFP8QuantizeGeneralG2SKernel + + +@cute.jit +def quantize_colwise_mxfp8_2d( + sX_tile: cute.Tensor, # (TILE_Y, TILE_X) input smem view, post-TMA + sO_col_tile: cute.Tensor, # (TILE_Y, TILE_X) fp8 smem view (colwise FP8 output) + mS_col_stage: cute.Tensor, # colwise scale tensor (1D swizzled, or 2D linear) + sBlockAmax: cute.Tensor, # receives the amax of block w at index w + MAX_NORM_RCP: cutlass.Constexpr[float], + tile_row_start: Int32, + tile_col_start: Int32, + M: Int32, + N: Int32, + DTYPE: cutlass.Constexpr[Type[cutlass.Numeric]], + FP8_DTYPE: cutlass.Constexpr[Type[cutlass.Numeric]], + SWIZZLE: cutlass.Constexpr[bool], + TILE_X: cutlass.Constexpr[int], + TILE_Y: cutlass.Constexpr[int], + SKIP_SCALE_BOUNDS: cutlass.Constexpr[bool], +): + """Quantize one SMEM tile colwise with one scale per 32x32 block; returns the block amax.""" + tidx, _, _ = cute.arch.thread_idx() + + # Each thread handles one vertical MXFP8 block, so we have 1 thread along the Y direction since one tile only has + # one block in the Y (column) direction + CTA_THREADS_Y = MXFP8_BLOCK_SCALING_SIZE // TILE_Y # 1 thread along the Y direction + _, tv_layout = cute.make_layout_tv( + thr_layout=cute.make_layout((CTA_THREADS_Y, TILE_X), stride=(TILE_X, 1)), + val_layout=cute.make_layout((MXFP8_BLOCK_SCALING_SIZE, 1), stride=(1, 1)), + ) + sX_thread = cute.composition(sX_tile, tv_layout)[tidx, None] + sO_thread = cute.composition(sO_col_tile, tv_layout)[tidx, None] + + # Gather this thread's 32 values from the MXFP8 block and its own amax. + # Note: TMA zero-fills past M and N, so the padding values never win when comparing amax. + in_c = [None] * MXFP8_BLOCK_SCALING_SIZE + if cutlass.const_expr(is_packed16(DTYPE)): + abs_max_scalar = abs_max_scalar_f16 if DTYPE is cutlass.Float16 else abs_max_scalar_bf16 + to_f32 = to_f32_f16 if DTYPE is cutlass.Float16 else to_f32_bf16 + in_c[0] = sX_thread[0] + amax_value = in_c[0] + for i in cutlass.range_constexpr(1, MXFP8_BLOCK_SCALING_SIZE): + in_c[i] = sX_thread[i] + amax_value = abs_max_scalar(amax_value, in_c[i]) + amax_c = fabs_f32(to_f32(amax_value)) + else: + to_f32 = None + in_c[0] = Float32(sX_thread[0]) + amax_c = fabs_f32(in_c[0]) + for i in cutlass.range_constexpr(1, MXFP8_BLOCK_SCALING_SIZE): + in_c[i] = Float32(sX_thread[i]) + amax_c = cute.arch.fmax(amax_c, fabs_f32(in_c[i])) + + # Reduce the amaxes of the 32 threads in this warp to get the block amax, and let the leader thread + # of each warp write it to SMEM. + # Use f32 redux if it is available (blackwell family); otherwise use butterfly shuffle for reduction + if cutlass.const_expr(device_compute_capability() in ((10, 0), (10, 3))): + block_amax = cute.arch.warp_redux_sync(amax_c, kind="fmax") + else: + block_amax = amax_c + for k in cutlass.range_constexpr(THREADS_PER_WARP.bit_length() - 1): + block_amax = cute.arch.fmax( + block_amax, cute.arch.shuffle_sync_bfly(block_amax, offset=1 << k) + ) + if tidx % THREADS_PER_WARP == 0: + sBlockAmax[tidx // THREADS_PER_WARP] = block_amax + + biased_exp_c = cvt_f32_to_fp8e8m0fnu(block_amax * MAX_NORM_RCP) + if cutlass.const_expr(SWIZZLE): + scale_coord = (0, tidx % 32, tidx // 32) + else: + scale_coord = (0, tidx) + if cutlass.const_expr(SKIP_SCALE_BOUNDS): + mS_col_stage[scale_coord] = biased_exp_c + elif tile_row_start < M and tile_col_start + tidx < N: + mS_col_stage[scale_coord] = biased_exp_c + + inv_scale_c = exp2f_rcp(biased_exp_c) + cvt_x2_func = get_cvt_f32x2_to_fp8x2_func(FP8_DTYPE) + sO_thread_fp8 = cute.make_tensor( + cute.recast_ptr(sO_thread.iterator, dtype=FP8_DTYPE), sO_thread.layout + ) + for j in cutlass.range_constexpr(MXFP8_BLOCK_SCALING_SIZE // 2): + lo, hi = 2 * j, 2 * j + 1 + x_lo, x_hi = in_c[lo], in_c[hi] + if cutlass.const_expr(to_f32 is not None): + x_lo, x_hi = to_f32(x_lo), to_f32(x_hi) + pair = cvt_x2_func(x_hi * inv_scale_c, x_lo * inv_scale_c) + sO_thread_fp8[lo] = Uint8(pair & Uint16(0xFF)).bitcast(FP8_DTYPE) + sO_thread_fp8[hi] = Uint8(pair >> Uint16(8)).bitcast(FP8_DTYPE) + + return block_amax + + +@cute.jit +def quantize_rowwise_mxfp8_2d( + sX_tile: cute.Tensor, # (TILE_Y, TILE_X) input smem view, post-TMA + sO_row_tile: cute.Tensor, # (TILE_Y, TILE_X) fp8 smem view (rowwise FP8 output) + mS_row_stage: cute.Tensor, # rowwise scale tensor (1D swizzled, or 2D linear) + sBlockAmax: cute.Tensor, # block amaxes from the colwise pass, or scratch for the reduction + MAX_NORM_RCP: cutlass.Constexpr[float], + tile_row_start: Int32, + tile_col_start: Int32, + M: Int32, + N: Int32, + DTYPE: cutlass.Constexpr[Type[cutlass.Numeric]], + FP8_DTYPE: cutlass.Constexpr[Type[cutlass.Numeric]], + TILE_X: cutlass.Constexpr[int], + TILE_Y: cutlass.Constexpr[int], + SKIP_SCALE_BOUNDS: cutlass.Constexpr[bool], + BLOCK_AMAX_IN_SMEM: cutlass.Constexpr[ + bool + ], # If colwise pass has already left the block amaxes in sBlockAmax +): + """Quantize one SMEM tile rowwise with one scale per 32x32 block; returns the block amax. + + With BLOCK_AMAX_IN_SMEM the colwise pass has already left the block amaxes in sBlockAmax + (and the caller synchronized); otherwise they are reduced here, which syncs the CTA. + """ + tidx, _, _ = cute.arch.thread_idx() + + # Same vectorized, bank-staggered loads as the general kernel's rowwise pass. + PACK_SIZE = 4 + WAVES = MXFP8_BLOCK_SCALING_SIZE // PACK_SIZE + THREADS_PER_BANK = 32 * 4 // MXFP8_BLOCK_SCALING_SIZE + + # Each thread covers one horizontal MXFP8 block, so we have TILE_X // MXFP8_BLOCK_SCALING_SIZE threads + # along X (row direction) + CTA_THREADS_X = TILE_X // MXFP8_BLOCK_SCALING_SIZE + NUM_WARPS = TILE_Y * CTA_THREADS_X // THREADS_PER_WARP + # A CTA consists of NUM_WARPS warps stacked together + thr_layout = cute.blocked_product( + cute.make_layout( + (THREADS_PER_WARP // CTA_THREADS_X, CTA_THREADS_X), stride=(CTA_THREADS_X, 1) + ), # Thread layout in a warp, (16, 2) + cute.make_layout((NUM_WARPS, 1)), # Warp layout in a CTA (2, 1) + ) + + _, tv_layout = cute.make_layout_tv( + thr_layout=thr_layout, + val_layout=cute.make_layout((1, MXFP8_BLOCK_SCALING_SIZE), stride=(0, 1)), + ) + + sX_thread = cute.composition(sX_tile, tv_layout)[tidx, None] + sO_thread = cute.composition(sO_row_tile, tv_layout)[tidx, None] + sO_thread_u32 = cute.make_tensor( + cute.recast_ptr(sO_thread.iterator, dtype=Uint32), + cute.make_layout((MXFP8_BLOCK_SCALING_SIZE // 4,), stride=(1,)), + ) + + # Coordinates in thr_layout: (lane_y, lane_x) within the warp, warps stacked along Y. + warp_id = tidx // THREADS_PER_WARP + lane_id = tidx % THREADS_PER_WARP + lane_y = lane_id // CTA_THREADS_X + lane_x = lane_id % CTA_THREADS_X + thread_y = warp_id * (THREADS_PER_WARP // CTA_THREADS_X) + lane_y + thread_x = lane_x + + # Each thread start reading from the specfic bank based on its thread ID so they can do their best to access different banks + # to avoid bank conflict. + bank_group = (tidx % THREADS_PER_WARP) // THREADS_PER_BANK + # The offset this thread should start reading from based on what's its first bank to access. + offset = bank_group * PACK_SIZE + + if cutlass.const_expr(is_packed16(DTYPE)): + sX_thread_rw_i64 = cute.make_tensor( + cute.recast_ptr(sX_thread.iterator, dtype=Int64), + cute.make_layout((1, MXFP8_BLOCK_SCALING_SIZE // 4), stride=(0, 1)), + ) + in_r = [[None, None] for _ in range(WAVES)] + for w in cutlass.range_constexpr(WAVES): + idx = (w + offset // 4) % (MXFP8_BLOCK_SCALING_SIZE // 4) + in_r[w][0], in_r[w][1] = unpack_i64_to_i32x2(sX_thread_rw_i64[0, idx]) + else: + sX_thread_rw = cute.make_tensor( + sX_thread.iterator, + cute.make_layout((1, MXFP8_BLOCK_SCALING_SIZE), stride=(0, 1)), + ) + in_r = [[None] * PACK_SIZE for _ in range(WAVES)] + for w in cutlass.range_constexpr(WAVES): + start = (w * PACK_SIZE + offset) % MXFP8_BLOCK_SCALING_SIZE + for i in cutlass.range_constexpr(PACK_SIZE): + in_r[w][i] = Float32(sX_thread_rw[0, start + i]) + + if cutlass.const_expr(BLOCK_AMAX_IN_SMEM): + # If colwise pass has already writes the block amaxes into sBlockAmax, just read it back. + block_amax = sBlockAmax[thread_x] + else: + # Otherwise we need to compute the block amaxes in this rowwise pass + if cutlass.const_expr(is_packed16(DTYPE)): + abs_max_x2 = abs_max_x2_f16 if DTYPE is cutlass.Float16 else abs_max_x2_bf16 + x2_lo_to_f32 = x2_lo_to_f32_f16 if DTYPE is cutlass.Float16 else x2_lo_to_f32_bf16 + x2_hi_to_f32 = x2_hi_to_f32_f16 if DTYPE is cutlass.Float16 else x2_hi_to_f32_bf16 + # Load the fisst wave's two values into amax_2x, then fold in the rest of the waves. + amax_2x = abs_max_x2(in_r[0][0], in_r[0][1]) + for w in cutlass.range_constexpr(1, WAVES): + amax_2x = abs_max_x2(amax_2x, in_r[w][0]) + amax_2x = abs_max_x2(amax_2x, in_r[w][1]) + amax_r = cute.arch.fmax( + fabs_f32(x2_lo_to_f32(amax_2x)), fabs_f32(x2_hi_to_f32(amax_2x)) + ) + else: + vals = [in_r[w][i] for w in range(WAVES) for i in range(PACK_SIZE)] + amax_r = fabs_f32(vals[0]) + for j in cutlass.range_constexpr(1, len(vals)): + amax_r = cute.arch.fmax(amax_r, fabs_f32(vals[j])) + + # Use butterfly shuffles to reduce the 32 threads' local amax in this warp to get 2 block amax + # for the left half and the right half of the warp (each warp with shape (16, 1)) + for k in cutlass.range_constexpr((THREADS_PER_WARP // CTA_THREADS_X).bit_length() - 1): + amax_r = cute.arch.fmax( + amax_r, cute.arch.shuffle_sync_bfly(amax_r, offset=CTA_THREADS_X << k) + ) + + # Write the half-warp block amaxes to SMEM, then let the first warp reduce them to get the final block amax. + sWarpBlockAmax = cute.make_tensor( + sBlockAmax.iterator, + cute.make_layout((NUM_WARPS, CTA_THREADS_X), stride=(CTA_THREADS_X, 1)), + ) + # Let the threads in the first row in each half warp write its partial warp amax to SMEM + # Warp shape is (16, 2) so each warp writes 2 partial amax for a (16, 32) block which will be reduced to produce + # the (32, 32) block amax + if lane_y == 0: + sWarpBlockAmax[(warp_id, lane_x)] = amax_r + cute.arch.sync_threads() + + # Reduce 2 warps' amax to the whole (32, 32) block's amax + block_amax = sWarpBlockAmax[(0, thread_x)] + for w in cutlass.range_constexpr(1, NUM_WARPS): + block_amax = cute.arch.fmax(block_amax, sWarpBlockAmax[(w, thread_x)]) + + biased_exp_r = cvt_f32_to_fp8e8m0fnu(block_amax * MAX_NORM_RCP) + if cutlass.const_expr(SKIP_SCALE_BOUNDS): + mS_row_stage[(thread_y, thread_x)] = biased_exp_r + elif tile_row_start + thread_y < M and tile_col_start + thread_x * MXFP8_BLOCK_SCALING_SIZE < N: + mS_row_stage[(thread_y, thread_x)] = biased_exp_r + + inv_scale_r = exp2f_rcp(biased_exp_r) + scale_2x = pack_f32x2(inv_scale_r, inv_scale_r) + if cutlass.const_expr(is_packed16(DTYPE)): + mul_cvt_x4_func = mul_f32x2_cvt_packed16x4_to_fp8x4(DTYPE, FP8_DTYPE) + else: + mul_cvt_x4_func = mul_f32x2_cvt_f32x4_to_fp8x4(FP8_DTYPE) + for w in cutlass.range_constexpr(WAVES): + idx = ((w * PACK_SIZE + offset) % MXFP8_BLOCK_SCALING_SIZE) // 4 + if cutlass.const_expr(is_packed16(DTYPE)): + sO_thread_u32[idx] = mul_cvt_x4_func(in_r[w][0], in_r[w][1], scale_2x) + else: + sO_thread_u32[idx] = mul_cvt_x4_func( + in_r[w][0], in_r[w][1], in_r[w][2], in_r[w][3], scale_2x + ) + + return block_amax + + +@cute.jit +def quantize_mxfp8_smem_tile_2d( # pylint: disable=unused-argument + sX_tile: cute.Tensor, + sActInput_tile: Optional[cute.Tensor], + sO_row_tile: Optional[cute.Tensor], + mS_row_stage: Optional[cute.Tensor], + sO_col_tile: Optional[cute.Tensor], + mS_col_stage: Optional[cute.Tensor], + rowwise_dbias_acc: Optional[cute.Tensor], + tile_row_start: Int32, + tile_col_start: Int32, + M: Int32, + N: Int32, + ROWWISE: cutlass.Constexpr[bool], + COLWISE: cutlass.Constexpr[bool], + WITH_AMAX: cutlass.Constexpr[bool], + MAX_NORM_RCP: cutlass.Constexpr[float], + DTYPE: cutlass.Constexpr[Type[cutlass.Numeric]], + FP8_DTYPE: cutlass.Constexpr[Type[cutlass.Numeric]], + WITH_GEMM_SWIZZLED_SCALES: cutlass.Constexpr[bool], + TILE_X: cutlass.Constexpr[int], + TILE_Y: cutlass.Constexpr[int], + SKIP_SCALE_BOUNDS: cutlass.Constexpr[bool], + smem: cutlass.memory.SmemAllocator, + **_fusion_args, # The general tile function's fusion knobs, all off for a plain cast. +): + """2D quantize a SMEM tile; returns (amax, None).""" + NUM_BLOCKS_X = TILE_X // MXFP8_BLOCK_SCALING_SIZE + NUM_WARPS = TILE_Y * NUM_BLOCKS_X // THREADS_PER_WARP + + # One amax per (warp, block) pair, for the rowwise-only cross-warp reduction. + @cute.struct + class BlockAmaxStorage: # pylint: disable=missing-class-docstring + sBlockAmax: cute.struct.MemRange[Float32, NUM_WARPS * NUM_BLOCKS_X] + + sBlockAmax = smem.allocate(BlockAmaxStorage).sBlockAmax.get_tensor( + cute.make_layout(NUM_WARPS * NUM_BLOCKS_X) + ) + + block_amax = Float32(0.0) + if cutlass.const_expr(COLWISE): + block_amax = quantize_colwise_mxfp8_2d( + sX_tile, + sO_col_tile, + mS_col_stage, + sBlockAmax, + MAX_NORM_RCP, + tile_row_start, + tile_col_start, + M, + N, + DTYPE=DTYPE, + FP8_DTYPE=FP8_DTYPE, + SWIZZLE=WITH_GEMM_SWIZZLED_SCALES, + TILE_X=TILE_X, + TILE_Y=TILE_Y, + SKIP_SCALE_BOUNDS=SKIP_SCALE_BOUNDS, + ) + if cutlass.const_expr(ROWWISE): + # Publish the block amaxes to the rowwise pass. + cute.arch.sync_threads() + if cutlass.const_expr(ROWWISE): + block_amax = quantize_rowwise_mxfp8_2d( + sX_tile, + sO_row_tile, + mS_row_stage, + sBlockAmax, + MAX_NORM_RCP, + tile_row_start, + tile_col_start, + M, + N, + DTYPE=DTYPE, + FP8_DTYPE=FP8_DTYPE, + TILE_X=TILE_X, + TILE_Y=TILE_Y, + SKIP_SCALE_BOUNDS=SKIP_SCALE_BOUNDS, + BLOCK_AMAX_IN_SMEM=COLWISE, + ) + amax = None + if cutlass.const_expr(WITH_AMAX): + amax = block_amax + return amax, None + + +class MXFP8Quantize2DKernel(MXFP8QuantizeGeneralG2SKernel): + """MXFP8 quantization with one scale per 32x32 block (plain cast only). + + Reuses the general G2S kernel's pipeline with its own tile function and tile configuration. + """ + + _quantize_smem_tile = staticmethod(quantize_mxfp8_smem_tile_2d) + + def __init__( + self, + cfg: MXFP8QuantizeConfig, + SKIP_INPUT_MASKING: bool = False, + SKIP_SCALE_BOUNDS: bool = False, + ): + assert cfg.USE_2D_QUANTIZATION + # 2D quantization is only supported for plain casts and no fusion should be requested + assert not (cfg.WITH_DBIAS or cfg.WITH_ACT or cfg.WITH_DACT) + + super().__init__(cfg, SKIP_INPUT_MASKING, SKIP_SCALE_BOUNDS) + + # Overrides the general kernel's tile configuration with the 2D kernel's + self._NUM_TILES = 2 + self._USE_STATIC_NUM_TILES = True + self._MIN_BLOCKS_PER_MP = 0 + + self._NUM_WARPS = 2 + self._THREADS_PER_CTA = self._NUM_WARPS * THREADS_PER_WARP + + # In rowwise pass, + self._TILE_ROWS = MXFP8_BLOCK_SCALING_SIZE + # In colwise pass, each thread covers one vertical MXFP8 block, and since THREADS_PER_WARP == MXFP8_BLOCK_SCALING_SIZE, + # two warps cover 2 * MXFP8_BLOCK_SCALING_SIZE columns + self._TILE_COLS = self._NUM_WARPS * MXFP8_BLOCK_SCALING_SIZE diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_common.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_common.py index 8a66b55e48f..6d72d50a88d 100644 --- a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_common.py +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_common.py @@ -139,8 +139,6 @@ def __init__( use_2d_quantization: bool = False, activation: Optional[str] = None, ): - if use_2d_quantization: - raise ValueError("2D block scaling is not implemented by the CuTeDSL MXFP8 kernels") if dtype is None or dtype not in ("Float32", "Float16", "BFloat16"): raise ValueError(f"unknown input dtype {dtype!r}; expected Float32|Float16|BFloat16") self.DTYPE = str_to_cutlass_dtype(dtype) @@ -193,6 +191,11 @@ def __init__( # uses a cross-thread smem reduction over THREADS_Y. Both mirror the CUDA # kernel's COLWISE_SCALING / rowwise dbias branches. self.WITH_DBIAS = with_dbias + # The C APIs that fuse dbias or an activation pass no quantization config, so 2D block + # scaling only reaches plain casts and fusion is not supported. + if use_2d_quantization and (with_dbias or with_dact or with_act): + raise ValueError("2D block scaling is only implemented for a plain cast") + self.USE_2D_QUANTIZATION = use_2d_quantization self.MAX_NORM_RCP = ( FP8E4M3_MAX_NORM_RCP if fp8_dtype == "Float8E4M3" else FP8E5M2_MAX_NORM_RCP ) @@ -203,7 +206,8 @@ def __str__(self): f"rowwise={self.ROWWISE}, colwise={self.COLWISE}, " f"swizzled={self.WITH_GEMM_SWIZZLED_SCALES}, with_amax={self.WITH_AMAX}, " f"with_dbias={self.WITH_DBIAS}, with_dact={self.WITH_DACT}, " - f"with_act={self.WITH_ACT}, activation={self.ACTIVATION})" + f"with_act={self.WITH_ACT}, use_2d_quantization={self.USE_2D_QUANTIZATION}, " + f"activation={self.ACTIVATION})" ) __repr__ = __str__ @@ -212,6 +216,14 @@ def is_plain_cast(self) -> bool: """Whether cfg is a plain cast: no amax, dbias or (d)activation fused in.""" return not (self.WITH_AMAX or self.WITH_DBIAS or self.WITH_DACT or self.WITH_ACT) + def act_needs_masking(self) -> bool: + """Whether TMA's zero padding must be masked after the activation, i.e. act(0) != 0. + + All activations supported by MXFP8 satisfy act(0) == 0, so this is a precaution for + activations added in the future. + """ + return self.WITH_ACT and self.ACTIVATION not in ("relu", "gelu", "silu", "qgelu", "srelu") + class MXFP8QuantizeKernelBase(abc.ABC): """Base class for MXFP8 quantize kernels.""" @@ -268,34 +280,20 @@ def __init__(self, cfg: MXFP8QuantizeConfig): from .quantize_mxfp8_g2s_bidim import MXFP8QuantizeBidimensionalG2SKernel from .quantize_mxfp8_g2r_rowwise_2lane import MXFP8QuantizeRowwise2LaneG2RKernel from .quantize_mxfp8_g2r_bidim import MXFP8QuantizeBidimensionalG2RKernel + from .quantize_mxfp8_2d import MXFP8Quantize2DKernel # pylint: enable=import-outside-toplevel self.cfg = cfg - # These activation functions satisfy f(0) = 0 so we don't need to mask with OOB regions - # (zeros filled by TMA are still zeros without applying activation to them) - # Note that as the time of writing, all activations supported by MXFP8 satisfy this property, so !ACT_NEED_MASKING is never reachable. - # This is only a precaution for future use cases if someone wants to add a new activation function. - self.ACT_NEED_MASKING = cfg.WITH_ACT and cfg.ACTIVATION not in ( - "relu", - "gelu", - "silu", - "qgelu", - "srelu", - ) - # Instantiate all possible kernels at compile time, - # and we will pick the right one at runtime based on the input shape and config. - self.general_skip_input_masking_kernel = MXFP8QuantizeGeneralG2SKernel( - cfg, SKIP_INPUT_MASKING=True - ) - self.general_skip_input_masking_and_scale_bounds_kernel = MXFP8QuantizeGeneralG2SKernel( - cfg, SKIP_INPUT_MASKING=True, SKIP_SCALE_BOUNDS=True + # 2D block scaling only swaps the general kernel's per-tile math. + general_cls = ( + MXFP8Quantize2DKernel if cfg.USE_2D_QUANTIZATION else MXFP8QuantizeGeneralG2SKernel ) - # We only need to mask when WITH_ACT is enabled because with activations applied zeros filled by TMA affect block statistics - self.general_kernel = MXFP8QuantizeGeneralG2SKernel(cfg) if self.ACT_NEED_MASKING else None - - # Specialized kernels in a specific dispatch priority order so we prefer to dispatch to kernels that come first in the list. - self.specialized_kernels = [ + # Candidate kernels in dispatch priority order; the first one whose runtime + # eligible_to_dispatch check passes is launched. The specialized kernels come first, + # then the general kernel without and with boundary handling. The last one is always + # eligible. + self.preferred_kernels = [ kernel_cls(cfg) for kernel_cls in ( MXFP8QuantizeRowwise2LaneG2RKernel, @@ -304,6 +302,9 @@ def __init__(self, cfg: MXFP8QuantizeConfig): MXFP8QuantizeBidimensionalG2SKernel, ) if kernel_cls.supports(cfg) + ] + [ + general_cls(cfg, SKIP_INPUT_MASKING=True, SKIP_SCALE_BOUNDS=True), + general_cls(cfg, SKIP_INPUT_MASKING=not cfg.act_needs_masking()), ] @cute.jit @@ -320,13 +321,13 @@ def __call__( mWorkspace: Optional[cute.Tensor], stream: CUstream, ): - dispatched_to_specialized = False - for kernel in self.specialized_kernels: - if not dispatched_to_specialized: + dispatched = False + for kernel in self.preferred_kernels: + if not dispatched: if kernel.eligible_to_dispatch( self.cfg, mX, mO_row, mS_row, mO_col, mS_col, mNoop, mDActInput ): - dispatched_to_specialized = True + dispatched = True kernel( mX, mO_row, @@ -339,69 +340,6 @@ def __call__( mWorkspace, stream, ) - # If not using a specialized kernel, fall back to the general kernel - if not dispatched_to_specialized: - # With _USE_STATIC_NUM_TILES, each CTA always processes _NUM_TILES tiles, including OOB ones, - # so the scale-bounds check can only be dropped if M divides TILE_ROWS * _NUM_TILES. - num_tiles_per_cta = ( - self.general_skip_input_masking_kernel._NUM_TILES - if self.general_skip_input_masking_kernel._USE_STATIC_NUM_TILES - else 1 - ) - # If the input shape can be perfectly tiled by the general kernel's tile size, we can skip some boundary check because - # we know we will not touch any out-of-bounds region. - shape_is_divisible = ( - mX.shape[0] - % (self.general_skip_input_masking_kernel._TILE_ROWS * num_tiles_per_cta) - == 0 - and mX.shape[1] % self.general_skip_input_masking_kernel._TILE_COLS == 0 - ) - # If shape is divisible, then we won't access OOB regions regardless whatever - if shape_is_divisible: - self.general_skip_input_masking_and_scale_bounds_kernel( - mX, - mO_row, - mS_row, - mO_col, - mS_col, - mAmax, - mNoop, - mDActInput, - mWorkspace, - stream, - ) - else: - # Otherwise we can't skip scale bound check, but we may still skip masking activation inputs with zero - if cutlass.const_expr(self.ACT_NEED_MASKING): - # Masking and the scale guards now share one condition, so this is a plain - # two-way choice: either the shape tiles exactly (drop both) or it does not. - self.general_kernel( - mX, - mO_row, - mS_row, - mO_col, - mS_col, - mAmax, - mNoop, - mDActInput, - mWorkspace, - stream, - ) - else: - # We still need to check the scale bounds, but we can skip some activations masking because their output is 0 - # for OOB regions where TMA fills with zeros - self.general_skip_input_masking_kernel( - mX, - mO_row, - mS_row, - mO_col, - mS_col, - mAmax, - mNoop, - mDActInput, - mWorkspace, - stream, - ) def compile_cutedsl_function_from_cfg(cfg): diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_bidim.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_bidim.py index 61598a7d37b..3cb611e775f 100644 --- a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_bidim.py +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_bidim.py @@ -609,7 +609,8 @@ class MXFP8QuantizeBidimensionalG2RKernel(MXFP8QuantizeKernelBase): def supports(cfg: MXFP8QuantizeConfig) -> bool: # BF16 packed math. return ( - not cfg.WITH_AMAX + not cfg.USE_2D_QUANTIZATION + and not cfg.WITH_AMAX and not cfg.WITH_DBIAS and cfg.DTYPE is cutlass.BFloat16 and cfg.COLWISE diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_rowwise_1lane.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_rowwise_1lane.py index 0c287f02abe..fb325f289b8 100644 --- a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_rowwise_1lane.py +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_rowwise_1lane.py @@ -219,7 +219,13 @@ class MXFP8QuantizeRowwise1LaneG2RKernel(MXFP8QuantizeKernelBase): @staticmethod def supports(cfg: MXFP8QuantizeConfig) -> bool: - return cfg.is_plain_cast() and is_packed16(cfg.DTYPE) and cfg.ROWWISE and not cfg.COLWISE + return ( + not cfg.USE_2D_QUANTIZATION + and cfg.is_plain_cast() + and is_packed16(cfg.DTYPE) + and cfg.ROWWISE + and not cfg.COLWISE + ) @staticmethod @cute.jit diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_rowwise_2lane.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_rowwise_2lane.py index 03d5d510ab8..19a879ce72c 100644 --- a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_rowwise_2lane.py +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_rowwise_2lane.py @@ -262,7 +262,8 @@ class MXFP8QuantizeRowwise2LaneG2RKernel(MXFP8QuantizeKernelBase): def supports(cfg: MXFP8QuantizeConfig) -> bool: # BF16 packed math. return ( - not cfg.WITH_AMAX + not cfg.USE_2D_QUANTIZATION + and not cfg.WITH_AMAX and not cfg.WITH_DBIAS and cfg.DTYPE is cutlass.BFloat16 and cfg.ROWWISE diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2s_bidim.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2s_bidim.py index 5f0b3d2aed5..5e769cc84c4 100644 --- a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2s_bidim.py +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2s_bidim.py @@ -255,7 +255,8 @@ class MXFP8QuantizeBidimensionalG2SKernel(MXFP8QuantizeKernelBase): def supports(cfg: MXFP8QuantizeConfig) -> bool: # This kernel supports swizzled scales but we don't dispatch to it for now. return ( - cfg.is_plain_cast() + not cfg.USE_2D_QUANTIZATION + and cfg.is_plain_cast() and is_packed16(cfg.DTYPE) and cfg.ROWWISE and cfg.COLWISE diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2s_general.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2s_general.py index 81148416fe9..da73da2d777 100644 --- a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2s_general.py +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2s_general.py @@ -29,7 +29,7 @@ import cutlass from cutlass import cute from cutlass import pipeline -from cutlass import Float32, Int32, Int64, Uint16, Uint32, Uint8 +from cutlass import Boolean, Float32, Int32, Int64, Uint16, Uint32, Uint8 from cuda.bindings.driver import CUstream # pylint: disable=no-name-in-module from transformer_engine.common.CuTeDSL.utils import ( @@ -576,6 +576,7 @@ def quantize_mxfp8_smem_tile_general( DBIAS_REDUCTION_IN_COLWISE: cutlass.Constexpr[bool], COLWISE_DBIAS_REDUCTION_ONLY: cutlass.Constexpr[bool], DBIAS_REDUCTION_IN_ROWWISE: cutlass.Constexpr[bool], + smem: Optional[cutlass.memory.SmemAllocator] = None, # pylint: disable=unused-argument ): """Quantize one SMEM tile in the directions enabled by ROWWISE / COLWISE; returns (amax, dbias). @@ -665,6 +666,9 @@ class MXFP8QuantizeGeneralG2SKernel(MXFP8QuantizeKernelBase): """ _NUM_STAGES = 2 # The pipeline depth is always 2 + _quantize_smem_tile = staticmethod( + quantize_mxfp8_smem_tile_general + ) # Function to quantize one SMEM tile def __init__( self, @@ -726,6 +730,38 @@ def __init__( and cfg.ACTIVATION != "relu" ) + @cute.jit + def eligible_to_dispatch( # pylint: disable=arguments-differ,unused-argument + self, + cfg: cutlass.Constexpr[MXFP8QuantizeConfig], + mX: cute.Tensor, + mO_row: Optional[cute.Tensor], + mS_row: Optional[cute.Tensor], + mO_col: Optional[cute.Tensor], + mS_col: Optional[cute.Tensor], + mNoop: cute.Pointer, + mDActInput: Optional[cute.Tensor], + ) -> Boolean: + """Runtime check: dropping the scale bounds checks, or the input masking an activation + with act(0) != 0 needs, is only safe when the CTAs tile the input exactly.""" + eligible = True + # If we skip scale OOB check, or we skip input masking with an activation that doesn't satisfy act(0)=0, + # then it is required to have the input shape perfectly divisible + if cutlass.const_expr( + self.SKIP_SCALE_BOUNDS or (self.SKIP_INPUT_MASKING and cfg.act_needs_masking()) + ): + M, N = mX.shape + # With _USE_STATIC_NUM_TILES, each CTA always processes _NUM_TILES tiles, including OOB ones, + # so the scale-bounds check can only be dropped if M divides TILE_ROWS * _NUM_TILES. + num_tiles_per_cta = self._NUM_TILES if self._USE_STATIC_NUM_TILES else 1 + # If the input shape can be perfectly tiled by the general kernel's tile size, we can skip some boundary check because + # we know we will not touch any out-of-bounds region. + shape_is_divisible = ( + M % (self._TILE_ROWS * num_tiles_per_cta) == 0 and N % self._TILE_COLS == 0 + ) + eligible = shape_is_divisible + return eligible + @cute.jit def __call__( self, @@ -1217,7 +1253,7 @@ class DactStorage: sO_col_tile = sO_col[(None, stage_idx)] mS_col_stage = cute.flatten(mS_col[(None, (tile_idx_y, tile_idx_x))]) - amax, dbias = quantize_mxfp8_smem_tile_general( + amax, dbias = self._quantize_smem_tile( sX_tile, sActInput_tile, sO_row_tile, @@ -1247,6 +1283,7 @@ class DactStorage: DBIAS_REDUCTION_IN_COLWISE=self.DBIAS_REDUCTION_IN_COLWISE, COLWISE_DBIAS_REDUCTION_ONLY=self.COLWISE_DBIAS_REDUCTION_ONLY, DBIAS_REDUCTION_IN_ROWWISE=self.DBIAS_REDUCTION_IN_ROWWISE, + smem=smem, ) if cutlass.const_expr(cfg.WITH_AMAX): per_thread_amax = cute.arch.fmax(per_thread_amax, amax) diff --git a/transformer_engine/common/CuTeDSL/utils.py b/transformer_engine/common/CuTeDSL/utils.py index 2200823d058..e38112689c5 100644 --- a/transformer_engine/common/CuTeDSL/utils.py +++ b/transformer_engine/common/CuTeDSL/utils.py @@ -107,6 +107,18 @@ def exp2f_rcp(scale_e8m0) -> Float32: return result +@cute.jit +def exp2f(scale_e8m0) -> Float32: + """2^(biased_exp - 127) for an e8m0 scale, matching ptx::exp2f in util/ptx.cuh.""" + biased_exp = Int32(scale_e8m0.bitcast(cutlass.Uint8)) + result = (biased_exp << Int32(FP32_MANTISSA_BITS)).bitcast(Float32) + # 2^-127 is subnormal, so it cannot be built by shifting into the exponent field. + for cmp_val, repl_bits in [(255, 0x7FFFFFFF), (0, 0x00400000)]: + if biased_exp == Int32(cmp_val): + result = Int32(repl_bits).bitcast(Float32) + return result + + @cute.jit def exp2_bf16x2_rcp(scale_e8m0) -> Uint32: """Pack two BF16 copies of the E8M0 reciprocal computed by exp2f_rcp.""" diff --git a/transformer_engine/common/cast/dispatch/dequantize.cuh b/transformer_engine/common/cast/dispatch/dequantize.cuh index bf4a0218118..bb2df9196c7 100644 --- a/transformer_engine/common/cast/dispatch/dequantize.cuh +++ b/transformer_engine/common/cast/dispatch/dequantize.cuh @@ -20,6 +20,10 @@ #include "../mxfp8/group_dequantize_mxfp8.cuh" #include "../nvfp4/dequantize_nvfp4.cuh" +#ifdef NVTE_WITH_CUTEDSL +#include "../mxfp8/dequantize_mxfp8_cutedsl.cuh" +#endif + namespace transformer_engine { namespace dispatch { @@ -41,7 +45,13 @@ inline void dequantize_helper(const Tensor &input, Tensor *output, cudaStream_t } case NVTE_MXFP8_1D_SCALING: { if (is_supported_by_CC_100()) { - mxfp8::dequantize(input, output, stream); + bool dequantized_with_cutedsl = false; +#ifdef NVTE_WITH_CUTEDSL + dequantized_with_cutedsl = cutedsl_backend::mxfp8_dequantize_cutedsl(input, output, stream); +#endif + if (!dequantized_with_cutedsl) { + mxfp8::dequantize(input, output, stream); + } } else { NVTE_ERROR("MXFP8 Dequantization is NOT supported by architectures < 10.0"); } diff --git a/transformer_engine/common/cast/mxfp8/dequantize_mxfp8_cutedsl.cuh b/transformer_engine/common/cast/mxfp8/dequantize_mxfp8_cutedsl.cuh new file mode 100644 index 00000000000..fd5e7545efb --- /dev/null +++ b/transformer_engine/common/cast/mxfp8/dequantize_mxfp8_cutedsl.cuh @@ -0,0 +1,146 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +#ifndef TRANSFORMER_ENGINE_COMMON_CAST_MXFP8_DEQUANTIZE_MXFP8_CUTEDSL_CUH_ +#define TRANSFORMER_ENGINE_COMMON_CAST_MXFP8_DEQUANTIZE_MXFP8_CUTEDSL_CUH_ + +#include