diff --git a/docs/api/pytorch.rst b/docs/api/pytorch.rst index 2d8cc87827e..4a7ea9377cb 100644 --- a/docs/api/pytorch.rst +++ b/docs/api/pytorch.rst @@ -6,6 +6,9 @@ PyTorch ======= +General-purpose layers +---------------------- + .. autoapiclass:: transformer_engine.pytorch.Linear(in_features, out_features, **kwargs) :members: forward, set_tensor_parallel_group @@ -43,6 +46,24 @@ PyTorch .. autoapiclass:: transformer_engine.pytorch.TransformerLayer(hidden_size, ffn_hidden_size, num_attention_heads, **kwargs) :members: forward, set_context_parallel_group, set_tensor_parallel_group +Model layers +------------ + +DeepSeek-V3 +~~~~~~~~~~~ + +.. autoapiclass:: transformer_engine.pytorch.models.DeepSeekV3Layer(hidden_size, num_attention_heads, **kwargs) + :members: forward + +.. autoapiclass:: transformer_engine.pytorch.models.DeepSeekV3MoE(hidden_size, moe_ffn_hidden_size, num_experts, **kwargs) + :members: forward, update_expert_bias + +.. autoapiclass:: transformer_engine.pytorch.models.MultiLatentAttention(hidden_size, num_attention_heads, **kwargs) + :members: forward + +Other +----- + .. autoapiclass:: transformer_engine.pytorch.dot_product_attention.inference.InferenceParams(max_batch_size, max_sequence_length) :members: reset, allocate_memory, pre_step, get_seqlens_pre_step, convert_paged_to_nonpaged, step @@ -71,6 +92,15 @@ PyTorch .. autoapifunction:: transformer_engine.pytorch.deinterleave_glu_tensor +MLA rotary position embeddings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoapifunction:: transformer_engine.pytorch.attention.mla_rope.build_rope_tables + +.. autoapifunction:: transformer_engine.pytorch.attention.mla_rope.apply_mla_rope_q + +.. autoapifunction:: transformer_engine.pytorch.attention.mla_rope.apply_mla_rope_kv + Data types ---------- diff --git a/docs/examples/jax/test_attention.py b/docs/examples/jax/test_attention.py index c86631dbc2a..f4caa35a950 100644 --- a/docs/examples/jax/test_attention.py +++ b/docs/examples/jax/test_attention.py @@ -59,9 +59,20 @@ def test_bshd_gqa_swa_matches_baseline(): @requires_hopper_or_newer -def test_single_gpu_benchmark(): +def test_single_gpu_benchmark(monkeypatch): import attention + # The benchmark synchronizes on the native baseline, which has the same memory problem + # as test_bshd_gqa_swa_matches_baseline. Benchmark the same smaller prefix; the tutorial script itself + # still runs at the full sequence length. + seq = 1024 + monkeypatch.setattr(attention, "qkv", tuple(t[:, :seq] for t in attention.qkv)) + monkeypatch.setattr(attention, "dout", attention.dout[:, :seq]) + monkeypatch.setattr( + attention, + "sequence_descriptor", + SequenceDescriptor.from_seqlens(jnp.full((attention.batch,), seq, dtype=jnp.int32)), + ) attention.run_single_gpu_bench() diff --git a/examples/pytorch/deepseek_v3/README.md b/examples/pytorch/deepseek_v3/README.md new file mode 100644 index 00000000000..c52fe579bee --- /dev/null +++ b/examples/pytorch/deepseek_v3/README.md @@ -0,0 +1,124 @@ +# DeepSeekV3Layer with expert parallelism + +A forward + backward benchmark of one `DeepSeekV3Layer`: RMSNorm, multi-latent attention, +and MoE with a shared expert. Routed experts are sharded across GPUs using NCCL EP. + +## Quick start + +Requires SM90+ GPUs with NVLink and an NCCL EP-enabled TE build; +see [requirements](#requirements). From this directory: + +```bash +bash run_deepseek_v3_layer_ep.sh --dsv3 +``` + +For MXFP8 with the fused grouped MLP (SM100-class GPUs): + +```bash +NVTE_CUTEDSL_FUSED_GROUPED_MLP=1 bash run_deepseek_v3_layer_ep.sh --dsv3 --recipe mxfp8 +``` + +## Requirements + +The following NCCL EP requirements apply to `--impl te`. The naive variants use +ordinary NCCL collectives and can also run on older GPUs without NVLink. + +- SM90 or newer GPUs connected with NVLink. +- NCCL >= 2.30.4, PyTorch >= 2.11 (symmetric memory), Transformer Engine built with NCCL EP + enabled (`NVTE_WITH_NCCL_EP`, on by default when an SM90+ architecture is targeted; + `NVTE_WITH_NCCL_EP=0` disables it). +- `NVTE_CUTEDSL_FUSED_GROUPED_MLP=1` additionally needs SM100-class GPUs and the CuTe DSL + (`nvidia-cutlass-dsl`) for the fused MXFP8 grouped MLP. + +## Running + +Run these commands from this directory. Single node, all local GPUs: + +```bash +bash run_deepseek_v3_layer_ep.sh # small dims, bf16 +bash run_deepseek_v3_layer_ep.sh --dsv3 # DeepSeek-V3 layer dims, bf16 +bash run_deepseek_v3_layer_ep.sh --dsv3 --recipe mxfp8 # MXFP8 experts (unfused grouped GEMM) +NVTE_CUTEDSL_FUSED_GROUPED_MLP=1 bash run_deepseek_v3_layer_ep.sh --dsv3 --recipe mxfp8 +bash run_deepseek_v3_layer_ep.sh --dsv3 --impl naive # all_to_all + loop over experts +bash run_deepseek_v3_layer_ep.sh --dsv3 --impl naive_grouped # all_to_all + TE grouped GEMM +``` + +`--impl` selects the MoE block inside the same layer (attention and norms are identical): + +- `te` (default): `DeepSeekV3MoE`, NCCL EP dispatch/combine, experts as one grouped GEMM. +- `naive`: MoE written with plain PyTorch, no TE MoE code: sigmoid top-k router with expert + bias, `all_to_all_single` dispatch and combine (two host syncs per layer for the split sizes), + a Python loop over the local experts with dense `F.linear` SwiGLU MLPs, `index_copy` / + `index_add` to place results, and a shared expert. This is what an EP MoE looks like before + any fused kernels. +- `naive_grouped`: the same all_to_all dispatch and combine, but the received rows are sorted by + local expert and run through one `te.ops.GroupedLinear` / `ScaledSwiGLU` / `GroupedLinear` + stack. Isolates the cost of the Python loop from the cost of the communication path. + +Multi-node: launch `torchrun` yourself, EP spans every rank: + +```bash +torchrun --nnodes=2 --nproc-per-node=4 --rdzv-backend=c10d --rdzv-endpoint=:29500 \ + deepseek_v3_layer_ep.py --dsv3 --recipe mxfp8 +``` + +Every rank owns `--num-local-experts` experts (default 8), so the expert count is +`8 * world_size`. Other knobs: `--tokens-per-rank`, `--topk`, `--hidden`, `--num-heads`, +`--moe-ffn`, the MLA dims (`--q-lora-rank`, `--kv-lora-rank`, `--qk-nope-head-dim`, +`--qk-rope-head-dim`, `--v-head-dim`), `--warmup`, `--iters`. +`--warmup 0` is supported; `--iters` must be positive and `--tokens-per-rank` must be a +positive multiple of four. MXFP8 requires `--impl te`. + +## Results + +GB200 and GB300 GPUs, 4096 tokens per rank, top-k 8, 8 local experts per GPU. +Times cover one layer's forward + backward; throughput is global, in millions of tokens/s. +Every number is the median of three independent runs (spread within 0.3 ms). +See [benchmark setup](#benchmark-setup) below for the full configuration. + +### TE vs. plain PyTorch (`--dsv3`) + +Same layer and dimensions, BF16 unless noted. + +All variants run the router backward and clear parameter gradients before each step. + +| MoE implementation | 4 GB200 (32 experts) | 4 GB300 (32 experts) | 8 GB300 (64 experts) | +|---|---:|---:|---:| +| `naive`: all_to_all + loop over experts | 28.61 ms | 27.21 ms | 27.70 ms | +| `naive_grouped`: all_to_all + TE grouped GEMM | 17.79 ms | 16.72 ms | 17.13 ms | +| `te`: NCCL EP + grouped GEMM | 13.68 ms | 12.37 ms | 14.09 ms | +| `te`, mxfp8 (unfused grouped GEMM) | 10.99 ms | 10.32 ms | 12.11 ms | +| `te`, mxfp8 fused | 10.11 ms | 9.49 ms | 10.45 ms | + +### TE throughput (`--dsv3`) + +| Precision | 4 GB200 · Mtok/s | 4 GB300 · Mtok/s | 8 GB300 · Mtok/s | +|---|---:|---:|---:| +| BF16 | 1.20 | 1.33 | 2.32 | +| MXFP8 | 1.49 | 1.59 | 2.71 | +| MXFP8 fused | 1.62 | 1.73 | 3.14 | + +4 GPUs = 1 node / 32 experts; 8 GPUs = 2 nodes / 64 experts. +Both nodes share one NVLink domain (MNNVL). + +“Fused” enables `NVTE_CUTEDSL_FUSED_GROUPED_MLP=1`. + +### Benchmark setup + +| Setting | Value | +|---|---| +| hardware | GB200 (SM100), one node; GB300 (SM103), one or two nodes; 4 GPUs per node, both GB300 nodes in one NVLink domain (MNNVL) | +| software | CUDA 13.3, NCCL 2.30.7, PyTorch 2.13.0a0+9186a08b2c.nv26.07, cuDNN 9.26, cuDNN Frontend 1.29.0, CUTLASS DSL 4.6.2 | +| NVIDIA driver | GB200: 580.173.02; GB300: 580.173.10 | +| `--dsv3` dims | hidden 7168, 128 heads, MLA q_lora 1536 / kv_lora 512 / nope 128 / rope 64 / v 128, expert ffn 2048, shared expert ffn 2048 | +| MoE | 8 local experts per rank (32 on 4 GPUs, 64 on 8), top-k 8, 4096 tokens per rank | +| precision | bf16 params and activations; `--recipe mxfp8` = MXFP8 block scaling for the expert GEMMs and dense projections | +| timing | fwd + bwd, 5 warmup, 10 timed iterations per run, median of 3 runs; wall-clock timing with CUDA synchronization, no CUDA graphs or profiler | +| environment | `OMP_NUM_THREADS=8`, `NVTE_GROUPED_LINEAR_SINGLE_PARAM=0`, `NVTE_ALLOW_NONDETERMINISTIC_ALGO=1`, `NVTE_FLASH_ATTN_V2=1`, `NVTE_FLASH_ATTN_V3=0`, `NVTE_FLASH_ATTN_V4=0`; EP buffer reused | + +Each run reuses a normally distributed random input and backpropagates an all-ones +output gradient. Parameter and input gradients are cleared before each iteration. + +`ep_bootstrap` is given the recv capacity returned by +`DeepSeekV3MoE.ep_recv_capacity(ep_size, tokens_per_rank, topk, num_local_experts)`, with +per-expert zones aligned to 256 rows. diff --git a/examples/pytorch/deepseek_v3/deepseek_v3_layer_ep.py b/examples/pytorch/deepseek_v3/deepseek_v3_layer_ep.py new file mode 100644 index 00000000000..e95a7ecba6d --- /dev/null +++ b/examples/pytorch/deepseek_v3/deepseek_v3_layer_ep.py @@ -0,0 +1,283 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. +"""DeepSeekV3Layer with expert parallelism over all ranks: forward + backward timing. + +One process per GPU, launched via run_deepseek_v3_layer_ep.sh (torchrun). Every rank +holds ``--num-local-experts`` routed experts. ``--impl te`` exchanges tokens with NCCL EP +and runs the experts as one grouped GEMM (DeepSeekV3MoE); ``--impl naive`` is a plain +PyTorch MoE (all_to_all_single + a Python loop over experts) dropped into the same layer; +``--impl naive_grouped`` keeps the all_to_all but runs the experts as one TE grouped GEMM. +Timed iterations run inside a ``torch.cuda.profiler`` window, so +``nsys profile -c cudaProfilerApi --capture-range-end=stop torchrun ...`` records +only them. +""" + +import argparse +import os +import sys +import time +from contextlib import nullcontext + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch.distributed.nn.functional import all_to_all_single + +import transformer_engine.pytorch as te +from transformer_engine.common import recipe as te_recipe +from transformer_engine.pytorch.ep import ep_bootstrap, ep_finalize, release_symm_mem_pool +from transformer_engine.pytorch.models import DeepSeekV3Layer, DeepSeekV3MoE + + +def _parse_args(argv=None): + p = argparse.ArgumentParser(description="DeepSeekV3Layer EP example (fwd + bwd)") + p.add_argument("--tokens-per-rank", type=int, default=4096) + p.add_argument("--hidden", type=int, default=2048) + p.add_argument("--num-heads", type=int, default=16) + p.add_argument("--moe-ffn", type=int, default=1024) + p.add_argument("--num-local-experts", type=int, default=8) + p.add_argument("--topk", type=int, default=8) + p.add_argument("--q-lora-rank", type=int, default=512) + p.add_argument("--kv-lora-rank", type=int, default=256) + p.add_argument("--qk-nope-head-dim", type=int, default=64) + p.add_argument("--qk-rope-head-dim", type=int, default=32) + p.add_argument("--v-head-dim", type=int, default=64) + p.add_argument( + "--dsv3", + action="store_true", + help=( + "DeepSeek-V3 layer dims (hidden 7168, 128 heads, MLA 1536/512/128/64/128, expert ffn" + " 2048)." + ), + ) + p.add_argument("--impl", choices=["te", "naive", "naive_grouped"], default="te") + p.add_argument("--recipe", choices=["none", "mxfp8"], default="none") + p.add_argument( + "--ep-buffer-per-call", + action="store_true", + help="Let the layer create a new EpBuffer per forward instead of reusing one.", + ) + p.add_argument("--warmup", type=int, default=5) + p.add_argument("--iters", type=int, default=10) + args = p.parse_args(argv) + if args.warmup < 0: + p.error("--warmup must be non-negative") + if args.iters <= 0: + p.error("--iters must be positive") + if args.tokens_per_rank <= 0 or args.tokens_per_rank % 4: + p.error("--tokens-per-rank must be a positive multiple of 4") + if args.impl != "te" and args.recipe != "none": + p.error("--recipe mxfp8 is only supported with --impl te") + if args.dsv3: + args.hidden, args.num_heads, args.moe_ffn = 7168, 128, 2048 + args.q_lora_rank, args.kv_lora_rank = 1536, 512 + args.qk_nope_head_dim, args.qk_rope_head_dim, args.v_head_dim = 128, 64, 128 + return args + + +def _autocast(name): + if name == "none": + return nullcontext() + return te.autocast(enabled=True, recipe=te_recipe.MXFP8BlockScaling()) + + +class NaiveMoE(torch.nn.Module): + """DeepSeek-style MoE with torch all_to_all dispatch/combine: sigmoid top-k router with + expert bias, a shared expert, and experts either as a Python loop of dense SwiGLU MLPs or + (``grouped=True``) as one TE grouped GEMM stack.""" + + def __init__(self, hidden, ffn, num_experts, topk, ep_group, shared_ffn, dtype, grouped=False): + super().__init__() + self.grouped = grouped + self.hidden, self.topk, self.group = hidden, topk, ep_group + self.ws, self.rank = dist.get_world_size(ep_group), dist.get_rank(ep_group) + self.num_experts, self.local = num_experts, num_experts // self.ws + self.gate = torch.nn.Linear(hidden, num_experts, bias=False, dtype=dtype, device="cuda") + self.register_buffer("expert_bias", torch.zeros(num_experts, device="cuda")) + std = hidden**-0.5 + if grouped: + self.experts = te.ops.Sequential( + te.ops.GroupedLinear(self.local, hidden, 2 * ffn, bias=False, dtype=dtype), + te.ops.ScaledSwiGLU(glu_interleave_size=32), + te.ops.GroupedLinear(self.local, ffn, hidden, bias=False, dtype=dtype), + ) + else: + self.w1 = torch.nn.Parameter( + torch.randn(self.local, 2 * ffn, hidden, dtype=dtype, device="cuda") * std + ) + self.w2 = torch.nn.Parameter( + torch.randn(self.local, hidden, ffn, dtype=dtype, device="cuda") * ffn**-0.5 + ) + self.shared_w1 = torch.nn.Linear( + hidden, 2 * shared_ffn, bias=False, dtype=dtype, device="cuda" + ) + self.shared_w2 = torch.nn.Linear(shared_ffn, hidden, bias=False, dtype=dtype, device="cuda") + + @staticmethod + def _swiglu(h): + a, g = h.chunk(2, dim=-1) + return F.silu(a) * g + + def forward(self, hidden_states): + x = hidden_states.reshape(-1, self.hidden) + scores = torch.sigmoid(self.gate(x).float()) + _, idx = torch.topk(scores + self.expert_bias, self.topk, dim=-1) + probs = scores.gather(1, idx) + probs = probs / probs.sum(-1, keepdim=True) * 2.5 + # Dispatch: sort (token, expert) pairs by destination rank, exchange counts, all_to_all. + flat_e, flat_p = idx.reshape(-1), probs.reshape(-1) + tok = torch.arange(x.shape[0], device=x.device).repeat_interleave(self.topk) + order = torch.argsort(flat_e // self.local, stable=True) + flat_e, flat_p, tok = flat_e[order], flat_p[order], tok[order] + send = torch.bincount(flat_e // self.local, minlength=self.ws) + recv = torch.empty_like(send) + dist.all_to_all_single(recv, send, group=self.group) + send, recv = send.tolist(), recv.tolist() + n_recv = sum(recv) + x_recv = all_to_all_single( + torch.empty(n_recv, self.hidden, dtype=x.dtype, device=x.device), + x[tok], + recv, + send, + group=self.group, + ) + e_recv = torch.empty(n_recv, dtype=flat_e.dtype, device=x.device) + p_recv = torch.empty(n_recv, dtype=flat_p.dtype, device=x.device) + dist.all_to_all_single(e_recv, flat_e.contiguous(), recv, send, group=self.group) + p_recv = all_to_all_single(p_recv, flat_p.contiguous(), recv, send, group=self.group) + local_e = e_recv - self.rank * self.local + if self.grouped: + # Experts: sort received rows by local expert, one grouped GEMM stack. + by_expert = torch.argsort(local_e, stable=True) + counts = torch.bincount(local_e, minlength=self.local) + y_sorted = self.experts( + x_recv[by_expert], counts, p_recv[by_expert].to(x.dtype), counts + ) + y_recv = torch.empty_like(x_recv).index_copy(0, by_expert, y_sorted) + else: + # Experts: one dense SwiGLU MLP per local expert. + y_recv = torch.zeros_like(x_recv) + for e in range(self.local): + sel = (local_e == e).nonzero().squeeze(1) + if sel.numel() == 0: + continue + h = self._swiglu(F.linear(x_recv[sel], self.w1[e])) * p_recv[sel, None].to(x.dtype) + y_recv = y_recv.index_copy(0, sel, F.linear(h, self.w2[e])) + # Combine: reverse all_to_all, sum the top-k contributions per token. + y = all_to_all_single(torch.empty_like(x[tok]), y_recv, send, recv, group=self.group) + out = torch.zeros_like(x).index_add(0, tok, y) + out = out + self.shared_w2(self._swiglu(self.shared_w1(x))) + return out.view_as(hidden_states) + + +def main(): + """Build the layer, run warmup + timed fwd/bwd iterations, print throughput on rank 0.""" + args = _parse_args() + local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local_rank) + dist.init_process_group("nccl", device_id=torch.device("cuda", local_rank)) + rank, world_size = dist.get_rank(), dist.get_world_size() + + major, minor = torch.cuda.get_device_capability() + if args.impl == "te" and major * 10 + minor < 90: + if rank == 0: + print(f"SKIPPED: NCCL EP requires SM>=90 (got SM{major}{minor})") + dist.destroy_process_group() + return 0 + + ep_group = dist.new_group(ranks=list(range(world_size)), backend="nccl") + dist.all_reduce(torch.zeros(1, device="cuda"), group=ep_group) + num_experts = args.num_local_experts * world_size + if args.impl == "te": + ep_bootstrap( + ep_group, + num_experts=num_experts, + max_tokens_per_rank=args.tokens_per_rank, + hidden_dim=args.hidden, + num_topk=args.topk, + recv_capacity_per_rank=DeepSeekV3MoE.ep_recv_capacity( + world_size, args.tokens_per_rank, args.topk, args.num_local_experts + ), + ) + + torch.manual_seed(0) + mlp_kwargs = dict( + num_experts=num_experts, + moe_ffn_hidden_size=args.moe_ffn, + shared_expert_ffn_hidden_size=args.moe_ffn, + topk=args.topk, + ep_group=ep_group, + ep_max_tokens_per_rank=args.tokens_per_rank, + ) + if args.impl != "te": + # Build the TE MoE without EP (replaced below); keeps the pre-MLP RMSNorm. + mlp_kwargs.pop("ep_group"), mlp_kwargs.pop("ep_max_tokens_per_rank") + layer = DeepSeekV3Layer( + args.hidden, + args.num_heads, + params_dtype=torch.bfloat16, + **mlp_kwargs, + q_lora_rank=args.q_lora_rank, + kv_lora_rank=args.kv_lora_rank, + qk_nope_head_dim=args.qk_nope_head_dim, + qk_rope_head_dim=args.qk_rope_head_dim, + v_head_dim=args.v_head_dim, + ) + if args.impl != "te": + layer.mlp = NaiveMoE( + args.hidden, + args.moe_ffn, + num_experts, + args.topk, + ep_group, + args.moe_ffn, + torch.bfloat16, + grouped=args.impl == "naive_grouped", + ) + seq = args.tokens_per_rank // 4 + x = torch.randn(seq, 4, args.hidden, dtype=torch.bfloat16, device="cuda", requires_grad=True) + + ep_buffer = None + if args.impl == "te" and not args.ep_buffer_per_call: + ep_buffer = layer.mlp.make_ep_buffer() + + def step(): + layer.zero_grad(set_to_none=True) + x.grad = None + with _autocast(args.recipe): + out = layer(x, ep_buffer=ep_buffer) + out.backward(torch.ones_like(out)) + + for _ in range(args.warmup): + step() + torch.cuda.synchronize() + dist.barrier() + + torch.cuda.profiler.start() + start = time.perf_counter() + for i in range(args.iters): + with torch.cuda.nvtx.range(f"iter{i}"): + step() + torch.cuda.synchronize() + ms = (time.perf_counter() - start) / args.iters * 1e3 + torch.cuda.profiler.stop() + dist.barrier() + + if rank == 0: + tok_s = args.tokens_per_rank * world_size / (ms / 1e3) + print( + f"DeepSeekV3Layer impl={args.impl}:" + f" ranks={world_size} experts={num_experts} topk={args.topk} tokens/rank={args.tokens_per_rank} hidden={args.hidden} recipe={args.recipe} fused_mlp={os.environ.get('NVTE_CUTEDSL_FUSED_GROUPED_MLP', '0')} ep_buffer={'per_call' if ep_buffer is None else 'reused'} fwd+bwd" + f" {ms:.3f} ms/iter ({tok_s / 1e6:.2f} Mtok/s)", + flush=True, + ) + if args.impl == "te": + ep_finalize() + release_symm_mem_pool() + dist.destroy_process_group() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/pytorch/deepseek_v3/run_deepseek_v3_layer_ep.sh b/examples/pytorch/deepseek_v3/run_deepseek_v3_layer_ep.sh new file mode 100755 index 00000000000..9341b7f14a6 --- /dev/null +++ b/examples/pytorch/deepseek_v3/run_deepseek_v3_layer_ep.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. +# +# Launcher for deepseek_v3_layer_ep.py on all local GPUs. Extra args go to the script: +# bash run_deepseek_v3_layer_ep.sh # bf16, small dims +# bash run_deepseek_v3_layer_ep.sh --dsv3 --recipe mxfp8 # DeepSeek-V3 dims, MXFP8 experts +# NVTE_CUTEDSL_FUSED_GROUPED_MLP=1 bash run_deepseek_v3_layer_ep.sh --dsv3 --recipe mxfp8 +# NSYS=1 bash run_deepseek_v3_layer_ep.sh --dsv3 # nsys report in results/ +# Multi-node: run torchrun yourself with --nnodes/--rdzv-endpoint; EP spans all ranks. + +set -uo pipefail + +DETECTED_GPUS=$(nvidia-smi -L 2>/dev/null | wc -l) +NUM_GPUS="${NUM_GPUS:-${DETECTED_GPUS}}" +if [ "${NUM_GPUS}" -lt 2 ]; then + echo "EP requires >= 2 GPUs (found ${NUM_GPUS}); SKIPPING." + exit 0 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +: ${NCCL_EP_JIT_CACHE_DIR:="${TMPDIR:-/tmp}/nccl_ep_jit_cache_$(id -u)"} +export NCCL_EP_JIT_CACHE_DIR +mkdir -p "$NCCL_EP_JIT_CACHE_DIR" + +PREFIX=() +if [ "${NSYS:-0}" = "1" ]; then + mkdir -p "${SCRIPT_DIR}/results" + PREFIX=(nsys profile -t cuda,nvtx,nccl -c cudaProfilerApi --capture-range-end=stop + --cuda-graph-trace=node -o "${SCRIPT_DIR}/results/deepseek_v3_layer_ep_%h") +fi + +"${PREFIX[@]}" torchrun --standalone --nnodes=1 --nproc-per-node="${NUM_GPUS}" \ + "${SCRIPT_DIR}/deepseek_v3_layer_ep.py" "$@" diff --git a/qa/L0_pytorch_unittest/test.sh b/qa/L0_pytorch_unittest/test.sh index 2aa02340018..f215114d679 100644 --- a/qa/L0_pytorch_unittest/test.sh +++ b/qa/L0_pytorch_unittest/test.sh @@ -42,6 +42,7 @@ PYTORCH_JIT=0 NVTE_TORCH_COMPILE=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 NVTE_FUSED PYTORCH_JIT=0 NVTE_TORCH_COMPILE=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 NVTE_FUSED_ATTN=0 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_cuda_graphs.xml $TE_PATH/tests/pytorch/test_cuda_graphs.py || test_fail "test_cuda_graphs.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_jit.xml $TE_PATH/tests/pytorch/test_jit.py || test_fail "test_jit.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_fused_rope.xml $TE_PATH/tests/pytorch/test_fused_rope.py || test_fail "test_fused_rope.py" +python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_mla_rope.xml $TE_PATH/tests/pytorch/attention/test_mla_rope.py || test_fail "test_mla_rope.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_nvfp4.xml $TE_PATH/tests/pytorch/nvfp4 || test_fail "test_nvfp4" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_mxfp8.xml $TE_PATH/tests/pytorch/mxfp8 || test_fail "test_mxfp8" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_strided_batched_gemm.xml $TE_PATH/tests/pytorch/test_strided_batched_gemm.py || test_fail "test_strided_batched_gemm.py" @@ -65,6 +66,7 @@ python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_backward_overrid python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_permutation.xml $TE_PATH/tests/pytorch/test_permutation.py || test_fail "test_permutation.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_cross_entropy.xml $TE_PATH/tests/pytorch/test_cross_entropy.py || test_fail "test_cross_entropy.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_cpu_offloading.xml $TE_PATH/tests/pytorch/test_cpu_offloading.py || test_fail "test_cpu_offloading.py" +python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_models.xml $TE_PATH/tests/pytorch/test_models.py || test_fail "test_models.py" NVTE_FLASH_ATTN=0 NVTE_CPU_OFFLOAD_V1=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_cpu_offloading_v1.xml $TE_PATH/tests/pytorch/test_cpu_offloading_v1.py || test_fail "test_cpu_offloading_v1.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_hybrid_quantization.xml $TE_PATH/tests/pytorch/test_hybrid_quantization.py || test_fail "test_hybrid_quantization.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_identity_quantizer.xml $TE_PATH/tests/pytorch/test_identity_quantizer.py || test_fail "test_identity_quantizer.py" diff --git a/qa/L1_pytorch_distributed_unittest/test.sh b/qa/L1_pytorch_distributed_unittest/test.sh index 41ab6aeb444..9b17315fabf 100644 --- a/qa/L1_pytorch_distributed_unittest/test.sh +++ b/qa/L1_pytorch_distributed_unittest/test.sh @@ -57,6 +57,7 @@ python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_cu_seqlens_cache.xml python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_cast_master_weights_to_fp8.xml $TE_PATH/tests/pytorch/distributed/test_cast_master_weights_to_fp8.py || test_fail "test_cast_master_weights_to_fp8.py" python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_newton_schulz.xml $TE_PATH/tests/pytorch/distributed/test_newton_schulz.py || test_fail "test_newton_schulz.py" python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_ep.xml $TE_PATH/tests/pytorch/distributed/test_ep.py || test_fail "test_ep.py" +python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_models.xml $TE_PATH/tests/pytorch/distributed/test_models.py || test_fail "distributed/test_models.py" # debug tests diff --git a/tests/pytorch/attention/test_linear_mxfp8_attention.py b/tests/pytorch/attention/test_linear_mxfp8_attention.py index e5cb1c8a655..f1926814f3a 100644 --- a/tests/pytorch/attention/test_linear_mxfp8_attention.py +++ b/tests/pytorch/attention/test_linear_mxfp8_attention.py @@ -35,7 +35,11 @@ _current_file = pathlib.Path(__file__).resolve() sys.path = [str(_current_file.parent.parent)] + sys.path from utils import ModelConfig, compare_and_assert, get_available_attention_backends -from mla_rope_utils import apply_mla_rope, build_rope_tables +from transformer_engine.pytorch.attention.mla_rope import ( + apply_mla_rope_kv, + apply_mla_rope_q, + build_rope_tables, +) try: @@ -179,6 +183,13 @@ def _run_projections( return q_flat, kv_flat, q, kv, k_pos_emb +def _apply_rope(q, kv, k_pos_emb, rope_tables): + cos, sin = rope_tables + q = apply_mla_rope_q(q, cos, sin, HEAD_DIM_NOPE, HEAD_DIM_ROPE) + k, v = apply_mla_rope_kv(kv, k_pos_emb, cos, sin, HEAD_DIM_NOPE, HEAD_DIM_ROPE, HEAD_DIM_V) + return q, k, v + + def _run_forward_bf16( modules: tuple, x: torch.Tensor, @@ -186,7 +197,7 @@ def _run_forward_bf16( ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: q_proj, kv_proj, dpa, out_linear = modules _, _, q, kv, k_pos_emb = _run_projections(q_proj, kv_proj, x) - q, k, v = apply_mla_rope(q, kv, k_pos_emb, cos_table=rope_tables[0], sin_table=rope_tables[1]) + q, k, v = _apply_rope(q, kv, k_pos_emb, rope_tables) attn_out = dpa(q, k, v, qkv_format="sbhd") return q, k, v, out_linear(attn_out.view(x.shape[0], x.shape[1], HIDDEN_SIZE)) @@ -208,13 +219,7 @@ def _run_forward_mxfp8( x, is_first_microbatch, ) - q, k, v = apply_mla_rope( - q, - kv, - k_pos_emb, - cos_table=rope_tables[0], - sin_table=rope_tables[1], - ) + q, k, v = _apply_rope(q, kv, k_pos_emb, rope_tables) attn_out = dpa(q, k, v, qkv_format="sbhd") out = out_linear( attn_out.view(x.shape[0], x.shape[1], HIDDEN_SIZE), @@ -288,7 +293,7 @@ def test_accuracy(self, batch_size: int, seq_len: int) -> None: _set_seed() baseline_modules, mxfp8_modules = _build_modules() x = torch.randn(seq_len, batch_size, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda") - rope_tables = build_rope_tables(seq_len, device=x.device) + rope_tables = build_rope_tables(seq_len, HEAD_DIM_ROPE, device=x.device) q_bf16, k_bf16, v_bf16, out_bf16 = _run_forward_bf16(baseline_modules, x, rope_tables) q_mxfp8, k_mxfp8, v_mxfp8, out_mxfp8 = _run_forward_mxfp8( @@ -374,7 +379,7 @@ def test_backward(self, batch_size: int, seq_len: int) -> None: device="cuda", requires_grad=True, ) - rope_tables = build_rope_tables(seq_len, device=x.device) + rope_tables = build_rope_tables(seq_len, HEAD_DIM_ROPE, device=x.device) *_, out_mxfp8 = _run_forward_mxfp8(mxfp8_modules, x, fp8_recipe, rope_tables) out_mxfp8.sum().backward() @@ -408,7 +413,7 @@ def test_performance(self, batch_size: int, seq_len: int) -> None: device="cuda", requires_grad=True, ) - rope_tables = build_rope_tables(seq_len, device=x.device) + rope_tables = build_rope_tables(seq_len, HEAD_DIM_ROPE, device=x.device) mxfp8_fprop_ms, mxfp8_bprop_ms = _benchmark_training_step( _run_forward_mxfp8, mxfp8_modules, x, fp8_recipe, rope_tables diff --git a/tests/pytorch/attention/test_mla_rope.py b/tests/pytorch/attention/test_mla_rope.py new file mode 100644 index 00000000000..2b35cec5cef --- /dev/null +++ b/tests/pytorch/attention/test_mla_rope.py @@ -0,0 +1,184 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +import math + +import pytest +import torch + +from transformer_engine.pytorch.attention import mla_rope +from transformer_engine.pytorch.module import LayerNormLinear + + +@pytest.mark.parametrize("nope,rope,vdim", [(64, 32, 64), (48, 32, 64), (64, 48, 64), (64, 32, 48)]) +def test_mla_rope_matches_pytorch(nope, rope, vdim): + if not mla_rope.HAVE_TRITON: + pytest.skip("Triton unavailable") + s, b, h = 64, 2, 4 + cos, sin = mla_rope.build_rope_tables(s, rope, device="cuda") + + torch.manual_seed(0) + q_leaf = torch.randn(s, b, h, nope + rope, device="cuda", requires_grad=True) + kv_leaf = torch.randn(s, b, h, nope + vdim, device="cuda", requires_grad=True) + pos_leaf = torch.randn(s, b, 1, rope, device="cuda", requires_grad=True) + grad_q = torch.randn(s, b, h, nope + rope, device="cuda") + grad_k = torch.randn(s, b, h, nope + rope, device="cuda") + grad_v = torch.randn(s, b, h, vdim, device="cuda") + + def run(fmt): + q, kv, pos = q_leaf * 1.0, kv_leaf * 1.0, pos_leaf * 1.0 + q_out = mla_rope.apply_mla_rope_q(q, cos, sin, nope, rope, fmt) + k_out, v_out = mla_rope.apply_mla_rope_kv(kv, pos, cos, sin, nope, rope, vdim, fmt) + torch.autograd.backward( + [q_out, k_out, v_out], [grad_q.clone(), grad_k.clone(), grad_v.clone()] + ) + grads = (q_leaf.grad.clone(), kv_leaf.grad.clone(), pos_leaf.grad.clone()) + q_leaf.grad = kv_leaf.grad = pos_leaf.grad = None + return (q_out.clone(), k_out, v_out), grads + + (q_t, k_t, v_t), grads_t = run("sbhd") + + seq_dim = 0 + q_ref = torch.cat( + ( + (q_leaf * 1.0)[..., :nope], + mla_rope._rotate_interleaved_to_neox((q_leaf * 1.0)[..., nope:], cos, sin, seq_dim), + ), + dim=-1, + ) + k_ref = torch.cat( + ( + (kv_leaf * 1.0)[..., :nope], + mla_rope._rotate_interleaved_to_neox(pos_leaf * 1.0, cos, sin, seq_dim).expand( + s, b, h, rope + ), + ), + dim=-1, + ) + v_ref = (kv_leaf * 1.0)[..., nope:] + torch.autograd.backward([q_ref, k_ref, v_ref], [grad_q.clone(), grad_k.clone(), grad_v.clone()]) + + torch.testing.assert_close(q_t, q_ref, rtol=1e-5, atol=1e-5) + torch.testing.assert_close(k_t, k_ref, rtol=1e-5, atol=1e-5) + torch.testing.assert_close(v_t, v_ref, rtol=1e-5, atol=1e-5) + torch.testing.assert_close(grads_t[0], q_leaf.grad, rtol=1e-5, atol=1e-5) + torch.testing.assert_close(grads_t[1], kv_leaf.grad, rtol=1e-5, atol=1e-5) + torch.testing.assert_close(grads_t[2], pos_leaf.grad, rtol=1e-5, atol=1e-5) + + +@pytest.mark.parametrize("compiled", [False, True]) +def test_mla_rope_q_preserves_input_and_gradient(compiled): + if not mla_rope.HAVE_TRITON: + pytest.skip("Triton unavailable") + s, b, h, nope, rope = 8, 2, 4, 64, 32 + cos, sin = mla_rope.build_rope_tables(s, rope, device="cuda") + q = torch.randn(s, b, h, nope + rope, device="cuda", requires_grad=True) + q_before = q.detach().clone() + incoming_grad = torch.randn_like(q) + grad_before = incoming_grad.clone() + apply_rope = lambda x: mla_rope.apply_mla_rope_q(x, cos, sin, nope, rope) + if compiled: + apply_rope = torch.compile(apply_rope, fullgraph=True) + + aux = (q * q).sum() + rotated = apply_rope(q) + assert rotated.data_ptr() != q.data_ptr() + torch.autograd.backward((rotated, aux), (incoming_grad, torch.ones_like(aux))) + + q_ref = q_before.requires_grad_() + rotated_ref = torch.cat( + (q_ref[..., :nope], mla_rope._rotate_interleaved_to_neox(q_ref[..., nope:], cos, sin, 0)), + dim=-1, + ) + torch.autograd.backward( + (rotated_ref, (q_ref * q_ref).sum()), (grad_before, torch.ones_like(aux)) + ) + torch.testing.assert_close(q, q_before) + torch.testing.assert_close(incoming_grad, grad_before) + torch.testing.assert_close(rotated, rotated_ref) + torch.testing.assert_close(q.grad, q_ref.grad) + + +def test_mla_rope_q_in_place_eager(): + if not mla_rope.HAVE_TRITON: + pytest.skip("Triton unavailable") + s, b, h, nope, rope = 8, 2, 4, 64, 32 + cos, sin = mla_rope.build_rope_tables(s, rope, device="cuda") + leaf = torch.randn(s, b, h, nope + rope, device="cuda", requires_grad=True) + q = (leaf * 1).view(s, b, h, nope + rope) + q_before = q.detach().clone() + incoming_grad = torch.randn_like(q) + grad_before = incoming_grad.clone() + rotated = mla_rope.apply_mla_rope_q(q, cos, sin, nope, rope, in_place=True) + assert rotated.data_ptr() == q.data_ptr() + torch.autograd.backward(rotated, incoming_grad) + + q_ref = q_before.requires_grad_() + rotated_ref = torch.cat( + (q_ref[..., :nope], mla_rope._rotate_interleaved_to_neox(q_ref[..., nope:], cos, sin, 0)), + dim=-1, + ) + torch.autograd.backward(rotated_ref, grad_before) + torch.testing.assert_close(rotated, rotated_ref) + torch.testing.assert_close(incoming_grad, grad_before) + torch.testing.assert_close(leaf.grad, q_ref.grad) + + +def test_mla_rope_q_in_place_rejects_compile(): + if not mla_rope.HAVE_TRITON: + pytest.skip("Triton unavailable") + s, b, h, nope, rope = 8, 2, 4, 64, 32 + cos, sin = mla_rope.build_rope_tables(s, rope, device="cuda") + q = torch.randn(s, b, h, nope + rope, device="cuda") + compiled = torch.compile( + lambda x: mla_rope.apply_mla_rope_q(x, cos, sin, nope, rope, in_place=True), + fullgraph=True, + ) + with pytest.raises(RuntimeError, match="in_place=True is not supported under torch.compile"): + compiled(q) + + +def test_mla_rope_q_layernormlinear_view(): + if not mla_rope.HAVE_TRITON: + pytest.skip("Triton unavailable") + s, b, h, nope, rope = 8, 2, 4, 64, 32 + cos, sin = mla_rope.build_rope_tables(s, rope, device="cuda") + projection = LayerNormLinear( + 64, h * (nope + rope), normalization="RMSNorm", params_dtype=torch.float32, device="cuda" + ) + x = torch.randn(s, b, 64, device="cuda", requires_grad=True) + q = projection(x).view(s, b, h, nope + rope) + q_before = q.detach().clone() + rotated = mla_rope.apply_mla_rope_q(q, cos, sin, nope, rope) + rotated.sum().backward() + + rotated_ref = torch.cat( + ( + q_before[..., :nope], + mla_rope._rotate_interleaved_to_neox(q_before[..., nope:], cos, sin, 0), + ), + dim=-1, + ) + torch.testing.assert_close(q, q_before) + torch.testing.assert_close(rotated, rotated_ref) + assert x.grad is not None + + +def test_rope_tables_yarn(): + s, rope = 8192, 64 + cos, sin = mla_rope.build_rope_tables(s, rope, device="cuda") + cos_none, sin_none = mla_rope.build_rope_tables(s, rope, device="cuda", scaling_factor=None) + assert torch.equal(cos, cos_none) and torch.equal(sin, sin_none) + + yarn = dict(scaling_factor=40.0, original_max_position_embeddings=4096) + cos_y, sin_y = mla_rope.build_rope_tables(s, rope, device="cuda", **yarn) + factor = mla_rope.yarn_concentration_factor(40.0, 1.0, 0.0) + assert factor == pytest.approx(0.1 * math.log(40.0) + 1.0) + # amplitude scaled by the concentration factor + torch.testing.assert_close(cos_y**2 + sin_y**2, torch.full_like(cos_y, factor**2)) + # high-frequency dims untouched, low-frequency dims interpolated by 1/scaling_factor + torch.testing.assert_close(cos_y[:, 0] / factor, cos[:, 0]) + angle_y = torch.atan2(sin_y[:, rope // 2 - 1], cos_y[:, rope // 2 - 1]) + angle = torch.atan2(sin[:, rope // 2 - 1], cos[:, rope // 2 - 1]) + torch.testing.assert_close(angle_y[:64], angle[:64] / 40.0, atol=1e-4, rtol=0) diff --git a/tests/pytorch/distributed/run_models.py b/tests/pytorch/distributed/run_models.py new file mode 100644 index 00000000000..e38a45f2672 --- /dev/null +++ b/tests/pytorch/distributed/run_models.py @@ -0,0 +1,241 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. +"""Multi-process tests for model-specific layers (te.models), launched via torchrun.""" + +import os +import sys + +import torch +import torch.distributed as dist + +from transformer_engine.pytorch.ep import ep_bootstrap, ep_finalize, release_symm_mem_pool +from transformer_engine.pytorch.models import DeepSeekV3Layer, DeepSeekV3MoE, MultiLatentAttention + +HIDDEN = 256 +MOE_FFN = 128 +SHARED_FFN = 128 +NUM_LOCAL_EXPERTS = 2 +TOP_K = 2 +TOKENS_PER_RANK = 64 +HEADS = 4 +DTYPE = torch.bfloat16 + +MLA_KWARGS = dict( + q_lora_rank=96, + kv_lora_rank=64, + qk_nope_head_dim=64, + qk_rope_head_dim=32, + v_head_dim=64, +) + + +def _device_sm() -> int: + major, minor = torch.cuda.get_device_capability() + return major * 10 + minor + + +def _recv_capacity(ep_size: int) -> int: + return DeepSeekV3MoE.ep_recv_capacity(ep_size, TOKENS_PER_RANK, TOP_K, NUM_LOCAL_EXPERTS) + + +def _broadcast_params(module: torch.nn.Module) -> None: + for t in list(module.parameters()) + list(module.buffers()): + dist.broadcast(t.detach(), src=0) + + +def _make_layer(ep_group, ep_size: int, num_experts: int) -> DeepSeekV3Layer: + ep = ep_group is not None + return DeepSeekV3Layer( + HIDDEN, + HEADS, + num_experts=num_experts, + moe_ffn_hidden_size=MOE_FFN, + topk=TOP_K, + shared_expert_ffn_hidden_size=SHARED_FFN, + params_dtype=DTYPE, + ep_group=ep_group, + ep_max_tokens_per_rank=TOKENS_PER_RANK if ep else None, + **MLA_KWARGS, + ) + + +def _copy_weights(ep_layer: DeepSeekV3Layer, ref: DeepSeekV3Layer, rank: int) -> None: + ref_params = dict(ref.named_parameters()) + ref_bufs = dict(ref.named_buffers()) + with torch.no_grad(): + for name, p in ep_layer.named_parameters(): + if not name.startswith("mlp.experts."): + p.copy_(ref_params[name]) + for name, b in ep_layer.named_buffers(): + if name in ref_bufs and b.shape == ref_bufs[name].shape: + b.copy_(ref_bufs[name]) + ep_fc1, _, ep_fc2 = ep_layer.mlp.experts[1:4] + ref_fc1, _, ref_fc2 = ref.mlp.experts + for local_e in range(NUM_LOCAL_EXPERTS): + global_e = rank * NUM_LOCAL_EXPERTS + local_e + getattr(ep_fc1, f"weight{local_e}").copy_(getattr(ref_fc1, f"weight{global_e}")) + getattr(ep_fc2, f"weight{local_e}").copy_(getattr(ref_fc2, f"weight{global_e}")) + + +def test_layer_ep_matches_local( + rank: int, ep_size: int, ep_group, num_microbatches: int = 1 +) -> None: + """Full DeepSeekV3Layer with EP must match the all-experts-local layer numerically.""" + num_experts = NUM_LOCAL_EXPERTS * ep_size + torch.manual_seed(0) + ref = _make_layer(None, ep_size, num_experts) + _broadcast_params(ref) + ep_layer = _make_layer(ep_group, ep_size, num_experts) + _copy_weights(ep_layer, ref, rank) + + torch.manual_seed(1234 + rank) + microbatches = [] + for mb in range(num_microbatches): + seq_len = TOKENS_PER_RANK // 2 - 8 * mb + x = torch.randn(seq_len, 2, HIDDEN, dtype=DTYPE, device="cuda") + x_ep = x.clone().requires_grad_(True) + x_ref = x.clone().requires_grad_(True) + out_ep = ep_layer(x_ep) + out_ref = ref(x_ref) + assert out_ep.shape == x.shape + torch.testing.assert_close(out_ep, out_ref, rtol=0.05, atol=0.05) + microbatches.append((x_ep, x_ref, out_ep, out_ref)) + + # All microbatches must retain their routing until their own backward. + for x_ep, x_ref, out_ep, out_ref in microbatches: + grad_out = torch.randn_like(out_ep) + out_ep.backward(grad_out.clone()) + out_ref.backward(grad_out.clone()) + torch.testing.assert_close(x_ep.grad, x_ref.grad, rtol=0.05, atol=0.05) + + ref_params = dict(ref.named_parameters()) + for name, p in ep_layer.named_parameters(): + if name.startswith("mlp.experts.") or p.grad is None: + continue + torch.testing.assert_close(p.grad, ref_params[name].grad, rtol=0.1, atol=0.1, msg=name) + + # A local expert's wgrad on its owner rank equals the sum of the + # reference wgrads over all ranks. all_reduce is collective, so every + # rank must reduce every expert's grad (in the same order). + ep_fc1, _, ep_fc2 = ep_layer.mlp.experts[1:4] + ref_fc1, _, ref_fc2 = ref.mlp.experts + for ep_fc, ref_fc in ((ep_fc1, ref_fc1), (ep_fc2, ref_fc2)): + ref_grads = [getattr(ref_fc, f"weight{e}").grad.float().clone() for e in range(num_experts)] + for g in ref_grads: + dist.all_reduce(g) + for local_e in range(NUM_LOCAL_EXPERTS): + global_e = rank * NUM_LOCAL_EXPERTS + local_e + ep_grad = getattr(ep_fc, f"weight{local_e}").grad.float() + torch.testing.assert_close(ep_grad, ref_grads[global_e], rtol=0.1, atol=0.1) + + counts = ep_layer.mlp._last_tokens_per_expert.clone() + dist.all_reduce(counts, group=ep_group) + last_num_tokens = microbatches[-1][0].numel() // HIDDEN + assert counts.sum().item() == ep_size * last_num_tokens * TOP_K + + expected_bias = ep_layer.mlp.expert_bias.clone() + expected_bias += ep_layer.mlp.expert_bias_update_rate * torch.sign( + counts.float().mean() - counts.float() + ) + ep_layer.mlp._last_tokens_per_expert = counts + ep_layer.mlp.update_expert_bias() + torch.testing.assert_close(ep_layer.mlp.expert_bias, expected_bias, rtol=0, atol=0) + biases = [torch.empty_like(expected_bias) for _ in range(ep_size)] + dist.all_gather(biases, ep_layer.mlp.expert_bias, group=ep_group) + for bias in biases: + torch.testing.assert_close(bias, expected_bias, rtol=0, atol=0) + + +def test_mla_tp_matches_local(rank: int, tp_size: int, tp_group) -> None: + """Compare sharded MLA outputs and gradients with an unsharded reference.""" + torch.backends.cuda.matmul.allow_tf32 = False + for fmt in ("sbhd", "bshd"): + for explicit_size in (False, True): + torch.manual_seed(0) + common = dict(params_dtype=torch.float32, qkv_format=fmt, **MLA_KWARGS) + ref = MultiLatentAttention(HIDDEN, 2 * tp_size, **common) + _broadcast_params(ref) + tp = MultiLatentAttention( + HIDDEN, + 2 * tp_size, + tp_group=tp_group, + **({"tp_size": tp_size} if explicit_size else {}), + **common, + ) + ref_params = dict(ref.named_parameters()) + + def shard(name, tensor): + if name in ("q_up_proj.weight", "kv_up_proj.weight"): + return tensor.chunk(tp_size, dim=0)[rank] + if name == "out_proj.weight": + return tensor.chunk(tp_size, dim=1)[rank] + return tensor + + with torch.no_grad(): + for name, param in tp.named_parameters(): + param.copy_(shard(name, ref_params[name])) + + torch.manual_seed(1234) + shape = (16, 2, HIDDEN) if fmt == "sbhd" else (2, 16, HIDDEN) + x = torch.randn(shape, device="cuda", requires_grad=True) + x_ref = x.detach().clone().requires_grad_() + out = tp(x) + out_ref = ref(x_ref) + torch.testing.assert_close(out, out_ref, rtol=1e-3, atol=1e-3) + grad = torch.randn_like(out) + out.backward(grad.clone()) + out_ref.backward(grad.clone()) + torch.testing.assert_close(x.grad, x_ref.grad, rtol=1e-3, atol=1e-3) + for name, param in tp.named_parameters(): + torch.testing.assert_close( + param.grad, + shard(name, ref_params[name].grad), + rtol=1e-3, + atol=1e-3, + msg=name, + ) + + +def main() -> int: + dist.init_process_group(backend="nccl") + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + if "--tp" in sys.argv: + test_mla_tp_matches_local(dist.get_rank(), dist.get_world_size(), dist.group.WORLD) + print(f"[rank {dist.get_rank()}] TP PASSED") + dist.destroy_process_group() + return 0 + from torch.distributed import _symmetric_memory as _symm_mem + + _symm_mem.set_backend("NCCL") + + rank = dist.get_rank() + ep_size = dist.get_world_size() + if _device_sm() < 90: + if rank == 0: + print(f"NCCL EP requires SM>=90 (got SM{_device_sm()}); skipping.") + dist.destroy_process_group() + return 0 + + ep_group = dist.new_group(ranks=list(range(ep_size)), backend="nccl") + ep_bootstrap( + ep_group, + num_experts=NUM_LOCAL_EXPERTS * ep_size, + max_tokens_per_rank=TOKENS_PER_RANK, + hidden_dim=HIDDEN, + num_topk=TOP_K, + recv_capacity_per_rank=None if "--eager" in sys.argv else _recv_capacity(ep_size), + ) + for num_microbatches in (1, 3): + test_layer_ep_matches_local(rank, ep_size, ep_group, num_microbatches) + print(f"[rank {rank}] PASSED") + + dist.barrier() + ep_finalize() + release_symm_mem_pool() + dist.destroy_process_group() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/pytorch/distributed/test_models.py b/tests/pytorch/distributed/test_models.py new file mode 100644 index 00000000000..d748b6bac76 --- /dev/null +++ b/tests/pytorch/distributed/test_models.py @@ -0,0 +1,46 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +import os +import subprocess +from pathlib import Path + +import pytest +import torch + +TEST_ROOT = Path(__file__).parent.resolve() +NUM_PROCS = min(8, torch.cuda.device_count()) +LAUNCH_CMD = ["torchrun", f"--nproc_per_node={NUM_PROCS}"] + + +def _has_nvlink() -> bool: + # NCCL EP falls back to the network transport and deadlocks on PCIe-only nodes. + out = subprocess.run( + ["nvidia-smi", "nvlink", "--status"], capture_output=True, text=True, check=False + ).stdout + return "GB/s" in out + + +@pytest.mark.skipif(NUM_PROCS < 2, reason="EP requires >= 2 GPUs") +@pytest.mark.skipif(not _has_nvlink(), reason="NCCL EP requires NVLink") +@pytest.mark.parametrize("eager", [False, True], ids=["fixed", "eager"]) +def test_deepseek_layer_ep(eager): + result = subprocess.run( + LAUNCH_CMD + [str(TEST_ROOT / "run_models.py")] + (["--eager"] if eager else []), + env=os.environ, + check=False, + timeout=300, + ) + assert result.returncode == 0 + + +@pytest.mark.skipif(NUM_PROCS < 2, reason="TP requires >= 2 GPUs") +def test_mla_tp(): + result = subprocess.run( + LAUNCH_CMD + [str(TEST_ROOT / "run_models.py"), "--tp"], + env=os.environ, + check=False, + timeout=300, + ) + assert result.returncode == 0 diff --git a/tests/pytorch/mxfp8/test_mxfp8_cutedsl_backend.py b/tests/pytorch/mxfp8/test_mxfp8_cutedsl_backend.py index 5d7feeb6ce6..57436eeb720 100644 --- a/tests/pytorch/mxfp8/test_mxfp8_cutedsl_backend.py +++ b/tests/pytorch/mxfp8/test_mxfp8_cutedsl_backend.py @@ -196,9 +196,14 @@ def fill(): return fill(), fill() -def run_quantize(method, act, x, ain, rowwise, columnwise, fp8_dtype, swizzled): +def run_quantize(method, act, x, ain, rowwise, columnwise, fp8_dtype, swizzled, use_2d_quant): """Quantize via the public dispatch; returns (mxfp8_tensor, dbias_or_None).""" - q = MXFP8Quantizer(fp8_dtype=fp8_dtype, rowwise=rowwise, columnwise=columnwise) + q = MXFP8Quantizer( + fp8_dtype=fp8_dtype, + rowwise=rowwise, + columnwise=columnwise, + with_2d_quantization=use_2d_quant, + ) # Emit scales in the GEMM-swizzled layout (MXFP8QuantConfig::swizzled). q.optimize_for_gemm = swizzled if method == "CAST_ONLY": @@ -216,7 +221,7 @@ def run_quantize(method, act, x, ain, rowwise, columnwise, fp8_dtype, swizzled): raise ValueError(f"unknown method {method!r}") -def get_cfg_key(method, act, in_dtype, fp8_dtype, rowwise, colwise, swizzled): +def get_cfg_key(method, act, in_dtype, fp8_dtype, rowwise, colwise, swizzled, use_2d_quant): """Mirror of MXFP8QuantConfig::to_key (quantize_mxfp8_cutedsl.cuh): the name the CuTeDSL backend registers its compiled kernel under for this config. Used to check if the CuTeDSL implementation is registered """ @@ -229,8 +234,7 @@ def get_cfg_key(method, act, in_dtype, fp8_dtype, rowwise, colwise, swizzled): elif with_dact: desc = f"d{act['desc']}" # with_amax is hardcoded to False for now because currently MXFP8 quantizers never allocate an amax - # The last False is for use_2d_quantization; these cases never request 2D block scaling - flags = (rowwise, colwise, swizzled, False, with_dbias, with_dact, with_act, False) + flags = (rowwise, colwise, swizzled, False, with_dbias, with_dact, with_act, use_2d_quant) major, minor = device_compute_capability() return ( f"cutedsl_mxfp8_sm{major * 10 + minor}_" @@ -271,7 +275,15 @@ def extract_quantized_output(out, rowwise, columnwise, swizzled): def run_test_case( - method, act, shape, block_size, in_dtype, fp8_dtype, swizzled=False, input_offset=0 + method, + act, + shape, + block_size, + in_dtype, + fp8_dtype, + swizzled=False, + input_offset=0, + use_2d_quant=False, ): """Assert the CuTeDSL and CUDA backends produce bit-identical outputs for the same input and config. input_offset (in elements) shifts x's data pointer past the @@ -287,14 +299,14 @@ def run_test_case( set_cutedsl_backend(False) out_cuda, dbias_cuda = run_quantize( - method, act, x, act_input, rowwise, columnwise, fp8_dtype, swizzled + method, act, x, act_input, rowwise, columnwise, fp8_dtype, swizzled, use_2d_quant ) cuda_output = extract_quantized_output(out_cuda, rowwise, columnwise, swizzled) set_cutedsl_backend(True) try: out_cutedsl, dbias_cutedsl = run_quantize( - method, act, x, act_input, rowwise, columnwise, fp8_dtype, swizzled + method, act, x, act_input, rowwise, columnwise, fp8_dtype, swizzled, use_2d_quant ) cutedsl_output = extract_quantized_output(out_cutedsl, rowwise, columnwise, swizzled) finally: @@ -304,7 +316,7 @@ def run_test_case( # CuTeDSL backend supports, so its kernel must have been registered under the # config key. If not, the backend rejected or missed the config and the # comparison above was CUDA vs CUDA. - key = get_cfg_key(method, act, in_dtype, fp8_dtype, rowwise, columnwise, swizzled) + key = get_cfg_key(method, act, in_dtype, fp8_dtype, rowwise, columnwise, swizzled, use_2d_quant) assert tvm_ffi.get_global_func(key, allow_missing=True) is not None, ( f"CuTeDSL kernel not registered for {key}; the CuTeDSL backend fell back " "to CUDA and this case compared CUDA against itself" @@ -314,6 +326,8 @@ def run_test_case( tag = ( f"{method}/{act['name']}/{M}x{N}/{DTYPE_TO_STR[in_dtype]}/{FP8_TO_STR[fp8_dtype]}/{layout}" ) + if use_2d_quant: + tag += "/2d" for name, cuda_bytes in cuda_output.items(): assert torch.equal( cutedsl_output[name], cuda_bytes @@ -334,6 +348,18 @@ def test_cast_only(swizzled, fp8_dtype, in_dtype, block_size, shape): run_test_case("CAST_ONLY", IDENTITY, shape, block_size, in_dtype, fp8_dtype, swizzled) +# 2D block scaling (one scale per 32x32 block) is only reachable as a plain cast. +@pytest.mark.parametrize("shape", MATRIX_SIZES, ids=get_shape_id) +@pytest.mark.parametrize("block_size", BLOCK_SIZES, ids=get_block_id) +@pytest.mark.parametrize("in_dtype", IN_DTYPES, ids=get_dtype_id) +@pytest.mark.parametrize("fp8_dtype", FP8_DTYPES, ids=get_fp8_id) +@pytest.mark.parametrize("swizzled", SWIZZLE_MODES, ids=get_swizzle_id) +def test_cast_only_2d(swizzled, fp8_dtype, in_dtype, block_size, shape): + run_test_case( + "CAST_ONLY", IDENTITY, shape, block_size, in_dtype, fp8_dtype, swizzled, use_2d_quant=True + ) + + def test_cast_only_rowwise_specialized_grid_y_overflow(): """Fall back when the specialized rowwise launch would exceed CUDA's grid.y limit.""" # The specialized kernel covers four rows per CTA, so this shape would require @@ -431,3 +457,43 @@ def test_specialized_kernels(shape, block_size, in_dtype, swizzled, input_offset swizzled, input_offset, ) + + +OUT_DTYPES = [torch.float32, torch.bfloat16, torch.float16] +DEQUANT_SHAPES = MATRIX_SIZES + [(100, 48)] + + +# Test cases for dequantization (mirrors C++ test's OperatorTest_DequantizeMXFP8). +# With both data copies present, dequantize reads the rowwise one. +@pytest.mark.parametrize("shape", DEQUANT_SHAPES, ids=get_shape_id) +@pytest.mark.parametrize("block_size", BLOCK_SIZES, ids=get_block_id) +@pytest.mark.parametrize("out_dtype", OUT_DTYPES, ids=get_dtype_id) +@pytest.mark.parametrize("fp8_dtype", FP8_DTYPES, ids=get_fp8_id) +@pytest.mark.parametrize("swizzled", SWIZZLE_MODES, ids=get_swizzle_id) +def test_dequantize(swizzled, fp8_dtype, out_dtype, block_size, shape): + M, N = shape + rowwise = block_size[1] != 1 + columnwise = block_size[0] != 1 + x, _ = generate_inputs(M, N, torch.bfloat16) + set_cutedsl_backend(False) + q = MXFP8Quantizer(fp8_dtype=fp8_dtype, rowwise=rowwise, columnwise=columnwise) + q.optimize_for_gemm = swizzled + qx = q(x) + out_cuda = qx.dequantize(dtype=out_dtype) + + set_cutedsl_backend(True) + try: + out_cutedsl = qx.dequantize(dtype=out_dtype) + finally: + set_cutedsl_backend(False) + + major, minor = device_compute_capability() + key = ( + f"cutedsl_mxfp8_dequant_sm{major * 10 + minor}_{FP8_TO_KEY[fp8_dtype]}_" + f"{DTYPE_TO_STR[out_dtype]}_{int(rowwise)}_{int(columnwise)}_{int(swizzled)}" + ) + assert tvm_ffi.get_global_func(key, allow_missing=True) is not None, ( + f"CuTeDSL kernel not registered for {key}; the CuTeDSL backend fell back " + "to CUDA and this case compared CUDA against itself" + ) + assert torch.equal(out_cutedsl, out_cuda) diff --git a/tests/pytorch/mxfp8/test_mxfp8_group_quantize_graph_safe.py b/tests/pytorch/mxfp8/test_mxfp8_group_quantize_graph_safe.py index 7a1edc071e9..26fb7c5c65b 100644 --- a/tests/pytorch/mxfp8/test_mxfp8_group_quantize_graph_safe.py +++ b/tests/pytorch/mxfp8/test_mxfp8_group_quantize_graph_safe.py @@ -804,6 +804,23 @@ def _requantize_setup(M: int = 1024, N: int = 256): return x, torch.tensor(split_sections, dtype=torch.int64, device="cuda"), len(split_sections) +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +@pytest.mark.parametrize("columnwise", [False, True]) +def test_prequantized_requantize_empty_gemm_metadata(columnwise): + """Empty dispatch results must still advertise GEMM-ready scale layout.""" + inputs, splits, num_groups = _requantize_setup(M=0, N=512) + wire = make_prequantized_wire_tensor(inputs, splits) + tex.group_requantize_inplace( + wire, make_op_quantizer(columnwise), num_groups, splits, te.DType.kBFloat16 + ) + assert wire._with_gemm_swizzled_scales + assert wire.rowwise_data.numel() == 0 + assert wire.scale_inv.numel() == 0 + if columnwise: + assert wire.columnwise_data is not None + assert wire.columnwise_data.numel() == 0 + + @pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) def test_prequantized_requantize_passes_through_gemm_ready_input(): """A tensor already GEMM-ready in both directions is left untouched.""" diff --git a/tests/pytorch/nvfp4/test_nvfp4_global_scale.py b/tests/pytorch/nvfp4/test_nvfp4_global_scale.py new file mode 100644 index 00000000000..945e6ff7bc1 --- /dev/null +++ b/tests/pytorch/nvfp4/test_nvfp4_global_scale.py @@ -0,0 +1,107 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. +"""Tests for the fused NVFP4 global-scale / GEMM-alpha Triton kernel. + +The kernel replaces the eager ``amax.float() / (6 * scale_max)`` divisions and the +product / square root of the two operand scales used by the grouped MLP. It must be +bit-identical to the eager ops, including with a broadcast (single-group) amax, and must +be safe to capture in a CUDA graph.""" + +import pytest +import torch + +from transformer_engine.pytorch.triton.nvfp4_global_scale import nvfp4_global_scale_pair + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") + +_DENOMS = [6.0 * 448.0, 6.0 * 256.0, 6.0 * 114688.0] + + +def _random_amax(n, device): + # Wide dynamic range plus exact zeros (empty experts) and a few special magnitudes. + amax = torch.exp2(torch.empty(n, device=device).uniform_(-40.0, 40.0)) + amax *= torch.rand(n, device=device) + 0.5 + if n >= 4: + amax[0] = 0.0 + amax[1] = 1.0 + amax[2] = torch.finfo(torch.float32).tiny + amax[3] = 448.0 * 6.0 + return amax + + +def _reference(amax_a, amax_b, denom_a, denom_b, combine): + scale_a = amax_a.to(torch.float32) / denom_a + scale_b = amax_b.to(torch.float32) / denom_b + if combine is None: + return scale_a, scale_b + if combine == "sqrt_product": + return torch.sqrt(scale_a * scale_b) + return (scale_a * scale_b).to(torch.float32) + + +def _assert_bitwise_equal(out, ref): + if isinstance(ref, tuple): + assert isinstance(out, tuple) and len(out) == len(ref) + for o, r in zip(out, ref): + _assert_bitwise_equal(o, r) + return + assert out.dtype == torch.float32 + assert out.shape == ref.shape + assert torch.equal(out.view(torch.int32), ref.contiguous().view(torch.int32)) + + +@pytest.mark.parametrize("n", [1, 3, 8, 32, 257, 4096]) +@pytest.mark.parametrize("combine", [None, "product", "sqrt_product"]) +@pytest.mark.parametrize("denoms", [(_DENOMS[0], _DENOMS[0]), (_DENOMS[1], _DENOMS[2])]) +@pytest.mark.parametrize("broadcast", ["none", "a", "b"]) +def test_matches_eager(n, combine, denoms, broadcast): + torch.manual_seed(n) + device = torch.device("cuda") + amax_a = _random_amax(1 if broadcast == "a" else n, device) + amax_b = _random_amax(1 if broadcast == "b" else n, device) + out = nvfp4_global_scale_pair(amax_a, amax_b, denoms[0], denoms[1], combine=combine) + if combine is None and amax_a.numel() != amax_b.numel(): + # Separate scales keep their own shapes, so the caller falls back to eager. + assert out is None + return + assert out is not None + _assert_bitwise_equal(out, _reference(amax_a, amax_b, denoms[0], denoms[1], combine)) + + +def test_falls_back_on_unsupported_inputs(): + device = torch.device("cuda") + amax = torch.ones(8, device=device) + assert nvfp4_global_scale_pair(amax.half(), amax, 1.0, 1.0) is None + assert nvfp4_global_scale_pair(amax, amax[:3], 1.0, 1.0) is None + assert nvfp4_global_scale_pair(amax.cpu(), amax.cpu(), 1.0, 1.0) is None + big = torch.ones(4097, device=device) + assert nvfp4_global_scale_pair(big, big, 1.0, 1.0) is None + + +@pytest.mark.parametrize("combine", [None, "product", "sqrt_product"]) +def test_cuda_graph_replay(combine): + device = torch.device("cuda") + n = 32 + denom = _DENOMS[0] + amax_a = _random_amax(n, device) + amax_b = _random_amax(n, device) + + # Warm up (compiles the kernel) on a side stream before capture. + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + nvfp4_global_scale_pair(amax_a, amax_b, denom, denom, combine=combine) + torch.cuda.current_stream().wait_stream(stream) + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = nvfp4_global_scale_pair(amax_a, amax_b, denom, denom, combine=combine) + + for seed in range(3): + torch.manual_seed(seed) + amax_a.copy_(_random_amax(n, device)) + amax_b.copy_(_random_amax(n, device)) + graph.replay() + torch.cuda.synchronize() + _assert_bitwise_equal(out, _reference(amax_a, amax_b, denom, denom, combine)) diff --git a/tests/pytorch/nvfp4/test_nvfp4_quantize_exact.py b/tests/pytorch/nvfp4/test_nvfp4_quantize_exact.py index 51b5b46c158..0947e4e744c 100644 --- a/tests/pytorch/nvfp4/test_nvfp4_quantize_exact.py +++ b/tests/pytorch/nvfp4/test_nvfp4_quantize_exact.py @@ -123,6 +123,7 @@ def check_quantization_nvfp4_versus_reference( nvfp4_4over6_err_mode: str = "MAE", nvfp4_4over6_err_use_fast_math: bool = False, scale_dtype: te.DType = te.DType.kFloat8E4M3, + check_amax_reset: bool = False, ) -> None: scale_dtype = te.DType.cast(scale_dtype) if nvfp4_e4m3_max not in NVFP4_SUPPORTED_SCALE_MAX_VALUES: @@ -252,6 +253,19 @@ def check_quantization_nvfp4_versus_reference( torch.testing.assert_close(qx_amax, ref_amax, atol=0.0, rtol=0.0) + if check_amax_reset: + assert not row_scaled_nvfp4 + # Reusing an output whose maximum decreases verifies that every amax buffer is reset. + for magnitude in (16.0, 0.5, 0.0, 2.0): + x = torch.full((M, N), -magnitude, dtype=x_dtype, device=device) + nvfp4_quantizer.update_quantized(x, x_nvfp4_sut) + expected_amax = x.float().abs().amax().reshape(1) + torch.testing.assert_close(x_nvfp4_sut._amax_rowwise, expected_amax, atol=0.0, rtol=0.0) + if return_transpose: + torch.testing.assert_close( + x_nvfp4_sut._amax_columnwise, expected_amax, atol=0.0, rtol=0.0 + ) + @pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) @pytest.mark.parametrize("return_transpose", [False, True], ids=["rowwise", "with_columnwise"]) @@ -359,6 +373,7 @@ def test_quantization_block_tiling_versus_reference( row_scaled_nvfp4: bool, nvfp4_4over6_config: NVFP44Over6TestConfig, ) -> None: + check_amax_reset = not row_scaled_nvfp4 and not nvfp4_4over6_config.use_4over6 check_quantization_nvfp4_versus_reference( x_dtype=x_dtype, M=M, @@ -372,6 +387,7 @@ def test_quantization_block_tiling_versus_reference( nvfp4_e4m3_max=nvfp4_4over6_config.e4m3_max, nvfp4_4over6_err_mode=nvfp4_4over6_config.err_mode, nvfp4_4over6_err_use_fast_math=nvfp4_4over6_config.err_use_fast_math, + check_amax_reset=check_amax_reset, ) @@ -807,29 +823,42 @@ def test_nvfp4_quantization_noncontiguous_inputs( @pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) @pytest.mark.parametrize( - "M, N", + "with_2d_quantization, M, N", [ - # Aligned tiles - (128, 128), - (256, 256), - (512, 512), - (2048, 2048), - # Padded tiles (non-multiple of kTileDim=128) - (256, 272), - (304, 304), - (320, 256), + # Existing 2D coverage: aligned and padded tiles. + (True, 128, 128), + (True, 256, 256), + (True, 512, 512), + (True, 2048, 2048), + (True, 256, 272), + (True, 304, 304), + (True, 320, 256), + # Focused 1D coverage for aligned and padded output reuse. + (False, 128, 128), + (False, 256, 272), + ], + ids=[ + "2d-128x128", + "2d-256x256", + "2d-512x512", + "2d-2048x2048", + "2d-256x272", + "2d-304x304", + "2d-320x256", + "1d-128x128", + "1d-256x272", ], ) @pytest.mark.parametrize("x_dtype", [torch.float32, torch.bfloat16], ids=str) -def test_nvfp4_2d_columnwise_only_matches_both_directions( +def test_nvfp4_columnwise_only_matches_both_directions( x_dtype: torch.dtype, + with_2d_quantization: bool, M: int, N: int, ): - """Bitwise check: 2D NVFP4 with columnwise-only must produce the same - columnwise data/scales as the columnwise half of (rowwise + columnwise) 2D. + """Columnwise-only NVFP4 must match the columnwise half of both-directions. - Covers both kernels depending on the (dtype, shape) routing: + The 2D cases cover both kernels depending on the (dtype, shape) routing: - bf16 with rows % 32 == 0 and cols % 32 == 0 routes to the optimized ``quantize_transpose_nvfp4_2D_kernel`` (instantiated with RETURN_ROWWISE=false), validating that gating the rowwise pass/store leaves the shared @@ -853,7 +882,7 @@ def _make_quantizer(*, rowwise: bool, columnwise: bool) -> NVFP4Quantizer: amax_reduction_group=None, with_rht=False, with_post_rht_amax=False, - with_2d_quantization=True, + with_2d_quantization=with_2d_quantization, row_scaled_nvfp4=False, ) @@ -861,7 +890,7 @@ def _make_quantizer(*, rowwise: bool, columnwise: bool) -> NVFP4Quantizer: q_both = _make_quantizer(rowwise=True, columnwise=True) out_both = q_both(x) - # SUT: produce columnwise only (the path that hits the new amax-only pass). + # SUT: produce columnwise only. q_col_only = _make_quantizer(rowwise=False, columnwise=True) out_col_only = q_col_only(x) @@ -901,5 +930,15 @@ def _make_quantizer(*, rowwise: bool, columnwise: bool) -> NVFP4Quantizer: out_col_only._amax_columnwise, out_both._amax_columnwise, atol=0, rtol=0 ) + if (M, N) in ((128, 128), (256, 272)): + # Reusing a columnwise-only output exercises the path where its amax is primary. + for magnitude in (16.0, 0.5, 0.0, 2.0): + x = torch.full((M, N), -magnitude, dtype=x_dtype, device=device) + q_col_only.update_quantized(x, out_col_only) + expected_amax = x.float().abs().amax().reshape(1) + torch.testing.assert_close( + out_col_only._amax_columnwise, expected_amax, atol=0.0, rtol=0.0 + ) + # Sanity: column-only path must not allocate a rowwise output. assert out_col_only._rowwise_data is None diff --git a/tests/pytorch/test_models.py b/tests/pytorch/test_models.py new file mode 100644 index 00000000000..9d677be2fe2 --- /dev/null +++ b/tests/pytorch/test_models.py @@ -0,0 +1,194 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +import math + +import pytest +import torch + +import transformer_engine.pytorch as te +from transformer_engine.pytorch.ops.fused.grouped_mlp import ( + GroupedMLP_CuTeGEMMGLU, + fuse_glu_ops, +) +from transformer_engine.pytorch.ops.fuser import OperationFuser +from transformer_engine.pytorch.utils import deinterleave_glu_tensor +from transformer_engine.pytorch.models import DeepSeekV3MoE, MultiLatentAttention +from utils import make_recipe, quantization_tols + +SEQ_LEN = 128 +BATCH = 2 +HIDDEN = 256 +HEADS = 4 +DTYPE = torch.bfloat16 + +MLA_KWARGS = dict( + q_lora_rank=96, + kv_lora_rank=64, + qk_nope_head_dim=64, + qk_rope_head_dim=32, + v_head_dim=64, +) + + +def _input(requires_grad=True): + torch.manual_seed(1234) + return torch.randn( + SEQ_LEN, BATCH, HIDDEN, dtype=DTYPE, device="cuda", requires_grad=requires_grad + ) + + +@pytest.mark.parametrize("mscale_all_dim", [0.0, 1.0]) +def test_mla_yarn_softmax_scale(mscale_all_dim): + mla = MultiLatentAttention( + HIDDEN, + HEADS, + params_dtype=DTYPE, + rope_scaling_factor=40.0, + original_max_position_embeddings=64, + mscale_all_dim=mscale_all_dim, + **MLA_KWARGS, + ) + m = 0.1 * mscale_all_dim * math.log(40.0) + 1.0 + qk_head_dim = MLA_KWARGS["qk_nope_head_dim"] + MLA_KWARGS["qk_rope_head_dim"] + assert mla.softmax_scale == pytest.approx(m * m / math.sqrt(qk_head_dim)) + + +@pytest.mark.parametrize("shared", [False, True], ids=["no_shared", "shared"]) +@pytest.mark.parametrize("grouped", [False, True], ids=["ungrouped", "grouped"]) +@pytest.mark.parametrize("topk", [2, 4]) +@pytest.mark.parametrize("quantization", [None, "fp8_block_scaling"]) +def test_moe_matches_dense_reference(shared, grouped, topk, quantization): + """Routed output must equal the prob-weighted sum of the selected expert MLPs.""" + if quantization is not None: + available, reason = te.is_fp8_block_scaling_available(return_reason=True) + if not available: + pytest.skip(reason) + torch.manual_seed(0) + num_experts = 4 + moe = DeepSeekV3MoE( + HIDDEN, + moe_ffn_hidden_size=128, + num_experts=num_experts, + topk=topk, + num_groups=2 if grouped else None, + group_topk=topk // 2 if grouped else None, + shared_expert_ffn_hidden_size=128 if shared else None, + params_dtype=DTYPE, + ) + x = _input() + with te.autocast(enabled=quantization is not None, recipe=make_recipe(quantization)): + out = moe(x) + assert out.shape == x.shape + assert torch.isfinite(out).all() + out.sum().backward() + assert torch.isfinite(x.grad).all() + for name, parameter in moe.named_parameters(): + if parameter.requires_grad: + assert parameter.grad is not None, name + assert torch.isfinite(parameter.grad).all(), name + + tokens = x.detach().reshape(-1, HIDDEN) + probs, _ = moe._route(moe.gate(tokens).float()) + assert (probs > 0).sum(dim=1).eq(topk).all() + assert moe._last_tokens_per_expert.sum().item() == tokens.shape[0] * topk + + fc1, _, fc2 = moe.experts + quantizers = [ + te.Float8BlockQuantizer( + fp8_dtype=te.DType.kFloat8E4M3, + rowwise=True, + columnwise=False, + block_scaling_dim=dim, + ) + for dim in (1, 2) + ] + + def qdq(tensor, block_dim): + if quantization is None: + return tensor + return quantizers[block_dim - 1](tensor).dequantize(dtype=DTYPE) + + ref_tokens = qdq(tokens, 1) + ref = torch.zeros_like(tokens) + for e in range(num_experts): + w1 = deinterleave_glu_tensor(qdq(getattr(fc1, f"weight{e}"), 2), 32) + w2 = qdq(getattr(fc2, f"weight{e}"), 2) + gate_part, lin_part = (ref_tokens @ w1.t()).chunk(2, dim=-1) + act = torch.nn.functional.silu(gate_part.float()) * lin_part.float() + act = qdq(act.to(DTYPE) * probs[:, e : e + 1].to(DTYPE), 1) + ref += act @ w2.t() + if shared: + with te.autocast(enabled=quantization is not None, recipe=make_recipe(quantization)): + ref += moe.shared_expert(tokens) + torch.testing.assert_close(out.reshape(-1, HIDDEN), ref, rtol=0.05, atol=0.05) + + bias_before = moe.expert_bias.clone() + moe.update_expert_bias() + assert torch.isfinite(moe.expert_bias).all() + if topk < num_experts: + assert not torch.equal(bias_before, moe.expert_bias) + + +@pytest.mark.parametrize("quantization", ["mxfp8", "nvfp4_rht"]) +def test_moe_fused_quantized_uneven_expert_rows(monkeypatch, quantization): + available_fn = te.is_mxfp8_available if quantization == "mxfp8" else te.is_nvfp4_available + available, reason = available_fn(return_reason=True) + if not available: + pytest.skip(reason) + + recipe = make_recipe(quantization) + torch.manual_seed(0) + moe = DeepSeekV3MoE(HIDDEN, 128, num_experts=2, topk=1, params_dtype=DTYPE) + fused_ops = fuse_glu_ops(list(moe.experts), recipe=recipe) + if ( + fuse_glu_ops not in OperationFuser.forward_backward_fusion_functions + or len(fused_ops) != 1 + or not isinstance(fused_ops[0], GroupedMLP_CuTeGEMMGLU) + ): + pytest.skip("requires the fused grouped MLP") + reference = DeepSeekV3MoE(HIDDEN, 128, num_experts=2, topk=1, params_dtype=DTYPE) + reference.load_state_dict(moe.state_dict()) + with torch.no_grad(): + for module in (moe, reference): + module.gate.weight.zero_() + module.gate.weight[0, 0] = 1 + module.gate.weight[1, 0] = -1 + + tokens = torch.empty(512, HIDDEN, device="cuda", dtype=DTYPE).uniform_(-0.25, 0.25) + tokens[:128, 0] = 2 + tokens[128:, 0] = -2 + x = tokens.detach().requires_grad_() + x_ref = tokens.detach().clone().requires_grad_() + grad = torch.empty_like(tokens).uniform_(-0.25, 0.25) + splits = [] + moe.experts.register_forward_pre_hook(lambda _module, args: splits.append(args[1].clone())) + + with te.autocast(enabled=True, recipe=recipe): + out = moe(x) + out.backward(grad.clone()) + assert torch.equal(splits[0], torch.tensor([256, 512], device="cuda")) + fused_op = moe.experts._module_groups[0]._forward_ops[0][0] + assert isinstance(fused_op, GroupedMLP_CuTeGEMMGLU) + + monkeypatch.setattr( + OperationFuser, + "forward_backward_fusion_functions", + [fn for fn in OperationFuser.forward_backward_fusion_functions if fn is not fuse_glu_ops], + ) + ref_splits = [] + reference.experts.register_forward_pre_hook( + lambda _module, args: ref_splits.append(args[1].clone()) + ) + with te.autocast(enabled=True, recipe=make_recipe(quantization)): + ref_out = reference(x_ref) + ref_out.backward(grad.clone()) + assert torch.equal(ref_splits[0], splits[0]) + + tols = quantization_tols(quantization) + torch.testing.assert_close(out, ref_out, **tols) + torch.testing.assert_close(x.grad, x_ref.grad, **tols) + ref_params = dict(reference.named_parameters()) + for name, param in moe.named_parameters(): + torch.testing.assert_close(param.grad, ref_params[name].grad, **tols) diff --git a/tests/pytorch/test_recipe.py b/tests/pytorch/test_recipe.py index 9558003eda6..e1447999db2 100644 --- a/tests/pytorch/test_recipe.py +++ b/tests/pytorch/test_recipe.py @@ -12,6 +12,7 @@ import transformer_engine.common.recipe import transformer_engine.pytorch as te +import transformer_engine.pytorch.quantization as quantization from transformer_engine.pytorch import ( Float8BlockQuantizer, MXFP8Quantizer, @@ -57,6 +58,37 @@ fp4_available, reason_for_no_fp4 = te.is_nvfp4_available(return_reason=True) +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +def test_autocast_reuse_after_failed_entry(monkeypatch): + monkeypatch.setattr(FP8GlobalStateManager, "quantization_state", quantization.FP8GlobalState()) + original_check = quantization.check_recipe_support + fail_next_check = True + + def fail_once(recipe): + nonlocal fail_next_check + if fail_next_check: + fail_next_check = False + raise RuntimeError("recipe validation failed") + original_check(recipe) + + monkeypatch.setattr(quantization, "check_recipe_support", fail_once) + context = te.autocast(recipe=Float8CurrentScaling()) + with te.autocast(enabled=False, calibrating=True, recipe=Float8CurrentScaling()): + saved_state = FP8GlobalStateManager.get_autocast_state() + saved_depth = FP8GlobalStateManager.quantization_state.autocast_depth + with pytest.raises(RuntimeError, match="recipe validation failed"): + with context: + pytest.fail("Autocast body ran after failed validation") + assert FP8GlobalStateManager.get_autocast_state() == saved_state + assert FP8GlobalStateManager.quantization_state.autocast_depth == saved_depth + + with context: + assert FP8GlobalStateManager.is_fp8_enabled() + assert FP8GlobalStateManager.quantization_state.autocast_depth == saved_depth + 1 + assert FP8GlobalStateManager.get_autocast_state() == saved_state + assert FP8GlobalStateManager.quantization_state.autocast_depth == saved_depth + + # FP8 per tensor delayed scaling @pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) class TestFP8Recipe: diff --git a/tests/pytorch/test_sanity.py b/tests/pytorch/test_sanity.py index 2627de023ac..0fe2bacc6cf 100644 --- a/tests/pytorch/test_sanity.py +++ b/tests/pytorch/test_sanity.py @@ -35,6 +35,7 @@ is_bf16_available, ) from transformer_engine.common import recipe +from transformer_engine.pytorch.models import DeepSeekV3Layer from transformer_engine.pytorch.cpp_extensions import general_gemm from transformer_engine.pytorch.tensor.utils import replace_raw_data from transformer_engine.pytorch.module import ( @@ -285,7 +286,7 @@ def _test_sanity_e2e_gradient_accumulation_fusion(block, dtype, config, fp8_reci ), f"grad_added_to_main_grad not set to True for {failed_grad_added_flags}." -def _test_sanity_e2e(block, dtype, config, fp8_recipe, skip_wgrad): +def _test_sanity_e2e(block, dtype, config, fp8_recipe, skip_wgrad, *, check_finite=False): te_inp_hidden_states = torch.randn( (config.max_seqlen_q, config.batch_size, config.hidden_size), dtype=dtype, @@ -303,6 +304,14 @@ def _test_sanity_e2e(block, dtype, config, fp8_recipe, skip_wgrad): loss.backward() torch.cuda.synchronize() + if check_finite: + assert torch.isfinite(te_out).all(), "Non-finite output" + assert torch.isfinite(te_inp_hidden_states.grad).all(), "Non-finite input gradient" + for name, parameter in block.named_parameters(): + if parameter.requires_grad: + assert parameter.grad is not None, name + assert torch.isfinite(parameter.grad).all(), name + def _test_sanity_e2e_bert(block, dtype, config, fp8_recipe, skip_wgrad): te_inp_hidden_states = torch.randn( @@ -858,6 +867,41 @@ def checked_norm(*args, _norm=norm_, _suffix=suffix, **kwargs): assert set(seen_norm_stages) == {"fwd", "bwd"} +@pytest.mark.parametrize("dtype", param_types) +@pytest.mark.parametrize("fp8_recipe", fp8_recipes, ids=recipe_id) +@pytest.mark.parametrize("moe", all_boolean, ids=["moe", "dense"]) +def test_sanity_deepseek_v3_layer(dtype, fp8_recipe, moe): + config = model_configs["small"] + + if fp8_recipe is not None: + if not is_fp8_supported(config): + pytest.skip("Model config does not support FP8") + if fp8_recipe.nvfp4() and dtype == torch.float16: + pytest.skip("FP16 output for NVFP4 not supported") + if fp8_recipe.nvfp4() and moe and dtype != torch.bfloat16: + pytest.skip("NVFP4 GroupedLinear requires BF16") + + mlp_kwargs = ( + dict(num_experts=4, topk=2, moe_ffn_hidden_size=64, shared_expert_ffn_hidden_size=64) + if moe + else dict(ffn_hidden_size=4 * config.hidden_size) + ) + block = DeepSeekV3Layer( + config.hidden_size, + config.num_heads, + q_lora_rank=64, + kv_lora_rank=64, + qk_nope_head_dim=32, + qk_rope_head_dim=32, + v_head_dim=32, + params_dtype=dtype, + device="cuda", + **mlp_kwargs, + ) + + _test_sanity_e2e(block, dtype, config, fp8_recipe, skip_wgrad=False, check_finite=True) + + @pytest.mark.parametrize("dtype", param_types) @pytest.mark.parametrize("fp8_recipe", fp8_recipes, ids=recipe_id) @pytest.mark.parametrize("model", ["small"]) diff --git a/transformer_engine/common/CuTeDSL/__init__.py b/transformer_engine/common/CuTeDSL/__init__.py index 15aae1aba2d..24f4b3202bc 100644 --- a/transformer_engine/common/CuTeDSL/__init__.py +++ b/transformer_engine/common/CuTeDSL/__init__.py @@ -10,6 +10,9 @@ import tvm_ffi +from transformer_engine.common.CuTeDSL.cast.mxfp8.dequantize_mxfp8 import ( + get_mxfp8_dequantization_function, +) from transformer_engine.common.CuTeDSL.cast.mxfp8.quantize_mxfp8_common import ( get_mxfp8_quantization_function, ) @@ -25,6 +28,9 @@ def register_cutedsl_backends(): tvm_ffi.register_global_func( "get_mxfp8_quantization_function", get_mxfp8_quantization_function, override=True ) + tvm_ffi.register_global_func( + "get_mxfp8_dequantization_function", get_mxfp8_dequantization_function, override=True + ) tvm_ffi.register_global_func( "get_mxfp8_requantization_function", get_mxfp8_requantization_function, override=True ) diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/__init__.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/__init__.py index 444d85234ec..95f3b89882e 100644 --- a/transformer_engine/common/CuTeDSL/cast/mxfp8/__init__.py +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/__init__.py @@ -2,6 +2,7 @@ # # See LICENSE for license information. -"""CuTeDSL MXFP8 quantization kernels.""" +"""CuTeDSL MXFP8 quantization and dequantization kernels.""" +from . import dequantize_mxfp8 from . import quantize_mxfp8_common diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/dequantize_mxfp8.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/dequantize_mxfp8.py new file mode 100644 index 00000000000..cdf5b435ef8 --- /dev/null +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/dequantize_mxfp8.py @@ -0,0 +1,455 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""CuTeDSL MXFP8 dequantization kernel, with its compilation and TVM-FFI registration.""" + +# Local @cute.struct classes are shared-memory storage descriptors. +# pylint: disable=missing-class-docstring + +import logging +import cutlass +from cutlass import cute +from cutlass import pipeline +from cutlass import Float8E8M0FNU, Float32, Int32 +from cuda.bindings.driver import CUstream # pylint: disable=no-name-in-module +import tvm_ffi + +from transformer_engine.common.CuTeDSL.utils import ( + device_compute_capability, + exp2f, + str_to_cutlass_dtype, +) +from .quantize_mxfp8_common import ( + CUTEDSL_DEBUG_LOGGING, + MXFP8_BLOCK_SCALING_SIZE, + SYM_N_DIVISIBILITY, + THREADS_PER_WARP, + derive_swizzled_scale_layout, +) + +logger = logging.getLogger("transformer_engine.cutedsl.mxfp8") + + +@cute.jit +def dequantize_mxfp8_smem_tile( + sQ: cute.Tensor, # Input quantized SMEM tile + sO: cute.Tensor, # Output dequantized SMEM tile + gS: cute.Tensor, # This tile's E8M0 scales in GMEM + TILE_ROWS: cutlass.Constexpr[int], # How many rows in the tile + TILE_COLS: cutlass.Constexpr[int], # How many columns in the tile + ELEMS_PER_THREAD: cutlass.Constexpr[int], # How many values each thread dequantizes + DEQUANTIZE_FROM_ROWWISE: cutlass.Constexpr[bool], # Which direction to dequantize from +): + """Dequantize a smem tile sQ into sO using the scales in gS. + + Requires CTA threads multiplied by ELEMS_PER_THREAD to cover the tile. + + sQ, sO: (TILE_ROWS, TILE_COLS) + gS: + - rowwise: (TILE_ROWS, max(1, TILE_COLS // MXFP8_BLOCK_SCALING_SIZE)) + - colwise: (max(1, TILE_ROWS // MXFP8_BLOCK_SCALING_SIZE), TILE_COLS) + """ + + tidx, _, _ = cute.arch.thread_idx() + + # MXFP8_BLOCK_SCALING_SIZE must be divisible by ELEMS_PER_THREAD + assert MXFP8_BLOCK_SCALING_SIZE % ELEMS_PER_THREAD == 0 + FRAGS_SHARING_SAME_SCALE = MXFP8_BLOCK_SCALING_SIZE // ELEMS_PER_THREAD + if cutlass.const_expr(DEQUANTIZE_FROM_ROWWISE): + assert TILE_COLS % ELEMS_PER_THREAD == 0 + THR_ROWS = TILE_ROWS + THR_COLS = TILE_COLS // ELEMS_PER_THREAD + # Row-major threads: a warp reads contiguous 16 B chunks along a row (no bank conflicts) + thr_layout = cute.make_layout((THR_ROWS, THR_COLS), stride=(THR_COLS, 1)) + val_layout = cute.make_layout((1, ELEMS_PER_THREAD), stride=(0, 1)) + scale_y = tidx // THR_COLS + scale_x = tidx % THR_COLS + # Every `FRAGS_SHARING_SAME_SCALE` threads along x direction share the same scale + scale_e8m0 = gS[scale_y, scale_x // FRAGS_SHARING_SAME_SCALE] + else: + assert TILE_ROWS % ELEMS_PER_THREAD == 0 and MXFP8_BLOCK_SCALING_SIZE % TILE_ROWS == 0 + THR_ROWS = TILE_ROWS // ELEMS_PER_THREAD + THR_COLS = TILE_COLS + thr_layout = cute.make_layout((THR_ROWS, THR_COLS)) + val_layout = cute.make_layout((ELEMS_PER_THREAD, 1), stride=(1, 0)) + scale_y = tidx % THR_ROWS + scale_x = tidx // THR_ROWS + # Every `FRAGS_SHARING_SAME_SCALE` threads along y direction share the same scale + scale_e8m0 = gS[scale_y // FRAGS_SHARING_SAME_SCALE, scale_x] + + _, tv_layout = cute.make_layout_tv( + thr_layout=thr_layout, + val_layout=val_layout, + ) + + sQ_frag = cute.composition(sQ, tv_layout)[tidx, None] + sO_frag = cute.composition(sO, tv_layout)[tidx, None] + + rQ_frag = cute.make_rmem_tensor(cute.make_layout(ELEMS_PER_THREAD), sQ.element_type) + rO_frag = cute.make_rmem_tensor(cute.make_layout(ELEMS_PER_THREAD), sO.element_type) + + cute.autovec_copy(sQ_frag, rQ_frag) + + scale_f32 = exp2f(scale_e8m0) + rO_frag.store((rQ_frag.load().to(Float32) * scale_f32).to(sO.element_type)) + + cute.autovec_copy(rO_frag, sO_frag) + + +class MXFP8DequantizeConfig: + """Compile-time configuration of the CuTeDSL MXFP8 dequantize kernel.""" + + def __init__( + self, + fp8_dtype: str, + out_dtype: str, + rowwise: bool, + colwise: bool, + with_gemm_swizzled_scales: bool, + ): + if not (rowwise or colwise): + raise ValueError("at least one of rowwise and columnwise data must be present") + if fp8_dtype not in ("Float8E4M3", "Float8E5M2"): + raise ValueError( + f"unknown FP8 dtype {fp8_dtype!r}; expected 'Float8E4M3' or 'Float8E5M2'" + ) + if out_dtype not in ("Float32", "Float16", "BFloat16"): + raise ValueError( + f"unknown output dtype {out_dtype!r}; expected Float32|Float16|BFloat16" + ) + self.FP8_DTYPE = str_to_cutlass_dtype(fp8_dtype) + self.FP8_DTYPE_STR = fp8_dtype + self.OUT_DTYPE = str_to_cutlass_dtype(out_dtype) + self.OUT_DTYPE_STR = out_dtype + # Which data copies (each with its scales) the caller passes. Both are (M, N) row-major. + self.ROWWISE = rowwise + self.COLWISE = colwise + self.WITH_GEMM_SWIZZLED_SCALES = with_gemm_swizzled_scales + + def __str__(self): + return ( + f"MXFP8DequantizeConfig(fp8_dtype={self.FP8_DTYPE_STR}, out_dtype={self.OUT_DTYPE_STR}," + f" rowwise={self.ROWWISE}, colwise={self.COLWISE}," + f" swizzled={self.WITH_GEMM_SWIZZLED_SCALES})" + ) + + __repr__ = __str__ + + +class MXFP8DequantizeKernel: + """MXFP8 dequantize with staged TMA I/O, a 1:1 port of dequantize_mxfp8_kernel in + cast/mxfp8/dequantize_mxfp8.cuh (same tiling, thread mapping, barriers and pipelining). + + Each CTA processes _NUM_TILES (_TILE_ROWS, _TILE_COLS) tiles stacked vertically through a + _NUM_STAGES-deep smem ring. The next tile's TMA load is issued before the current one is + consumed, and the output is written back with a TMA store. + """ + + _NUM_STAGES = 2 + _NUM_TILES = 8 # Each CTA handles 8 tiles stacked vertically + + _THREADS_PER_CTA = 128 + _NUM_WARPS = _THREADS_PER_CTA // THREADS_PER_WARP + # How many rows and columns are processed by a CTA at a time + # Rowwise: each thread owns _ELEMS_PER_THREAD elements in a row + # Colwise: each thread owns _ELEMS_PER_THREAD elements in a column + _TILE_ROWS = 16 + _TILE_COLS = _THREADS_PER_CTA + _ELEMS_PER_THREAD = _TILE_ROWS * _TILE_COLS // _THREADS_PER_CTA # 16 + + # How many rows and columns are processed by a CTA in total + CTA_BLOCK_ROWS = _TILE_ROWS * _NUM_TILES + CTA_BLOCK_COLS = _TILE_COLS + + def __init__(self, cfg: MXFP8DequantizeConfig): + self.cfg = cfg + # Prefer using rowwise quantization result to dequantize + self.DEQUANTIZE_FROM_ROWWISE = cfg.ROWWISE + + @cute.jit + def __call__( + self, + mX_row: cute.Tensor, + mS_row: cute.Tensor, + mX_col: cute.Tensor, + mS_col: cute.Tensor, + mO: cute.Tensor, + stream: CUstream, + ): + if cutlass.const_expr(CUTEDSL_DEBUG_LOGGING): + cute.printf(f"[CuTeDSL] MXFP8DequantizeKernel.__call__() with config: {self.cfg}\n") + + mX = mX_row if cutlass.const_expr(self.DEQUANTIZE_FROM_ROWWISE) else mX_col + mS = mS_row if cutlass.const_expr(self.DEQUANTIZE_FROM_ROWWISE) else mS_col + M, N = mX.shape + if cutlass.const_expr(self.cfg.WITH_GEMM_SWIZZLED_SCALES): + if cutlass.const_expr(self.DEQUANTIZE_FROM_ROWWISE): + mS, _ = derive_swizzled_scale_layout(M, N, True, False, mS, None) + else: + _, mS = derive_swizzled_scale_layout(M, N, False, True, None, mS) + + # Unswizzled boxes, like create_2D_tensor_map's default. + tiler = (self._TILE_ROWS, self._TILE_COLS) + smem_tile_layout = cute.make_ordered_layout(tiler, order=(1, 0)) + tma_atom_in, tma_in = cute.nvgpu.cpasync.make_tiled_tma_atom( + cute.nvgpu.cpasync.CopyBulkTensorTileG2SOp(), mX, smem_tile_layout, tiler + ) + tma_atom_out, tma_out = cute.nvgpu.cpasync.make_tiled_tma_atom( + cute.nvgpu.cpasync.CopyBulkTensorTileS2GOp(), mO, smem_tile_layout, tiler + ) + + grid = [ + cute.ceil_div(Int32(N), self.CTA_BLOCK_COLS), + cute.ceil_div(M, self.CTA_BLOCK_ROWS), + 1, + ] + self.kernel(mS, tma_atom_in, tma_in, tma_atom_out, tma_out).launch( + grid=grid, block=[self._THREADS_PER_CTA, 1, 1], stream=stream + ) + + @cute.kernel + def kernel( + self, + mS: cute.Tensor, + tma_atom_in: cute.CopyAtom, + tma_in: cute.Tensor, + tma_atom_out: cute.CopyAtom, + tma_out: cute.Tensor, + ): + """Device entry: out = exp2(scale - 127) * x, computed in FP32.""" + + TILE_SIZE = self._TILE_ROWS * self._TILE_COLS + + bidx, bidy, _ = cute.arch.block_idx() + warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) + + # This CTA's block, split into _NUM_TILES tiles: ((_TILE_ROWS, _TILE_COLS), (_NUM_TILES, 1)). + cta_block_tiler = (self.CTA_BLOCK_ROWS, self.CTA_BLOCK_COLS) + tiler = (self._TILE_ROWS, self._TILE_COLS) + + # Select the block of tiles that belongs to this CTA + tma_in = cute.local_tile(tma_in, cta_block_tiler, (bidy, bidx)) + tma_out = cute.local_tile(tma_out, cta_block_tiler, (bidy, bidx)) + + gQ = cute.zipped_divide(tma_in, tiler) + gO = cute.zipped_divide(tma_out, tiler) + + if cutlass.const_expr(self.DEQUANTIZE_FROM_ROWWISE): + scale_cta_block_tiler = ( + self.CTA_BLOCK_ROWS, + self.CTA_BLOCK_COLS // MXFP8_BLOCK_SCALING_SIZE, + ) + # self._TILE_COLS // MXFP8_BLOCK_SCALING_SIZE might be fractional meaning a tile only owns a + # partial MXFP8 block. In which case this tile still own at least 1 scale on that direction + scale_tiler = ( + self._TILE_ROWS, + max(1, self._TILE_COLS // MXFP8_BLOCK_SCALING_SIZE), + ) + # Each data row has its own scales, so tiles never share them + tiles_sharing_same_scale = 1 + else: + scale_cta_block_tiler = ( + self.CTA_BLOCK_ROWS // MXFP8_BLOCK_SCALING_SIZE, + self.CTA_BLOCK_COLS, + ) + # self._TILE_ROWS // MXFP8_BLOCK_SCALING_SIZE might be fractional meaning a tile only owns a + # partial MXFP8 block. In which case this tile still own at least 1 scale on that direction + scale_tiler = ( + max(1, self._TILE_ROWS // MXFP8_BLOCK_SCALING_SIZE), + self._TILE_COLS, + ) + # A tile shorter than an MXFP8 block shares its scale row with the other tiles in it + tiles_sharing_same_scale = max(1, MXFP8_BLOCK_SCALING_SIZE // self._TILE_ROWS) + # If tiles_sharing_same_scale > 1, then this tile owns only a fraction of the MXFP8 block and + # shares its scale with the other tiles, so we need to index with tile_idx // tiles_sharing_same_scale + # to obtain the scale slice that belongs to this tile + mS = cute.local_tile(mS, scale_cta_block_tiler, (bidy, bidx)) + + @cute.struct + class SharedStorage: + mbar: cute.struct.MemRange[cute.Int64, 2 * self._NUM_STAGES] + sQ: cute.struct.Align[ + cute.struct.MemRange[self.cfg.FP8_DTYPE, self._NUM_STAGES * TILE_SIZE], 128 + ] + sO: cute.struct.Align[ + cute.struct.MemRange[self.cfg.OUT_DTYPE, self._NUM_STAGES * TILE_SIZE], 128 + ] + + storage = cutlass.memory.SmemAllocator().allocate(SharedStorage) + ring_layout = cute.zipped_product( + cute.make_ordered_layout(tiler, order=(1, 0)), cute.make_layout(self._NUM_STAGES) + ) + sQ = storage.sQ.get_tensor(ring_layout) + sO = storage.sO.get_tensor(ring_layout) + + tXs_in, tXg_in = cute.nvgpu.cpasync.tma_partition( + tma_atom_in, 0, cute.make_layout(1), sQ, gQ + ) + tXs_out, tXg_out = cute.nvgpu.cpasync.tma_partition( + tma_atom_out, 0, cute.make_layout(1), sO, gO + ) + + # Only warp 0 is the producer (issues TMA); every warp is a consumer. + mainloop_pipeline = pipeline.PipelineTmaAsync.create( + barrier_storage=storage.mbar.data_ptr(), + num_stages=self._NUM_STAGES, + producer_group=pipeline.CooperativeGroup(pipeline.Agent.Thread, 1), + consumer_group=pipeline.CooperativeGroup(pipeline.Agent.Thread, self._NUM_WARPS), + tx_count=TILE_SIZE * self.cfg.FP8_DTYPE.width // 8, + cta_layout_vmnk=None, + ) + prod_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Producer, self._NUM_STAGES + ) + cons_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, self._NUM_STAGES + ) + + # Prologue: fill the pipeline. + if warp_idx == 0: + for it in cutlass.range_constexpr(self._NUM_STAGES): + mainloop_pipeline.producer_acquire(prod_state) + cute.copy( + tma_atom_in, + tXg_in[(None, it)], + tXs_in[(None, prod_state.index)], + tma_bar_ptr=mainloop_pipeline.producer_get_barrier(prod_state), + ) + mainloop_pipeline.producer_commit(prod_state) + prod_state.advance() + + for tile_idx in cutlass.range_constexpr(self._NUM_TILES): + stage = cons_state.index + # The first _NUM_STAGES tiles don't have to wait for previous TMA read because they are not used yet + if cutlass.const_expr(tile_idx >= self._NUM_STAGES): + if warp_idx == 0: + # Wait for previous TMA to finish their read so we can write to this output tile + cute.arch.cp_async_bulk_wait_group(self._NUM_STAGES - 1, read=True) + cute.arch.sync_threads() + + mainloop_pipeline.consumer_wait(cons_state) + + gS = cute.local_tile(mS, scale_tiler, (tile_idx // tiles_sharing_same_scale, 0)) + + dequantize_mxfp8_smem_tile( + sQ[(None, None), stage], + sO[(None, None), stage], + gS, + self._TILE_ROWS, + self._TILE_COLS, + self._ELEMS_PER_THREAD, + self.DEQUANTIZE_FROM_ROWWISE, + ) + + # Make the smem writes visible to the TMA engine before the store. + cute.arch.fence_proxy("async.shared", space="cta") + cute.arch.sync_threads() + mainloop_pipeline.consumer_release(cons_state) + cons_state.advance() + + if warp_idx == 0: + cute.copy( + tma_atom_out, + tXs_out[(None, stage)], + tXg_out[(None, tile_idx)], + ) + cute.arch.cp_async_bulk_commit_group() + if cutlass.const_expr(tile_idx + self._NUM_STAGES < self._NUM_TILES): + mainloop_pipeline.producer_acquire(prod_state) + cute.copy( + tma_atom_in, + tXg_in[(None, tile_idx + self._NUM_STAGES)], + tXs_in[(None, prod_state.index)], + tma_bar_ptr=mainloop_pipeline.producer_get_barrier(prod_state), + ) + mainloop_pipeline.producer_commit(prod_state) + prod_state.advance() + + if warp_idx == 0: + cute.arch.cp_async_bulk_wait_group(0, read=False) + cute.arch.sync_threads() + + +def compile_dequantize_function_from_cfg(cfg: MXFP8DequantizeConfig): + """Compile the dequantize kernel for cfg with symbolic shapes and a TVM-FFI entrypoint.""" + sym_M = cute.sym_int32() + sym_N = cute.sym_int32(divisibility=SYM_N_DIVISIBILITY) + # Scales are allocated at the padded shapes described in compile_cutedsl_function_from_cfg. + scale_row_shape = (cute.sym_int32(divisibility=128), cute.sym_int32(divisibility=4)) + scale_col_shape = (cute.sym_int32(divisibility=4), cute.sym_int32(divisibility=128)) + + def fake(dtype, shape, align, present=True): + if not present: + return None + return cute.runtime.make_fake_compact_tensor( + dtype, shape, stride_order=(1, 0), memspace=cute.AddressSpace.gmem, assumed_align=align + ) + + return cute.compile( + MXFP8DequantizeKernel(cfg), + fake(cfg.FP8_DTYPE, (sym_M, sym_N), 16, cfg.ROWWISE), # mX_row + fake(Float8E8M0FNU, scale_row_shape, 4, cfg.ROWWISE), # mS_row + fake(cfg.FP8_DTYPE, (sym_M, sym_N), 16, cfg.COLWISE), # mX_col + fake(Float8E8M0FNU, scale_col_shape, 4, cfg.COLWISE), # mS_col + fake(cfg.OUT_DTYPE, (sym_M, sym_N), 16), # mO + cute.runtime.make_fake_stream(), + options="--enable-tvm-ffi", + ) + + +def get_mxfp8_dequantization_function( + fn_name: str, + fp8_dtype: str, + out_dtype: str, + rowwise: bool, + colwise: bool, + with_gemm_swizzled_scales: bool, +) -> bool: + """Compile the MXFP8 dequantize kernel for this config and register it in the TVM-FFI global + registry under `fn_name`. rowwise/colwise say which data copies the C++ caller passes; the + direction actually dequantized is chosen in MXFP8DequantizeConfig. Returns False if the config is unsupported, so the C++ dispatcher + falls back to the CUDA kernel. Mirrors get_mxfp8_quantization_function. + """ + try: + if tvm_ffi.get_global_func(fn_name, allow_missing=True) is not None: + return True + + major, minor = device_compute_capability() + if major < 10: + logger.warning( + "CuTeDSL MXFP8 backend requires compute capability >= 10.0 (Blackwell), " + "but detected %d.%d; falling back to the CUDA C++ kernel.", + major, + minor, + ) + return False + + try: + cfg = MXFP8DequantizeConfig( + fp8_dtype, out_dtype, rowwise, colwise, with_gemm_swizzled_scales + ) + except ValueError as e: + logger.warning( + "CuTeDSL MXFP8 backend does not support this dequantize config, " + "falling back to the CUDA C++ kernel: %s", + e, + ) + return False + + logger.debug("Compiling CuTeDSL MXFP8 dequantization kernel for %s", cfg) + compiled = compile_dequantize_function_from_cfg(cfg) + # Register the native function to skip the Python argument-parsing wrapper + # (see get_mxfp8_quantization_function). + native = getattr(compiled, "__tvm_ffi_object__", lambda: None)() + tvm_ffi.register_global_func( + fn_name, native if native is not None else compiled, override=True + ) + return True + except Exception as e: # pylint: disable=broad-exception-caught + logger.error( + "CuTeDSL MXFP8 dequantize kernel compilation & registration failed, falling back to" + " the CUDA C++ kernel: %s", + e, + ) + return False diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_2d.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_2d.py new file mode 100644 index 00000000000..fdca0825d44 --- /dev/null +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_2d.py @@ -0,0 +1,431 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""MXFP8 2D block-scaling quantization kernel implemented in CuTeDSL. + +Ports the kIs2DBlockScaling path of quantize_mxfp8_kernel (quantize_mxfp8.cuh): every 32x32 +block of the input shares one E8M0 scale, which the rowwise scales repeat down the block's 32 +rows and the colwise scales repeat across its 32 columns. + +The TMA pipeline is the general G2S kernel's; only the per-tile math differs, so this subclasses +it and swaps in its own tile function. A tile is 32 rows by TILE_COLS, i.e. TILE_COLS // 32 +blocks side by side, and the colwise pass maps warp w onto columns [32w, 32w + 32), which is +exactly block w, so a warp reduction of the per-column amaxes yields the block amax. The rowwise +pass reads it back from SMEM, or, when there is no colwise pass, reduces its per-row amaxes over +the block's 32 rows itself. + +Only a plain cast is implemented: the C APIs that fuse dbias or an activation pass no +quantization config, so only plain casts (nvte_quantize_v2, nvte_multi_tensor_quantize) can +request 2D scaling. +""" + +from typing import Optional, Type + +import cutlass +from cutlass import cute +from cutlass import Float32, Int32, Int64, Uint16, Uint32, Uint8 + +from transformer_engine.common.CuTeDSL.utils import ( + device_compute_capability, + is_packed16, + abs_max_x2_bf16, + abs_max_x2_f16, + abs_max_scalar_bf16, + abs_max_scalar_f16, + to_f32_bf16, + to_f32_f16, + x2_lo_to_f32_bf16, + x2_lo_to_f32_f16, + x2_hi_to_f32_bf16, + x2_hi_to_f32_f16, + fabs_f32, + exp2f_rcp, + pack_f32x2, + unpack_i64_to_i32x2, +) +from transformer_engine.common.CuTeDSL.utils_fp8 import ( + get_cvt_f32x2_to_fp8x2_func, + cvt_f32_to_fp8e8m0fnu, + mul_f32x2_cvt_f32x4_to_fp8x4, + mul_f32x2_cvt_packed16x4_to_fp8x4, +) +from .quantize_mxfp8_common import ( + MXFP8QuantizeConfig, + MXFP8_BLOCK_SCALING_SIZE, + THREADS_PER_WARP, +) +from .quantize_mxfp8_g2s_general import MXFP8QuantizeGeneralG2SKernel + + +@cute.jit +def quantize_colwise_mxfp8_2d( + sX_tile: cute.Tensor, # (TILE_Y, TILE_X) input smem view, post-TMA + sO_col_tile: cute.Tensor, # (TILE_Y, TILE_X) fp8 smem view (colwise FP8 output) + mS_col_stage: cute.Tensor, # colwise scale tensor (1D swizzled, or 2D linear) + sBlockAmax: cute.Tensor, # receives the amax of block w at index w + MAX_NORM_RCP: cutlass.Constexpr[float], + tile_row_start: Int32, + tile_col_start: Int32, + M: Int32, + N: Int32, + DTYPE: cutlass.Constexpr[Type[cutlass.Numeric]], + FP8_DTYPE: cutlass.Constexpr[Type[cutlass.Numeric]], + SWIZZLE: cutlass.Constexpr[bool], + TILE_X: cutlass.Constexpr[int], + TILE_Y: cutlass.Constexpr[int], + SKIP_SCALE_BOUNDS: cutlass.Constexpr[bool], +): + """Quantize one SMEM tile colwise with one scale per 32x32 block; returns the block amax.""" + tidx, _, _ = cute.arch.thread_idx() + + # Each thread handles one vertical MXFP8 block, so we have 1 thread along the Y direction since one tile only has + # one block in the Y (column) direction + CTA_THREADS_Y = MXFP8_BLOCK_SCALING_SIZE // TILE_Y # 1 thread along the Y direction + _, tv_layout = cute.make_layout_tv( + thr_layout=cute.make_layout((CTA_THREADS_Y, TILE_X), stride=(TILE_X, 1)), + val_layout=cute.make_layout((MXFP8_BLOCK_SCALING_SIZE, 1), stride=(1, 1)), + ) + sX_thread = cute.composition(sX_tile, tv_layout)[tidx, None] + sO_thread = cute.composition(sO_col_tile, tv_layout)[tidx, None] + + # Gather this thread's 32 values from the MXFP8 block and its own amax. + # Note: TMA zero-fills past M and N, so the padding values never win when comparing amax. + in_c = [None] * MXFP8_BLOCK_SCALING_SIZE + if cutlass.const_expr(is_packed16(DTYPE)): + abs_max_scalar = abs_max_scalar_f16 if DTYPE is cutlass.Float16 else abs_max_scalar_bf16 + to_f32 = to_f32_f16 if DTYPE is cutlass.Float16 else to_f32_bf16 + in_c[0] = sX_thread[0] + amax_value = in_c[0] + for i in cutlass.range_constexpr(1, MXFP8_BLOCK_SCALING_SIZE): + in_c[i] = sX_thread[i] + amax_value = abs_max_scalar(amax_value, in_c[i]) + amax_c = fabs_f32(to_f32(amax_value)) + else: + to_f32 = None + in_c[0] = Float32(sX_thread[0]) + amax_c = fabs_f32(in_c[0]) + for i in cutlass.range_constexpr(1, MXFP8_BLOCK_SCALING_SIZE): + in_c[i] = Float32(sX_thread[i]) + amax_c = cute.arch.fmax(amax_c, fabs_f32(in_c[i])) + + # Reduce the amaxes of the 32 threads in this warp to get the block amax, and let the leader thread + # of each warp write it to SMEM. + # Use f32 redux if it is available (blackwell family); otherwise use butterfly shuffle for reduction + if cutlass.const_expr(device_compute_capability() in ((10, 0), (10, 3))): + block_amax = cute.arch.warp_redux_sync(amax_c, kind="fmax") + else: + block_amax = amax_c + for k in cutlass.range_constexpr(THREADS_PER_WARP.bit_length() - 1): + block_amax = cute.arch.fmax( + block_amax, cute.arch.shuffle_sync_bfly(block_amax, offset=1 << k) + ) + if tidx % THREADS_PER_WARP == 0: + sBlockAmax[tidx // THREADS_PER_WARP] = block_amax + + biased_exp_c = cvt_f32_to_fp8e8m0fnu(block_amax * MAX_NORM_RCP) + if cutlass.const_expr(SWIZZLE): + scale_coord = (0, tidx % 32, tidx // 32) + else: + scale_coord = (0, tidx) + if cutlass.const_expr(SKIP_SCALE_BOUNDS): + mS_col_stage[scale_coord] = biased_exp_c + elif tile_row_start < M and tile_col_start + tidx < N: + mS_col_stage[scale_coord] = biased_exp_c + + inv_scale_c = exp2f_rcp(biased_exp_c) + cvt_x2_func = get_cvt_f32x2_to_fp8x2_func(FP8_DTYPE) + sO_thread_fp8 = cute.make_tensor( + cute.recast_ptr(sO_thread.iterator, dtype=FP8_DTYPE), sO_thread.layout + ) + for j in cutlass.range_constexpr(MXFP8_BLOCK_SCALING_SIZE // 2): + lo, hi = 2 * j, 2 * j + 1 + x_lo, x_hi = in_c[lo], in_c[hi] + if cutlass.const_expr(to_f32 is not None): + x_lo, x_hi = to_f32(x_lo), to_f32(x_hi) + pair = cvt_x2_func(x_hi * inv_scale_c, x_lo * inv_scale_c) + sO_thread_fp8[lo] = Uint8(pair & Uint16(0xFF)).bitcast(FP8_DTYPE) + sO_thread_fp8[hi] = Uint8(pair >> Uint16(8)).bitcast(FP8_DTYPE) + + return block_amax + + +@cute.jit +def quantize_rowwise_mxfp8_2d( + sX_tile: cute.Tensor, # (TILE_Y, TILE_X) input smem view, post-TMA + sO_row_tile: cute.Tensor, # (TILE_Y, TILE_X) fp8 smem view (rowwise FP8 output) + mS_row_stage: cute.Tensor, # rowwise scale tensor (1D swizzled, or 2D linear) + sBlockAmax: cute.Tensor, # block amaxes from the colwise pass, or scratch for the reduction + MAX_NORM_RCP: cutlass.Constexpr[float], + tile_row_start: Int32, + tile_col_start: Int32, + M: Int32, + N: Int32, + DTYPE: cutlass.Constexpr[Type[cutlass.Numeric]], + FP8_DTYPE: cutlass.Constexpr[Type[cutlass.Numeric]], + TILE_X: cutlass.Constexpr[int], + TILE_Y: cutlass.Constexpr[int], + SKIP_SCALE_BOUNDS: cutlass.Constexpr[bool], + BLOCK_AMAX_IN_SMEM: cutlass.Constexpr[ + bool + ], # If colwise pass has already left the block amaxes in sBlockAmax +): + """Quantize one SMEM tile rowwise with one scale per 32x32 block; returns the block amax. + + With BLOCK_AMAX_IN_SMEM the colwise pass has already left the block amaxes in sBlockAmax + (and the caller synchronized); otherwise they are reduced here, which syncs the CTA. + """ + tidx, _, _ = cute.arch.thread_idx() + + # Same vectorized, bank-staggered loads as the general kernel's rowwise pass. + PACK_SIZE = 4 + WAVES = MXFP8_BLOCK_SCALING_SIZE // PACK_SIZE + THREADS_PER_BANK = 32 * 4 // MXFP8_BLOCK_SCALING_SIZE + + # Each thread covers one horizontal MXFP8 block, so we have TILE_X // MXFP8_BLOCK_SCALING_SIZE threads + # along X (row direction) + CTA_THREADS_X = TILE_X // MXFP8_BLOCK_SCALING_SIZE + NUM_WARPS = TILE_Y * CTA_THREADS_X // THREADS_PER_WARP + # A CTA consists of NUM_WARPS warps stacked together + thr_layout = cute.blocked_product( + cute.make_layout( + (THREADS_PER_WARP // CTA_THREADS_X, CTA_THREADS_X), stride=(CTA_THREADS_X, 1) + ), # Thread layout in a warp, (16, 2) + cute.make_layout((NUM_WARPS, 1)), # Warp layout in a CTA (2, 1) + ) + + _, tv_layout = cute.make_layout_tv( + thr_layout=thr_layout, + val_layout=cute.make_layout((1, MXFP8_BLOCK_SCALING_SIZE), stride=(0, 1)), + ) + + sX_thread = cute.composition(sX_tile, tv_layout)[tidx, None] + sO_thread = cute.composition(sO_row_tile, tv_layout)[tidx, None] + sO_thread_u32 = cute.make_tensor( + cute.recast_ptr(sO_thread.iterator, dtype=Uint32), + cute.make_layout((MXFP8_BLOCK_SCALING_SIZE // 4,), stride=(1,)), + ) + + # Coordinates in thr_layout: (lane_y, lane_x) within the warp, warps stacked along Y. + warp_id = tidx // THREADS_PER_WARP + lane_id = tidx % THREADS_PER_WARP + lane_y = lane_id // CTA_THREADS_X + lane_x = lane_id % CTA_THREADS_X + thread_y = warp_id * (THREADS_PER_WARP // CTA_THREADS_X) + lane_y + thread_x = lane_x + + # Each thread start reading from the specfic bank based on its thread ID so they can do their best to access different banks + # to avoid bank conflict. + bank_group = (tidx % THREADS_PER_WARP) // THREADS_PER_BANK + # The offset this thread should start reading from based on what's its first bank to access. + offset = bank_group * PACK_SIZE + + if cutlass.const_expr(is_packed16(DTYPE)): + sX_thread_rw_i64 = cute.make_tensor( + cute.recast_ptr(sX_thread.iterator, dtype=Int64), + cute.make_layout((1, MXFP8_BLOCK_SCALING_SIZE // 4), stride=(0, 1)), + ) + in_r = [[None, None] for _ in range(WAVES)] + for w in cutlass.range_constexpr(WAVES): + idx = (w + offset // 4) % (MXFP8_BLOCK_SCALING_SIZE // 4) + in_r[w][0], in_r[w][1] = unpack_i64_to_i32x2(sX_thread_rw_i64[0, idx]) + else: + sX_thread_rw = cute.make_tensor( + sX_thread.iterator, + cute.make_layout((1, MXFP8_BLOCK_SCALING_SIZE), stride=(0, 1)), + ) + in_r = [[None] * PACK_SIZE for _ in range(WAVES)] + for w in cutlass.range_constexpr(WAVES): + start = (w * PACK_SIZE + offset) % MXFP8_BLOCK_SCALING_SIZE + for i in cutlass.range_constexpr(PACK_SIZE): + in_r[w][i] = Float32(sX_thread_rw[0, start + i]) + + if cutlass.const_expr(BLOCK_AMAX_IN_SMEM): + # If colwise pass has already writes the block amaxes into sBlockAmax, just read it back. + block_amax = sBlockAmax[thread_x] + else: + # Otherwise we need to compute the block amaxes in this rowwise pass + if cutlass.const_expr(is_packed16(DTYPE)): + abs_max_x2 = abs_max_x2_f16 if DTYPE is cutlass.Float16 else abs_max_x2_bf16 + x2_lo_to_f32 = x2_lo_to_f32_f16 if DTYPE is cutlass.Float16 else x2_lo_to_f32_bf16 + x2_hi_to_f32 = x2_hi_to_f32_f16 if DTYPE is cutlass.Float16 else x2_hi_to_f32_bf16 + # Load the fisst wave's two values into amax_2x, then fold in the rest of the waves. + amax_2x = abs_max_x2(in_r[0][0], in_r[0][1]) + for w in cutlass.range_constexpr(1, WAVES): + amax_2x = abs_max_x2(amax_2x, in_r[w][0]) + amax_2x = abs_max_x2(amax_2x, in_r[w][1]) + amax_r = cute.arch.fmax( + fabs_f32(x2_lo_to_f32(amax_2x)), fabs_f32(x2_hi_to_f32(amax_2x)) + ) + else: + vals = [in_r[w][i] for w in range(WAVES) for i in range(PACK_SIZE)] + amax_r = fabs_f32(vals[0]) + for j in cutlass.range_constexpr(1, len(vals)): + amax_r = cute.arch.fmax(amax_r, fabs_f32(vals[j])) + + # Use butterfly shuffles to reduce the 32 threads' local amax in this warp to get 2 block amax + # for the left half and the right half of the warp (each warp with shape (16, 1)) + for k in cutlass.range_constexpr((THREADS_PER_WARP // CTA_THREADS_X).bit_length() - 1): + amax_r = cute.arch.fmax( + amax_r, cute.arch.shuffle_sync_bfly(amax_r, offset=CTA_THREADS_X << k) + ) + + # Write the half-warp block amaxes to SMEM, then let the first warp reduce them to get the final block amax. + sWarpBlockAmax = cute.make_tensor( + sBlockAmax.iterator, + cute.make_layout((NUM_WARPS, CTA_THREADS_X), stride=(CTA_THREADS_X, 1)), + ) + # Let the threads in the first row in each half warp write its partial warp amax to SMEM + # Warp shape is (16, 2) so each warp writes 2 partial amax for a (16, 32) block which will be reduced to produce + # the (32, 32) block amax + if lane_y == 0: + sWarpBlockAmax[(warp_id, lane_x)] = amax_r + cute.arch.sync_threads() + + # Reduce 2 warps' amax to the whole (32, 32) block's amax + block_amax = sWarpBlockAmax[(0, thread_x)] + for w in cutlass.range_constexpr(1, NUM_WARPS): + block_amax = cute.arch.fmax(block_amax, sWarpBlockAmax[(w, thread_x)]) + + biased_exp_r = cvt_f32_to_fp8e8m0fnu(block_amax * MAX_NORM_RCP) + if cutlass.const_expr(SKIP_SCALE_BOUNDS): + mS_row_stage[(thread_y, thread_x)] = biased_exp_r + elif tile_row_start + thread_y < M and tile_col_start + thread_x * MXFP8_BLOCK_SCALING_SIZE < N: + mS_row_stage[(thread_y, thread_x)] = biased_exp_r + + inv_scale_r = exp2f_rcp(biased_exp_r) + scale_2x = pack_f32x2(inv_scale_r, inv_scale_r) + if cutlass.const_expr(is_packed16(DTYPE)): + mul_cvt_x4_func = mul_f32x2_cvt_packed16x4_to_fp8x4(DTYPE, FP8_DTYPE) + else: + mul_cvt_x4_func = mul_f32x2_cvt_f32x4_to_fp8x4(FP8_DTYPE) + for w in cutlass.range_constexpr(WAVES): + idx = ((w * PACK_SIZE + offset) % MXFP8_BLOCK_SCALING_SIZE) // 4 + if cutlass.const_expr(is_packed16(DTYPE)): + sO_thread_u32[idx] = mul_cvt_x4_func(in_r[w][0], in_r[w][1], scale_2x) + else: + sO_thread_u32[idx] = mul_cvt_x4_func( + in_r[w][0], in_r[w][1], in_r[w][2], in_r[w][3], scale_2x + ) + + return block_amax + + +@cute.jit +def quantize_mxfp8_smem_tile_2d( # pylint: disable=unused-argument + sX_tile: cute.Tensor, + sActInput_tile: Optional[cute.Tensor], + sO_row_tile: Optional[cute.Tensor], + mS_row_stage: Optional[cute.Tensor], + sO_col_tile: Optional[cute.Tensor], + mS_col_stage: Optional[cute.Tensor], + rowwise_dbias_acc: Optional[cute.Tensor], + tile_row_start: Int32, + tile_col_start: Int32, + M: Int32, + N: Int32, + ROWWISE: cutlass.Constexpr[bool], + COLWISE: cutlass.Constexpr[bool], + WITH_AMAX: cutlass.Constexpr[bool], + MAX_NORM_RCP: cutlass.Constexpr[float], + DTYPE: cutlass.Constexpr[Type[cutlass.Numeric]], + FP8_DTYPE: cutlass.Constexpr[Type[cutlass.Numeric]], + WITH_GEMM_SWIZZLED_SCALES: cutlass.Constexpr[bool], + TILE_X: cutlass.Constexpr[int], + TILE_Y: cutlass.Constexpr[int], + SKIP_SCALE_BOUNDS: cutlass.Constexpr[bool], + smem: cutlass.memory.SmemAllocator, + **_fusion_args, # The general tile function's fusion knobs, all off for a plain cast. +): + """2D quantize a SMEM tile; returns (amax, None).""" + NUM_BLOCKS_X = TILE_X // MXFP8_BLOCK_SCALING_SIZE + NUM_WARPS = TILE_Y * NUM_BLOCKS_X // THREADS_PER_WARP + + # One amax per (warp, block) pair, for the rowwise-only cross-warp reduction. + @cute.struct + class BlockAmaxStorage: # pylint: disable=missing-class-docstring + sBlockAmax: cute.struct.MemRange[Float32, NUM_WARPS * NUM_BLOCKS_X] + + sBlockAmax = smem.allocate(BlockAmaxStorage).sBlockAmax.get_tensor( + cute.make_layout(NUM_WARPS * NUM_BLOCKS_X) + ) + + block_amax = Float32(0.0) + if cutlass.const_expr(COLWISE): + block_amax = quantize_colwise_mxfp8_2d( + sX_tile, + sO_col_tile, + mS_col_stage, + sBlockAmax, + MAX_NORM_RCP, + tile_row_start, + tile_col_start, + M, + N, + DTYPE=DTYPE, + FP8_DTYPE=FP8_DTYPE, + SWIZZLE=WITH_GEMM_SWIZZLED_SCALES, + TILE_X=TILE_X, + TILE_Y=TILE_Y, + SKIP_SCALE_BOUNDS=SKIP_SCALE_BOUNDS, + ) + if cutlass.const_expr(ROWWISE): + # Publish the block amaxes to the rowwise pass. + cute.arch.sync_threads() + if cutlass.const_expr(ROWWISE): + block_amax = quantize_rowwise_mxfp8_2d( + sX_tile, + sO_row_tile, + mS_row_stage, + sBlockAmax, + MAX_NORM_RCP, + tile_row_start, + tile_col_start, + M, + N, + DTYPE=DTYPE, + FP8_DTYPE=FP8_DTYPE, + TILE_X=TILE_X, + TILE_Y=TILE_Y, + SKIP_SCALE_BOUNDS=SKIP_SCALE_BOUNDS, + BLOCK_AMAX_IN_SMEM=COLWISE, + ) + amax = None + if cutlass.const_expr(WITH_AMAX): + amax = block_amax + return amax, None + + +class MXFP8Quantize2DKernel(MXFP8QuantizeGeneralG2SKernel): + """MXFP8 quantization with one scale per 32x32 block (plain cast only). + + Reuses the general G2S kernel's pipeline with its own tile function and tile configuration. + """ + + _quantize_smem_tile = staticmethod(quantize_mxfp8_smem_tile_2d) + + def __init__( + self, + cfg: MXFP8QuantizeConfig, + SKIP_INPUT_MASKING: bool = False, + SKIP_SCALE_BOUNDS: bool = False, + ): + assert cfg.USE_2D_QUANTIZATION + # 2D quantization is only supported for plain casts and no fusion should be requested + assert not (cfg.WITH_DBIAS or cfg.WITH_ACT or cfg.WITH_DACT) + + super().__init__(cfg, SKIP_INPUT_MASKING, SKIP_SCALE_BOUNDS) + + # Overrides the general kernel's tile configuration with the 2D kernel's + self._NUM_TILES = 2 + self._USE_STATIC_NUM_TILES = True + self._MIN_BLOCKS_PER_MP = 0 + + self._NUM_WARPS = 2 + self._THREADS_PER_CTA = self._NUM_WARPS * THREADS_PER_WARP + + # In rowwise pass, + self._TILE_ROWS = MXFP8_BLOCK_SCALING_SIZE + # In colwise pass, each thread covers one vertical MXFP8 block, and since THREADS_PER_WARP == MXFP8_BLOCK_SCALING_SIZE, + # two warps cover 2 * MXFP8_BLOCK_SCALING_SIZE columns + self._TILE_COLS = self._NUM_WARPS * MXFP8_BLOCK_SCALING_SIZE diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_common.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_common.py index 8a66b55e48f..6d72d50a88d 100644 --- a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_common.py +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_common.py @@ -139,8 +139,6 @@ def __init__( use_2d_quantization: bool = False, activation: Optional[str] = None, ): - if use_2d_quantization: - raise ValueError("2D block scaling is not implemented by the CuTeDSL MXFP8 kernels") if dtype is None or dtype not in ("Float32", "Float16", "BFloat16"): raise ValueError(f"unknown input dtype {dtype!r}; expected Float32|Float16|BFloat16") self.DTYPE = str_to_cutlass_dtype(dtype) @@ -193,6 +191,11 @@ def __init__( # uses a cross-thread smem reduction over THREADS_Y. Both mirror the CUDA # kernel's COLWISE_SCALING / rowwise dbias branches. self.WITH_DBIAS = with_dbias + # The C APIs that fuse dbias or an activation pass no quantization config, so 2D block + # scaling only reaches plain casts and fusion is not supported. + if use_2d_quantization and (with_dbias or with_dact or with_act): + raise ValueError("2D block scaling is only implemented for a plain cast") + self.USE_2D_QUANTIZATION = use_2d_quantization self.MAX_NORM_RCP = ( FP8E4M3_MAX_NORM_RCP if fp8_dtype == "Float8E4M3" else FP8E5M2_MAX_NORM_RCP ) @@ -203,7 +206,8 @@ def __str__(self): f"rowwise={self.ROWWISE}, colwise={self.COLWISE}, " f"swizzled={self.WITH_GEMM_SWIZZLED_SCALES}, with_amax={self.WITH_AMAX}, " f"with_dbias={self.WITH_DBIAS}, with_dact={self.WITH_DACT}, " - f"with_act={self.WITH_ACT}, activation={self.ACTIVATION})" + f"with_act={self.WITH_ACT}, use_2d_quantization={self.USE_2D_QUANTIZATION}, " + f"activation={self.ACTIVATION})" ) __repr__ = __str__ @@ -212,6 +216,14 @@ def is_plain_cast(self) -> bool: """Whether cfg is a plain cast: no amax, dbias or (d)activation fused in.""" return not (self.WITH_AMAX or self.WITH_DBIAS or self.WITH_DACT or self.WITH_ACT) + def act_needs_masking(self) -> bool: + """Whether TMA's zero padding must be masked after the activation, i.e. act(0) != 0. + + All activations supported by MXFP8 satisfy act(0) == 0, so this is a precaution for + activations added in the future. + """ + return self.WITH_ACT and self.ACTIVATION not in ("relu", "gelu", "silu", "qgelu", "srelu") + class MXFP8QuantizeKernelBase(abc.ABC): """Base class for MXFP8 quantize kernels.""" @@ -268,34 +280,20 @@ def __init__(self, cfg: MXFP8QuantizeConfig): from .quantize_mxfp8_g2s_bidim import MXFP8QuantizeBidimensionalG2SKernel from .quantize_mxfp8_g2r_rowwise_2lane import MXFP8QuantizeRowwise2LaneG2RKernel from .quantize_mxfp8_g2r_bidim import MXFP8QuantizeBidimensionalG2RKernel + from .quantize_mxfp8_2d import MXFP8Quantize2DKernel # pylint: enable=import-outside-toplevel self.cfg = cfg - # These activation functions satisfy f(0) = 0 so we don't need to mask with OOB regions - # (zeros filled by TMA are still zeros without applying activation to them) - # Note that as the time of writing, all activations supported by MXFP8 satisfy this property, so !ACT_NEED_MASKING is never reachable. - # This is only a precaution for future use cases if someone wants to add a new activation function. - self.ACT_NEED_MASKING = cfg.WITH_ACT and cfg.ACTIVATION not in ( - "relu", - "gelu", - "silu", - "qgelu", - "srelu", - ) - # Instantiate all possible kernels at compile time, - # and we will pick the right one at runtime based on the input shape and config. - self.general_skip_input_masking_kernel = MXFP8QuantizeGeneralG2SKernel( - cfg, SKIP_INPUT_MASKING=True - ) - self.general_skip_input_masking_and_scale_bounds_kernel = MXFP8QuantizeGeneralG2SKernel( - cfg, SKIP_INPUT_MASKING=True, SKIP_SCALE_BOUNDS=True + # 2D block scaling only swaps the general kernel's per-tile math. + general_cls = ( + MXFP8Quantize2DKernel if cfg.USE_2D_QUANTIZATION else MXFP8QuantizeGeneralG2SKernel ) - # We only need to mask when WITH_ACT is enabled because with activations applied zeros filled by TMA affect block statistics - self.general_kernel = MXFP8QuantizeGeneralG2SKernel(cfg) if self.ACT_NEED_MASKING else None - - # Specialized kernels in a specific dispatch priority order so we prefer to dispatch to kernels that come first in the list. - self.specialized_kernels = [ + # Candidate kernels in dispatch priority order; the first one whose runtime + # eligible_to_dispatch check passes is launched. The specialized kernels come first, + # then the general kernel without and with boundary handling. The last one is always + # eligible. + self.preferred_kernels = [ kernel_cls(cfg) for kernel_cls in ( MXFP8QuantizeRowwise2LaneG2RKernel, @@ -304,6 +302,9 @@ def __init__(self, cfg: MXFP8QuantizeConfig): MXFP8QuantizeBidimensionalG2SKernel, ) if kernel_cls.supports(cfg) + ] + [ + general_cls(cfg, SKIP_INPUT_MASKING=True, SKIP_SCALE_BOUNDS=True), + general_cls(cfg, SKIP_INPUT_MASKING=not cfg.act_needs_masking()), ] @cute.jit @@ -320,13 +321,13 @@ def __call__( mWorkspace: Optional[cute.Tensor], stream: CUstream, ): - dispatched_to_specialized = False - for kernel in self.specialized_kernels: - if not dispatched_to_specialized: + dispatched = False + for kernel in self.preferred_kernels: + if not dispatched: if kernel.eligible_to_dispatch( self.cfg, mX, mO_row, mS_row, mO_col, mS_col, mNoop, mDActInput ): - dispatched_to_specialized = True + dispatched = True kernel( mX, mO_row, @@ -339,69 +340,6 @@ def __call__( mWorkspace, stream, ) - # If not using a specialized kernel, fall back to the general kernel - if not dispatched_to_specialized: - # With _USE_STATIC_NUM_TILES, each CTA always processes _NUM_TILES tiles, including OOB ones, - # so the scale-bounds check can only be dropped if M divides TILE_ROWS * _NUM_TILES. - num_tiles_per_cta = ( - self.general_skip_input_masking_kernel._NUM_TILES - if self.general_skip_input_masking_kernel._USE_STATIC_NUM_TILES - else 1 - ) - # If the input shape can be perfectly tiled by the general kernel's tile size, we can skip some boundary check because - # we know we will not touch any out-of-bounds region. - shape_is_divisible = ( - mX.shape[0] - % (self.general_skip_input_masking_kernel._TILE_ROWS * num_tiles_per_cta) - == 0 - and mX.shape[1] % self.general_skip_input_masking_kernel._TILE_COLS == 0 - ) - # If shape is divisible, then we won't access OOB regions regardless whatever - if shape_is_divisible: - self.general_skip_input_masking_and_scale_bounds_kernel( - mX, - mO_row, - mS_row, - mO_col, - mS_col, - mAmax, - mNoop, - mDActInput, - mWorkspace, - stream, - ) - else: - # Otherwise we can't skip scale bound check, but we may still skip masking activation inputs with zero - if cutlass.const_expr(self.ACT_NEED_MASKING): - # Masking and the scale guards now share one condition, so this is a plain - # two-way choice: either the shape tiles exactly (drop both) or it does not. - self.general_kernel( - mX, - mO_row, - mS_row, - mO_col, - mS_col, - mAmax, - mNoop, - mDActInput, - mWorkspace, - stream, - ) - else: - # We still need to check the scale bounds, but we can skip some activations masking because their output is 0 - # for OOB regions where TMA fills with zeros - self.general_skip_input_masking_kernel( - mX, - mO_row, - mS_row, - mO_col, - mS_col, - mAmax, - mNoop, - mDActInput, - mWorkspace, - stream, - ) def compile_cutedsl_function_from_cfg(cfg): diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_bidim.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_bidim.py index 61598a7d37b..3cb611e775f 100644 --- a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_bidim.py +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_bidim.py @@ -609,7 +609,8 @@ class MXFP8QuantizeBidimensionalG2RKernel(MXFP8QuantizeKernelBase): def supports(cfg: MXFP8QuantizeConfig) -> bool: # BF16 packed math. return ( - not cfg.WITH_AMAX + not cfg.USE_2D_QUANTIZATION + and not cfg.WITH_AMAX and not cfg.WITH_DBIAS and cfg.DTYPE is cutlass.BFloat16 and cfg.COLWISE diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_rowwise_1lane.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_rowwise_1lane.py index 0c287f02abe..fb325f289b8 100644 --- a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_rowwise_1lane.py +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_rowwise_1lane.py @@ -219,7 +219,13 @@ class MXFP8QuantizeRowwise1LaneG2RKernel(MXFP8QuantizeKernelBase): @staticmethod def supports(cfg: MXFP8QuantizeConfig) -> bool: - return cfg.is_plain_cast() and is_packed16(cfg.DTYPE) and cfg.ROWWISE and not cfg.COLWISE + return ( + not cfg.USE_2D_QUANTIZATION + and cfg.is_plain_cast() + and is_packed16(cfg.DTYPE) + and cfg.ROWWISE + and not cfg.COLWISE + ) @staticmethod @cute.jit diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_rowwise_2lane.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_rowwise_2lane.py index 03d5d510ab8..19a879ce72c 100644 --- a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_rowwise_2lane.py +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2r_rowwise_2lane.py @@ -262,7 +262,8 @@ class MXFP8QuantizeRowwise2LaneG2RKernel(MXFP8QuantizeKernelBase): def supports(cfg: MXFP8QuantizeConfig) -> bool: # BF16 packed math. return ( - not cfg.WITH_AMAX + not cfg.USE_2D_QUANTIZATION + and not cfg.WITH_AMAX and not cfg.WITH_DBIAS and cfg.DTYPE is cutlass.BFloat16 and cfg.ROWWISE diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2s_bidim.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2s_bidim.py index 5f0b3d2aed5..5e769cc84c4 100644 --- a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2s_bidim.py +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2s_bidim.py @@ -255,7 +255,8 @@ class MXFP8QuantizeBidimensionalG2SKernel(MXFP8QuantizeKernelBase): def supports(cfg: MXFP8QuantizeConfig) -> bool: # This kernel supports swizzled scales but we don't dispatch to it for now. return ( - cfg.is_plain_cast() + not cfg.USE_2D_QUANTIZATION + and cfg.is_plain_cast() and is_packed16(cfg.DTYPE) and cfg.ROWWISE and cfg.COLWISE diff --git a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2s_general.py b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2s_general.py index 81148416fe9..da73da2d777 100644 --- a/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2s_general.py +++ b/transformer_engine/common/CuTeDSL/cast/mxfp8/quantize_mxfp8_g2s_general.py @@ -29,7 +29,7 @@ import cutlass from cutlass import cute from cutlass import pipeline -from cutlass import Float32, Int32, Int64, Uint16, Uint32, Uint8 +from cutlass import Boolean, Float32, Int32, Int64, Uint16, Uint32, Uint8 from cuda.bindings.driver import CUstream # pylint: disable=no-name-in-module from transformer_engine.common.CuTeDSL.utils import ( @@ -576,6 +576,7 @@ def quantize_mxfp8_smem_tile_general( DBIAS_REDUCTION_IN_COLWISE: cutlass.Constexpr[bool], COLWISE_DBIAS_REDUCTION_ONLY: cutlass.Constexpr[bool], DBIAS_REDUCTION_IN_ROWWISE: cutlass.Constexpr[bool], + smem: Optional[cutlass.memory.SmemAllocator] = None, # pylint: disable=unused-argument ): """Quantize one SMEM tile in the directions enabled by ROWWISE / COLWISE; returns (amax, dbias). @@ -665,6 +666,9 @@ class MXFP8QuantizeGeneralG2SKernel(MXFP8QuantizeKernelBase): """ _NUM_STAGES = 2 # The pipeline depth is always 2 + _quantize_smem_tile = staticmethod( + quantize_mxfp8_smem_tile_general + ) # Function to quantize one SMEM tile def __init__( self, @@ -726,6 +730,38 @@ def __init__( and cfg.ACTIVATION != "relu" ) + @cute.jit + def eligible_to_dispatch( # pylint: disable=arguments-differ,unused-argument + self, + cfg: cutlass.Constexpr[MXFP8QuantizeConfig], + mX: cute.Tensor, + mO_row: Optional[cute.Tensor], + mS_row: Optional[cute.Tensor], + mO_col: Optional[cute.Tensor], + mS_col: Optional[cute.Tensor], + mNoop: cute.Pointer, + mDActInput: Optional[cute.Tensor], + ) -> Boolean: + """Runtime check: dropping the scale bounds checks, or the input masking an activation + with act(0) != 0 needs, is only safe when the CTAs tile the input exactly.""" + eligible = True + # If we skip scale OOB check, or we skip input masking with an activation that doesn't satisfy act(0)=0, + # then it is required to have the input shape perfectly divisible + if cutlass.const_expr( + self.SKIP_SCALE_BOUNDS or (self.SKIP_INPUT_MASKING and cfg.act_needs_masking()) + ): + M, N = mX.shape + # With _USE_STATIC_NUM_TILES, each CTA always processes _NUM_TILES tiles, including OOB ones, + # so the scale-bounds check can only be dropped if M divides TILE_ROWS * _NUM_TILES. + num_tiles_per_cta = self._NUM_TILES if self._USE_STATIC_NUM_TILES else 1 + # If the input shape can be perfectly tiled by the general kernel's tile size, we can skip some boundary check because + # we know we will not touch any out-of-bounds region. + shape_is_divisible = ( + M % (self._TILE_ROWS * num_tiles_per_cta) == 0 and N % self._TILE_COLS == 0 + ) + eligible = shape_is_divisible + return eligible + @cute.jit def __call__( self, @@ -1217,7 +1253,7 @@ class DactStorage: sO_col_tile = sO_col[(None, stage_idx)] mS_col_stage = cute.flatten(mS_col[(None, (tile_idx_y, tile_idx_x))]) - amax, dbias = quantize_mxfp8_smem_tile_general( + amax, dbias = self._quantize_smem_tile( sX_tile, sActInput_tile, sO_row_tile, @@ -1247,6 +1283,7 @@ class DactStorage: DBIAS_REDUCTION_IN_COLWISE=self.DBIAS_REDUCTION_IN_COLWISE, COLWISE_DBIAS_REDUCTION_ONLY=self.COLWISE_DBIAS_REDUCTION_ONLY, DBIAS_REDUCTION_IN_ROWWISE=self.DBIAS_REDUCTION_IN_ROWWISE, + smem=smem, ) if cutlass.const_expr(cfg.WITH_AMAX): per_thread_amax = cute.arch.fmax(per_thread_amax, amax) diff --git a/transformer_engine/common/CuTeDSL/utils.py b/transformer_engine/common/CuTeDSL/utils.py index 2200823d058..e38112689c5 100644 --- a/transformer_engine/common/CuTeDSL/utils.py +++ b/transformer_engine/common/CuTeDSL/utils.py @@ -107,6 +107,18 @@ def exp2f_rcp(scale_e8m0) -> Float32: return result +@cute.jit +def exp2f(scale_e8m0) -> Float32: + """2^(biased_exp - 127) for an e8m0 scale, matching ptx::exp2f in util/ptx.cuh.""" + biased_exp = Int32(scale_e8m0.bitcast(cutlass.Uint8)) + result = (biased_exp << Int32(FP32_MANTISSA_BITS)).bitcast(Float32) + # 2^-127 is subnormal, so it cannot be built by shifting into the exponent field. + for cmp_val, repl_bits in [(255, 0x7FFFFFFF), (0, 0x00400000)]: + if biased_exp == Int32(cmp_val): + result = Int32(repl_bits).bitcast(Float32) + return result + + @cute.jit def exp2_bf16x2_rcp(scale_e8m0) -> Uint32: """Pack two BF16 copies of the E8M0 reciprocal computed by exp2f_rcp.""" diff --git a/transformer_engine/common/cast/dispatch/dequantize.cuh b/transformer_engine/common/cast/dispatch/dequantize.cuh index bf4a0218118..bb2df9196c7 100644 --- a/transformer_engine/common/cast/dispatch/dequantize.cuh +++ b/transformer_engine/common/cast/dispatch/dequantize.cuh @@ -20,6 +20,10 @@ #include "../mxfp8/group_dequantize_mxfp8.cuh" #include "../nvfp4/dequantize_nvfp4.cuh" +#ifdef NVTE_WITH_CUTEDSL +#include "../mxfp8/dequantize_mxfp8_cutedsl.cuh" +#endif + namespace transformer_engine { namespace dispatch { @@ -41,7 +45,13 @@ inline void dequantize_helper(const Tensor &input, Tensor *output, cudaStream_t } case NVTE_MXFP8_1D_SCALING: { if (is_supported_by_CC_100()) { - mxfp8::dequantize(input, output, stream); + bool dequantized_with_cutedsl = false; +#ifdef NVTE_WITH_CUTEDSL + dequantized_with_cutedsl = cutedsl_backend::mxfp8_dequantize_cutedsl(input, output, stream); +#endif + if (!dequantized_with_cutedsl) { + mxfp8::dequantize(input, output, stream); + } } else { NVTE_ERROR("MXFP8 Dequantization is NOT supported by architectures < 10.0"); } diff --git a/transformer_engine/common/cast/mxfp8/dequantize_mxfp8_cutedsl.cuh b/transformer_engine/common/cast/mxfp8/dequantize_mxfp8_cutedsl.cuh new file mode 100644 index 00000000000..fd5e7545efb --- /dev/null +++ b/transformer_engine/common/cast/mxfp8/dequantize_mxfp8_cutedsl.cuh @@ -0,0 +1,146 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +#ifndef TRANSFORMER_ENGINE_COMMON_CAST_MXFP8_DEQUANTIZE_MXFP8_CUTEDSL_CUH_ +#define TRANSFORMER_ENGINE_COMMON_CAST_MXFP8_DEQUANTIZE_MXFP8_CUTEDSL_CUH_ + +#include +#include + +#include +#include +#include + +#include "../../common.h" +#include "../../tvm_ffi_bridge.h" + +namespace transformer_engine { +namespace cutedsl_backend { + +using namespace tvm_ffi_bridge; + +struct MXFP8DequantConfig { + static constexpr const char *kEntrypointName = "get_mxfp8_dequantization_function"; + + DType fp8_dtype; // The fp8 input format + DType out_dtype; // The high-precision output format + bool rowwise; // If rowwise data and scales are passed + bool colwise; // If columnwise data and scales are passed + bool swizzled; // If the scales are in the GEMM-swizzled layout + uint32_t sm_arch = static_cast(cuda::sm_arch()); + + // Bit layout: fp8_dtype [3:0], out_dtype [7:4], flags [10:8], SM architecture [19:11]. + uint32_t to_id() const { + static_assert(static_cast(DType::kNumTypes) <= 16, + "DType no longer fits in the 4 bits to_id() gives it."); + NVTE_CHECK(sm_arch < 512, "SM architecture no longer fits in the 9 bits to_id() gives it."); + return static_cast(fp8_dtype) | (static_cast(out_dtype) << 4) | + (static_cast(rowwise) << 8) | (static_cast(colwise) << 9) | + (static_cast(swizzled) << 10) | (sm_arch << 11); + } + + std::optional get_kernel() const { + static TVMFFIConfigCache &cache = TVMFFIConfigCache::create(); + return cache.get_or_load(*this); + } + + std::string to_key() const { + std::string key; + key.reserve(64); // longest: cutedsl_mxfp8_dequant_smXXX_Float8E4M3_BFloat16_1_1_1 + key.append("cutedsl_mxfp8_dequant_sm") + .append(std::to_string(sm_arch)) + .append("_") + .append(to_string(fp8_dtype)) + .append("_") + .append(to_string(out_dtype)) + .append("_") + .append(rowwise ? "1" : "0") + .append("_") + .append(colwise ? "1" : "0") + .append("_") + .append(swizzled ? "1" : "0"); + return key; + } + + bool retrieve_func_from_python(const std::string &fn_name) const { + auto entrypoint = tvm::ffi::Function::GetGlobal(kEntrypointName); + if (!entrypoint.has_value()) { + return false; + } + tvm::ffi::Any result = + (*entrypoint)(tvm::ffi::String(fn_name), tvm::ffi::String(to_string(fp8_dtype)), + tvm::ffi::String(to_string(out_dtype)), rowwise, colwise, swizzled); + return result.try_cast().value_or(false); + } +}; + +// Signature mirrors mxfp8::dequantize. Returns false to fall back to the CUDA kernel. +inline bool mxfp8_dequantize_cutedsl(const Tensor &input, Tensor *output, cudaStream_t stream) { + if (!tvm_ffi_bridge::TVMFFICentral::getInstance().get_cutedsl_backend_enabled()) { + return false; + } + // Pass every data copy present; the Python side chooses which direction to dequantize. + const bool rowwise = input.has_data(); + const bool colwise = input.has_columnwise_data(); + + // Sanity checks, mirroring mxfp8::dequantize in dequantize_mxfp8.cuh + NVTE_CHECK(input.dim() >= 2, "Input must have at least 2 dimensions."); + NVTE_CHECK(rowwise || colwise, "Cannot dequantize tensor without data."); + if (rowwise) { + NVTE_CHECK(is_fp8_dtype(input.data.dtype), "Input must have FP8 type."); + } + if (colwise) { + NVTE_CHECK(is_fp8_dtype(input.columnwise_data.dtype), "Input must have FP8 type."); + } + NVTE_CHECK(!(rowwise && colwise) || input.data.dtype == input.columnwise_data.dtype, + "Rowwise and columnwise data must have the same FP8 type."); + NVTE_CHECK(!is_fp8_dtype(output->data.dtype), "Output must be in higher precision."); + NVTE_CHECK(output->shape() == input.shape(), "Input and output shapes need to match."); + + if (input.flat_last_dim() % 16 != 0) { + report_cutedsl_not_chosen("the last dimension of input is not a multiple of 16."); + return false; + } + const bool misaligned = !is_aligned_ptr(output->data.dptr, 16) || + (rowwise && (!is_aligned_ptr(input.data.dptr, 16) || + !is_aligned_ptr(input.scale_inv.dptr, 4))) || + (colwise && (!is_aligned_ptr(input.columnwise_data.dptr, 16) || + !is_aligned_ptr(input.columnwise_scale_inv.dptr, 4))); + if (misaligned) { + report_cutedsl_not_chosen("the input, output or scale pointer is misaligned."); + return false; + } + + checkCuDriverContext(stream); + const MXFP8DequantConfig config{/*fp8_dtype=*/input.dtype(), + /*out_dtype=*/output->data.dtype, + /*rowwise=*/rowwise, + /*colwise=*/colwise, + /*swizzled=*/input.with_gemm_swizzled_scales}; + std::optional func = config.get_kernel(); + if (!func.has_value()) { + return false; + } + + const int32_t device_index = transformer_engine::cuda::current_device(); + tvm_ffi_bridge::DLTensorWrapper mX_row, mS_row, mX_col, mS_col; + if (rowwise) { + mX_row = tvm_ffi_bridge::DLTensorWrapper(input.data, true, device_index); + mS_row = tvm_ffi_bridge::DLTensorWrapper(input.scale_inv, true, device_index); + } + if (colwise) { + mX_col = tvm_ffi_bridge::DLTensorWrapper(input.columnwise_data, true, device_index); + mS_col = tvm_ffi_bridge::DLTensorWrapper(input.columnwise_scale_inv, true, device_index); + } + tvm_ffi_bridge::DLTensorWrapper mO(output->data, true, device_index); + (*func)(&mX_row, &mS_row, &mX_col, &mS_col, &mO, static_cast(stream)); + return true; +} + +} // namespace cutedsl_backend +} // namespace transformer_engine + +#endif // TRANSFORMER_ENGINE_COMMON_CAST_MXFP8_DEQUANTIZE_MXFP8_CUTEDSL_CUH_ diff --git a/transformer_engine/common/cast/mxfp8/quantize_mxfp8_cutedsl.cuh b/transformer_engine/common/cast/mxfp8/quantize_mxfp8_cutedsl.cuh index 6380dda21da..4356b365e9c 100644 --- a/transformer_engine/common/cast/mxfp8/quantize_mxfp8_cutedsl.cuh +++ b/transformer_engine/common/cast/mxfp8/quantize_mxfp8_cutedsl.cuh @@ -231,11 +231,6 @@ bool mxfp8_quantize_cutedsl(const Tensor *input_tensor, const Tensor *act_input_ if (!tvm_ffi_bridge::TVMFFICentral::getInstance().get_cutedsl_backend_enabled()) { return false; } - // TODO(kainingz): port 2D quantization to CuTeDSL - if (use_2d_quantization) { - // report_cutedsl_not_chosen("2D quantization is not supported."); - return false; - } checkCuDriverContext(stream); const MXFP8QuantConfig config{/*dtype=*/input_tensor->dtype(), /*fp8_dtype=*/output_tensor->dtype(), diff --git a/transformer_engine/common/include/transformer_engine/recipe.h b/transformer_engine/common/include/transformer_engine/recipe.h index 98c5eed0985..1c9af7d4fcf 100644 --- a/transformer_engine/common/include/transformer_engine/recipe.h +++ b/transformer_engine/common/include/transformer_engine/recipe.h @@ -76,7 +76,9 @@ void nvte_delayed_scaling_recipe_amax_and_scale_update_after_reduction( /*! \brief Compute an FP8 tensor's amax. * * The amax (maximum absolute value) of the input tensor is computed - * and written to the amax buffer of the output tensor. + * and written to the amax buffer of the output tensor. For a Tensor + * with distinct row-wise and column-wise amax buffers, the + * amax is written to both. * * \param[in] input Input tensor. Must be unquantized. * \param[in,out] output Output tensor. Must be an FP8 tensor with per-tensor scaling. @@ -88,7 +90,8 @@ void nvte_compute_amax(const NVTETensor input, NVTETensor output, cudaStream_t s * * The amax (maximum absolute value) of the input tensor is computed * and written to the amax buffer of the output tensor, using the provided - * quantization configuration. + * quantization configuration. For a Tensor with distinct row-wise + * and column-wise amax buffers, the amax is written to both. * One useful config is the noop tensor, which is needed by cuda graph. * * \param[in] input Input tensor. Must be unquantized. diff --git a/transformer_engine/common/recipe/current_scaling.cu b/transformer_engine/common/recipe/current_scaling.cu index 3ba51dba754..2a623baf4de 100644 --- a/transformer_engine/common/recipe/current_scaling.cu +++ b/transformer_engine/common/recipe/current_scaling.cu @@ -21,16 +21,20 @@ namespace { constexpr int amax_kernel_threads = 512; -__launch_bounds__(1) __global__ void zero_amax_kernel(float *amax_ptr, const float *noop_ptr) { +__launch_bounds__(1) __global__ + void zero_amax_kernel(float *rowwise_amax_ptr, float *colwise_amax_ptr, const float *noop_ptr) { if (noop_ptr != nullptr && noop_ptr[0] == 1.0f) { return; } - *amax_ptr = 0; + *rowwise_amax_ptr = 0; + if (colwise_amax_ptr != nullptr) { + *colwise_amax_ptr = 0; + } } template __launch_bounds__(amax_kernel_threads) __global__ - void amax_kernel(const InputType *input, float *amax, const size_t N, + void amax_kernel(const InputType *input, float *amax, float *columnwise_amax, const size_t N, const size_t num_aligned_elements, const float *noop_ptr) { if (noop_ptr != nullptr && noop_ptr[0] == 1.0f) { return; @@ -66,14 +70,17 @@ __launch_bounds__(amax_kernel_threads) __global__ max = reduce_max(max, warp_id); if (threadIdx.x == 0) { atomicMaxFloat(amax, max); + if (columnwise_amax != nullptr) { + atomicMaxFloat(columnwise_amax, max); + } } } template void launch_amax_kernel(const InputType *input, float *amax, const size_t N, const float *noop_ptr, - cudaStream_t stream) { - // Zero out amax so we can update with atomic max - zero_amax_kernel<<<1, 1, 0, stream>>>(amax, noop_ptr); + cudaStream_t stream, float *columnwise_amax = nullptr) { + // Zero out amax (and the optional columnwise_amax) so we can update with atomic max + zero_amax_kernel<<<1, 1, 0, stream>>>(amax, columnwise_amax, noop_ptr); NVTE_CHECK_CUDA(cudaGetLastError()); // Return immediately if tensor is empty @@ -94,18 +101,18 @@ void launch_amax_kernel(const InputType *input, float *amax, const size_t N, con // Launch kernel switch (align) { case Alignment::SAME_ALIGNED: - amax_kernel - <<>>(input, amax, N, num_aligned_elements, noop_ptr); + amax_kernel<<>>( + input, amax, columnwise_amax, N, num_aligned_elements, noop_ptr); break; case Alignment::SAME_UNALIGNED: - amax_kernel - <<>>(input, amax, N, num_aligned_elements, noop_ptr); + amax_kernel<<>>( + input, amax, columnwise_amax, N, num_aligned_elements, noop_ptr); break; case Alignment::DIFFERENT: { // This case is a logic error, since there is only one pointer (input) // in the alignment check. Still safe to process without vectorization. amax_kernel<1, true, InputType> - <<>>(input, amax, N, N, noop_ptr); + <<>>(input, amax, columnwise_amax, N, N, noop_ptr); break; } } @@ -170,10 +177,16 @@ void compute_amax_impl(const NVTETensor input_, const NVTETensor output_, cudaSt // Compute amax float *amax_ptr = reinterpret_cast( (output.amax.dptr != nullptr) ? output.amax.dptr : output.columnwise_amax.dptr); + // Tensor with a distinct column-wise amax get the same value written there as well + float *columnwise_amax = nullptr; + if (output.scaling_mode == NVTE_NVFP4_1D_SCALING && output.columnwise_amax.dptr != nullptr && + output.columnwise_amax.dptr != amax_ptr) { + columnwise_amax = reinterpret_cast(output.columnwise_amax.dptr); + } TRANSFORMER_ENGINE_TYPE_SWITCH_INPUT( input.data.dtype, IType, constexpr int nvec = 32 / sizeof(IType); launch_amax_kernel( reinterpret_cast(input.data.dptr), amax_ptr, input.data.numel(), noop_ptr, - stream);); // NOLINT(*) + stream, columnwise_amax);); // NOLINT(*) } void group_compute_amax_impl(const NVTEGroupedTensor input_, NVTEGroupedTensor output_, diff --git a/transformer_engine/common/triton/nvfp4_global_scale.py b/transformer_engine/common/triton/nvfp4_global_scale.py new file mode 100644 index 00000000000..aee88be855e --- /dev/null +++ b/transformer_engine/common/triton/nvfp4_global_scale.py @@ -0,0 +1,53 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Triton kernel for the per-group NVFP4 global scales of a GEMM operand pair.""" + +import triton +import triton.language as tl + +# Output modes of _nvfp4_global_scale_pair_kernel. +NVFP4_GLOBAL_SCALE_PAIR = 0 # out_a = amax_a / denom_a, out_b = amax_b / denom_b +NVFP4_GLOBAL_SCALE_PRODUCT = 1 # out_a = (amax_a / denom_a) * (amax_b / denom_b) +NVFP4_GLOBAL_SCALE_SQRT_PRODUCT = 2 # out_a = sqrt of the product above + + +@triton.jit +def _nvfp4_global_scale_pair_kernel( + amax_a_ptr, + amax_b_ptr, + out_a_ptr, + out_b_ptr, + n, + a_broadcast, + b_broadcast, + denom_a, + denom_b, + MODE: tl.constexpr, + BLOCK: tl.constexpr, +): + """Per-group NVFP4 global scales of two operands, and optionally their product. + + One program handles all ``n`` groups. ``a_broadcast`` / ``b_broadcast`` mark an operand + with a single amax shared by every group. The arithmetic matches the eager PyTorch ops + bit for bit: dividing a CUDA tensor by a Python scalar multiplies by the FP32 reciprocal, + and the product and square root are IEEE round-to-nearest. + """ + idx = tl.arange(0, BLOCK) + mask = idx < n + amax_a = tl.load(amax_a_ptr + tl.where(a_broadcast, 0, idx), mask=mask, other=0.0) + amax_b = tl.load(amax_b_ptr + tl.where(b_broadcast, 0, idx), mask=mask, other=0.0) + one = tl.full([BLOCK], 1.0, tl.float32) + rcp_a = tl.math.div_rn(one, tl.full([BLOCK], denom_a, tl.float32)) + rcp_b = tl.math.div_rn(one, tl.full([BLOCK], denom_b, tl.float32)) + scale_a = amax_a.to(tl.float32) * rcp_a + scale_b = amax_b.to(tl.float32) * rcp_b + if MODE == 0: + tl.store(out_a_ptr + idx, scale_a, mask=mask) + tl.store(out_b_ptr + idx, scale_b, mask=mask) + else: + product = scale_a * scale_b + if MODE == 2: + product = tl.math.sqrt_rn(product) + tl.store(out_a_ptr + idx, product, mask=mask) diff --git a/transformer_engine/pytorch/__init__.py b/transformer_engine/pytorch/__init__.py index c408de1e52c..27535de9f97 100644 --- a/transformer_engine/pytorch/__init__.py +++ b/transformer_engine/pytorch/__init__.py @@ -46,6 +46,7 @@ from transformer_engine.pytorch.attention import InferenceParams from transformer_engine.pytorch.attention import RotaryPositionEmbedding from transformer_engine.pytorch.transformer import TransformerLayer +from transformer_engine.pytorch import models from transformer_engine.pytorch.permutation import ( moe_permute, moe_permute_with_probs, diff --git a/tests/pytorch/attention/mla_rope_utils.py b/transformer_engine/pytorch/attention/mla_rope.py similarity index 53% rename from tests/pytorch/attention/mla_rope_utils.py rename to transformer_engine/pytorch/attention/mla_rope.py index 90eebfc66ae..8b0e4ba747b 100644 --- a/tests/pytorch/attention/mla_rope_utils.py +++ b/transformer_engine/pytorch/attention/mla_rope.py @@ -2,16 +2,18 @@ # # See LICENSE for license information. -"""MLA RoPE for DSv3 671B - Triton forward and backward kernels. +"""Fused MLA RoPE kernels (DeepSeekV3-style decoupled RoPE/NoPE). -Source: Megatron-LM megatron/core/fusions/fused_mla_yarn_rope_apply.py -Falls back to pure PyTorch when Triton is unavailable. +The query kernel rotates the trailing ``head_dim_rope`` slice; the KV +kernel builds the key (nope | broadcast-rotated shared rope head) and value +tensors in a single pass. Falls back to pure PyTorch when Triton is unavailable +or for the ``bshd`` layout. -Note: DSv3 uses YaRN-scaled RoPE for long-context extrapolation. This test -intentionally uses plain RoPE (base=10000) because it only validates MXFP8 -attention path wiring, tensor shapes, forward/backward flow, and relative BF16 -vs MXFP8 behavior. Both reference and MXFP8 paths use the same RoPE tables. -""" +The rope slice is read interleaved (checkpoint layout) and written in NeoX +half-split layout.""" + +import math +from typing import Optional, Tuple import torch @@ -23,25 +25,106 @@ except ImportError: HAVE_TRITON = False -HEAD_DIM_ROPE = 64 -HEAD_DIM_NOPE = 128 -HEAD_DIM_V = 128 -ROTARY_BASE = 10000 +__all__ = [ + "build_rope_tables", + "apply_mla_rope_q", + "apply_mla_rope_kv", + "yarn_mscale", + "yarn_concentration_factor", +] + + +def _yarn_correction_dim(num_rotations, dim, base, max_pos): + return (dim * math.log(max_pos / (num_rotations * 2 * math.pi))) / (2 * math.log(base)) + + +def _yarn_correction_range(beta_fast, beta_slow, dim, base, max_pos, round_to_int=True): + low = _yarn_correction_dim(beta_fast, dim, base, max_pos) + high = _yarn_correction_dim(beta_slow, dim, base, max_pos) + if round_to_int: + low, high = math.floor(low), math.ceil(high) + return max(low, 0), min(high, dim - 1) + + +def _yarn_linear_ramp(low, high, dim, device): + if low == high: + high += 0.001 + ramp = (torch.arange(dim, dtype=torch.float32, device=device) - low) / (high - low) + return torch.clamp(ramp, 0, 1) + + +def yarn_mscale(scale: float, mscale: float = 1.0) -> float: + """YaRN attention temperature factor ``0.1 * mscale * ln(scale) + 1`` (1 for scale <= 1).""" + if scale <= 1: + return 1.0 + return 0.1 * mscale * math.log(scale) + 1.0 + + +def yarn_concentration_factor(scaling_factor: float, mscale: float, mscale_all_dim: float) -> float: + """Factor multiplied into cos/sin tables.""" + return yarn_mscale(scaling_factor, mscale) / yarn_mscale(scaling_factor, mscale_all_dim) def build_rope_tables( seq_len: int, - emb_dim: int = HEAD_DIM_ROPE, - base: int = ROTARY_BASE, - device: torch.device = None, -) -> tuple[torch.Tensor, torch.Tensor]: - inv_freq = 1.0 / ( - base ** (torch.arange(0, emb_dim, 2, dtype=torch.float32, device=device) / emb_dim) - ) + emb_dim: int, + base: float = 10000.0, + device: Optional[torch.device] = None, + scaling_factor: Optional[float] = None, + original_max_position_embeddings: int = 4096, + beta_fast: float = 32.0, + beta_slow: float = 1.0, + mscale: float = 1.0, + mscale_all_dim: float = 0.0, +) -> Tuple[torch.Tensor, torch.Tensor]: + r"""Build cosine/sine tables for :func:`apply_mla_rope_q` and :func:`apply_mla_rope_kv`. + + With :math:`d` = ``emb_dim``, row :math:`p` holds :math:`\cos\theta_{p,j}` and + :math:`\sin\theta_{p,j}` for :math:`j < d`, where + :math:`\theta_{p,j} = p\,\omega_{j \bmod d/2}` and :math:`\omega_i = \mathrm{base}^{-2i/d}`. + YaRN rescales :math:`\omega` and multiplies the tables by a magnitude factor. + + Parameters + ---------- + seq_len : int + Number of sequence positions. + emb_dim : int + Positive, even number of rotary channels; excludes the non-rotary channels. + base : float, default = 10000.0 + Base for the rotary frequencies. + device : torch.device, optional + Table device; ``None`` uses the default PyTorch device. + scaling_factor : float, optional + YaRN context-extension factor; ``None`` selects unscaled RoPE. + original_max_position_embeddings : int, default = 4096 + Original context length used to determine the YaRN frequency ramp. + beta_fast, beta_slow : float, default = 32.0, 1.0 + Rotation-count boundaries of the YaRN interpolation ramp. + mscale, mscale_all_dim : float, default = 1.0, 0.0 + YaRN magnitude coefficients. Tables are multiplied by the ratio of + ``yarn_mscale(scaling_factor, mscale)`` to + ``yarn_mscale(scaling_factor, mscale_all_dim)``. + + Returns + ------- + tuple of torch.Tensor + Contiguous FP32 ``(cos_table, sin_table)``, each of shape + ``[seq_len, emb_dim]`` with duplicated halves for NeoX layout. + """ + exponent = torch.arange(0, emb_dim, 2, dtype=torch.float32, device=device) / emb_dim + inv_freq = 1.0 / (base**exponent) + factor = 1.0 + if scaling_factor is not None: + low, high = _yarn_correction_range( + beta_fast, beta_slow, emb_dim, base, original_max_position_embeddings + ) + extra_mask = 1.0 - _yarn_linear_ramp(low, high, emb_dim // 2, device) + inv_freq = (inv_freq / scaling_factor) * (1 - extra_mask) + inv_freq * extra_mask + factor = yarn_concentration_factor(scaling_factor, mscale, mscale_all_dim) t = torch.arange(seq_len, device=device, dtype=torch.float32) freqs = torch.outer(t, inv_freq) freqs = torch.cat([freqs, freqs], dim=-1) - return torch.cos(freqs).contiguous(), torch.sin(freqs).contiguous() + return (torch.cos(freqs) * factor).contiguous(), (torch.sin(freqs) * factor).contiguous() if HAVE_TRITON: @@ -69,20 +152,9 @@ def _get_thd_token_idx(cu_seqlens, pid_m, seq_num, cp_rank, cp_size): ) * this_seq_len // 2 return token_idx - @triton.autotune( - configs=[ - triton.Config({"BLOCK_H": 1}), - triton.Config({"BLOCK_H": 2}), - triton.Config({"BLOCK_H": 4}), - triton.Config({"BLOCK_H": 8}), - triton.Config({"BLOCK_H": 16}), - triton.Config({"BLOCK_H": 32}), - triton.Config({"BLOCK_H": 64}), - triton.Config({"BLOCK_H": 128}), - ], - key=["emb_dim", "head_num"], - restore_value=["Q"], - ) + _AUTOTUNE_CONFIGS = [triton.Config({"BLOCK_H": h}) for h in (1, 2, 4, 8, 16, 32, 64, 128)] + + @triton.autotune(configs=_AUTOTUNE_CONFIGS, key=["emb_dim", "head_num"], restore_value=["Q"]) @triton.jit def rotary_fwd_q_kernel( Q, @@ -100,6 +172,7 @@ def rotary_fwd_q_kernel( cp_size, BLOCK_H: tl.constexpr, ): + """In-place RoPE fwd on the trailing rope slice of q.""" pid_m = tl.program_id(axis=0) pid_head = tl.program_id(axis=1) if cu_seqlens_q is None: @@ -129,20 +202,7 @@ def rotary_fwd_q_kernel( tl.store(Q + x_left_off, x_left, mask=mask) tl.store(Q + x_right_off, x_right, mask=mask) - @triton.autotune( - configs=[ - triton.Config({"BLOCK_H": 1}), - triton.Config({"BLOCK_H": 2}), - triton.Config({"BLOCK_H": 4}), - triton.Config({"BLOCK_H": 8}), - triton.Config({"BLOCK_H": 16}), - triton.Config({"BLOCK_H": 32}), - triton.Config({"BLOCK_H": 64}), - triton.Config({"BLOCK_H": 128}), - ], - key=["emb_dim", "head_num"], - restore_value=["DO"], - ) + @triton.autotune(configs=_AUTOTUNE_CONFIGS, key=["emb_dim", "head_num"], restore_value=["DO"]) @triton.jit def rotary_bwd_q_kernel( DO, @@ -160,6 +220,7 @@ def rotary_bwd_q_kernel( cp_size, BLOCK_H: tl.constexpr, ): + """In-place RoPE bwd on the trailing rope slice of dq.""" pid_m = tl.program_id(axis=0) pid_head = tl.program_id(axis=1) if cu_seqlens_q is None: @@ -189,19 +250,7 @@ def rotary_bwd_q_kernel( tl.store(DO + x_1_off, x_1, mask=mask) tl.store(DO + x_2_off, x_2, mask=mask) - @triton.autotune( - configs=[ - triton.Config({"BLOCK_H": 1}), - triton.Config({"BLOCK_H": 2}), - triton.Config({"BLOCK_H": 4}), - triton.Config({"BLOCK_H": 8}), - triton.Config({"BLOCK_H": 16}), - triton.Config({"BLOCK_H": 32}), - triton.Config({"BLOCK_H": 64}), - triton.Config({"BLOCK_H": 128}), - ], - key=["emb_dim", "k_dim", "v_dim", "head_num"], - ) + @triton.autotune(configs=_AUTOTUNE_CONFIGS, key=["emb_dim", "k_dim", "v_dim", "head_num"]) @triton.jit def rotary_fwd_kv_kernel( KV, @@ -228,6 +277,7 @@ def rotary_fwd_kv_kernel( cp_size, BLOCK_H: tl.constexpr, ): + """Fwd: build (key, value) from kv and the shared rotated rope head.""" pid_m = tl.program_id(axis=0) pid_head = tl.program_id(axis=1) if cu_seqlens_kv is None: @@ -268,19 +318,7 @@ def rotary_fwd_kv_kernel( tl.store(K_ptr + x_left_off, x_left, mask=mask) tl.store(K_ptr + x_right_off, x_right, mask=mask) - @triton.autotune( - configs=[ - triton.Config({"BLOCK_H": 1}), - triton.Config({"BLOCK_H": 2}), - triton.Config({"BLOCK_H": 4}), - triton.Config({"BLOCK_H": 8}), - triton.Config({"BLOCK_H": 16}), - triton.Config({"BLOCK_H": 32}), - triton.Config({"BLOCK_H": 64}), - triton.Config({"BLOCK_H": 128}), - ], - key=["emb_dim", "k_dim", "v_dim", "head_num"], - ) + @triton.autotune(configs=_AUTOTUNE_CONFIGS, key=["emb_dim", "k_dim", "v_dim", "head_num"]) @triton.jit def rotary_bwd_kv_kernel( dK, @@ -307,6 +345,7 @@ def rotary_bwd_kv_kernel( cp_size, BLOCK_H: tl.constexpr, ): + """Bwd: scatter (dk, dv) into dkv and reduce rope-slice grads into demb.""" pid_m = tl.program_id(axis=0) pid_head = tl.program_id(axis=1) if cu_seqlens_kv is None: @@ -357,19 +396,25 @@ def rotary_bwd_kv_kernel( tl.store(dEMB_ptr + tl.arange(0, emb_dim // 2) * 2, x_1) tl.store(dEMB_ptr + tl.arange(0, emb_dim // 2) * 2 + 1, x_2) - def _flattened_token_stride(tensor: torch.Tensor) -> int: - if tensor.dim() == 4: - return tensor.stride(1) - return tensor.stride(0) + def _token_stride(tensor: torch.Tensor) -> int: + return tensor.stride(1) if tensor.dim() == 4 else tensor.stride(0) class _MLARoPEQTriton(torch.autograd.Function): + """RoPE on the trailing rope slice of q [s, b, h, nope+rope].""" + @staticmethod - def forward(ctx, q, cos, sin, head_dim_nope, head_dim_rope): + def forward(ctx, q, cos, sin, head_dim_nope, head_dim_rope, in_place): + """Rotate the rope slice of q.""" + if in_place: + ctx.mark_dirty(q) + else: + q = q.clone(memory_format=torch.contiguous_format) s, b, nheads, _ = q.shape - total = s * b - grid_q = lambda META: (total, triton.cdiv(nheads, META["BLOCK_H"])) - rotary_fwd_q_kernel[grid_q]( + def grid(meta): + return (s * b, triton.cdiv(nheads, meta["BLOCK_H"])) + + rotary_fwd_q_kernel[grid]( q, cos, sin, @@ -379,54 +424,58 @@ def forward(ctx, q, cos, sin, head_dim_nope, head_dim_rope): b, None, None, - _flattened_token_stride(q), + _token_stride(q), q.stride(2), 0, 1, ) - ctx.save_for_backward(cos, sin) - ctx.head_dim_nope = head_dim_nope - ctx.head_dim_rope = head_dim_rope - ctx.nheads = nheads - ctx.s = s - ctx.b = b + ctx.dims = (s, b, nheads, head_dim_nope, head_dim_rope) return q @staticmethod def backward(ctx, dq): + """Counter-rotate the rope slice of dq.""" cos, sin = ctx.saved_tensors - s, b, nheads = ctx.s, ctx.b, ctx.nheads - total = s * b + dq = dq.clone(memory_format=torch.contiguous_format) + s, b, nheads, head_dim_nope, head_dim_rope = ctx.dims + + def grid(meta): + return (s * b, triton.cdiv(nheads, meta["BLOCK_H"])) - grid_q = lambda META: (total, triton.cdiv(nheads, META["BLOCK_H"])) - rotary_bwd_q_kernel[grid_q]( + rotary_bwd_q_kernel[grid]( dq, cos, sin, - ctx.head_dim_nope, - ctx.head_dim_rope, + head_dim_nope, + head_dim_rope, nheads, b, None, None, - _flattened_token_stride(dq), + _token_stride(dq), dq.stride(2), 0, 1, ) - return dq, None, None, None, None + return dq, None, None, None, None, None class _MLARoPEKVTriton(torch.autograd.Function): + """kv [s, b, h, nope+v] + shared rope head [s, b, 1, rope] -> (k, v).""" + @staticmethod def forward(ctx, kv, k_pos_emb, cos, sin, head_dim_nope, head_dim_rope, head_dim_v): + """Build (k, v) from kv and the shared rope head.""" + if not kv.is_contiguous(): + kv = kv.contiguous() s, b, nheads, _ = kv.shape - total = s * b - o_key = kv.new_empty(s, b, nheads, head_dim_nope + head_dim_rope) o_value = kv.new_empty(s, b, nheads, head_dim_v) - grid_kv = lambda META: (total, triton.cdiv(nheads, META["BLOCK_H"])) - rotary_fwd_kv_kernel[grid_kv]( + + def grid(meta): + return (s * b, triton.cdiv(nheads, meta["BLOCK_H"])) + + rotary_fwd_kv_kernel[grid]( kv, k_pos_emb, o_key, @@ -440,37 +489,34 @@ def forward(ctx, kv, k_pos_emb, cos, sin, head_dim_nope, head_dim_rope, head_dim b, None, None, - _flattened_token_stride(kv), + _token_stride(kv), kv.stride(2), - _flattened_token_stride(k_pos_emb), - _flattened_token_stride(o_key), + _token_stride(k_pos_emb), + _token_stride(o_key), o_key.stride(2), - _flattened_token_stride(o_value), + _token_stride(o_value), o_value.stride(2), 0, 1, ) - ctx.save_for_backward(cos, sin) - ctx.head_dim_nope = head_dim_nope - ctx.head_dim_rope = head_dim_rope - ctx.head_dim_v = head_dim_v - ctx.nheads = nheads - ctx.s = s - ctx.b = b + ctx.dims = (s, b, nheads, head_dim_nope, head_dim_rope, head_dim_v) return o_key, o_value @staticmethod def backward(ctx, dk_out, dv_out): + """Gradients for (kv, k_pos_emb) from (dk, dv).""" cos, sin = ctx.saved_tensors - s, b, nheads = ctx.s, ctx.b, ctx.nheads - ndp, ndr, ndv = ctx.head_dim_nope, ctx.head_dim_rope, ctx.head_dim_v - total = s * b - + s, b, nheads, ndp, ndr, ndv = ctx.dims + dk_out = dk_out.contiguous() + dv_out = dv_out.contiguous() d_kv = dk_out.new_empty(s, b, nheads, ndp + ndv) d_emb = dk_out.new_empty(s, b, 1, ndr) - grid_kv = lambda META: (total, triton.cdiv(nheads, META["BLOCK_H"])) - rotary_bwd_kv_kernel[grid_kv]( + + def grid(meta): + return (s * b, triton.cdiv(nheads, meta["BLOCK_H"])) + + rotary_bwd_kv_kernel[grid]( dk_out, dv_out, d_kv, @@ -484,185 +530,170 @@ def backward(ctx, dk_out, dv_out): b, None, None, - _flattened_token_stride(dk_out), + _token_stride(dk_out), dk_out.stride(2), - _flattened_token_stride(dv_out), + _token_stride(dv_out), dv_out.stride(2), - _flattened_token_stride(d_kv), + _token_stride(d_kv), d_kv.stride(2), - _flattened_token_stride(d_emb), + _token_stride(d_emb), 0, 1, ) return d_kv, d_emb, None, None, None, None, None -def _apply_mla_rope_q_with_tables( +def _rotate_interleaved_to_neox(x, cos_table, sin_table, seq_dim): + shape = [1, 1, 1, cos_table.shape[-1]] + shape[seq_dim] = cos_table.shape[0] + cos_ = cos_table.view(shape).to(x.dtype) + sin_ = sin_table.view(shape).to(x.dtype) + half = x.shape[-1] // 2 + x_1 = x[..., 0::2] + x_2 = x[..., 1::2] + x_left = x_1 * cos_[..., :half] - x_2 * sin_[..., :half] + x_right = x_2 * cos_[..., half:] + x_1 * sin_[..., half:] + return torch.cat((x_left, x_right), dim=-1) + + +def apply_mla_rope_q( q: torch.Tensor, cos_table: torch.Tensor, sin_table: torch.Tensor, - head_dim_nope: int = HEAD_DIM_NOPE, - head_dim_rope: int = HEAD_DIM_ROPE, + head_dim_nope: int, + head_dim_rope: int, + tensor_format: str = "sbhd", + in_place: bool = False, ) -> torch.Tensor: - if HAVE_TRITON: + r"""Rotate the trailing query channels, preserving the non-rotary prefix. + + With :math:`d` = ``head_dim_rope``, each interleaved pair :math:`(x_{2i}, x_{2i+1})` of + the rotary channels at position :math:`p` is rotated and written in half-split order: + + .. math:: + + y_i = x_{2i}\cos\theta_{p,i} - x_{2i+1}\sin\theta_{p,i}, \qquad + y_{i+d/2} = x_{2i+1}\cos\theta_{p,i} + x_{2i}\sin\theta_{p,i} + + where :math:`\cos\theta_{p,i}` and :math:`\sin\theta_{p,i}` are entry :math:`i` of + row :math:`p` of the tables. + + Parameters + ---------- + q : torch.Tensor + CUDA query tensor of shape ``[s, b, h, head_dim_nope + head_dim_rope]`` + for ``sbhd``, or ``[b, s, h, head_dim_nope + head_dim_rope]`` for ``bshd``. + cos_table, sin_table : torch.Tensor + Contiguous FP32 tables of shape ``[s, head_dim_rope]`` on the same device + as ``q``, as returned by :func:`build_rope_tables`. + head_dim_nope, head_dim_rope : int + Number of non-rotary and rotary channels per head. + tensor_format : {"sbhd", "bshd"}, default = "sbhd" + Input and output layout. + in_place : bool, default = False + Rotate ``q`` in place instead of allocating an output. Requires a contiguous + ``sbhd`` input and a power-of-two rotary dimension of at least two. + Supported in eager mode only. + + Returns + ------- + torch.Tensor + Rotated queries with the same shape and dtype as ``q``. + + Notes + ----- + Triton is used for ``sbhd`` when ``head_dim_rope`` is a power of two and + at least two; other supported layouts/dimensions use PyTorch operations. + The Triton path computes gradients for ``q`` only; treat the tables as constants. + + The default path leaves ``q`` and incoming backward gradients unchanged. + With ``in_place=True``, ``q`` must have no other consumers that need its + original values, and PyTorch's usual in-place autograd rules apply. In + particular, views of custom autograd Function outputs cannot be mutated. + """ + use_triton = ( + HAVE_TRITON + and tensor_format == "sbhd" + and head_dim_rope >= 2 + and _is_power_of_two(head_dim_rope) + ) + if in_place and (not use_triton or not q.is_contiguous()): + raise ValueError("in_place=True requires contiguous sbhd input and Triton RoPE support") + if in_place and torch.compiler.is_compiling(): + raise RuntimeError("in_place=True is not supported under torch.compile") + if use_triton: return _MLARoPEQTriton.apply( - q, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, + q, cos_table, sin_table, head_dim_nope, head_dim_rope, in_place ) - return _apply_pytorch_q(q, cos_table, sin_table, head_dim_nope, head_dim_rope) + seq_dim = 0 if tensor_format == "sbhd" else 1 + q_rope = _rotate_interleaved_to_neox(q[..., head_dim_nope:], cos_table, sin_table, seq_dim) + return torch.cat((q[..., :head_dim_nope], q_rope), dim=-1) -def _apply_mla_rope_kv_with_tables( +def apply_mla_rope_kv( kv: torch.Tensor, k_pos_emb: torch.Tensor, cos_table: torch.Tensor, sin_table: torch.Tensor, - head_dim_nope: int = HEAD_DIM_NOPE, - head_dim_rope: int = HEAD_DIM_ROPE, - head_dim_v: int = HEAD_DIM_V, -) -> tuple[torch.Tensor, torch.Tensor]: - if HAVE_TRITON: + head_dim_nope: int, + head_dim_rope: int, + head_dim_v: int, + tensor_format: str = "sbhd", +) -> Tuple[torch.Tensor, torch.Tensor]: + """Assemble keys and values using a shared rotary key head. + + ``k_pos_emb`` is rotated as the rotary channels in :func:`apply_mla_rope_q` and + appended to the non-rotary key channels of every head. + + Parameters + ---------- + kv : torch.Tensor + CUDA tensor of shape ``[s, b, h, head_dim_nope + head_dim_v]`` for + ``sbhd``, or ``[b, s, h, head_dim_nope + head_dim_v]`` for ``bshd``. + Each head contains the non-rotary key channels followed by value channels. + k_pos_emb : torch.Tensor + Shared rotary key channels of shape ``[s, b, 1, head_dim_rope]`` for + ``sbhd``, or ``[b, s, 1, head_dim_rope]`` for ``bshd``, matching the + device and dtype of ``kv``. Triton requires a contiguous last dimension + and ``stride(0) == b * stride(1)``. + cos_table, sin_table : torch.Tensor + Contiguous FP32 tables of shape ``[s, head_dim_rope]`` on the same device + as ``kv``, as returned by :func:`build_rope_tables`. + head_dim_nope, head_dim_rope, head_dim_v : int + Number of non-rotary key, rotary key and value channels per head. + tensor_format : {"sbhd", "bshd"}, default = "sbhd" + Input and output layout. + + Returns + ------- + tuple of torch.Tensor + Contiguous ``(k, v)`` in the selected layout and input dtype. The last + dimensions are ``head_dim_nope + head_dim_rope`` for ``k`` and + ``head_dim_v`` for ``v``. The rotated shared key head is broadcast to all heads. + + Notes + ----- + Triton is used for ``sbhd`` when all three head dimensions are powers of two + and ``head_dim_rope >= 2``; other supported layouts/dimensions use PyTorch. + Inputs are preserved. The Triton path computes gradients for ``kv`` and + ``k_pos_emb`` only; treat the tables as constants. + """ + if ( + HAVE_TRITON + and tensor_format == "sbhd" + and head_dim_rope >= 2 + and all(_is_power_of_two(dim) for dim in (head_dim_nope, head_dim_rope, head_dim_v)) + ): return _MLARoPEKVTriton.apply( - kv, - k_pos_emb, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - head_dim_v, - ) - return _apply_pytorch_kv( - kv, - k_pos_emb, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - head_dim_v, - ) - - -def apply_mla_rope_q( - q: torch.Tensor, - head_dim_nope: int = HEAD_DIM_NOPE, - head_dim_rope: int = HEAD_DIM_ROPE, - base: int = ROTARY_BASE, - cos_table: torch.Tensor | None = None, - sin_table: torch.Tensor | None = None, -) -> torch.Tensor: - if cos_table is None or sin_table is None: - s = q.shape[0] - cos_table, sin_table = build_rope_tables( - s, - emb_dim=head_dim_rope, - base=base, - device=q.device, - ) - return _apply_mla_rope_q_with_tables( - q, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - ) - - -def apply_mla_rope_kv( - kv: torch.Tensor, - k_pos_emb: torch.Tensor, - head_dim_nope: int = HEAD_DIM_NOPE, - head_dim_rope: int = HEAD_DIM_ROPE, - head_dim_v: int = HEAD_DIM_V, - base: int = ROTARY_BASE, - cos_table: torch.Tensor | None = None, - sin_table: torch.Tensor | None = None, -) -> tuple[torch.Tensor, torch.Tensor]: - if cos_table is None or sin_table is None: - s = kv.shape[0] - cos_table, sin_table = build_rope_tables( - s, - emb_dim=head_dim_rope, - base=base, - device=kv.device, - ) - return _apply_mla_rope_kv_with_tables( - kv, - k_pos_emb, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - head_dim_v, - ) - - -def apply_mla_rope( - q: torch.Tensor, - kv: torch.Tensor, - k_pos_emb: torch.Tensor, - head_dim_nope: int = HEAD_DIM_NOPE, - head_dim_rope: int = HEAD_DIM_ROPE, - head_dim_v: int = HEAD_DIM_V, - base: int = ROTARY_BASE, - cos_table: torch.Tensor | None = None, - sin_table: torch.Tensor | None = None, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - if cos_table is None or sin_table is None: - s = q.shape[0] - cos_table, sin_table = build_rope_tables( - s, - emb_dim=head_dim_rope, - base=base, - device=q.device, + kv, k_pos_emb, cos_table, sin_table, head_dim_nope, head_dim_rope, head_dim_v ) - q = _apply_mla_rope_q_with_tables(q, cos_table, sin_table, head_dim_nope, head_dim_rope) - k, v = _apply_mla_rope_kv_with_tables( - kv, - k_pos_emb, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - head_dim_v, - ) - return q, k, v - - -def _rotate_interleaved_to_neox( - x: torch.Tensor, cos_table: torch.Tensor, sin_table: torch.Tensor -) -> torch.Tensor: - cos_ = cos_table[:, None, None, :].to(x.dtype) - sin_ = sin_table[:, None, None, :].to(x.dtype) - half_dim = x.shape[-1] // 2 - x_1 = x[..., 0::2] - x_2 = x[..., 1::2] - x_left = x_1 * cos_[..., :half_dim] - x_2 * sin_[..., :half_dim] - x_right = x_2 * cos_[..., half_dim:] + x_1 * sin_[..., half_dim:] - return torch.cat((x_left, x_right), dim=-1) - - -def _apply_pytorch_q(q, cos_table, sin_table, head_dim_nope, head_dim_rope): - q_nope = q[..., :head_dim_nope] - q_rope = q[..., head_dim_nope : head_dim_nope + head_dim_rope] - q_rope = _rotate_interleaved_to_neox(q_rope, cos_table, sin_table) - return torch.cat((q_nope, q_rope), dim=-1) - - -def _apply_pytorch_kv( - kv, - k_pos_emb, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - head_dim_v, -): + seq_dim = 0 if tensor_format == "sbhd" else 1 k_nope = kv[..., :head_dim_nope] v = kv[..., head_dim_nope : head_dim_nope + head_dim_v] - k_rope = _rotate_interleaved_to_neox(k_pos_emb, cos_table, sin_table).expand( - -1, -1, kv.shape[2], -1 - ) - return torch.cat((k_nope, k_rope), dim=-1), v + k_rope = _rotate_interleaved_to_neox(k_pos_emb, cos_table, sin_table, seq_dim) + k_rope = k_rope.expand(*k_nope.shape[:-1], -1) + return torch.cat((k_nope, k_rope), dim=-1), v.contiguous() + + +def _is_power_of_two(value: int) -> bool: + return value > 0 and value & (value - 1) == 0 diff --git a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp index f2445697b71..3172549be78 100644 --- a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp +++ b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp @@ -35,11 +35,14 @@ void reset_tensor_data(transformer_engine::TensorWrapper &tensor, bool rowwise, } } -bool is_empty_grouped_tensor_param(const NVTEBasicTensor &t) { - if (t.data_ptr == nullptr) { - return true; +bool is_empty_grouped_tensor_param(const GroupedTensorWrapper &input) { + const auto logical_shape = input.logical_shape(); + for (size_t dimension = 0; dimension < logical_shape.ndim; ++dimension) { + if (logical_shape.data[dimension] == 0) { + return true; + } } - return t.shape.ndim == 1 && t.shape.data[0] == 0; + return false; } } // namespace @@ -501,15 +504,18 @@ std::optional maybe_swizzle_grouped_tensor(GroupedTensorW if (input.scaling_mode() != NVTE_MXFP8_1D_SCALING) { return std::nullopt; } + const auto row_scales = input.get_rowwise_scale_inv(); + const auto col_scales = input.get_columnwise_scale_inv(); + const bool empty_input = is_empty_grouped_tensor_param(input); + const bool swizzle_rowwise = rowwise_usage && !empty_input; + const bool swizzle_columnwise = columnwise_usage && !empty_input; if (input.get_with_gemm_swizzled_scales()) { return std::nullopt; } - - const auto row_scales = input.get_rowwise_scale_inv(); - const auto col_scales = input.get_columnwise_scale_inv(); - const bool swizzle_rowwise = rowwise_usage && !is_empty_grouped_tensor_param(row_scales); - const bool swizzle_columnwise = columnwise_usage && !is_empty_grouped_tensor_param(col_scales); if (!swizzle_rowwise && !swizzle_columnwise) { + if (empty_input) { + input.set_with_gemm_swizzled_scales(true); + } return std::nullopt; } @@ -643,8 +649,8 @@ void grouped_swizzle_for_gemm(py::handle &tensor, bool rowwise, bool columnwise) } else { tensor.attr("columnwise_scale_inv") = py::none(); } - tensor.attr("_with_gemm_swizzled_scales") = py::cast(true); } + tensor.attr("_with_gemm_swizzled_scales") = py::cast(tensor_nvte.get_with_gemm_swizzled_scales()); } namespace { diff --git a/transformer_engine/pytorch/csrc/quantizer.cpp b/transformer_engine/pytorch/csrc/quantizer.cpp index 6251c85743a..eccebf62acb 100644 --- a/transformer_engine/pytorch/csrc/quantizer.cpp +++ b/transformer_engine/pytorch/csrc/quantizer.cpp @@ -2753,22 +2753,13 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou void* amax_ptr = rowwise_amax_ptr != nullptr ? rowwise_amax_ptr : columnwise_amax_ptr; NVTE_CHECK(amax_ptr != nullptr, "Could not find amax pointer"); - // Compute amax of input tensor + // Compute amax of input tensor. When both the row-wise and the column-wise amax are + // allocated, the amax kernel updates both (identical values) kernel_out.set_amax(amax_ptr, DType::kFloat32, std::vector{1}); NVTE_SCOPED_GIL_RELEASE({ nvte_compute_amax_with_config(input.data(), kernel_out.data(), quant_config, stream); }); kernel_out.set_amax(rowwise_amax_ptr, DType::kFloat32, std::vector{1}); - - // Make sure row-wise and column-wise amaxes match - if (rowwise_amax_ptr != amax_ptr && rowwise_amax_ptr != nullptr) { - NVTE_CHECK_CUDA(cudaMemcpyAsync(rowwise_amax_ptr, amax_ptr, sizeof(float), - cudaMemcpyDeviceToDevice, stream)); - } - if (columnwise_amax_ptr != amax_ptr && columnwise_amax_ptr != nullptr) { - NVTE_CHECK_CUDA(cudaMemcpyAsync(columnwise_amax_ptr, amax_ptr, sizeof(float), - cudaMemcpyDeviceToDevice, stream)); - } } } diff --git a/transformer_engine/pytorch/models/__init__.py b/transformer_engine/pytorch/models/__init__.py new file mode 100644 index 00000000000..bee5474c816 --- /dev/null +++ b/transformer_engine/pytorch/models/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Model-specific transformer layers composed from Transformer Engine modules.""" + +from transformer_engine.pytorch.models.deepseek_v3 import ( + DeepSeekV3Layer, + DeepSeekV3MoE, + MultiLatentAttention, +) + +__all__ = ["DeepSeekV3Layer", "DeepSeekV3MoE", "MultiLatentAttention"] diff --git a/transformer_engine/pytorch/models/deepseek_v3/__init__.py b/transformer_engine/pytorch/models/deepseek_v3/__init__.py new file mode 100644 index 00000000000..a7cbb50ae2f --- /dev/null +++ b/transformer_engine/pytorch/models/deepseek_v3/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""DeepSeekV3 transformer layer built from Transformer Engine MoE building blocks.""" + +from transformer_engine.pytorch.models.deepseek_v3.multi_latent_attention import ( + MultiLatentAttention, +) +from transformer_engine.pytorch.models.deepseek_v3.moe import DeepSeekV3MoE +from transformer_engine.pytorch.models.deepseek_v3.transformer_layer import DeepSeekV3Layer + +__all__ = ["DeepSeekV3Layer", "DeepSeekV3MoE", "MultiLatentAttention"] diff --git a/transformer_engine/pytorch/models/deepseek_v3/moe.py b/transformer_engine/pytorch/models/deepseek_v3/moe.py new file mode 100644 index 00000000000..9ad63ba6d42 --- /dev/null +++ b/transformer_engine/pytorch/models/deepseek_v3/moe.py @@ -0,0 +1,328 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""DeepSeekV3 MoE block: sigmoid router with aux-loss-free bias, shared + +routed experts.""" + +from typing import Optional, Union + +import torch + +import transformer_engine.pytorch.ops as te_ops +from transformer_engine.pytorch.router import fused_topk_with_score_function +from transformer_engine.pytorch.permutation import ( + moe_permute_and_pad_with_probs, + moe_permute_with_probs, + moe_unpermute, +) +from transformer_engine.pytorch.quantization import ( + FP8GlobalStateManager, + get_align_size_for_quantization, +) + +__all__ = ["DeepSeekV3MoE"] + + +_EP_ALIGNMENT = 256 +_FUSED_MLP_ROWS = 256 + + +def _make_swiglu_mlp(hidden_size, ffn_hidden_size, dtype, device, num_experts=None): + """Dense SwiGLU MLP, or a grouped one (probs applied inside the activation) per expert. + + The grouped variant fuses into a single CuTe grouped MLP on supported hardware. + """ + common = {"bias": False, "dtype": dtype, "device": device} + if num_experts is None: + return te_ops.Sequential( + te_ops.Linear(hidden_size, 2 * ffn_hidden_size, **common), + te_ops.SwiGLU(), + te_ops.Linear(ffn_hidden_size, hidden_size, **common), + ) + return te_ops.Sequential( + te_ops.GroupedLinear(num_experts, hidden_size, 2 * ffn_hidden_size, **common), + te_ops.ScaledSwiGLU(glu_interleave_size=32), + te_ops.GroupedLinear(num_experts, ffn_hidden_size, hidden_size, **common), + ) + + +class DeepSeekV3MoE(torch.nn.Module): + """ + DeepSeekV3 Mixture-of-Experts block. + + Each token is scored by a sigmoid router with a non-trainable expert bias + updated by ``update_expert_bias()`` (aux-loss-free load balancing) and, + optionally, group-limited routing: experts are split into ``num_groups`` + groups, the top ``group_topk`` groups are selected by their summed scores, + and the final ``topk`` experts are chosen only from those groups. Selected + tokens run through the routed experts, a SwiGLU MLP shared across experts + as a grouped GEMM, with the routing probability applied inside the MLP. An + optional shared expert (dense SwiGLU MLP) is added to every token. On + hardware that supports it the expert MLP runs as a single fused + grouped-GEMM kernel. MXFP8 and NVFP4 pad each expert's token count to a + multiple of 256, including unfused execution; FP8 block scaling uses 128. + + Without ``ep_group`` all experts live on the local device. With + ``ep_group`` the experts are split across the group and tokens are + exchanged over NCCL; this requires ``ep_bootstrap`` to be called once per + process before constructing the module, and bfloat16 inputs. + + Parameters + ---------- + hidden_size : int + size of each input sample. + moe_ffn_hidden_size : int + ffn size of each routed expert. + num_experts : int + total number of routed experts. + topk : int, default = 8 + number of experts per token. + num_groups : int, optional + number of expert groups for node-limited routing; requires ``group_topk``. + group_topk : int, optional + number of groups each token is limited to; requires ``num_groups``. + routed_scaling_factor : float, default = 2.5 + scaling applied to the routing probabilities. + shared_expert_ffn_hidden_size : int, optional + ffn size of the shared expert; ``None`` + disables the shared expert. + expert_bias_update_rate : float, default = 1e-3 + step size of the aux-loss-free bias update + (see :meth:`update_expert_bias`). + params_dtype : torch.dtype, optional + dtype of module parameters. + ep_group : ProcessGroup, optional + expert-parallel process group; enables the NCCL EP path. + ep_max_tokens_per_rank : int, optional + max local tokens per forward (required with EP). + """ + + def __init__( + self, + hidden_size: int, + moe_ffn_hidden_size: int, + num_experts: int, + topk: int = 8, + num_groups: Optional[int] = None, + group_topk: Optional[int] = None, + routed_scaling_factor: float = 2.5, + shared_expert_ffn_hidden_size: Optional[int] = None, + expert_bias_update_rate: float = 1e-3, + params_dtype: Optional[torch.dtype] = None, + device: Union[torch.device, str] = "cuda", + ep_group: Optional[torch.distributed.ProcessGroup] = None, + ep_max_tokens_per_rank: Optional[int] = None, + ) -> None: + super().__init__() + + if num_experts <= 0: + raise ValueError("num_experts must be positive.") + if not 1 <= topk <= num_experts: + raise ValueError("topk must be in [1, num_experts].") + if (num_groups is None) != (group_topk is None): + raise ValueError("num_groups and group_topk must be provided together.") + if num_groups is not None: + if num_groups <= 0 or num_experts % num_groups != 0: + raise ValueError("num_groups must be positive and divide num_experts.") + if not 1 <= group_topk <= num_groups: + raise ValueError("group_topk must be in [1, num_groups].") + if topk % group_topk != 0: + raise ValueError("topk must be divisible by group_topk.") + if topk // group_topk > num_experts // num_groups: + raise ValueError("topk per group must not exceed the number of experts per group.") + + dtype = params_dtype if params_dtype is not None else torch.get_default_dtype() + self.hidden_size = hidden_size + self.num_experts = num_experts + self.topk = topk + self.num_groups = num_groups + self.group_topk = group_topk + self.routed_scaling_factor = routed_scaling_factor + self.expert_bias_update_rate = expert_bias_update_rate + + self.gate = torch.nn.Linear( + hidden_size, num_experts, bias=False, dtype=dtype, device=device + ) + self.register_buffer( + "expert_bias", torch.zeros(num_experts, dtype=torch.float32, device=device) + ) + self._last_tokens_per_expert: Optional[torch.Tensor] = None + + self.ep_group = ep_group + self.ep_size = 1 if ep_group is None else torch.distributed.get_world_size(ep_group) + assert num_experts % self.ep_size == 0 + num_local_experts = num_experts // self.ep_size + + expert_mlp = _make_swiglu_mlp( + hidden_size, moe_ffn_hidden_size, dtype, device, num_experts=num_local_experts + ) + + self._ep_buffer_kwargs = None + if ep_group is not None: + from transformer_engine.pytorch.ep import EpConfig, get_ep_drop_on_overflow + + assert ep_max_tokens_per_rank is not None, "EP requires ep_max_tokens_per_rank." + drop_on_overflow = get_ep_drop_on_overflow() + if drop_on_overflow is None: + raise RuntimeError("EP requires ep_bootstrap before constructing DeepSeekV3MoE.") + cap = self.ep_recv_capacity( + self.ep_size, ep_max_tokens_per_rank, topk, num_local_experts + ) + self._ep_buffer_kwargs = { + "top_k": topk, + "max_tokens_per_rank": ep_max_tokens_per_rank, + "hidden_dim": hidden_size, + "num_local_experts": num_local_experts, + "recv_capacity_per_rank": cap, + "alignment": _EP_ALIGNMENT, + } + config = EpConfig( + ep_group=ep_group, + drop_on_overflow=drop_on_overflow, + **self._ep_buffer_kwargs, + ) + dispatch = te_ops.MoeDispatch(config) + combine = te_ops.MoeCombine(config) + fc1, activation, fc2 = expert_mlp + dispatch.set_extra_output_channel(0, "tokens_per_expert", output_to_caller=False) + dispatch.set_extra_output_channel(1, "routing_weights", output_to_caller=False) + fc1.set_extra_input_channel(0, "tokens_per_expert") + activation.set_extra_input_channel(0, "routing_weights") + fc2.set_extra_input_channel(0, "tokens_per_expert") + self.experts = te_ops.Sequential( + {"dispatch": dispatch, "0": fc1, "1": activation, "2": fc2, "combine": combine} + ) + else: + self.experts = expert_mlp + + self.shared_expert = None + if shared_expert_ffn_hidden_size is not None: + self.shared_expert = _make_swiglu_mlp( + hidden_size, shared_expert_ffn_hidden_size, dtype, device + ) + + @staticmethod + def ep_recv_capacity( + ep_size: int, max_tokens_per_rank: int, topk: int, num_local_experts: int + ) -> int: + """Worst-case receive capacity including per-expert alignment padding.""" + cap = ep_size * max_tokens_per_rank * topk + cap += num_local_experts * _EP_ALIGNMENT + return -(-cap // _FUSED_MLP_ROWS) * _FUSED_MLP_ROWS + + def _route(self, logits: torch.Tensor, topk_indices: Optional[torch.Tensor] = None): + return fused_topk_with_score_function( + logits=logits, + topk=self.topk, + use_pre_softmax=False, + num_groups=self.num_groups, + group_topk=self.group_topk, + scaling_factor=self.routed_scaling_factor, + score_function="sigmoid", + expert_bias=self.expert_bias, + topk_indices=topk_indices, + ) + + def _forward_local(self, tokens: torch.Tensor) -> torch.Tensor: + probs, routing_map = self._route(self.gate(tokens).float()) + tokens_per_expert = routing_map.sum(dim=0) + self._last_tokens_per_expert = tokens_per_expert.detach() + + # Quantized grouped GEMMs need every expert's row count aligned. + align = 1 + if FP8GlobalStateManager.is_fp8_enabled(): + recipe = FP8GlobalStateManager.get_fp8_recipe() + align = get_align_size_for_quantization(recipe) + if recipe.mxfp8() or recipe.nvfp4(): + align = max(align, _FUSED_MLP_ROWS) + elif recipe.float8_block_scaling(): + align = max(align, 128) + if align > 1: + permuted, permuted_probs, row_id_map, pad_offsets, tokens_per_expert = ( + moe_permute_and_pad_with_probs(tokens, probs, routing_map, tokens_per_expert, align) + ) + else: + permuted, permuted_probs, row_id_map = moe_permute_with_probs( + tokens, probs, routing_map, num_out_tokens=tokens.shape[0] * self.topk + ) + pad_offsets = None + + # The fused grouped MLP requires the total row count to be a multiple + # of 128; rows beyond sum(tokens_per_expert) fall outside every group. + num_rows = permuted.shape[0] + pad = (-num_rows) % 128 + if pad: + permuted = torch.nn.functional.pad(permuted, (0, 0, 0, pad)) + permuted_probs = torch.nn.functional.pad(permuted_probs, (0, pad)) + + out = self.experts( + permuted, tokens_per_expert, permuted_probs.to(tokens.dtype), tokens_per_expert + ) + return moe_unpermute( + out[:num_rows], row_id_map, restore_shape=tokens.shape, pad_offsets=pad_offsets + ) + + def make_ep_buffer(self, device: Optional[torch.device] = None): + """EP buffer for this module's routing config. ``forward`` creates one per call unless + given one; a buffer holds one call's routing state until its backward runs, so reuse + it only across calls whose backward has completed.""" + from transformer_engine.pytorch.ep import EpBuffer + + assert self._ep_buffer_kwargs is not None, "make_ep_buffer requires ep_group." + if device is None: + device = torch.device("cuda", torch.cuda.current_device()) + return EpBuffer(**self._ep_buffer_kwargs, device=device) + + def _forward_ep(self, tokens: torch.Tensor, ep_buffer=None) -> torch.Tensor: + assert tokens.dtype == torch.bfloat16, "The EP path requires bfloat16 inputs." + buffer = ep_buffer if ep_buffer is not None else self.make_ep_buffer(tokens.device) + topk_idx = torch.empty( + (tokens.shape[0], self.topk), dtype=torch.int64, device=tokens.device + ) + probs, topk_idx = self._route(self.gate(tokens).float(), topk_indices=topk_idx) + flat_idx = topk_idx.flatten() + self._last_tokens_per_expert = torch.zeros( + self.num_experts, dtype=torch.long, device=tokens.device + ).scatter_add_(0, flat_idx, torch.ones_like(flat_idx)) + topk_weights = probs.gather(1, topk_idx) + + return self.experts( + tokens, + topk_idx, + topk_weights, + topk_idx, + op_kwargs={self.experts[0]: {"buffer": buffer}, self.experts[-1]: {"buffer": buffer}}, + ) + + def forward(self, hidden_states: torch.Tensor, ep_buffer=None) -> torch.Tensor: + """ + Parameters + ---------- + hidden_states : torch.Tensor + input of shape ``[..., hidden_size]``. + ep_buffer : EpBuffer, optional + buffer from :meth:`make_ep_buffer` to reuse instead of creating one + per call (EP only). + """ + tokens = hidden_states.reshape(-1, self.hidden_size) + if self.ep_group is not None: + out = self._forward_ep(tokens, ep_buffer) + else: + out = self._forward_local(tokens) + if self.shared_expert is not None: + out = out + self.shared_expert(tokens) + return out.view_as(hidden_states) + + @torch.no_grad() + def update_expert_bias(self) -> None: + """Aux-loss-free bias update from the last forward's routing counts. + + With data/expert parallelism, all-reduce ``_last_tokens_per_expert`` + across ranks before calling (or call on identically-routed ranks). + """ + counts = self._last_tokens_per_expert + if counts is None: + return + err = counts.float().mean() - counts.float() + self.expert_bias += self.expert_bias_update_rate * torch.sign(err) diff --git a/transformer_engine/pytorch/models/deepseek_v3/multi_latent_attention.py b/transformer_engine/pytorch/models/deepseek_v3/multi_latent_attention.py new file mode 100644 index 00000000000..efc69f1594e --- /dev/null +++ b/transformer_engine/pytorch/models/deepseek_v3/multi_latent_attention.py @@ -0,0 +1,281 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Multi-Latent Attention (MLA) block as used in DeepSeekV3.""" + +import math +from typing import Optional, Union + +import torch + +from transformer_engine.pytorch.module import Linear, LayerNormLinear +from transformer_engine.pytorch.attention import DotProductAttention +from transformer_engine.pytorch.distributed import allreduce, get_distributed_world_size +from transformer_engine.pytorch.attention.mla_rope import ( + apply_mla_rope_kv, + apply_mla_rope_q, + build_rope_tables, + yarn_mscale, +) + +__all__ = ["MultiLatentAttention"] + + +class _ReduceGrad(torch.autograd.Function): + """Replicate an input across TP ranks and sum its gradients.""" + + @staticmethod + def forward(ctx, inp, tp_group): + """Return the replicated input unchanged.""" + ctx.tp_group = tp_group + return inp + + @staticmethod + def backward(ctx, grad_output): + """Sum gradients from each rank's attention heads.""" + grad_input = grad_output.clone(memory_format=torch.contiguous_format) + grad_input, _ = allreduce(grad_input, ctx.tp_group) + return grad_input, None + + +class MultiLatentAttention(torch.nn.Module): + """ + Multi-Latent Attention as used in DeepSeekV3. + + Queries and key-values are projected through low-rank latents + (``q_lora_rank``, ``kv_lora_rank``); RMSNorm on each latent is fused into + the up-projection (:class:`LayerNormLinear` with RMSNorm). Each query/key + head is split into a ``qk_nope_head_dim`` part and a ``qk_rope_head_dim`` + part; RoPE is applied only to the rope part, and the key rope part comes + from a single shared head broadcast to all heads. Attention runs through + :class:`DotProductAttention` with asymmetric head dims + ``kv_channels=(qk_nope_head_dim + qk_rope_head_dim, v_head_dim)``, which + supports the cuDNN fused attention backend. + + RoPE uses the fused MLA kernels from + :mod:`transformer_engine.pytorch.attention.mla_rope` (single-pass key/value + assembly); the rope slice follows + the DeepSeekV3 checkpoint convention (interleaved weights, NeoX output). + + Parameters + ---------- + hidden_size : int + size of each input sample. + num_attention_heads : int + number of attention heads. + q_lora_rank : int, default = 1536 + rank of the query latent. + kv_lora_rank : int, default = 512 + rank of the key-value latent. + qk_nope_head_dim : int, default = 128 + per-head dim of the non-rotary query/key part. + qk_rope_head_dim : int, default = 64 + per-head dim of the rotary query/key part. + v_head_dim : int, default = 128 + per-head dim of the values. + attention_dropout : float, default = 0.0 + dropout probability on attention scores. + attn_mask_type : str, default = "causal" + attention mask type passed to :class:`DotProductAttention`. + layernorm_epsilon : float, default = 1e-6 + epsilon of the latent RMSNorms (matches DeepSeekV3). + rotary_base : float, default = 10000.0 + RoPE base. + rope_scaling_factor : float, optional + YaRN context-extension factor; ``None`` disables YaRN. + original_max_position_embeddings : int, default = 4096 + pre-extension context length (YaRN). + beta_fast : float, default = 32.0 + YaRN high-frequency rotation bound. + beta_slow : float, default = 1.0 + YaRN low-frequency rotation bound. + mscale : float, default = 1.0 + YaRN mscale of the rope part. + mscale_all_dim : float, default = 0.0 + YaRN mscale of all dims; sets the default softmax scale to + ``m**2 / sqrt(qk head dim)`` with ``m = 0.1 * mscale_all_dim * ln(factor) + 1``. + softmax_scale : float, optional + softmax scale; defaults to ``1/sqrt(qk head dim)`` (times the YaRN + ``m**2`` when YaRN is enabled). + qkv_format : str, default = "sbhd" + layout of the input/output tensors. + params_dtype : torch.dtype, optional + dtype of module parameters. + tp_group : ProcessGroup, optional + tensor-parallel process group for the up/output projections. + tp_size : int, default = 1 + tensor-parallel world size. + """ + + def __init__( + self, + hidden_size: int, + num_attention_heads: int, + q_lora_rank: int = 1536, + kv_lora_rank: int = 512, + qk_nope_head_dim: int = 128, + qk_rope_head_dim: int = 64, + v_head_dim: int = 128, + attention_dropout: float = 0.0, + attn_mask_type: str = "causal", + layernorm_epsilon: float = 1e-6, + rotary_base: float = 10000.0, + rope_scaling_factor: Optional[float] = None, + original_max_position_embeddings: int = 4096, + beta_fast: float = 32.0, + beta_slow: float = 1.0, + mscale: float = 1.0, + mscale_all_dim: float = 0.0, + softmax_scale: Optional[float] = None, + qkv_format: str = "sbhd", + params_dtype: Optional[torch.dtype] = None, + tp_group: Optional[torch.distributed.ProcessGroup] = None, + tp_size: int = 1, + device: Union[torch.device, str] = "cuda", + ) -> None: + super().__init__() + + if tp_group is not None: + tp_size = get_distributed_world_size(tp_group) + assert qkv_format in ("sbhd", "bshd"), "MultiLatentAttention supports sbhd/bshd formats." + assert num_attention_heads % tp_size == 0 + + self.qkv_format = qkv_format + self.num_attention_heads = num_attention_heads + self.num_attention_heads_per_partition = num_attention_heads // tp_size + self.qk_nope_head_dim = qk_nope_head_dim + self.qk_rope_head_dim = qk_rope_head_dim + self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim + self.v_head_dim = v_head_dim + self.kv_lora_rank = kv_lora_rank + + common = {"bias": False, "params_dtype": params_dtype, "device": device} + tp = {"tp_group": tp_group, "tp_size": tp_size} + + self.q_down_proj = Linear(hidden_size, q_lora_rank, **common) + self.q_up_proj = LayerNormLinear( + q_lora_rank, + num_attention_heads * self.qk_head_dim, + normalization="RMSNorm", + eps=layernorm_epsilon, + parallel_mode="column" if tp_size > 1 else None, + **tp, + **common, + ) + self.kv_down_proj = Linear(hidden_size, kv_lora_rank + qk_rope_head_dim, **common) + self.kv_up_proj = LayerNormLinear( + kv_lora_rank, + num_attention_heads * (qk_nope_head_dim + v_head_dim), + normalization="RMSNorm", + eps=layernorm_epsilon, + parallel_mode="column" if tp_size > 1 else None, + **tp, + **common, + ) + self.out_proj = Linear( + num_attention_heads * v_head_dim, + hidden_size, + parallel_mode="row" if tp_size > 1 else None, + **tp, + **common, + ) + + self.rotary_base = rotary_base + self._yarn_kwargs = { + "scaling_factor": rope_scaling_factor, + "original_max_position_embeddings": original_max_position_embeddings, + "beta_fast": beta_fast, + "beta_slow": beta_slow, + "mscale": mscale, + "mscale_all_dim": mscale_all_dim, + } + self._rope_tables: Optional[tuple] = None + + if softmax_scale is None and rope_scaling_factor is not None: + m = yarn_mscale(rope_scaling_factor, mscale_all_dim) + softmax_scale = m * m / math.sqrt(self.qk_head_dim) + self.softmax_scale = softmax_scale + + self.core_attention = DotProductAttention( + num_attention_heads, + kv_channels=(self.qk_head_dim, v_head_dim), + attention_dropout=attention_dropout, + qkv_format=qkv_format, + attn_mask_type=attn_mask_type, + softmax_scale=softmax_scale, + tp_group=tp_group, + tp_size=tp_size, + ) + + def _rope_tables_for(self, seq_len: int, device: torch.device): + if ( + self._rope_tables is None + or self._rope_tables[0].shape[0] < seq_len + or self._rope_tables[0].device != device + ): + self._rope_tables = build_rope_tables( + seq_len, + self.qk_rope_head_dim, + base=self.rotary_base, + device=device, + **self._yarn_kwargs, + ) + cos, sin = self._rope_tables + return cos[:seq_len], sin[:seq_len] + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + attn_mask_type: Optional[str] = None, + ) -> torch.Tensor: + """ + Parameters + ---------- + hidden_states : torch.Tensor + input of shape ``[sq, b, h]`` (sbhd) or ``[b, sq, h]`` (bshd). + attention_mask : torch.Tensor, optional + boolean mask passed to :class:`DotProductAttention`. + attn_mask_type : str, optional + override of the constructor's mask type. + """ + seq_dim = 0 if self.qkv_format == "sbhd" else 1 + seq_len = hidden_states.shape[seq_dim] + heads = self.num_attention_heads_per_partition + + q = self.q_up_proj(self.q_down_proj(hidden_states)) + q = q.view(*q.shape[:-1], heads, self.qk_head_dim) + + kv_down = self.kv_down_proj(hidden_states) + kv_latent, k_pos = torch.split(kv_down, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1) + if self.kv_up_proj.tp_size > 1: + # k_pos is shared across TP ranks; its attention gradients must be summed. + k_pos = _ReduceGrad.apply(k_pos, self.kv_up_proj.tp_group) + kv = self.kv_up_proj(kv_latent) + kv = kv.view(*kv.shape[:-1], heads, self.qk_nope_head_dim + self.v_head_dim) + + cos, sin = self._rope_tables_for(seq_len, hidden_states.device) + q = apply_mla_rope_q( + q, cos, sin, self.qk_nope_head_dim, self.qk_rope_head_dim, self.qkv_format + ) + k, v = apply_mla_rope_kv( + kv, + k_pos.unsqueeze(-2), + cos, + sin, + self.qk_nope_head_dim, + self.qk_rope_head_dim, + self.v_head_dim, + self.qkv_format, + ) + + context = self.core_attention( + q, + k, + v, + attention_mask=attention_mask, + qkv_format=self.qkv_format, + attn_mask_type=attn_mask_type, + ) + return self.out_proj(context) diff --git a/transformer_engine/pytorch/models/deepseek_v3/transformer_layer.py b/transformer_engine/pytorch/models/deepseek_v3/transformer_layer.py new file mode 100644 index 00000000000..8ac8b93f8e9 --- /dev/null +++ b/transformer_engine/pytorch/models/deepseek_v3/transformer_layer.py @@ -0,0 +1,180 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""DeepSeekV3 transformer layer.""" + +from typing import Optional, Union + +import torch + +from transformer_engine.pytorch.module import LayerNormMLP, RMSNorm +from transformer_engine.pytorch.models.deepseek_v3.multi_latent_attention import ( + MultiLatentAttention, +) +from transformer_engine.pytorch.models.deepseek_v3.moe import DeepSeekV3MoE + +__all__ = ["DeepSeekV3Layer"] + + +class DeepSeekV3Layer(torch.nn.Module): + """ + A full DeepSeekV3 transformer layer, analogous to + :class:`TransformerLayer`: pre-RMSNorm + :class:`MultiLatentAttention`, + then either a dense SwiGLU MLP (:class:`LayerNormMLP` with RMSNorm, used + for the first dense layers of DeepSeekV3) or :class:`DeepSeekV3MoE`, each + with a residual connection. + + Parameters + ---------- + hidden_size : int + size of each input sample. + num_attention_heads : int + number of attention heads. + ffn_hidden_size : int + ffn size of the dense MLP (used when ``num_experts`` is + ``None``). + num_experts : int, optional + number of routed experts; ``None`` makes this a dense layer. + moe_ffn_hidden_size : int, optional + ffn size of each routed expert (required with MoE). + hidden_dropout : float, default = 0.0 + dropout probability on the residual branches. + **kwargs + kwargs common to the submodules (``q_lora_rank``, ``kv_lora_rank``, + ``qk_nope_head_dim``, ``qk_rope_head_dim``, ``v_head_dim``, + ``attention_dropout``, ``attn_mask_type``, ``qkv_format``, ``topk``, + ``num_groups``, ``group_topk``, ``routed_scaling_factor``, + ``shared_expert_ffn_hidden_size``, EP options, ...), forwarded to + :class:`MultiLatentAttention` and :class:`DeepSeekV3MoE`. + """ + + _MLA_KWARGS = frozenset( + { + "q_lora_rank", + "kv_lora_rank", + "qk_nope_head_dim", + "qk_rope_head_dim", + "v_head_dim", + "attention_dropout", + "attn_mask_type", + "rotary_base", + "rope_scaling_factor", + "original_max_position_embeddings", + "beta_fast", + "beta_slow", + "mscale", + "mscale_all_dim", + "softmax_scale", + "qkv_format", + "tp_group", + "tp_size", + } + ) + _MOE_KWARGS = frozenset( + { + "topk", + "num_groups", + "group_topk", + "routed_scaling_factor", + "shared_expert_ffn_hidden_size", + "expert_bias_update_rate", + "ep_group", + "ep_max_tokens_per_rank", + } + ) + + def __init__( + self, + hidden_size: int, + num_attention_heads: int, + ffn_hidden_size: Optional[int] = None, + num_experts: Optional[int] = None, + moe_ffn_hidden_size: Optional[int] = None, + hidden_dropout: float = 0.0, + layernorm_epsilon: float = 1e-5, + params_dtype: Optional[torch.dtype] = None, + device: Union[torch.device, str] = "cuda", + **kwargs, + ) -> None: + super().__init__() + + unknown = set(kwargs) - self._MLA_KWARGS - self._MOE_KWARGS + if unknown: + raise TypeError(f"Unexpected keyword arguments: {sorted(unknown)}") + mla_kwargs = {k: v for k, v in kwargs.items() if k in self._MLA_KWARGS} + moe_kwargs = {k: v for k, v in kwargs.items() if k in self._MOE_KWARGS} + + self.hidden_dropout = hidden_dropout + + self.input_layernorm = RMSNorm( + hidden_size, eps=layernorm_epsilon, device=device, dtype=params_dtype + ) + self.self_attention = MultiLatentAttention( + hidden_size, + num_attention_heads, + layernorm_epsilon=layernorm_epsilon, + params_dtype=params_dtype, + device=device, + **mla_kwargs, + ) + + if num_experts is None: + assert ffn_hidden_size is not None, "Dense layers require ffn_hidden_size." + self.pre_mlp_layernorm = None + self.mlp = LayerNormMLP( + hidden_size, + ffn_hidden_size, + eps=layernorm_epsilon, + normalization="RMSNorm", + activation="swiglu", + bias=False, + params_dtype=params_dtype, + device=device, + ) + else: + assert moe_ffn_hidden_size is not None, "MoE layers require moe_ffn_hidden_size." + self.pre_mlp_layernorm = RMSNorm( + hidden_size, eps=layernorm_epsilon, device=device, dtype=params_dtype + ) + self.mlp = DeepSeekV3MoE( + hidden_size, + moe_ffn_hidden_size, + num_experts, + params_dtype=params_dtype, + device=device, + **moe_kwargs, + ) + + def _residual_add(self, out: torch.Tensor, residual: torch.Tensor) -> torch.Tensor: + out = torch.nn.functional.dropout(out, p=self.hidden_dropout, training=self.training) + return residual + out + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + ep_buffer=None, + ) -> torch.Tensor: + """ + Parameters + ---------- + hidden_states : torch.Tensor + input of shape ``[sq, b, h]`` (sbhd) or ``[b, sq, h]`` (bshd). + attention_mask : torch.Tensor, optional + boolean attention mask. + ep_buffer : EpBuffer, optional + forwarded to :meth:`DeepSeekV3MoE.forward` (MoE layers with EP). + """ + attention_out = self.self_attention( + self.input_layernorm(hidden_states), + attention_mask=attention_mask, + ) + hidden_states = self._residual_add(attention_out, hidden_states) + + mlp_kwargs = {"ep_buffer": ep_buffer} if ep_buffer is not None else {} + if self.pre_mlp_layernorm is not None: + mlp_out = self.mlp(self.pre_mlp_layernorm(hidden_states), **mlp_kwargs) + else: + mlp_out = self.mlp(hidden_states) + return self._residual_add(mlp_out, hidden_states) diff --git a/transformer_engine/pytorch/ops/basic/combine.py b/transformer_engine/pytorch/ops/basic/combine.py index 9f9825d8b28..bca6d49ee68 100644 --- a/transformer_engine/pytorch/ops/basic/combine.py +++ b/transformer_engine/pytorch/ops/basic/combine.py @@ -102,16 +102,12 @@ def fuser_forward( basic_op_kwargs: list[dict[str, Any]], ) -> tuple[torch.Tensor, list[tuple[()]]]: # NCCL EP reads the routing state from the EpBuffer, so topk_idx is unused. - del ( - basic_op_extra_inputs, - prev_op_grad_output_quantizer, - next_op_input_quantizer, - basic_op_kwargs, - ) + del basic_op_extra_inputs, prev_op_grad_output_quantizer, next_op_input_quantizer # Only BF16 combine forward is supported for now. input_ = maybe_dequantize(input_, torch.bfloat16) ctx = basic_op_ctxs[0] - buffer = validate_ep_buffer("MoeCombine", self.config, self.buffer) + kwargs = basic_op_kwargs[0] + buffer = validate_ep_buffer("MoeCombine", self.config, kwargs.get("buffer", self.buffer)) _validate_combine_inputs(input_, buffer) result, combine_state = _ep_combine_fwd( input_, diff --git a/transformer_engine/pytorch/ops/basic/dispatch.py b/transformer_engine/pytorch/ops/basic/dispatch.py index 4613be2c689..b9eee731346 100644 --- a/transformer_engine/pytorch/ops/basic/dispatch.py +++ b/transformer_engine/pytorch/ops/basic/dispatch.py @@ -103,10 +103,11 @@ def fuser_forward( next_op_input_quantizer: Optional[Quantizer], basic_op_kwargs: list[dict[str, Any]], ) -> tuple[torch.Tensor, Iterable[Iterable[torch.Tensor]]]: - del next_op_input_quantizer, basic_op_kwargs + del next_op_input_quantizer topk_idx, topk_weights = basic_op_extra_inputs[0] ctx = basic_op_ctxs[0] - buffer = validate_ep_buffer("MoeDispatch", self.config, self.buffer) + kwargs = basic_op_kwargs[0] + buffer = validate_ep_buffer("MoeDispatch", self.config, kwargs.get("buffer", self.buffer)) input_shape = _validate_dispatch_input(input_, buffer) buffer.num_local_tokens = input_shape[0] _validate_routing_inputs( diff --git a/transformer_engine/pytorch/ops/fused/grouped_mlp.py b/transformer_engine/pytorch/ops/fused/grouped_mlp.py index 84edc056fb7..9291736370e 100644 --- a/transformer_engine/pytorch/ops/fused/grouped_mlp.py +++ b/transformer_engine/pytorch/ops/fused/grouped_mlp.py @@ -36,6 +36,7 @@ from ...tensor.mxfp8_tensor import MXFP8Quantizer, MXFP8Tensor from ...tensor.storage.grouped_tensor_storage import GroupedTensorStorage from ...triton.grouped_dbias_dscales import compute_grouped_dbias, compute_grouped_dbias_dscales +from ...triton.nvfp4_global_scale import nvfp4_global_scale_pair from ...utils import ( ceil_div, clear_tensor_data, @@ -398,6 +399,50 @@ def _nvfp4_global_scale( return _nvfp4_amax(tensors, columnwise=columnwise).to(torch.float32) / denom +def _nvfp4_global_scale_pair( + tensors_a: GroupedTensor | Iterable[NVFP4TensorStorage], + quantizer_a: Quantizer, + columnwise_a: bool, + tensors_b: GroupedTensor | Iterable[NVFP4TensorStorage], + quantizer_b: Quantizer, + columnwise_b: bool, + *, + num_groups: int, + device: torch.device, + combine: Optional[Literal["product", "sqrt_product"]] = None, +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + """Per-group NVFP4 global scales of a GEMM operand pair. + + Returns both scales, or with ``combine`` their product (the GEMM alpha) or its square + root. When both operands have a second-level scale, a single Triton kernel computes the + result instead of several small eager kernels; its output is bit-identical to + ``_nvfp4_global_scale`` and the eager combination. + """ + if not getattr(quantizer_a, "disable_second_level_scale", False) and not getattr( + quantizer_b, "disable_second_level_scale", False + ): + fused = nvfp4_global_scale_pair( + _nvfp4_amax(tensors_a, columnwise=columnwise_a), + _nvfp4_amax(tensors_b, columnwise=columnwise_b), + 6.0 * _nvfp4_scale_max(quantizer_a), + 6.0 * _nvfp4_scale_max(quantizer_b), + combine=combine, + ) + if fused is not None: + return fused + scale_a = _nvfp4_global_scale( + tensors_a, quantizer_a, columnwise=columnwise_a, num_groups=num_groups, device=device + ) + scale_b = _nvfp4_global_scale( + tensors_b, quantizer_b, columnwise=columnwise_b, num_groups=num_groups, device=device + ) + if combine is None: + return scale_a, scale_b + if combine == "sqrt_product": + return torch.sqrt(scale_a * scale_b) + return (scale_a * scale_b).to(torch.float32) + + def _single_quantized_tensor_from_grouped( grouped: GroupedTensor, quantizer: Optional[MXFP8Quantizer | NVFP4Quantizer] = None, @@ -698,17 +743,16 @@ def _cudnn_compute_wgrad( device=device, ) else: - common_wgrad_kwargs["global_scale_a"] = _nvfp4_global_scale( + ( + common_wgrad_kwargs["global_scale_a"], + common_wgrad_kwargs["global_scale_b"], + ) = _nvfp4_global_scale_pair( grouped_dy, dy_quantizer, - columnwise=True, - num_groups=num_groups, - device=device, - ) - common_wgrad_kwargs["global_scale_b"] = _nvfp4_global_scale( + True, grouped_x, x_quantizer, - columnwise=True, + True, num_groups=num_groups, device=device, ) @@ -1589,31 +1633,26 @@ def fuser_forward( fc1_prob_tensor = None if not unit_activation_scale: - fc1_prob_tensor = ( - scales.detach() - .to(dtype=torch.float32 if use_nvfp4 else dtype) - .reshape((-1,) if canonical else (-1, 1, 1)) - ) + fc1_prob_tensor = scales.detach() + if use_nvfp4: + # The cuDNN NVFP4 kernel only accepts FP32 router probability scales. + fc1_prob_tensor = fc1_prob_tensor.to(dtype=torch.float32) + fc1_prob_tensor = fc1_prob_tensor.reshape((-1,) if canonical else (-1, 1, 1)) if use_nvfp4: # cuDNN receives NVFP4 block-scaled inputs without TE's per-group # global scale factors, so alpha supplies the product of the two # operand global scales. - fc1_alpha_tensor = ( - _nvfp4_global_scale( - grouped_fc1_x, - fc1_input_quantizer, - columnwise=False, - num_groups=num_groups, - device=device, - ) - * _nvfp4_global_scale( - grouped_fc1_weight, - fc1_weight_quantizer, - columnwise=False, - num_groups=num_groups, - device=device, - ) - ).to(torch.float32) + fc1_alpha_tensor = _nvfp4_global_scale_pair( + grouped_fc1_x, + fc1_input_quantizer, + False, + grouped_fc1_weight, + fc1_weight_quantizer, + False, + num_groups=num_groups, + device=device, + combine="product", + ) else: fc1_alpha_tensor = alpha_tensor @@ -2420,34 +2459,20 @@ def fuser_backward( fc2_d_dtype = torch.bfloat16 if use_nvfp4 else torch.float8_e4m3fn if use_nvfp4: - fc2_dy_global_scale = _nvfp4_global_scale( + # DSReLU applies alpha once, so it takes the full product of the two + # operand global scales. DGLU applies alpha to both gate branches, so + # the wrapper expects sqrt(product) to recover the same factor. + fc2_alpha_tensor = _nvfp4_global_scale_pair( grouped_fc2_dy, fc2_grad_output_quantizer, - columnwise=False, - num_groups=num_groups, - device=device, - ) - fc2_weight_col_global_scale = _nvfp4_global_scale( + False, grouped_fc2_weight, fc2_weight_quantizer, - columnwise=True, + True, num_groups=num_groups, device=device, - ) - if activation_is_srelu: - # DSReLU applies alpha once, so pass the full product of the - # two operand global scales. - fc2_alpha_tensor = ( - (fc2_dy_global_scale * fc2_weight_col_global_scale) - .to(torch.float32) - .expand(num_groups) - ) - else: - # DGLU applies alpha to both gate branches, so the wrapper - # expects sqrt(product) to recover the same global-scale factor. - fc2_alpha_tensor = torch.sqrt( - fc2_dy_global_scale * fc2_weight_col_global_scale - ).expand(num_groups) + combine="product" if activation_is_srelu else "sqrt_product", + ).expand(num_groups) fc2_beta_tensor = get_cached_ones_tensor(num_groups, torch.float32, device) fc2_norm_const_tensor = None else: @@ -2701,9 +2726,6 @@ def fuser_backward( else: fc2_bias_grads = [fc2_dbias_packed[idx] for idx in range(num_groups)] - if grad_scales is not None: - grad_scales = grad_scales.to(dtype=dtype) - fc1_bias_grads: Optional[list[Optional[torch.Tensor]]] = None fc1_bias_grad_packed: Optional[torch.Tensor] = None if fc1_op.has_bias: diff --git a/transformer_engine/pytorch/quantization.py b/transformer_engine/pytorch/quantization.py index fe537d5792c..1c29914da86 100644 --- a/transformer_engine/pytorch/quantization.py +++ b/transformer_engine/pytorch/quantization.py @@ -246,7 +246,7 @@ def _check_support() -> Tuple[bool, str]: def check_recipe_support(recipe: Recipe) -> None: """Check if the given recipe is supported.""" - if torch.compiler.is_compiling() and isinstance(recipe, DelayedScaling): + if isinstance(recipe, DelayedScaling) and torch.compiler.is_compiling(): raise RuntimeError( "DelayedScaling is not supported under torch.compile. Please use other recipes instead." ) @@ -275,6 +275,11 @@ def check_recipe_support(recipe: Recipe) -> None: ) elif isinstance(recipe, MXFP8BlockScaling): recipe_supported, unsupported_reason = check_mxfp8_support() + elif isinstance(recipe, NVFP4BlockScaling): + recipe_supported, unsupported_reason = check_nvfp4_support() + else: + # By default, check FP8 support. + recipe_supported, unsupported_reason = check_fp8_support() if not recipe_supported: raise RuntimeError(unsupported_reason) @@ -798,12 +803,15 @@ def autocast_enter( """Set state and tracking variables for entry into FP8 region.""" fp8_recipe = get_default_fp8_recipe() if fp8_recipe is None else fp8_recipe - autocast_key = cls.get_unique_autocast_key(fp8_recipe, fp8_group) + if enabled: + check_recipe_support(fp8_recipe) qstate = cls.quantization_state - qstate.autocast_arguments[autocast_key] = ( - fp8_recipe, - fp8_group, - ) + if fp8_recipe.delayed(): + autocast_key = cls.get_unique_autocast_key(fp8_recipe, fp8_group) + qstate.autocast_arguments[autocast_key] = ( + fp8_recipe, + fp8_group, + ) qstate.fp8_enabled = enabled qstate.fp8_calibration = calibrating @@ -815,19 +823,6 @@ def autocast_enter( qstate.is_first_fp8_module = True qstate.autocast_depth += 1 - if enabled: - fp8_available, reason_for_no_fp8 = cls.is_fp8_available() - assert fp8_available, reason_for_no_fp8 - if isinstance(fp8_recipe, MXFP8BlockScaling): - mxfp8_available, reason_for_no_mxfp8 = cls.is_mxfp8_available() - assert mxfp8_available, reason_for_no_mxfp8 - if isinstance(fp8_recipe, Float8BlockScaling): - fp8_block_available, reason_for_no_fp8_block = cls.is_fp8_block_scaling_available() - assert fp8_block_available, reason_for_no_fp8_block - if isinstance(fp8_recipe, NVFP4BlockScaling): - nvfp4_available, reason_for_no_nvfp4 = cls.is_nvfp4_available() - assert nvfp4_available, reason_for_no_nvfp4 - @classmethod def autocast_exit(cls, enabled: bool, _graph: bool) -> None: """Set state and tracking variables for exit from FP8 region.""" @@ -836,9 +831,13 @@ def autocast_exit(cls, enabled: bool, _graph: bool) -> None: # Reduce only the non-FP8 weight modules here. # FP8 weight modules are reduced at the end of the optimizer # step after the weight amax is populated. - if enabled and qstate.autocast_depth == 0 and not _graph and torch.is_grad_enabled(): - # delayed scaling only function, for other recipes (current scaling with any granularity), - # this is noop for other recipes because cls.global_amax_buffer is empty list + if ( + enabled + and qstate.autocast_depth == 0 + and not _graph + and qstate.global_amax_buffer # Only delayed scaling needs this + and torch.is_grad_enabled() + ): cls.reduce_and_update_fp8_tensors(forward=True) @classmethod @@ -1100,10 +1099,8 @@ def __enter__(self) -> "autocast": raise RuntimeError( "autocast context manager cannot be entered more than once concurrently" ) - if self._enabled: - check_recipe_support(self._recipe) # Save current state so we always restore it on exit. - self._fp8_state = FP8GlobalStateManager.get_autocast_state() + fp8_state = FP8GlobalStateManager.get_autocast_state() FP8GlobalStateManager.autocast_enter( enabled=self._enabled, calibrating=self._calibrating, @@ -1111,6 +1108,7 @@ def __enter__(self) -> "autocast": fp8_group=self._amax_reduction_group, _graph=self._graph, ) + self._fp8_state = fp8_state return self def __exit__(self, exc_type, exc_val, exc_tb) -> None: diff --git a/transformer_engine/pytorch/triton/nvfp4_global_scale.py b/transformer_engine/pytorch/triton/nvfp4_global_scale.py new file mode 100644 index 00000000000..4efa261ff9b --- /dev/null +++ b/transformer_engine/pytorch/triton/nvfp4_global_scale.py @@ -0,0 +1,82 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""PyTorch wrapper for the fused NVFP4 global-scale Triton kernel.""" + +from typing import Optional, Tuple, Union + +import torch +import triton + +from transformer_engine.common.triton.nvfp4_global_scale import ( + NVFP4_GLOBAL_SCALE_PAIR, + NVFP4_GLOBAL_SCALE_PRODUCT, + NVFP4_GLOBAL_SCALE_SQRT_PRODUCT, + _nvfp4_global_scale_pair_kernel, +) + +# One program covers every group; larger group counts take the eager path. +_MAX_GROUPS = 4096 + +_MODES = { + None: NVFP4_GLOBAL_SCALE_PAIR, + "product": NVFP4_GLOBAL_SCALE_PRODUCT, + "sqrt_product": NVFP4_GLOBAL_SCALE_SQRT_PRODUCT, +} + + +def nvfp4_global_scale_pair( + amax_a: torch.Tensor, + amax_b: torch.Tensor, + denom_a: float, + denom_b: float, + combine: Optional[str] = None, +) -> Optional[Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]]: + """Per-group NVFP4 global scales ``amax / denom`` of two operands in one launch. + + Equivalent, bit for bit, to ``amax_a.float() / denom_a`` and ``amax_b.float() / denom_b`` + followed by the optional combination. ``amax_a`` and ``amax_b`` hold one FP32 amax per + group; with ``combine``, either may instead hold a single amax shared by all groups. + + Args: + amax_a, amax_b: per-group (or single) FP32 amaxes on the same CUDA device. + denom_a, denom_b: the operands' ``6 * scale_max`` denominators. + combine: ``None`` returns both scales, ``"product"`` their product and + ``"sqrt_product"`` the square root of the product. + + Returns: + The scale tensor(s), or ``None`` when the inputs are outside what the kernel + handles (non-FP32 amaxes, mismatched group counts, a broadcast amax without + ``combine``, more than 4096 groups); the caller then computes the scales eagerly. + """ + if amax_a.dtype != torch.float32 or amax_b.dtype != torch.float32: + return None + if amax_a.device != amax_b.device or not amax_a.is_cuda: + return None + num_a, num_b = amax_a.numel(), amax_b.numel() + n = max(num_a, num_b) + if n == 0 or n > _MAX_GROUPS or num_a not in (1, n) or num_b not in (1, n): + return None + mode = _MODES[combine] + if mode == NVFP4_GLOBAL_SCALE_PAIR and num_a != num_b: + # Keep each scale's shape: only the combined modes broadcast. + return None + out_a = torch.empty(n, dtype=torch.float32, device=amax_a.device) + out_b = torch.empty_like(out_a) if mode == NVFP4_GLOBAL_SCALE_PAIR else out_a + _nvfp4_global_scale_pair_kernel[(1,)]( + amax_a.contiguous(), + amax_b.contiguous(), + out_a, + out_b, + n, + num_a == 1 and n > 1, + num_b == 1 and n > 1, + float(denom_a), + float(denom_b), + MODE=mode, + BLOCK=max(16, triton.next_power_of_2(n)), + ) + if mode == NVFP4_GLOBAL_SCALE_PAIR: + return out_a, out_b + return out_a