From 628485cea5c7cb268098acf99cfa79bde9e35829 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Wed, 23 Sep 2026 09:51:16 -0700 Subject: [PATCH 01/37] Add DSv4 sparse attention bindings to TE DSv4 HCA and CSA need joint attention over local and compressed KV rows, while CSA also selects compressed rows. Expose the cuDNN Frontend compressor, selector, and sparse attention operations through a small parameter-free TE module so model-owned projections and positional handling remain reusable. Signed-off-by: Sudhakar Singh --- tests/pytorch/test_dsv4_attention.py | 114 ++++++ transformer_engine/pytorch/__init__.py | 1 + .../pytorch/attention/__init__.py | 4 + .../attention/sparse_attention/__init__.py | 14 + .../sparse_attention/_dsv4_backend.py | 381 ++++++++++++++++++ .../attention/sparse_attention/dsv4.py | 82 ++++ 6 files changed, 596 insertions(+) create mode 100644 tests/pytorch/test_dsv4_attention.py create mode 100644 transformer_engine/pytorch/attention/sparse_attention/__init__.py create mode 100644 transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py create mode 100644 transformer_engine/pytorch/attention/sparse_attention/dsv4.py diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py new file mode 100644 index 00000000000..bd637a6b31d --- /dev/null +++ b/tests/pytorch/test_dsv4_attention.py @@ -0,0 +1,114 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""DSv4 core against a dense oracle, including packed sequence boundaries.""" + +import pytest +import torch + +from transformer_engine.pytorch.attention.sparse_attention import dsv4 + + +@pytest.mark.parametrize("variant", ["hca", "csa"]) +@pytest.mark.parametrize("head_dim", [512, 576]) +def test_dsv4_forward_backward(variant, head_dim): + if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 0): + pytest.skip("DSv4 requires SM100") + cudnn = pytest.importorskip("cudnn") + dsa = getattr(cudnn, "DSA", None) + if not all( + hasattr(dsa, operation) + for operation in ( + "sparse_attention_forward_wrapper", + "sparse_attention_backward_wrapper", + ) + ): + pytest.skip("DSv4 requires cuDNN Frontend 1.29 sparse attention APIs") + torch.manual_seed(17) + ratio = 128 if variant == "hca" else 4 + lengths = [256, 128] + comp_lengths = [n // ratio for n in lengths] + total, total_comp = sum(lengths), sum(comp_lengths) + cu = torch.tensor([0, 256, total], device="cuda", dtype=torch.int32) + cu_comp = torch.tensor( + [0, comp_lengths[0], total_comp], device="cuda", dtype=torch.int32 + ) + query = ( + torch.randn(total, 64, head_dim, device="cuda", dtype=torch.bfloat16) * 0.1 + ).requires_grad_() + local = ( + torch.randn(total, head_dim, device="cuda", dtype=torch.bfloat16) * 0.1 + ).requires_grad_() + compressed = ( + torch.randn(total_comp, head_dim, device="cuda", dtype=torch.bfloat16) * 0.1 + ).requires_grad_() + sink = torch.randn(64, device="cuda", requires_grad=True) + leaves = (query, local, compressed, sink) + reference_leaves = tuple(x.detach().float().requires_grad_() for x in leaves) + indices = None + if variant == "csa": + # Select alternating compressed entries; distinct global IDs expose + # mistakes at the second sequence's compressed offset. + indices = torch.full((total, 32), -1, device="cuda", dtype=torch.int32) + start = comp_start = 0 + for length, count in zip(lengths, comp_lengths): + for position in range(length): + ids = torch.arange( + comp_start, comp_start + (position + 1) // ratio, 2, device="cuda" + ) + indices[start + position, : ids.numel()] = ids + start += length + comp_start += count + + output = dsv4.DSv4Attention(window_size=32, ratio=ratio)( + *leaves, + cu, + cu_comp, + indices=indices, + max_compressed_seqlen=max(comp_lengths) if indices is None else None, + ) + q, k, c, s = reference_leaves + reference = [] + start = comp_start = 0 + for length, count in zip(lengths, comp_lengths): + kv = torch.cat((k[start : start + length], c[comp_start : comp_start + count])) + pos = torch.arange(length, device="cuda") + local_mask = (pos[None, :] <= pos[:, None]) & (pos[None, :] > pos[:, None] - 32) + comp_ids = torch.arange(count, device="cuda") + comp_mask = (comp_ids[None, :] + 1) * ratio <= pos[:, None] + 1 + if indices is not None: + comp_mask &= (comp_ids % 2 == 0)[None, :] + mask = torch.cat((local_mask, comp_mask), dim=-1) + logits = torch.einsum("thd,kd->thk", q[start : start + length], kv) * ( + head_dim**-0.5 + ) + logits = logits.masked_fill(~mask[:, None, :], float("-inf")) + probabilities = torch.cat( + (logits, s[None, :, None].expand(length, -1, -1)), -1 + ).softmax(-1) + reference.append( + torch.einsum("thk,kd->thd", probabilities[..., :-1], kv[..., :512]) + ) + start += length + comp_start += count + expected = torch.cat(reference) + torch.testing.assert_close(output.float(), expected, atol=1e-3, rtol=1e-2) + gradient = torch.randn_like(output) + output.backward(gradient) + expected.backward(gradient.float()) + for actual, ref in zip(leaves, reference_leaves): + assert actual.grad is not None and torch.isfinite(actual.grad).all() + # Relative L2 error tolerates BF16 reduction rounding near zero entries. + relative_error = ( + actual.grad.float() - ref.grad + ).norm() / ref.grad.norm().clamp_min(1e-8) + assert relative_error < 0.02, relative_error.item() + + +def test_dsv4_rejects_unsupported_metadata(): + with pytest.raises(ValueError, match="positive"): + dsv4.DSv4Attention(window_size=0, ratio=4) + core = dsv4.DSv4Attention(window_size=32, ratio=4) + with pytest.raises(ValueError, match="query"): + core(torch.empty(1, 4, 512), None, None, None, None, None) diff --git a/transformer_engine/pytorch/__init__.py b/transformer_engine/pytorch/__init__.py index 576fb57c5e5..4d3d8b8ce59 100644 --- a/transformer_engine/pytorch/__init__.py +++ b/transformer_engine/pytorch/__init__.py @@ -29,6 +29,7 @@ from transformer_engine.pytorch.module import destroy_ub from transformer_engine.pytorch.module import UserBufferQuantizationMode from transformer_engine.pytorch.attention import DotProductAttention +from transformer_engine.pytorch.attention import DSv4Attention, sparse_attention from transformer_engine.pytorch.attention import GatedDeltaNetAttention from transformer_engine.pytorch.attention import FusedMLAQUpProjFunction, FusedMLAQUpProjRopeQuant from transformer_engine.pytorch.attention import MultiheadAttention diff --git a/transformer_engine/pytorch/attention/__init__.py b/transformer_engine/pytorch/attention/__init__.py index 977b2b1b0e9..cbc215a2004 100644 --- a/transformer_engine/pytorch/attention/__init__.py +++ b/transformer_engine/pytorch/attention/__init__.py @@ -5,6 +5,8 @@ """Python interface for attention""" from .dot_product_attention import DotProductAttention +from . import sparse_attention +from .sparse_attention import DSv4Attention from .linear_attention import GatedDeltaNetAttention from .fused_mla_q_uproj import FusedMLAQUpProjFunction, FusedMLAQUpProjRopeQuant from .multi_head_attention import MultiheadAttention @@ -13,6 +15,8 @@ __all__ = [ "DotProductAttention", + "DSv4Attention", + "sparse_attention", "GatedDeltaNetAttention", "FusedMLAQUpProjFunction", "FusedMLAQUpProjRopeQuant", diff --git a/transformer_engine/pytorch/attention/sparse_attention/__init__.py b/transformer_engine/pytorch/attention/sparse_attention/__init__.py new file mode 100644 index 00000000000..75b8e83476a --- /dev/null +++ b/transformer_engine/pytorch/attention/sparse_attention/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Experimental sparse attention variants. + +Each variant owns its tensor/index contract and preparation helpers. Optional +kernels load on execution. Add future variants as siblings of ``dsv4``. +""" + +from . import dsv4 +from .dsv4 import DSv4Attention + +__all__ = ["dsv4", "DSv4Attention"] diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py new file mode 100644 index 00000000000..ea0c3b190fd --- /dev/null +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py @@ -0,0 +1,381 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Experimental PyTorch bindings for cuDNN Frontend 1.29.0 DSv4 primitives. + +Compression, selection, sparse attention forward, and backward call cuDNN +Frontend APIs. There are no numerical fallbacks. Inputs use packed sequences; +model projections/norm/RoPE are external. +""" + +from importlib import import_module +from typing import Optional + +import torch +from torch.autograd.function import once_differentiable + +__all__ = ["compress", "select_blocks", "attention"] + + +def _namespace(name): + try: + return getattr(import_module("cudnn"), name) + except (ImportError, AttributeError) as exc: + raise ImportError( + "DSv4 requires nvidia-cudnn-frontend>=1.29.0 " + f"with the cudnn.{name} namespace." + ) from exc + + +def _sparse_attention_forward(): + try: + return getattr(_namespace("DSA"), "sparse_attention_forward_wrapper") + except AttributeError as exc: + raise ImportError( + "DSv4 attention requires cudnn.DSA.sparse_attention_forward_wrapper " + "from nvidia-cudnn-frontend>=1.29.0." + ) from exc + + +def _validate_tensors(reference, bf16, fp32, cu_seqlens, cu_seqlens_comp): + """Check metadata without copying packed sequence offsets to the CPU. + + Callers own prefix contents: start at zero, monotonic, final offsets equal + valid row counts, and compressed lengths floor(length / ratio). + """ + if not reference.is_cuda or torch.cuda.get_device_capability(reference.device) != ( + 10, + 0, + ): + raise ValueError("This DSv4 implementation requires an SM100 GPU.") + for tensor in (reference, *bf16, *fp32, cu_seqlens, cu_seqlens_comp): + if tensor.device != reference.device or not tensor.is_contiguous(): + raise ValueError("DSv4 inputs must be contiguous on the same CUDA device.") + if any(t.dtype != torch.bfloat16 for t in (reference, *bf16)): + raise TypeError("DSv4 activations must be BF16.") + if any(t.dtype != torch.float32 for t in fp32): + raise TypeError("DSv4 position biases and sinks must be FP32.") + for offsets in (cu_seqlens, cu_seqlens_comp): + if offsets.dtype != torch.int32 or offsets.ndim != 1 or offsets.numel() < 2: + raise ValueError("Sequence prefixes must be INT32 [batch+1].") + if cu_seqlens.shape != cu_seqlens_comp.shape: + raise ValueError("Local and compressed prefixes must describe the same batch.") + + +class _Compress(torch.autograd.Function): + @staticmethod + def forward( + ctx, kv, score, ape, cu_seqlens, cu_seqlens_comp, ratio, overlap, total_comp + ): + coff = 2 if overlap else 1 + if kv.ndim != 2 or kv.shape[-1] % coff: + raise ValueError("kv must have shape [tokens, coff * head_dim].") + head_dim = kv.shape[-1] // coff + result = _namespace("CSA").csa_compressor_forward_wrapper( + kv, + score, + ape, + cu_seqlens, + cu_seqlens_comp, + ratio=ratio, + head_dim=head_dim, + coff=coff, + total_comp=total_comp, + ) + ctx.save_for_backward(kv, score, ape, cu_seqlens, cu_seqlens_comp) + ctx.config = ratio, head_dim, coff + return result["out"] + + @staticmethod + @once_differentiable + def backward(ctx, grad_out): + ratio, head_dim, coff = ctx.config + # cuDNN owns scratch allocation and dAPE zeroing. Autograd may supply + # a strided/expanded gradient, while this kernel requires contiguous data. + result = _namespace("CSA").csa_compressor_backward_wrapper( + *ctx.saved_tensors, + grad_out.contiguous(), + ratio=ratio, + head_dim=head_dim, + coff=coff, + ) + return ( + result["grad_kv"], + result["grad_score"], + result["grad_ape"], + None, + None, + None, + None, + None, + ) + + +def compress( + kv: torch.Tensor, + score: torch.Tensor, + ape: torch.Tensor, + cu_seqlens: torch.Tensor, + cu_seqlens_comp: torch.Tensor, + *, + ratio: int, + overlap: bool, + total_comp: int, +) -> torch.Tensor: + """Gated pooling, with cuDNN forward and first-order backward. + + kv/score: contiguous BF16 [T, coff*D]; ape: FP32 [ratio, coff*D]. + coff is 2 for overlapping pooling, 1 otherwise. Sequence prefixes are + CUDA INT32 [B+1]; compressed segment lengths are floor(length / ratio). + Returns BF16 [total_comp, D]. Per-segment tails are dropped by cuDNN. + + Pass total_comp explicitly because inferring it from a CUDA prefix would + synchronize. It may be a capacity; only rows below cu_seqlens_comp[-1] + are valid. Remaining kernel-specific constraints are checked by cuDNN. + """ + if ratio <= 0 or total_comp < 0: + raise ValueError("ratio must be positive and total_comp nonnegative.") + if kv.ndim != 2 or score.shape != kv.shape or ape.shape != (ratio, kv.shape[-1]): + raise ValueError("Expected kv/score [T,coff*D] and ape [ratio,coff*D].") + _validate_tensors(kv, (score,), (ape,), cu_seqlens, cu_seqlens_comp) + return _Compress.apply( + kv, + score, + ape, + cu_seqlens, + cu_seqlens_comp, + ratio, + overlap, + total_comp, + ) + + +@torch.no_grad() +def select_blocks( + query: torch.Tensor, + compressed_key: torch.Tensor, + weights: torch.Tensor, + cu_seqlens: torch.Tensor, + cu_seqlens_comp: torch.Tensor, + *, + top_k: int, + ratio: int, + max_seqlen: int, + max_compressed_seqlen: int, + scale: Optional[float] = None, +) -> torch.Tensor: + """CSA selection using cuDNN's SM100 combined score + Top-K operation. + + This is the indexer, with its own projected Q/K/weights, not attention Q/KV. + BF16 query [T,H,128], compressed_key [Tc,128], weights [T,H]; H=32 or 64. + Full packed-sequence prefill only: causal timeline offsets start at zero. + Returns INT32 [T,top_k] indices into compressed_key's global packed rows; + -1 denotes an invalid slot. These are NOT IDs into local+compressed KV. + + Selection is discrete. Scale defaults to 1/sqrt(index head dim); apply any + separate head-reduction scaling to weights before this call. This helper + does not implement the separate indexer auxiliary training loss or promise + autograd through the selected IDs. + """ + if query.ndim != 3 or query.shape[1] not in (32, 64) or query.shape[2] != 128: + raise ValueError("CSA index query must have shape [T,32 or 64,128].") + if compressed_key.ndim != 2 or compressed_key.shape[1] != 128: + raise ValueError("CSA index key must have shape [Tc,128].") + if weights.shape != query.shape[:2]: + raise ValueError("CSA index weights must have shape [T,H].") + if ratio <= 0 or top_k <= 0 or max_seqlen <= 0 or max_compressed_seqlen <= 0: + raise ValueError( + "CSA ratio, top_k and maximum sequence lengths must be positive." + ) + _validate_tensors(query, (compressed_key, weights), (), cu_seqlens, cu_seqlens_comp) + if scale is None: + scale = query.shape[-1] ** -0.5 + result = _namespace("DSA").indexer_forward_top_k_wrapper( + query, + compressed_key.unsqueeze(1), + weights, + top_k=top_k, + ratio=ratio, + cu_seqlens_q=cu_seqlens, + cu_seqlens_k=cu_seqlens_comp, + max_seqlen_q=max_seqlen, + max_seqlen_k=max_compressed_seqlen, + precision="bf16", + topk_indices_global=True, + return_softmax=False, + sm_scale=scale, + ) + return result["indices"] + + +def _attention_indices( + query: torch.Tensor, + compressed_kv: torch.Tensor, + cu_seqlens: torch.Tensor, + cu_seqlens_comp: torch.Tensor, + window_size: int, + ratio: int, + indices: Optional[torch.Tensor], + max_compressed_seqlen: Optional[int], +) -> torch.Tensor: + """Map local and compressed entries into one concatenated KV address space.""" + total_q = query.shape[0] + device = query.device + lengths = cu_seqlens[1:] - cu_seqlens[:-1] + seq_ids = torch.repeat_interleave( + torch.arange(lengths.numel(), device=device), + lengths, + output_size=total_q, + ) + token = torch.arange(total_q, device=device) + seq_start = cu_seqlens[seq_ids] + position = token - seq_start + + local_position = position[:, None] - torch.arange( + window_size - 1, + -1, + -1, + device=device, + ) + local = seq_start[:, None] + local_position + local = torch.where(local_position >= 0, local, -1) + + comp_start = cu_seqlens_comp[seq_ids] + comp_end = cu_seqlens_comp[seq_ids + 1] + if indices is None: + if max_compressed_seqlen is None: + raise ValueError("max_compressed_seqlen is required for HCA attention.") + comp_position = torch.arange(max_compressed_seqlen, device=device)[None, :] + compressed = comp_start[:, None] + comp_position + valid = (compressed < comp_end[:, None]) & ( + (comp_position + 1) * ratio <= position[:, None] + 1 + ) + else: + if indices.ndim != 2 or indices.shape[0] != total_q: + raise ValueError("CSA indices must have shape [total_tokens, top_k].") + compressed = indices + comp_position = compressed - comp_start[:, None] + valid = ( + (compressed >= comp_start[:, None]) + & (compressed < comp_end[:, None]) + & ((comp_position + 1) * ratio <= position[:, None] + 1) + ) + compressed = torch.where(valid, compressed + query.shape[0], -1) + # cuDNN accepts an arbitrary logical width and pads its internal kernel + # layout, so preserve only the model-visible local + compressed slots. + return torch.cat((local, compressed), dim=1).to(torch.int32).contiguous() + + +class _Attention(torch.autograd.Function): + @staticmethod + def forward(ctx, query, kv, indices, sink, scale): + result = _sparse_attention_forward()( + query, + kv, + indices, + attn_sink=sink, + softmax_scale=scale, + # Prefix LSE is only needed by the optional indexer-loss path. + indexer_topk=0, + ) + out, lse = result["out"], result["lse"] + ctx.save_for_backward(query, kv, out, lse, sink, indices) + ctx.scale = scale + return out + + @staticmethod + @once_differentiable + def backward(ctx, grad_out): + query, kv, out, lse, sink, indices = ctx.saved_tensors + result = _namespace("DSA").sparse_attention_backward_wrapper( + query, + kv, + out, + grad_out.contiguous(), + lse, + sink, + indices, + softmax_scale=ctx.scale, + ) + return result["dq"], result["dkv"], None, result["d_sink"], None + + +def attention( + query: torch.Tensor, + local_kv: torch.Tensor, + compressed_kv: torch.Tensor, + sink: torch.Tensor, + cu_seqlens: torch.Tensor, + cu_seqlens_comp: torch.Tensor, + *, + window_size: int, + ratio: int, + indices: Optional[torch.Tensor] = None, + scale: Optional[float] = None, + max_compressed_seqlen: Optional[int] = None, +) -> torch.Tensor: + """Joint local + compressed DSv4 attention with first-order autograd. + + Full packed-sequence prefill: query [T,H,D], local_kv [T,D], compressed_kv + [Tc,D], sink [H]. Q/KV have already had the model's norm/RoPE applied. + cu_seqlens describes both query and local KV; cu_seqlens_comp describes + valid compressed rows, excluding capacity padding. Values are the first + 512 channels of each shared KV row (Q/K width may also be 576). + + Local positions satisfy max(0,q-window_size+1) <= k <= q per sequence. + Compressed block j is eligible when (j+1)*ratio <= q+1. With indices (CSA), + use its selected, valid, causal compressed rows; without indices (HCA), + use every eligible compressed row. IDs refer to compressed_kv only. + + One cuDNN sparse-attention call normalizes the combined visible entries with + the sink and provides the KV-only LSE consumed by cuDNN backward. Output is + [T,H,512], before output unrotation/projection. Scale defaults to D**-0.5. + No dense attention mask is part of this interface. HCA must provide + max_compressed_seqlen to bound its dense compressed-history metadata without + reading a CUDA prefix on host. + """ + if query.ndim != 3 or query.shape[1] != 64: + raise ValueError("DSv4 attention requires query [T,64,D].") + if query.shape[-1] not in (512, 576): + raise ValueError("attention head dimension must be 512 or 576.") + if local_kv.ndim != 2 or compressed_kv.ndim != 2: + raise ValueError("local_kv and compressed_kv must be 2D packed tensors.") + if ( + local_kv.shape[-1] != query.shape[-1] + or compressed_kv.shape[-1] != query.shape[-1] + ): + raise ValueError("Q and both KV tensors must have the same QK dimension.") + if local_kv.shape[0] != query.shape[0]: + raise ValueError("Full-sequence attention requires one local KV row per query.") + if sink.shape != (query.shape[1],): + raise ValueError("sink must have shape [64].") + if window_size <= 0 or ratio <= 0: + raise ValueError("window_size and ratio must be positive.") + if indices is None and (max_compressed_seqlen is None or max_compressed_seqlen < 0): + raise ValueError("HCA requires a nonnegative max_compressed_seqlen.") + _validate_tensors( + query, (local_kv, compressed_kv), (sink,), cu_seqlens, cu_seqlens_comp + ) + if indices is not None: + if indices.dtype != torch.int32 or indices.device != query.device: + raise ValueError("CSA indices must be CUDA INT32 on the query device.") + if scale is None: + scale = query.shape[-1] ** -0.5 + selected = _attention_indices( + query, + compressed_kv, + cu_seqlens, + cu_seqlens_comp, + window_size, + ratio, + indices, + max_compressed_seqlen, + ) + return _Attention.apply( + query.contiguous(), + torch.cat((local_kv, compressed_kv), dim=0), + selected, + sink.contiguous(), + scale, + ) diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4.py new file mode 100644 index 00000000000..bfaf6557a0b --- /dev/null +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4.py @@ -0,0 +1,82 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""DSv4 core and preparation helpers, composed with model-owned projections.""" + +from typing import Optional + +import torch + +from ._dsv4_backend import attention as _attention +from ._dsv4_backend import compress, select_blocks + +__all__ = ["DSv4Attention", "compress", "select_blocks"] + + +class DSv4Attention(torch.nn.Module): + """Experimental causal joint local + compressed attention for DSv4. + + Parameter-free core: the caller owns projections, compression, normalization, + RoPE, and the learned sink. Use :func:`compress` for gated pooling and + :func:`select_blocks` for the CSA indexer. Pass selected indices for CSA; + omit them and supply ``max_compressed_seqlen`` for HCA. + + Inputs are contiguous CUDA BF16 query [T,64,D], local KV [T,D], compressed + KV [Tc,D], FP32 sink [64], and CUDA INT32 sequence prefixes [B+1]. D is 512 + or 576; values use the first 512 KV channels. Output is [T,64,512], retaining + the head axis for DSv4's output unrotation. Prefixes start at zero and end + at valid row counts; compressed lengths are floor(sequence length / ratio). + CSA indices are distinct global packed compressed-row IDs, or -1 for padding. + + Only full sequences with positions starting at zero are supported. Local + keys satisfy max(0,q-window_size+1) <= k <= q. Compressed block j becomes + visible when (j+1)*ratio <= q+1. All visible local and compressed entries + share one softmax with the sink; its value contribution is zero. + + Requires SM100 and cuDNN Frontend 1.29.0. Forward and first-order backward + use cuDNN's DSA namespace. No dropout, arbitrary masks, cache/decode, FP8, + distributed attention, or higher-order gradients. This module does not use + DPA backend-selection flags. cuDNN operations are loaded on execution. + """ + + def __init__(self, *, window_size: int, ratio: int): + super().__init__() + if window_size <= 0 or ratio <= 0: + raise ValueError("window_size and ratio must be positive.") + self.window_size = window_size + self.ratio = ratio + + def forward( + self, + query: torch.Tensor, + local_kv: torch.Tensor, + compressed_kv: torch.Tensor, + sink: torch.Tensor, + cu_seqlens: torch.Tensor, + cu_seqlens_comp: torch.Tensor, + *, + indices: Optional[torch.Tensor] = None, + scale: Optional[float] = None, + max_compressed_seqlen: Optional[int] = None, + ) -> torch.Tensor: + """Evaluate the joint attention, with scale defaulting to D**-0.5.""" + from transformer_engine.pytorch.quantization import FP8GlobalStateManager + + if FP8GlobalStateManager.is_fp8_enabled(): + raise NotImplementedError( + "DSv4Attention supports BF16 only; disable TE FP8 autocast." + ) + return _attention( + query, + local_kv, + compressed_kv, + sink, + cu_seqlens, + cu_seqlens_comp, + window_size=self.window_size, + ratio=self.ratio, + indices=indices, + scale=scale, + max_compressed_seqlen=max_compressed_seqlen, + ) From 07b210f8693d3b1b9706a52687c8c4be02e28108 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Wed, 30 Sep 2026 22:53:02 -0700 Subject: [PATCH 02/37] Compose DSv4 hybrid attention from sparse primitives Build the BSD-to-BSD CSA/HCA layer over the existing cuDNN-backed compressor, indexer, and attention operations so callers can supply hidden states directly. Keep the shared RoPE preparation and indexer in small modules, and cache FP32 frequencies across forwards. Signed-off-by: Sudhakar Singh --- transformer_engine/pytorch/__init__.py | 6 +- .../pytorch/attention/__init__.py | 3 +- .../attention/sparse_attention/README.md | 123 +++++++++++ .../attention/sparse_attention/__init__.py | 3 +- .../sparse_attention/_dsv4_backend.py | 26 +-- .../sparse_attention/_dsv4_compressor.py | 61 ++++++ .../sparse_attention/_dsv4_indexer.py | 63 ++++++ .../attention/sparse_attention/_dsv4_rope.py | 61 ++++++ .../sparse_attention/dsv4_hybrid_attention.py | 195 ++++++++++++++++++ 9 files changed, 525 insertions(+), 16 deletions(-) create mode 100644 transformer_engine/pytorch/attention/sparse_attention/README.md create mode 100644 transformer_engine/pytorch/attention/sparse_attention/_dsv4_compressor.py create mode 100644 transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py create mode 100644 transformer_engine/pytorch/attention/sparse_attention/_dsv4_rope.py create mode 100644 transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py diff --git a/transformer_engine/pytorch/__init__.py b/transformer_engine/pytorch/__init__.py index 4d3d8b8ce59..b4e6eb65359 100644 --- a/transformer_engine/pytorch/__init__.py +++ b/transformer_engine/pytorch/__init__.py @@ -29,7 +29,11 @@ from transformer_engine.pytorch.module import destroy_ub from transformer_engine.pytorch.module import UserBufferQuantizationMode from transformer_engine.pytorch.attention import DotProductAttention -from transformer_engine.pytorch.attention import DSv4Attention, sparse_attention +from transformer_engine.pytorch.attention import ( + DSv4Attention, + DSv4HybridAttention, + sparse_attention, +) from transformer_engine.pytorch.attention import GatedDeltaNetAttention from transformer_engine.pytorch.attention import FusedMLAQUpProjFunction, FusedMLAQUpProjRopeQuant from transformer_engine.pytorch.attention import MultiheadAttention diff --git a/transformer_engine/pytorch/attention/__init__.py b/transformer_engine/pytorch/attention/__init__.py index cbc215a2004..6a7708e6b5b 100644 --- a/transformer_engine/pytorch/attention/__init__.py +++ b/transformer_engine/pytorch/attention/__init__.py @@ -6,7 +6,7 @@ from .dot_product_attention import DotProductAttention from . import sparse_attention -from .sparse_attention import DSv4Attention +from .sparse_attention import DSv4Attention, DSv4HybridAttention from .linear_attention import GatedDeltaNetAttention from .fused_mla_q_uproj import FusedMLAQUpProjFunction, FusedMLAQUpProjRopeQuant from .multi_head_attention import MultiheadAttention @@ -16,6 +16,7 @@ __all__ = [ "DotProductAttention", "DSv4Attention", + "DSv4HybridAttention", "sparse_attention", "GatedDeltaNetAttention", "FusedMLAQUpProjFunction", diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md new file mode 100644 index 00000000000..170ac5b4e06 --- /dev/null +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -0,0 +1,123 @@ +# DSv4 model integration + +For stateless, unpadded BF16 CSA/HCA layers, `DSv4HybridAttention` owns the +DSv4 projections, norms, RoPE, compressors, indexer, sink, and grouped output +projection. Its input and output are both `[batch, sequence, hidden_size]`: + +```python +import torch +from transformer_engine.pytorch.attention.sparse_attention import DSv4HybridAttention + +attention = DSv4HybridAttention( + hidden_size=4096, q_lora_rank=1536, + layer_type="compressed_sparse_attention", head_dim=512, rope_head_dim=64, + sliding_window=128, compression_ratio=4, o_groups=8, o_lora_rank=1024, + index_n_heads=32, index_topk=32, max_seqlen=2048, + params_dtype=torch.bfloat16, +) +output = attention(hidden_states) # [B, S, 4096] -> [B, S, 4096] +``` + +```text +BSD ── TE query/local-KV projections + norms ───────────────┐ + ├── TE compressor projections/norm ── cuDNN compression ──┼── cuDNN attention ── TE grouped output ── BSD + └── (CSA) TE index projections/norm ── cuDNN compression ── cuDNN selection ──┘ +``` + +The high-level layer uses 64 heads of width 512 and plain interleaved partial +RoPE with `rope_theta`; it does not implement other RoPE scaling policies. It +supports full sequences starting at position zero, equal unpadded lengths within +a batch, and at least one complete compression window. Sliding-only layers, +cache/decode, padding, TP/CP, FP8, and the separate indexer auxiliary loss +remain outside this first pass. +In CSA, the language-model loss therefore does not train `indexer.q_proj` or the +index-key/weight slices of `indexer.compressor.fused_proj`: top-k selection has no gradient. + +`_dsv4_rope.py` caches FP32 token frequencies (prebuilt when `max_seqlen` is +provided) and shares window-start slices with the CSA `_Indexer`. The indexer +owns its projections, index-key compressor, and block selection; neither helper +changes the cuDNN call contracts. + +The default projection layout groups operations that read the same input: one +TE `Linear` for query-down plus local KV, one for compressor KV plus gates, and +in CSA one for index-compressor KV plus gates plus per-head weights. Each weight +is contiguous; output slices feed the existing norms and cuDNN calls. The +current wrapper validates 64 attention heads. `_fuse_projections=False` retains +separate Linear calls as a comparison path. + +The cuDNN calls used by this layer are: + +| Stage | HCA | CSA | +| --- | --- | --- | +| Compressor forward/backward | Once | Twice: attention KV and index keys | +| Fused indexer score + top-k | — | Once | +| Sparse attention forward/backward | Once | Once | + +cuDNN also offers separate dense indexer scoring/top-k and sparse/dense +score-recompute plus indexer-backward operations. The first two are alternative +ways to select blocks; the latter operations train the indexer through a +separate auxiliary loss. This BSD-only API does not expose that loss. + +The lower-level `dsv4` calls expose three stages when a model owns its own +projections, RMSNorm, RoPE, sink, and output projection: + +```text +model projected KV + gates ── dsv4.compress ── model norm/RoPE ── compressed KV +model index projections ───── dsv4.compress ── model norm/RoPE ── index key +model index query + key + weights ───────────── dsv4.select_blocks ── IDs (CSA) +model query + local KV + compressed KV + IDs ─ dsv4.DSv4Attention ── head output +``` + +The model calls the same stages for HCA, omitting index compression and selection. +It passes `indices=None` and `max_compressed_seqlen` to attention. CSA calls +`compress` twice because its attention memory and index keys use separate +projections. `select_blocks` returns global IDs into the packed **compressed** +rows; the attention core maps them into its combined local/compressed KV space. + +```python +from transformer_engine.pytorch.attention.sparse_attention import dsv4 + +# Model code supplies BF16 projected tensors and applies its own norm/RoPE. +pooled = dsv4.compress(kv, gates, position_bias.float(), cu, cu_comp, + ratio=ratio, overlap=is_csa, total_comp=total_comp) +compressed_kv = model.finish_compressed(pooled) + +indices = None +if is_csa: + index_pooled = dsv4.compress(index_kv, index_gates, + index_position_bias.float(), cu, cu_comp, + ratio=ratio, overlap=True, total_comp=total_comp) + index_key = model.finish_index_key(index_pooled) + indices = dsv4.select_blocks( + index_query, index_key, index_weights, cu, cu_comp, + top_k=top_k, ratio=ratio, max_seqlen=max_seqlen, + max_compressed_seqlen=max_compressed_seqlen, + scale=(index_head_dim * index_n_heads) ** -0.5, + ) + +output = dsv4.DSv4Attention(window_size=window_size, ratio=ratio)( + query, local_kv, compressed_kv, sink.float(), cu, cu_comp, + indices=indices, + max_compressed_seqlen=max_compressed_seqlen if indices is None else None, +) +output = model.finish_output(output) +``` + +This is a wiring sketch: `model.finish_*` are model-owned transforms, not TE APIs. +The model must apply its own normalization and RoPE to `query`, `local_kv`, +`compressed_kv`, `index_query`, and `index_key` before their TE calls. `cu` and +`cu_comp` are CUDA INT32 packed-sequence prefixes; `total_comp` is the valid +number of compressed rows. All Q/KV tensors passed to these stages are +contiguous CUDA BF16. For CSA, pass the raw BF16 `index_weights` projection and +put both positive index scaling factors in `scale` to preserve the FP32 score +scale without an extra BF16 weight rounding. The cuDNN compressor pools with +FP32 intermediates and one BF16 output cast, so keys can differ slightly from +an eager model that rounds its softmax weights or products to BF16. + +The current core supports 64 query heads and head width 512 or 576 on SM100; +the selector supports 32 or 64 index heads of width 128. It handles +full-sequence prefill starting at position zero, with exact packed row counts +in the prefix arrays. A model using segment-local compressed IDs must convert +them to the global packed IDs expected here. Cache/decode, padded-capacity +attention rows, context parallelism, and the separate indexer auxiliary loss +are outside this API. diff --git a/transformer_engine/pytorch/attention/sparse_attention/__init__.py b/transformer_engine/pytorch/attention/sparse_attention/__init__.py index 75b8e83476a..a410f5e628e 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/__init__.py +++ b/transformer_engine/pytorch/attention/sparse_attention/__init__.py @@ -10,5 +10,6 @@ from . import dsv4 from .dsv4 import DSv4Attention +from .dsv4_hybrid_attention import DSv4HybridAttention -__all__ = ["dsv4", "DSv4Attention"] +__all__ = ["dsv4", "DSv4Attention", "DSv4HybridAttention"] diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py index ea0c3b190fd..f2bf615464e 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py @@ -28,13 +28,12 @@ def _namespace(name): ) from exc -def _sparse_attention_forward(): +def _dsa_op(name): try: - return getattr(_namespace("DSA"), "sparse_attention_forward_wrapper") + return getattr(_namespace("DSA"), name) except AttributeError as exc: raise ImportError( - "DSv4 attention requires cudnn.DSA.sparse_attention_forward_wrapper " - "from nvidia-cudnn-frontend>=1.29.0." + f"DSv4 requires cudnn.DSA.{name} from nvidia-cudnn-frontend>=1.29.0." ) from exc @@ -173,10 +172,11 @@ def select_blocks( Returns INT32 [T,top_k] indices into compressed_key's global packed rows; -1 denotes an invalid slot. These are NOT IDs into local+compressed KV. - Selection is discrete. Scale defaults to 1/sqrt(index head dim); apply any - separate head-reduction scaling to weights before this call. This helper - does not implement the separate indexer auxiliary training loss or promise - autograd through the selected IDs. + Selection is discrete. Scale defaults to 1/sqrt(index head dim). If the + model also scales by 1/sqrt(index heads), fold that positive factor into + scale and pass raw BF16 projected weights. Scaling in FP32 then rounding + weights to BF16 can change close top-k boundaries. This helper does not + implement the separate indexer auxiliary loss or autograd through IDs. """ if query.ndim != 3 or query.shape[1] not in (32, 64) or query.shape[2] != 128: raise ValueError("CSA index query must have shape [T,32 or 64,128].") @@ -191,7 +191,7 @@ def select_blocks( _validate_tensors(query, (compressed_key, weights), (), cu_seqlens, cu_seqlens_comp) if scale is None: scale = query.shape[-1] ** -0.5 - result = _namespace("DSA").indexer_forward_top_k_wrapper( + result = _dsa_op("indexer_forward_top_k_wrapper")( query, compressed_key.unsqueeze(1), weights, @@ -267,10 +267,10 @@ def _attention_indices( return torch.cat((local, compressed), dim=1).to(torch.int32).contiguous() -class _Attention(torch.autograd.Function): +class _SparseAttention(torch.autograd.Function): @staticmethod def forward(ctx, query, kv, indices, sink, scale): - result = _sparse_attention_forward()( + result = _dsa_op("sparse_attention_forward_wrapper")( query, kv, indices, @@ -288,7 +288,7 @@ def forward(ctx, query, kv, indices, sink, scale): @once_differentiable def backward(ctx, grad_out): query, kv, out, lse, sink, indices = ctx.saved_tensors - result = _namespace("DSA").sparse_attention_backward_wrapper( + result = _dsa_op("sparse_attention_backward_wrapper")( query, kv, out, @@ -372,7 +372,7 @@ def attention( indices, max_compressed_seqlen, ) - return _Attention.apply( + return _SparseAttention.apply( query.contiguous(), torch.cat((local_kv, compressed_kv), dim=0), selected, diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_compressor.py b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_compressor.py new file mode 100644 index 00000000000..e5aa252a8eb --- /dev/null +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_compressor.py @@ -0,0 +1,61 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Learned DSv4 projections and norm around cuDNN compression.""" + +import torch + +from transformer_engine.pytorch.module import Linear, RMSNorm + +from .dsv4 import compress + + +class _Compressor(torch.nn.Module): + """Project KV/gates and normalize the cuDNN-pooled rows.""" + + def __init__( + self, hidden_size, width, ratio, overlap, eps, device, params_dtype, + *, fused=False, weight_width=0, + ): + super().__init__() + projected = width * (2 if overlap else 1) + kw = dict(bias=False, device=device, params_dtype=params_dtype) + if fused: + # Fuse the KV, gate, and optional index-weight projections of x. + # Keep one contiguous weight for the shared projection call. + self.fused_proj = Linear(hidden_size, 2 * projected + weight_width, **kw) + else: + self.kv_proj = Linear(hidden_size, projected, **kw) + self.gate_proj = Linear(hidden_size, projected, **kw) + self.position_bias = torch.nn.Parameter( + torch.zeros(ratio, projected, device=device, dtype=params_dtype) + ) + self.kv_norm = RMSNorm(width, eps=eps, device=device, dtype=params_dtype) + self.ratio, self.overlap = ratio, overlap + self.projected, self.weight_width = projected, weight_width + + def forward(self, x, cu_seqlens, cu_seqlens_comp): + if hasattr(self, "fused_proj"): + projected = self.fused_proj(x) + widths = ( + (self.projected, self.projected, self.weight_width) + if self.weight_width else (self.projected, self.projected) + ) + parts = projected.split(widths, dim=-1) + kv, gate = parts[:2] + else: + kv, gate = self.kv_proj(x), self.gate_proj(x) + pooled = compress( + kv.flatten(0, 1).contiguous(), + gate.flatten(0, 1).contiguous(), + # cuDNN requires FP32 bias; the cast preserves gradients to the parameter. + self.position_bias.float(), + cu_seqlens, + cu_seqlens_comp, + ratio=self.ratio, + overlap=self.overlap, + total_comp=x.shape[0] * (x.shape[1] // self.ratio), + ) + compressed = self.kv_norm(pooled) + return (compressed, parts[2]) if self.weight_width else compressed diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py new file mode 100644 index 00000000000..705a3ce3cdb --- /dev/null +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py @@ -0,0 +1,63 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""CSA index projections, compression, and cuDNN block selection.""" + +import torch + +from transformer_engine.pytorch.module import Linear + +from ._dsv4_compressor import _Compressor +from ._dsv4_rope import apply_rotary +from .dsv4 import select_blocks + + +class _Indexer(torch.nn.Module): + """Select compressed rows using the indexer's own Q/K/weight projections.""" + + def __init__( + self, hidden_size, q_lora_rank, head_dim, n_heads, top_k, ratio, + eps, device, params_dtype, *, fused, + ): + super().__init__() + self.compressor = _Compressor( + hidden_size, head_dim, ratio, True, eps, device, params_dtype, + fused=fused, weight_width=n_heads if fused else 0, + ) + kw = dict(bias=False, device=device, params_dtype=params_dtype) + self.q_proj = Linear(q_lora_rank, n_heads * head_dim, **kw) + if not fused: + self.weights_proj = Linear(hidden_size, n_heads, **kw) + self.head_dim, self.n_heads = head_dim, n_heads + self.top_k, self.ratio, self.fused = top_k, ratio, fused + + def forward(self, hidden_states, q_residual, cu, cu_comp, token_rope, compressed_rope): + batch, seq, _ = hidden_states.shape + n_comp = seq // self.ratio + index_compressed = self.compressor(hidden_states, cu, cu_comp) + if self.fused: + index_compressed, weights = index_compressed + else: + weights = self.weights_proj(hidden_states) + # The singleton axis is the shared index-key head, not a tunable head count. + key = apply_rotary( + index_compressed.reshape(batch, n_comp, 1, self.head_dim), *compressed_rope + ).reshape(batch * n_comp, self.head_dim).contiguous() + query = self.q_proj(q_residual).reshape(batch, seq, self.n_heads, self.head_dim) + query = apply_rotary(query, *token_rope).reshape( + batch * seq, self.n_heads, self.head_dim + ).contiguous() + # Top-k IDs are discrete; training index projections needs a separate loss. + return select_blocks( + query, + key, + weights.reshape(batch * seq, self.n_heads).contiguous(), + cu, + cu_comp, + top_k=min(self.top_k, n_comp), + ratio=self.ratio, + max_seqlen=seq, + max_compressed_seqlen=n_comp, + scale=(self.head_dim * self.n_heads) ** -0.5, + ) diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_rope.py b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_rope.py new file mode 100644 index 00000000000..3a86dd03248 --- /dev/null +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_rope.py @@ -0,0 +1,61 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""DSv4's trailing-channel, interleaved RoPE for full-sequence attention.""" + +import torch + + +def rotary_embeddings(seq, ratio, width, theta, device): + """Return token and compressed-window (cos, sin) pairs in FP32.""" + inv_freq = theta ** ( + -torch.arange(0, width, 2, device=device, dtype=torch.float32) / width + ) + angles = torch.outer(torch.arange(seq, device=device, dtype=torch.float32), inv_freq) + cos = angles.cos().repeat_interleave(2, dim=-1)[None, :, None, :] + sin = angles.sin().repeat_interleave(2, dim=-1)[None, :, None, :] + window_positions = torch.arange(seq // ratio, device=device) * ratio + return (cos, sin), (cos[:, window_positions], sin[:, window_positions]) + + +class _DSv4RotaryEmbedding(torch.nn.Module): + """Cache shared FP32 token frequencies; compressed positions are window starts.""" + + def __init__(self, ratio, width, theta, device, max_seqlen=None): + super().__init__() + self.ratio, self.width, self.theta = ratio, width, theta + cos, sin = (None, None) + if max_seqlen is not None: + (cos, sin), _ = rotary_embeddings(max_seqlen, ratio, width, theta, device) + self.register_buffer("cos", cos, persistent=False) + self.register_buffer("sin", sin, persistent=False) + + def _apply(self, fn): + cos, sin = self.cos, self.sin + super()._apply(fn) + if cos is not None: + # Module.to(bfloat16) must not round the reusable frequencies. + self.cos = cos.to(device=self.cos.device) + self.sin = sin.to(device=self.sin.device) + return self + + def forward(self, seq, device): + if self.cos is None or self.cos.shape[1] < seq: + (self.cos, self.sin), _ = rotary_embeddings( + seq, self.ratio, self.width, self.theta, device + ) + n_comp = seq // self.ratio + return (self.cos[:, :seq], self.sin[:, :seq]), ( + self.cos[:, :n_comp * self.ratio:self.ratio], + self.sin[:, :n_comp * self.ratio:self.ratio], + ) + + +def apply_rotary(x, cos, sin): + """Rotate trailing interleaved pairs in FP32, then restore input dtype.""" + width = cos.shape[-1] + tail = x[..., -width:] + pair = torch.stack((-tail[..., 1::2], tail[..., 0::2]), -1).flatten(-2) + rotated = (tail.float() * cos + pair.float() * sin).to(x.dtype) + return torch.cat((x[..., :-width], rotated), -1) diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py new file mode 100644 index 00000000000..db47ac4e66e --- /dev/null +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py @@ -0,0 +1,195 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Unpadded BSD-to-BSD DSv4 CSA/HCA attention.""" + +from typing import Optional + +import torch + +from transformer_engine.pytorch.module import Linear, RMSNorm + +from ._dsv4_compressor import _Compressor +from ._dsv4_indexer import _Indexer +from ._dsv4_rope import _DSv4RotaryEmbedding, apply_rotary +from .dsv4 import DSv4Attention + + +class DSv4HybridAttention(torch.nn.Module): + """DSv4 CSA/HCA attention from unpadded ``[B,S,D]`` to ``[B,S,D]``. + + This first-pass layer handles full-sequence BF16 attention forward/backward + on SM100. By default, projections that share an input use one contiguous + TE Linear weight and split its output before the separate DSv4 operations. + It uses TE Linear/RMSNorm, the cuDNN DSv4 compressor and selector, and the + existing parameter-free cuDNN attention core. Sequences in a batch have + the same length and start at position zero. The model has 64 heads of width + 512; CSA index heads have width 128, matching the current cuDNN kernels. + The low-level core's 576-wide Q/K mode returns only 512 value channels, + which has no verified output-unrotation contract here. Cache/decode, + padding, TP/CP, FP8, sliding-only attention, and the indexer auxiliary + loss are unsupported. In CSA, top-k is discrete; the indexer projections + do not receive gradients from the returned hidden states alone. + """ + + def __init__( + self, + hidden_size: int, + q_lora_rank: int, + *, + layer_type: str, + head_dim: int, + rope_head_dim: int, + sliding_window: int, + compression_ratio: int, + o_groups: int, + o_lora_rank: int, + index_n_heads: int = 64, + index_topk: int = 512, + rope_theta: float = 160000.0, + rms_norm_eps: float = 1e-6, + max_seqlen: Optional[int] = None, + device: str = "cuda", + params_dtype: torch.dtype = torch.float32, + _fuse_projections: bool = True, + ): + super().__init__() + # Keep the current kernel geometry explicit without exposing untested sizes. + self.num_heads = 64 + self.index_head_dim = 128 + if layer_type not in ("compressed_sparse_attention", "heavily_compressed_attention"): + raise ValueError("Only DSv4 CSA and HCA are supported.") + if ( + head_dim != 512 + or rope_head_dim < 2 + or rope_head_dim % 2 + or rope_head_dim > head_dim + ): + raise ValueError("head_dim must be 512; rope_head_dim must be even and fit.") + if any( + v <= 0 + for v in ( + hidden_size, q_lora_rank, sliding_window, + compression_ratio, o_groups, o_lora_rank, + ) + ): + raise ValueError("DSv4 dimensions, window size and ratio must be positive.") + if self.num_heads * head_dim % o_groups: + raise ValueError("o_groups must divide the attention output channels.") + is_csa = layer_type == "compressed_sparse_attention" + if is_csa and ( + index_n_heads not in (32, 64) + or index_topk <= 0 + or rope_head_dim > self.index_head_dim + ): + raise ValueError( + "CSA requires 32 or 64 index heads, positive top-k, and RoPE width <= 128." + ) + + kw = dict(bias=False, device=device, params_dtype=params_dtype) + if _fuse_projections: + # Query-down and local-KV projections read the same hidden states. + # Keep slices explicit below while one contiguous weight feeds the GEMM. + self.q_a_kv_proj = Linear( + hidden_size, q_lora_rank + head_dim, **kw, + ) + else: + self.q_a_proj = Linear(hidden_size, q_lora_rank, **kw) + # LayerNormLinear would normalize before projection; DSv4 normalizes after it. + self.q_a_norm = RMSNorm(q_lora_rank, eps=rms_norm_eps, device=device, dtype=params_dtype) + self.q_b_proj = Linear(q_lora_rank, self.num_heads * head_dim, **kw) + if not _fuse_projections: + self.kv_proj = Linear(hidden_size, head_dim, **kw) + self.kv_norm = RMSNorm(head_dim, eps=rms_norm_eps, device=device, dtype=params_dtype) + self.compressor = _Compressor( + hidden_size, head_dim, compression_ratio, is_csa, rms_norm_eps, + device, params_dtype, fused=_fuse_projections, + ) + if is_csa: + self.indexer = _Indexer( + hidden_size, q_lora_rank, self.index_head_dim, index_n_heads, + index_topk, compression_ratio, rms_norm_eps, device, params_dtype, + fused=_fuse_projections, + ) + # HF/Megatron use one independent block per output group. A normal TE + # Linear forward mixes groups, so use its weight in the block multiply. + self.o_a_proj = Linear( + self.num_heads * head_dim // o_groups, o_groups * o_lora_rank, **kw + ) + self.o_b_proj = Linear(o_groups * o_lora_rank, hidden_size, **kw) + self.sinks = torch.nn.Parameter( + torch.zeros(self.num_heads, device=device, dtype=params_dtype) + ) + self.core_attention = DSv4Attention(window_size=sliding_window, ratio=compression_ratio) + self.rope = _DSv4RotaryEmbedding( + compression_ratio, rope_head_dim, rope_theta, device, max_seqlen + ) + self.hidden_size, self.q_lora_rank = hidden_size, q_lora_rank + self.fused_projections = _fuse_projections + self.head_dim, self.rope_head_dim = head_dim, rope_head_dim + self.rope_theta, self.rms_norm_eps = rope_theta, rms_norm_eps + self.compression_ratio = compression_ratio + self.o_groups, self.o_lora_rank = o_groups, o_lora_rank + self.is_csa = is_csa + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + if hidden_states.ndim != 3 or hidden_states.shape[-1] != self.hidden_size: + raise ValueError("hidden_states must have shape [batch, sequence, hidden_size].") + batch, seq, _ = hidden_states.shape + n_comp = seq // self.compression_ratio + if not n_comp: + raise ValueError("Sequence must contain at least one complete compression window.") + cu = torch.arange(batch + 1, device=hidden_states.device, dtype=torch.int32) * seq + cu_comp = torch.arange(batch + 1, device=hidden_states.device, dtype=torch.int32) * n_comp + + token_rope, compressed_rope = self.rope(seq, hidden_states.device) + + if self.fused_projections: + q_a, local_kv_projected = self.q_a_kv_proj(hidden_states).split( + (self.q_lora_rank, self.head_dim), dim=-1 + ) + else: + q_a = self.q_a_proj(hidden_states) + local_kv_projected = self.kv_proj(hidden_states) + q_residual = self.q_a_norm(q_a) + q = self.q_b_proj(q_residual).reshape(batch, seq, self.num_heads, self.head_dim) + # Query-up norm is unweighted; TE RMSNorm would add a learned scale. + q = q * torch.rsqrt( + q.float().square().mean(-1, keepdim=True) + self.rms_norm_eps + ).to(q.dtype) + q = apply_rotary(q, *token_rope).reshape( + batch * seq, self.num_heads, self.head_dim + ).contiguous() + local_kv = apply_rotary( + self.kv_norm(local_kv_projected).unsqueeze(2), *token_rope + ) + local_kv = local_kv.reshape(batch * seq, self.head_dim).contiguous() + compressed_kv = apply_rotary( + self.compressor(hidden_states, cu, cu_comp).reshape( + batch, n_comp, 1, self.head_dim + ), + *compressed_rope, + ) + compressed_kv = compressed_kv.reshape(batch * n_comp, self.head_dim).contiguous() + + indices = None + if self.is_csa: + # Discrete top-k trains the attention path, but not the indexer + # tower; cuDNN's separate score-loss/backward path is not wired here. + indices = self.indexer( + hidden_states, q_residual, cu, cu_comp, token_rope, compressed_rope, + ) + + output = self.core_attention( + q, local_kv, compressed_kv, self.sinks.float(), cu, cu_comp, + indices=indices, max_compressed_seqlen=None if self.is_csa else n_comp, + ).reshape(batch, seq, self.num_heads, self.head_dim) + cos, sin = token_rope + output = apply_rotary(output, cos, -sin).reshape(batch, seq, self.o_groups, -1) + weight = self.o_a_proj.weight.reshape(self.o_groups, self.o_lora_rank, -1) + grouped = torch.einsum("bsgd,grd->bsgr", output, weight).flatten(2) + return self.o_b_proj(grouped) + + +__all__ = ["DSv4HybridAttention"] From 73a2c338260ae628066b31c3e47aef1b308c6bd9 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Thu, 1 Oct 2026 13:04:53 -0700 Subject: [PATCH 03/37] Expose cuDNN dense indexer training primitives for CSA Megatron trains its CSA indexer with a separate dense objective, while discrete top-k selection leaves the indexer without language-model gradients. Expose the live indexer tensors and narrow cuDNN score/backward calls so model code can retain ownership of KL reduction and loss attachment. Allow a caller RoPE module to match compressed YaRN without copying that policy into TE; preserve the default BSD return. Signed-off-by: Sudhakar Singh --- .../attention/sparse_attention/README.md | 27 +++++--- .../sparse_attention/_dsv4_backend.py | 68 ++++++++++++++++++- .../sparse_attention/_dsv4_indexer.py | 11 ++- .../attention/sparse_attention/dsv4.py | 9 ++- .../sparse_attention/dsv4_hybrid_attention.py | 38 ++++++++--- 5 files changed, 129 insertions(+), 24 deletions(-) diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index 170ac5b4e06..fb8ac82607f 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -24,13 +24,14 @@ BSD ── TE query/local-KV projections + norms ────────── └── (CSA) TE index projections/norm ── cuDNN compression ── cuDNN selection ──┘ ``` -The high-level layer uses 64 heads of width 512 and plain interleaved partial -RoPE with `rope_theta`; it does not implement other RoPE scaling policies. It -supports full sequences starting at position zero, equal unpadded lengths within +The high-level layer uses 64 heads of width 512 and defaults to plain interleaved +partial RoPE with `rope_theta`; callers may inject a `rope` module returning token +and compressed `(cos, sin)` pairs. +It supports full sequences starting at position zero, equal unpadded lengths within a batch, and at least one complete compression window. Sliding-only layers, -cache/decode, padding, TP/CP, FP8, and the separate indexer auxiliary loss -remain outside this first pass. -In CSA, the language-model loss therefore does not train `indexer.q_proj` or the +cache/decode, padding, TP/CP, and FP8 remain outside this first pass. The +indexer auxiliary loss and its attachment remain caller-owned; without one, the +language-model loss does not train `indexer.q_proj` or the index-key/weight slices of `indexer.compressor.fused_proj`: top-k selection has no gradient. `_dsv4_rope.py` caches FP32 token frequencies (prebuilt when `max_seqlen` is @@ -53,10 +54,16 @@ The cuDNN calls used by this layer are: | Fused indexer score + top-k | — | Once | | Sparse attention forward/backward | Once | Once | -cuDNN also offers separate dense indexer scoring/top-k and sparse/dense -score-recompute plus indexer-backward operations. The first two are alternative -ways to select blocks; the latter operations train the indexer through a -separate auxiliary loss. This BSD-only API does not expose that loss. +For CSA, `return_indexer_context=True` returns `(BSD_output, context)`; HCA rejects +it. The live BF16 index Q/K/W have shapes `[B*S,H,128]`, `[B*S/ratio,128]`, and +`[B*S,H]`; selected IDs are `[B*S,topk]` global compressed-row IDs. Detached +attention Q/local KV/compressed KV/sink have shapes `[B*S,64,512]`, +`[B*S,512]`, `[B*S/ratio,512]`, and `[64]`. The opt-in context and dense +helpers currently support B1 BF16; the helpers take BSHD views plus a +caller-supplied FP32 full LSE `[1,S,64]` (local +window, all eligible compressed keys, and sink). Score helpers are not autograd +operations; the caller owns the mean-reduced scalar KL, coefficient, and loss +attachment. Per-token reduction is untested. The lower-level `dsv4` calls expose three stages when a model owns its own projections, RMSNorm, RoPE, sink, and output projection: diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py index f2bf615464e..c1539190cb7 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py @@ -15,7 +15,10 @@ import torch from torch.autograd.function import once_differentiable -__all__ = ["compress", "select_blocks", "attention"] +__all__ = [ + "compress", "select_blocks", "attention", "dense_indexer_loss_scores", + "dense_indexer_loss_backward", +] def _namespace(name): @@ -209,6 +212,69 @@ def select_blocks( return result["indices"] +def _pad_index_heads(query, weights): + if query.ndim != 4 or query.shape[0] != 1 or query.shape[2] not in (32, 64): + raise ValueError("Dense indexer loss requires B1 [B,S,32 or 64,128] queries.") + if query.shape[-1] != 128 or weights.shape != query.shape[:3]: + raise ValueError("Dense indexer weights must match B1 index queries.") + if query.dtype != torch.bfloat16 or weights.dtype != torch.bfloat16: + raise TypeError("Dense indexer Q and weights must be BF16.") + pad = 64 - query.shape[2] + if pad: + return torch.nn.functional.pad(query, (0, 0, 0, pad)), torch.nn.functional.pad( + weights, (0, pad) + ) + return query, weights + + +def dense_indexer_loss_scores( + index_q, index_k, index_w, attn_q, compressed_kv, full_lse, + *, ratio: int, index_scale: float, attention_scale: float, +): + """Return cuDNN dense index/teacher score dictionaries for B1 BF16 prefill. + + full_lse is FP32 [1,S,64] and must include the local window, every eligible + compressed key, and the sink. The sparse-attention saved LSE covers only + selected keys and excludes the sink, so it cannot serve this dense teacher. + """ + if any(t.dtype != torch.bfloat16 for t in (index_k, attn_q, compressed_kv)): + raise TypeError("Dense indexer keys and attention Q/K must be BF16.") + if full_lse.dtype != torch.float32 or full_lse.shape != attn_q.shape[:3]: + raise ValueError("full_lse must be FP32 [B,S,attention_heads].") + padded_q, padded_w = _pad_index_heads(index_q, index_w) + index_score = _dsa_op("dense_indexer_score_recompute_wrapper")( + padded_q, index_k.unsqueeze(2), padded_w, sm_scale=index_scale, ratio=ratio, + ) + attn_score = _dsa_op("dense_attn_score_recompute_wrapper")( + attn_q, compressed_kv.unsqueeze(2), full_lse, attention_scale, ratio=ratio, + ) + return index_score, attn_score + + +def dense_indexer_loss_backward( + index_q, index_k, index_w, index_score, attn_score, grad_loss, + *, ratio: int, index_scale: float, loss_coeff: float, +): + """Return dQ/dK/dW for B1 BF16 mean-reduced KL, padding H32 at cuDNN. + + The tested scalar grad_loss/loss_coeff contract is Megatron's default + calculate_per_token_loss=False reduction; per-token reduction is untested. + """ + padded_q, padded_w = _pad_index_heads(index_q, index_w) + result = _dsa_op("dense_indexer_backward_wrapper")( + padded_q, padded_w, index_k, attn_score["out"].clone(), attn_score["denom"], + index_score["out"].clone(), index_score["denom"], + grad_loss=grad_loss.contiguous(), sm_scale=index_scale, + loss_coeff=loss_coeff, ratio=ratio, + ) + heads = index_q.shape[2] + return ( + result["d_index_q"][:, :, :heads].contiguous(), + result["d_index_k"], + result["d_weights"][:, :, :heads].contiguous(), + ) + + def _attention_indices( query: torch.Tensor, compressed_kv: torch.Tensor, diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py index 705a3ce3cdb..9566e9dc12e 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py @@ -32,7 +32,10 @@ def __init__( self.head_dim, self.n_heads = head_dim, n_heads self.top_k, self.ratio, self.fused = top_k, ratio, fused - def forward(self, hidden_states, q_residual, cu, cu_comp, token_rope, compressed_rope): + def forward( + self, hidden_states, q_residual, cu, cu_comp, token_rope, compressed_rope, + *, return_context=False, + ): batch, seq, _ = hidden_states.shape n_comp = seq // self.ratio index_compressed = self.compressor(hidden_states, cu, cu_comp) @@ -49,10 +52,11 @@ def forward(self, hidden_states, q_residual, cu, cu_comp, token_rope, compressed batch * seq, self.n_heads, self.head_dim ).contiguous() # Top-k IDs are discrete; training index projections needs a separate loss. - return select_blocks( + weights = weights.reshape(batch * seq, self.n_heads).contiguous() + indices = select_blocks( query, key, - weights.reshape(batch * seq, self.n_heads).contiguous(), + weights, cu, cu_comp, top_k=min(self.top_k, n_comp), @@ -61,3 +65,4 @@ def forward(self, hidden_states, q_residual, cu, cu_comp, token_rope, compressed max_compressed_seqlen=n_comp, scale=(self.head_dim * self.n_heads) ** -0.5, ) + return (indices, query, key, weights) if return_context else indices diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4.py index bfaf6557a0b..f3d66fe90d1 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4.py @@ -9,9 +9,14 @@ import torch from ._dsv4_backend import attention as _attention -from ._dsv4_backend import compress, select_blocks +from ._dsv4_backend import ( + compress, dense_indexer_loss_backward, dense_indexer_loss_scores, select_blocks, +) -__all__ = ["DSv4Attention", "compress", "select_blocks"] +__all__ = [ + "DSv4Attention", "compress", "select_blocks", + "dense_indexer_loss_scores", "dense_indexer_loss_backward", +] class DSv4Attention(torch.nn.Module): diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py index db47ac4e66e..5e8a9fc2513 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py @@ -48,6 +48,7 @@ def __init__( index_n_heads: int = 64, index_topk: int = 512, rope_theta: float = 160000.0, + rope: Optional[torch.nn.Module] = None, rms_norm_eps: float = 1e-6, max_seqlen: Optional[int] = None, device: str = "cuda", @@ -122,21 +123,29 @@ def __init__( torch.zeros(self.num_heads, device=device, dtype=params_dtype) ) self.core_attention = DSv4Attention(window_size=sliding_window, ratio=compression_ratio) - self.rope = _DSv4RotaryEmbedding( + # A model can supply its RoPE frequencies (for example YaRN) without + # duplicating that policy here. It must return the same token/compressed + # (cos, sin) pairs as _DSv4RotaryEmbedding.forward. + self.rope = rope if rope is not None else _DSv4RotaryEmbedding( compression_ratio, rope_head_dim, rope_theta, device, max_seqlen ) self.hidden_size, self.q_lora_rank = hidden_size, q_lora_rank self.fused_projections = _fuse_projections self.head_dim, self.rope_head_dim = head_dim, rope_head_dim - self.rope_theta, self.rms_norm_eps = rope_theta, rms_norm_eps + self.rope_theta = rope_theta if rope is None else None + self.rms_norm_eps = rms_norm_eps self.compression_ratio = compression_ratio self.o_groups, self.o_lora_rank = o_groups, o_lora_rank self.is_csa = is_csa - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = False): if hidden_states.ndim != 3 or hidden_states.shape[-1] != self.hidden_size: raise ValueError("hidden_states must have shape [batch, sequence, hidden_size].") + if return_indexer_context and not self.is_csa: + raise ValueError("Only CSA has an indexer context.") batch, seq, _ = hidden_states.shape + if return_indexer_context and batch != 1: + raise ValueError("The CSA indexer context is currently supported for batch=1.") n_comp = seq // self.compression_ratio if not n_comp: raise ValueError("Sequence must contain at least one complete compression window.") @@ -175,11 +184,16 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: indices = None if self.is_csa: - # Discrete top-k trains the attention path, but not the indexer - # tower; cuDNN's separate score-loss/backward path is not wired here. - indices = self.indexer( - hidden_states, q_residual, cu, cu_comp, token_rope, compressed_rope, + # Megatron trains this tower with a separate loss, without sending + # its gradients into the shared hidden/query projections. + index_result = self.indexer( + hidden_states.detach(), q_residual.detach(), cu, cu_comp, + token_rope, compressed_rope, return_context=return_indexer_context, ) + if return_indexer_context: + indices, index_q, index_k, index_w = index_result + else: + indices = index_result output = self.core_attention( q, local_kv, compressed_kv, self.sinks.float(), cu, cu_comp, @@ -189,7 +203,15 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: output = apply_rotary(output, cos, -sin).reshape(batch, seq, self.o_groups, -1) weight = self.o_a_proj.weight.reshape(self.o_groups, self.o_lora_rank, -1) grouped = torch.einsum("bsgd,grd->bsgr", output, weight).flatten(2) - return self.o_b_proj(grouped) + result = self.o_b_proj(grouped) + if return_indexer_context: + return result, { + "index_q": index_q, "index_k": index_k, "index_w": index_w, + "indices": indices, + "attn_q": q.detach(), "local_kv": local_kv.detach(), + "compressed_kv": compressed_kv.detach(), "sink": self.sinks.float().detach(), + } + return result __all__ = ["DSv4HybridAttention"] From 0c9f73eb02f22036bcde18c68f262479bfdb2cc9 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Thu, 1 Oct 2026 13:10:15 -0700 Subject: [PATCH 04/37] Format DSv4 Python additions for repository checks The repository Black configuration reformats the newly added DSv4 files. Apply it so lint can assess the implementation without style-only failures; this commit changes no behavior. Signed-off-by: Sudhakar Singh --- .../sparse_attention/_dsv4_backend.py | 76 ++++++++----- .../sparse_attention/_dsv4_indexer.py | 54 +++++++-- .../attention/sparse_attention/dsv4.py | 16 ++- .../sparse_attention/dsv4_hybrid_attention.py | 105 +++++++++++------- 4 files changed, 168 insertions(+), 83 deletions(-) diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py index c1539190cb7..126e87b2b47 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py @@ -16,7 +16,10 @@ from torch.autograd.function import once_differentiable __all__ = [ - "compress", "select_blocks", "attention", "dense_indexer_loss_scores", + "compress", + "select_blocks", + "attention", + "dense_indexer_loss_scores", "dense_indexer_loss_backward", ] @@ -26,8 +29,7 @@ def _namespace(name): return getattr(import_module("cudnn"), name) except (ImportError, AttributeError) as exc: raise ImportError( - "DSv4 requires nvidia-cudnn-frontend>=1.29.0 " - f"with the cudnn.{name} namespace." + f"DSv4 requires nvidia-cudnn-frontend>=1.29.0 with the cudnn.{name} namespace." ) from exc @@ -67,9 +69,7 @@ def _validate_tensors(reference, bf16, fp32, cu_seqlens, cu_seqlens_comp): class _Compress(torch.autograd.Function): @staticmethod - def forward( - ctx, kv, score, ape, cu_seqlens, cu_seqlens_comp, ratio, overlap, total_comp - ): + def forward(ctx, kv, score, ape, cu_seqlens, cu_seqlens_comp, ratio, overlap, total_comp): coff = 2 if overlap else 1 if kv.ndim != 2 or kv.shape[-1] % coff: raise ValueError("kv must have shape [tokens, coff * head_dim].") @@ -188,9 +188,7 @@ def select_blocks( if weights.shape != query.shape[:2]: raise ValueError("CSA index weights must have shape [T,H].") if ratio <= 0 or top_k <= 0 or max_seqlen <= 0 or max_compressed_seqlen <= 0: - raise ValueError( - "CSA ratio, top_k and maximum sequence lengths must be positive." - ) + raise ValueError("CSA ratio, top_k and maximum sequence lengths must be positive.") _validate_tensors(query, (compressed_key, weights), (), cu_seqlens, cu_seqlens_comp) if scale is None: scale = query.shape[-1] ** -0.5 @@ -228,8 +226,16 @@ def _pad_index_heads(query, weights): def dense_indexer_loss_scores( - index_q, index_k, index_w, attn_q, compressed_kv, full_lse, - *, ratio: int, index_scale: float, attention_scale: float, + index_q, + index_k, + index_w, + attn_q, + compressed_kv, + full_lse, + *, + ratio: int, + index_scale: float, + attention_scale: float, ): """Return cuDNN dense index/teacher score dictionaries for B1 BF16 prefill. @@ -243,17 +249,33 @@ def dense_indexer_loss_scores( raise ValueError("full_lse must be FP32 [B,S,attention_heads].") padded_q, padded_w = _pad_index_heads(index_q, index_w) index_score = _dsa_op("dense_indexer_score_recompute_wrapper")( - padded_q, index_k.unsqueeze(2), padded_w, sm_scale=index_scale, ratio=ratio, + padded_q, + index_k.unsqueeze(2), + padded_w, + sm_scale=index_scale, + ratio=ratio, ) attn_score = _dsa_op("dense_attn_score_recompute_wrapper")( - attn_q, compressed_kv.unsqueeze(2), full_lse, attention_scale, ratio=ratio, + attn_q, + compressed_kv.unsqueeze(2), + full_lse, + attention_scale, + ratio=ratio, ) return index_score, attn_score def dense_indexer_loss_backward( - index_q, index_k, index_w, index_score, attn_score, grad_loss, - *, ratio: int, index_scale: float, loss_coeff: float, + index_q, + index_k, + index_w, + index_score, + attn_score, + grad_loss, + *, + ratio: int, + index_scale: float, + loss_coeff: float, ): """Return dQ/dK/dW for B1 BF16 mean-reduced KL, padding H32 at cuDNN. @@ -262,10 +284,17 @@ def dense_indexer_loss_backward( """ padded_q, padded_w = _pad_index_heads(index_q, index_w) result = _dsa_op("dense_indexer_backward_wrapper")( - padded_q, padded_w, index_k, attn_score["out"].clone(), attn_score["denom"], - index_score["out"].clone(), index_score["denom"], - grad_loss=grad_loss.contiguous(), sm_scale=index_scale, - loss_coeff=loss_coeff, ratio=ratio, + padded_q, + padded_w, + index_k, + attn_score["out"].clone(), + attn_score["denom"], + index_score["out"].clone(), + index_score["denom"], + grad_loss=grad_loss.contiguous(), + sm_scale=index_scale, + loss_coeff=loss_coeff, + ratio=ratio, ) heads = index_q.shape[2] return ( @@ -407,10 +436,7 @@ def attention( raise ValueError("attention head dimension must be 512 or 576.") if local_kv.ndim != 2 or compressed_kv.ndim != 2: raise ValueError("local_kv and compressed_kv must be 2D packed tensors.") - if ( - local_kv.shape[-1] != query.shape[-1] - or compressed_kv.shape[-1] != query.shape[-1] - ): + if local_kv.shape[-1] != query.shape[-1] or compressed_kv.shape[-1] != query.shape[-1]: raise ValueError("Q and both KV tensors must have the same QK dimension.") if local_kv.shape[0] != query.shape[0]: raise ValueError("Full-sequence attention requires one local KV row per query.") @@ -420,9 +446,7 @@ def attention( raise ValueError("window_size and ratio must be positive.") if indices is None and (max_compressed_seqlen is None or max_compressed_seqlen < 0): raise ValueError("HCA requires a nonnegative max_compressed_seqlen.") - _validate_tensors( - query, (local_kv, compressed_kv), (sink,), cu_seqlens, cu_seqlens_comp - ) + _validate_tensors(query, (local_kv, compressed_kv), (sink,), cu_seqlens, cu_seqlens_comp) if indices is not None: if indices.dtype != torch.int32 or indices.device != query.device: raise ValueError("CSA indices must be CUDA INT32 on the query device.") diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py index 9566e9dc12e..7a6478ad58b 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py @@ -17,13 +17,30 @@ class _Indexer(torch.nn.Module): """Select compressed rows using the indexer's own Q/K/weight projections.""" def __init__( - self, hidden_size, q_lora_rank, head_dim, n_heads, top_k, ratio, - eps, device, params_dtype, *, fused, + self, + hidden_size, + q_lora_rank, + head_dim, + n_heads, + top_k, + ratio, + eps, + device, + params_dtype, + *, + fused, ): super().__init__() self.compressor = _Compressor( - hidden_size, head_dim, ratio, True, eps, device, params_dtype, - fused=fused, weight_width=n_heads if fused else 0, + hidden_size, + head_dim, + ratio, + True, + eps, + device, + params_dtype, + fused=fused, + weight_width=n_heads if fused else 0, ) kw = dict(bias=False, device=device, params_dtype=params_dtype) self.q_proj = Linear(q_lora_rank, n_heads * head_dim, **kw) @@ -33,8 +50,15 @@ def __init__( self.top_k, self.ratio, self.fused = top_k, ratio, fused def forward( - self, hidden_states, q_residual, cu, cu_comp, token_rope, compressed_rope, - *, return_context=False, + self, + hidden_states, + q_residual, + cu, + cu_comp, + token_rope, + compressed_rope, + *, + return_context=False, ): batch, seq, _ = hidden_states.shape n_comp = seq // self.ratio @@ -44,13 +68,19 @@ def forward( else: weights = self.weights_proj(hidden_states) # The singleton axis is the shared index-key head, not a tunable head count. - key = apply_rotary( - index_compressed.reshape(batch, n_comp, 1, self.head_dim), *compressed_rope - ).reshape(batch * n_comp, self.head_dim).contiguous() + key = ( + apply_rotary( + index_compressed.reshape(batch, n_comp, 1, self.head_dim), *compressed_rope + ) + .reshape(batch * n_comp, self.head_dim) + .contiguous() + ) query = self.q_proj(q_residual).reshape(batch, seq, self.n_heads, self.head_dim) - query = apply_rotary(query, *token_rope).reshape( - batch * seq, self.n_heads, self.head_dim - ).contiguous() + query = ( + apply_rotary(query, *token_rope) + .reshape(batch * seq, self.n_heads, self.head_dim) + .contiguous() + ) # Top-k IDs are discrete; training index projections needs a separate loss. weights = weights.reshape(batch * seq, self.n_heads).contiguous() indices = select_blocks( diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4.py index f3d66fe90d1..c656a0b72b1 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4.py @@ -10,12 +10,18 @@ from ._dsv4_backend import attention as _attention from ._dsv4_backend import ( - compress, dense_indexer_loss_backward, dense_indexer_loss_scores, select_blocks, + compress, + dense_indexer_loss_backward, + dense_indexer_loss_scores, + select_blocks, ) __all__ = [ - "DSv4Attention", "compress", "select_blocks", - "dense_indexer_loss_scores", "dense_indexer_loss_backward", + "DSv4Attention", + "compress", + "select_blocks", + "dense_indexer_loss_scores", + "dense_indexer_loss_backward", ] @@ -69,9 +75,7 @@ def forward( from transformer_engine.pytorch.quantization import FP8GlobalStateManager if FP8GlobalStateManager.is_fp8_enabled(): - raise NotImplementedError( - "DSv4Attention supports BF16 only; disable TE FP8 autocast." - ) + raise NotImplementedError("DSv4Attention supports BF16 only; disable TE FP8 autocast.") return _attention( query, local_kv, diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py index 5e8a9fc2513..b6a28ba2e9f 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py @@ -61,18 +61,17 @@ def __init__( self.index_head_dim = 128 if layer_type not in ("compressed_sparse_attention", "heavily_compressed_attention"): raise ValueError("Only DSv4 CSA and HCA are supported.") - if ( - head_dim != 512 - or rope_head_dim < 2 - or rope_head_dim % 2 - or rope_head_dim > head_dim - ): + if head_dim != 512 or rope_head_dim < 2 or rope_head_dim % 2 or rope_head_dim > head_dim: raise ValueError("head_dim must be 512; rope_head_dim must be even and fit.") if any( v <= 0 for v in ( - hidden_size, q_lora_rank, sliding_window, - compression_ratio, o_groups, o_lora_rank, + hidden_size, + q_lora_rank, + sliding_window, + compression_ratio, + o_groups, + o_lora_rank, ) ): raise ValueError("DSv4 dimensions, window size and ratio must be positive.") @@ -80,9 +79,7 @@ def __init__( raise ValueError("o_groups must divide the attention output channels.") is_csa = layer_type == "compressed_sparse_attention" if is_csa and ( - index_n_heads not in (32, 64) - or index_topk <= 0 - or rope_head_dim > self.index_head_dim + index_n_heads not in (32, 64) or index_topk <= 0 or rope_head_dim > self.index_head_dim ): raise ValueError( "CSA requires 32 or 64 index heads, positive top-k, and RoPE width <= 128." @@ -93,7 +90,9 @@ def __init__( # Query-down and local-KV projections read the same hidden states. # Keep slices explicit below while one contiguous weight feeds the GEMM. self.q_a_kv_proj = Linear( - hidden_size, q_lora_rank + head_dim, **kw, + hidden_size, + q_lora_rank + head_dim, + **kw, ) else: self.q_a_proj = Linear(hidden_size, q_lora_rank, **kw) @@ -104,20 +103,31 @@ def __init__( self.kv_proj = Linear(hidden_size, head_dim, **kw) self.kv_norm = RMSNorm(head_dim, eps=rms_norm_eps, device=device, dtype=params_dtype) self.compressor = _Compressor( - hidden_size, head_dim, compression_ratio, is_csa, rms_norm_eps, - device, params_dtype, fused=_fuse_projections, + hidden_size, + head_dim, + compression_ratio, + is_csa, + rms_norm_eps, + device, + params_dtype, + fused=_fuse_projections, ) if is_csa: self.indexer = _Indexer( - hidden_size, q_lora_rank, self.index_head_dim, index_n_heads, - index_topk, compression_ratio, rms_norm_eps, device, params_dtype, + hidden_size, + q_lora_rank, + self.index_head_dim, + index_n_heads, + index_topk, + compression_ratio, + rms_norm_eps, + device, + params_dtype, fused=_fuse_projections, ) # HF/Megatron use one independent block per output group. A normal TE # Linear forward mixes groups, so use its weight in the block multiply. - self.o_a_proj = Linear( - self.num_heads * head_dim // o_groups, o_groups * o_lora_rank, **kw - ) + self.o_a_proj = Linear(self.num_heads * head_dim // o_groups, o_groups * o_lora_rank, **kw) self.o_b_proj = Linear(o_groups * o_lora_rank, hidden_size, **kw) self.sinks = torch.nn.Parameter( torch.zeros(self.num_heads, device=device, dtype=params_dtype) @@ -126,8 +136,12 @@ def __init__( # A model can supply its RoPE frequencies (for example YaRN) without # duplicating that policy here. It must return the same token/compressed # (cos, sin) pairs as _DSv4RotaryEmbedding.forward. - self.rope = rope if rope is not None else _DSv4RotaryEmbedding( - compression_ratio, rope_head_dim, rope_theta, device, max_seqlen + self.rope = ( + rope + if rope is not None + else _DSv4RotaryEmbedding( + compression_ratio, rope_head_dim, rope_theta, device, max_seqlen + ) ) self.hidden_size, self.q_lora_rank = hidden_size, q_lora_rank self.fused_projections = _fuse_projections @@ -164,20 +178,18 @@ def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = q_residual = self.q_a_norm(q_a) q = self.q_b_proj(q_residual).reshape(batch, seq, self.num_heads, self.head_dim) # Query-up norm is unweighted; TE RMSNorm would add a learned scale. - q = q * torch.rsqrt( - q.float().square().mean(-1, keepdim=True) + self.rms_norm_eps - ).to(q.dtype) - q = apply_rotary(q, *token_rope).reshape( - batch * seq, self.num_heads, self.head_dim - ).contiguous() - local_kv = apply_rotary( - self.kv_norm(local_kv_projected).unsqueeze(2), *token_rope + q = q * torch.rsqrt(q.float().square().mean(-1, keepdim=True) + self.rms_norm_eps).to( + q.dtype + ) + q = ( + apply_rotary(q, *token_rope) + .reshape(batch * seq, self.num_heads, self.head_dim) + .contiguous() ) + local_kv = apply_rotary(self.kv_norm(local_kv_projected).unsqueeze(2), *token_rope) local_kv = local_kv.reshape(batch * seq, self.head_dim).contiguous() compressed_kv = apply_rotary( - self.compressor(hidden_states, cu, cu_comp).reshape( - batch, n_comp, 1, self.head_dim - ), + self.compressor(hidden_states, cu, cu_comp).reshape(batch, n_comp, 1, self.head_dim), *compressed_rope, ) compressed_kv = compressed_kv.reshape(batch * n_comp, self.head_dim).contiguous() @@ -187,8 +199,13 @@ def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = # Megatron trains this tower with a separate loss, without sending # its gradients into the shared hidden/query projections. index_result = self.indexer( - hidden_states.detach(), q_residual.detach(), cu, cu_comp, - token_rope, compressed_rope, return_context=return_indexer_context, + hidden_states.detach(), + q_residual.detach(), + cu, + cu_comp, + token_rope, + compressed_rope, + return_context=return_indexer_context, ) if return_indexer_context: indices, index_q, index_k, index_w = index_result @@ -196,8 +213,14 @@ def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = indices = index_result output = self.core_attention( - q, local_kv, compressed_kv, self.sinks.float(), cu, cu_comp, - indices=indices, max_compressed_seqlen=None if self.is_csa else n_comp, + q, + local_kv, + compressed_kv, + self.sinks.float(), + cu, + cu_comp, + indices=indices, + max_compressed_seqlen=None if self.is_csa else n_comp, ).reshape(batch, seq, self.num_heads, self.head_dim) cos, sin = token_rope output = apply_rotary(output, cos, -sin).reshape(batch, seq, self.o_groups, -1) @@ -206,10 +229,14 @@ def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = result = self.o_b_proj(grouped) if return_indexer_context: return result, { - "index_q": index_q, "index_k": index_k, "index_w": index_w, + "index_q": index_q, + "index_k": index_k, + "index_w": index_w, "indices": indices, - "attn_q": q.detach(), "local_kv": local_kv.detach(), - "compressed_kv": compressed_kv.detach(), "sink": self.sinks.float().detach(), + "attn_q": q.detach(), + "local_kv": local_kv.detach(), + "compressed_kv": compressed_kv.detach(), + "sink": self.sinks.float().detach(), } return result From 39d4c95b2b4713411f1cdccaf05c48e698da7527 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:56:20 +0000 Subject: [PATCH 05/37] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/pytorch/test_dsv4_attention.py | 20 +++++-------------- .../sparse_attention/_dsv4_compressor.py | 16 ++++++++++++--- .../attention/sparse_attention/_dsv4_rope.py | 8 +++----- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index bd637a6b31d..9b189b67770 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -31,9 +31,7 @@ def test_dsv4_forward_backward(variant, head_dim): comp_lengths = [n // ratio for n in lengths] total, total_comp = sum(lengths), sum(comp_lengths) cu = torch.tensor([0, 256, total], device="cuda", dtype=torch.int32) - cu_comp = torch.tensor( - [0, comp_lengths[0], total_comp], device="cuda", dtype=torch.int32 - ) + cu_comp = torch.tensor([0, comp_lengths[0], total_comp], device="cuda", dtype=torch.int32) query = ( torch.randn(total, 64, head_dim, device="cuda", dtype=torch.bfloat16) * 0.1 ).requires_grad_() @@ -80,16 +78,10 @@ def test_dsv4_forward_backward(variant, head_dim): if indices is not None: comp_mask &= (comp_ids % 2 == 0)[None, :] mask = torch.cat((local_mask, comp_mask), dim=-1) - logits = torch.einsum("thd,kd->thk", q[start : start + length], kv) * ( - head_dim**-0.5 - ) + logits = torch.einsum("thd,kd->thk", q[start : start + length], kv) * (head_dim**-0.5) logits = logits.masked_fill(~mask[:, None, :], float("-inf")) - probabilities = torch.cat( - (logits, s[None, :, None].expand(length, -1, -1)), -1 - ).softmax(-1) - reference.append( - torch.einsum("thk,kd->thd", probabilities[..., :-1], kv[..., :512]) - ) + probabilities = torch.cat((logits, s[None, :, None].expand(length, -1, -1)), -1).softmax(-1) + reference.append(torch.einsum("thk,kd->thd", probabilities[..., :-1], kv[..., :512])) start += length comp_start += count expected = torch.cat(reference) @@ -100,9 +92,7 @@ def test_dsv4_forward_backward(variant, head_dim): for actual, ref in zip(leaves, reference_leaves): assert actual.grad is not None and torch.isfinite(actual.grad).all() # Relative L2 error tolerates BF16 reduction rounding near zero entries. - relative_error = ( - actual.grad.float() - ref.grad - ).norm() / ref.grad.norm().clamp_min(1e-8) + relative_error = (actual.grad.float() - ref.grad).norm() / ref.grad.norm().clamp_min(1e-8) assert relative_error < 0.02, relative_error.item() diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_compressor.py b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_compressor.py index e5aa252a8eb..3da46fbff02 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_compressor.py +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_compressor.py @@ -15,8 +15,17 @@ class _Compressor(torch.nn.Module): """Project KV/gates and normalize the cuDNN-pooled rows.""" def __init__( - self, hidden_size, width, ratio, overlap, eps, device, params_dtype, - *, fused=False, weight_width=0, + self, + hidden_size, + width, + ratio, + overlap, + eps, + device, + params_dtype, + *, + fused=False, + weight_width=0, ): super().__init__() projected = width * (2 if overlap else 1) @@ -40,7 +49,8 @@ def forward(self, x, cu_seqlens, cu_seqlens_comp): projected = self.fused_proj(x) widths = ( (self.projected, self.projected, self.weight_width) - if self.weight_width else (self.projected, self.projected) + if self.weight_width + else (self.projected, self.projected) ) parts = projected.split(widths, dim=-1) kv, gate = parts[:2] diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_rope.py b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_rope.py index 3a86dd03248..cf879b8a746 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_rope.py +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_rope.py @@ -9,9 +9,7 @@ def rotary_embeddings(seq, ratio, width, theta, device): """Return token and compressed-window (cos, sin) pairs in FP32.""" - inv_freq = theta ** ( - -torch.arange(0, width, 2, device=device, dtype=torch.float32) / width - ) + inv_freq = theta ** (-torch.arange(0, width, 2, device=device, dtype=torch.float32) / width) angles = torch.outer(torch.arange(seq, device=device, dtype=torch.float32), inv_freq) cos = angles.cos().repeat_interleave(2, dim=-1)[None, :, None, :] sin = angles.sin().repeat_interleave(2, dim=-1)[None, :, None, :] @@ -47,8 +45,8 @@ def forward(self, seq, device): ) n_comp = seq // self.ratio return (self.cos[:, :seq], self.sin[:, :seq]), ( - self.cos[:, :n_comp * self.ratio:self.ratio], - self.sin[:, :n_comp * self.ratio:self.ratio], + self.cos[:, : n_comp * self.ratio : self.ratio], + self.sin[:, : n_comp * self.ratio : self.ratio], ) From 540507e71bf936685a7c196b712cd0813d08bac2 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Thu, 1 Oct 2026 14:34:22 -0700 Subject: [PATCH 06/37] Clear DSv4 Python lint findings The PR Python check reports DSv4 docstring, literal, unused-argument, and local-initialization findings. Resolve them without changing the attention or indexer contracts so CI can validate the implementation. Signed-off-by: Sudhakar Singh --- .../pytorch/attention/sparse_attention/_dsv4_backend.py | 6 ++++-- .../pytorch/attention/sparse_attention/_dsv4_compressor.py | 3 ++- .../pytorch/attention/sparse_attention/_dsv4_indexer.py | 3 ++- .../pytorch/attention/sparse_attention/_dsv4_rope.py | 1 + .../attention/sparse_attention/dsv4_hybrid_attention.py | 4 +++- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py index 126e87b2b47..f7df45ab518 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py @@ -70,6 +70,7 @@ def _validate_tensors(reference, bf16, fp32, cu_seqlens, cu_seqlens_comp): class _Compress(torch.autograd.Function): @staticmethod def forward(ctx, kv, score, ape, cu_seqlens, cu_seqlens_comp, ratio, overlap, total_comp): + """Run cuDNN compression and save its backward inputs.""" coff = 2 if overlap else 1 if kv.ndim != 2 or kv.shape[-1] % coff: raise ValueError("kv must have shape [tokens, coff * head_dim].") @@ -92,6 +93,7 @@ def forward(ctx, kv, score, ape, cu_seqlens, cu_seqlens_comp, ratio, overlap, to @staticmethod @once_differentiable def backward(ctx, grad_out): + """Propagate gradients through cuDNN compression.""" ratio, head_dim, coff = ctx.config # cuDNN owns scratch allocation and dAPE zeroing. Autograd may supply # a strided/expanded gradient, while this kernel requires contiguous data. @@ -306,7 +308,6 @@ def dense_indexer_loss_backward( def _attention_indices( query: torch.Tensor, - compressed_kv: torch.Tensor, cu_seqlens: torch.Tensor, cu_seqlens_comp: torch.Tensor, window_size: int, @@ -365,6 +366,7 @@ def _attention_indices( class _SparseAttention(torch.autograd.Function): @staticmethod def forward(ctx, query, kv, indices, sink, scale): + """Run cuDNN sparse prefill and save its backward inputs.""" result = _dsa_op("sparse_attention_forward_wrapper")( query, kv, @@ -382,6 +384,7 @@ def forward(ctx, query, kv, indices, sink, scale): @staticmethod @once_differentiable def backward(ctx, grad_out): + """Propagate output gradients through cuDNN sparse attention.""" query, kv, out, lse, sink, indices = ctx.saved_tensors result = _dsa_op("sparse_attention_backward_wrapper")( query, @@ -454,7 +457,6 @@ def attention( scale = query.shape[-1] ** -0.5 selected = _attention_indices( query, - compressed_kv, cu_seqlens, cu_seqlens_comp, window_size, diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_compressor.py b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_compressor.py index 3da46fbff02..2349965a733 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_compressor.py +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_compressor.py @@ -29,7 +29,7 @@ def __init__( ): super().__init__() projected = width * (2 if overlap else 1) - kw = dict(bias=False, device=device, params_dtype=params_dtype) + kw = {"bias": False, "device": device, "params_dtype": params_dtype} if fused: # Fuse the KV, gate, and optional index-weight projections of x. # Keep one contiguous weight for the shared projection call. @@ -45,6 +45,7 @@ def __init__( self.projected, self.weight_width = projected, weight_width def forward(self, x, cu_seqlens, cu_seqlens_comp): + """Project and compress KV rows, returning optional fused index weights.""" if hasattr(self, "fused_proj"): projected = self.fused_proj(x) widths = ( diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py index 7a6478ad58b..fdc8e769d6b 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py @@ -42,7 +42,7 @@ def __init__( fused=fused, weight_width=n_heads if fused else 0, ) - kw = dict(bias=False, device=device, params_dtype=params_dtype) + kw = {"bias": False, "device": device, "params_dtype": params_dtype} self.q_proj = Linear(q_lora_rank, n_heads * head_dim, **kw) if not fused: self.weights_proj = Linear(hidden_size, n_heads, **kw) @@ -60,6 +60,7 @@ def forward( *, return_context=False, ): + """Select compressed rows and optionally expose live tensors for the indexer loss.""" batch, seq, _ = hidden_states.shape n_comp = seq // self.ratio index_compressed = self.compressor(hidden_states, cu, cu_comp) diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_rope.py b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_rope.py index cf879b8a746..7165064322a 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_rope.py +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_rope.py @@ -39,6 +39,7 @@ def _apply(self, fn): return self def forward(self, seq, device): + """Return cached token and compressed-window frequency pairs.""" if self.cos is None or self.cos.shape[1] < seq: (self.cos, self.sin), _ = rotary_embeddings( seq, self.ratio, self.width, self.theta, device diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py index b6a28ba2e9f..1443cbd42ea 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py @@ -85,7 +85,7 @@ def __init__( "CSA requires 32 or 64 index heads, positive top-k, and RoPE width <= 128." ) - kw = dict(bias=False, device=device, params_dtype=params_dtype) + kw = {"bias": False, "device": device, "params_dtype": params_dtype} if _fuse_projections: # Query-down and local-KV projections read the same hidden states. # Keep slices explicit below while one contiguous weight feeds the GEMM. @@ -153,6 +153,7 @@ def __init__( self.is_csa = is_csa def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = False): + """Run BSD-to-BSD attention and optionally expose the CSA loss context.""" if hidden_states.ndim != 3 or hidden_states.shape[-1] != self.hidden_size: raise ValueError("hidden_states must have shape [batch, sequence, hidden_size].") if return_indexer_context and not self.is_csa: @@ -195,6 +196,7 @@ def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = compressed_kv = compressed_kv.reshape(batch * n_comp, self.head_dim).contiguous() indices = None + index_q = index_k = index_w = None if self.is_csa: # Megatron trains this tower with a separate loss, without sending # its gradients into the shared hidden/query projections. From 8b1bf898b988f2d3f5b905aa8094ad0c0e3ebed2 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Thu, 1 Oct 2026 16:44:56 -0700 Subject: [PATCH 07/37] Default DSv4 hybrid attention to sequence-major tensors Megatron passes sequence-major hidden states to attention. Accept and return that layout by default so the batch-one path can enter the existing packed-row core through a view without an extra public BSD round trip. Keep explicit BSD input for batch-major callers; larger batches still repack for the present core layout. Signed-off-by: Sudhakar Singh --- .../attention/sparse_attention/README.md | 13 ++++++++----- .../sparse_attention/dsv4_hybrid_attention.py | 18 ++++++++++++++---- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index fb8ac82607f..1e38e17b0b6 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -2,7 +2,7 @@ For stateless, unpadded BF16 CSA/HCA layers, `DSv4HybridAttention` owns the DSv4 projections, norms, RoPE, compressors, indexer, sink, and grouped output -projection. Its input and output are both `[batch, sequence, hidden_size]`: +projection. Its default input and output are both `[sequence, batch, hidden_size]`: ```python import torch @@ -15,18 +15,21 @@ attention = DSv4HybridAttention( index_n_heads=32, index_topk=32, max_seqlen=2048, params_dtype=torch.bfloat16, ) -output = attention(hidden_states) # [B, S, 4096] -> [B, S, 4096] +output = attention(hidden_states) # [S, B, 4096] -> [S, B, 4096] ``` ```text -BSD ── TE query/local-KV projections + norms ───────────────┐ - ├── TE compressor projections/norm ── cuDNN compression ──┼── cuDNN attention ── TE grouped output ── BSD +SBD ── TE query/local-KV projections + norms ───────────────┐ + ├── TE compressor projections/norm ── cuDNN compression ──┼── cuDNN attention ── TE grouped output ── SBD └── (CSA) TE index projections/norm ── cuDNN compression ── cuDNN selection ──┘ ``` The high-level layer uses 64 heads of width 512 and defaults to plain interleaved partial RoPE with `rope_theta`; callers may inject a `rope` module returning token and compressed `(cos, sin)` pairs. +Use `input_format="bsd"` for batch-major callers. The SBD boundary conversion +is a view when batch size is one; larger batches require a repack for the current +cuDNN packed-row contract. It supports full sequences starting at position zero, equal unpadded lengths within a batch, and at least one complete compression window. Sliding-only layers, cache/decode, padding, TP/CP, and FP8 remain outside this first pass. The @@ -54,7 +57,7 @@ The cuDNN calls used by this layer are: | Fused indexer score + top-k | — | Once | | Sparse attention forward/backward | Once | Once | -For CSA, `return_indexer_context=True` returns `(BSD_output, context)`; HCA rejects +For CSA, `return_indexer_context=True` returns `(SBD_output, context)` by default; HCA rejects it. The live BF16 index Q/K/W have shapes `[B*S,H,128]`, `[B*S/ratio,128]`, and `[B*S,H]`; selected IDs are `[B*S,topk]` global compressed-row IDs. Detached attention Q/local KV/compressed KV/sink have shapes `[B*S,64,512]`, diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py index 1443cbd42ea..0d1df63353a 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py @@ -2,7 +2,7 @@ # # See LICENSE for license information. -"""Unpadded BSD-to-BSD DSv4 CSA/HCA attention.""" +"""Unpadded DSv4 CSA/HCA attention.""" from typing import Optional @@ -17,7 +17,7 @@ class DSv4HybridAttention(torch.nn.Module): - """DSv4 CSA/HCA attention from unpadded ``[B,S,D]`` to ``[B,S,D]``. + """DSv4 CSA/HCA attention from unpadded ``[S,B,D]`` to ``[S,B,D]``. This first-pass layer handles full-sequence BF16 attention forward/backward on SM100. By default, projections that share an input use one contiguous @@ -53,6 +53,7 @@ def __init__( max_seqlen: Optional[int] = None, device: str = "cuda", params_dtype: torch.dtype = torch.float32, + input_format: str = "sbd", _fuse_projections: bool = True, ): super().__init__() @@ -61,6 +62,8 @@ def __init__( self.index_head_dim = 128 if layer_type not in ("compressed_sparse_attention", "heavily_compressed_attention"): raise ValueError("Only DSv4 CSA and HCA are supported.") + if input_format not in ("sbd", "bsd"): + raise ValueError("input_format must be 'sbd' or 'bsd'.") if head_dim != 512 or rope_head_dim < 2 or rope_head_dim % 2 or rope_head_dim > head_dim: raise ValueError("head_dim must be 512; rope_head_dim must be even and fit.") if any( @@ -151,13 +154,18 @@ def __init__( self.compression_ratio = compression_ratio self.o_groups, self.o_lora_rank = o_groups, o_lora_rank self.is_csa = is_csa + self.input_format = input_format def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = False): - """Run BSD-to-BSD attention and optionally expose the CSA loss context.""" + """Run attention in the selected format and optionally expose CSA context.""" if hidden_states.ndim != 3 or hidden_states.shape[-1] != self.hidden_size: - raise ValueError("hidden_states must have shape [batch, sequence, hidden_size].") + raise ValueError("hidden_states must have the selected format and hidden_size.") if return_indexer_context and not self.is_csa: raise ValueError("Only CSA has an indexer context.") + if self.input_format == "sbd": + # cuDNN consumes batch-major packed rows. B=1 needs only a view; + # larger batches require a physical repack with the current core. + hidden_states = hidden_states.transpose(0, 1).contiguous() batch, seq, _ = hidden_states.shape if return_indexer_context and batch != 1: raise ValueError("The CSA indexer context is currently supported for batch=1.") @@ -229,6 +237,8 @@ def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = weight = self.o_a_proj.weight.reshape(self.o_groups, self.o_lora_rank, -1) grouped = torch.einsum("bsgd,grd->bsgr", output, weight).flatten(2) result = self.o_b_proj(grouped) + if self.input_format == "sbd": + result = result.transpose(0, 1).contiguous() if return_indexer_context: return result, { "index_q": index_q, From 4b01ff2354610af569fad678617ffa9e2dcb94b3 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Thu, 1 Oct 2026 22:57:45 -0700 Subject: [PATCH 08/37] Use weightless RMS norm for DSv4 query projection The eager FP32 square/mean path materializes full-size query intermediates. At B4/S32768/BF16 on B200, weightless F.rms_norm lowers HCA and CSA TE peak allocated memory by about 45 GiB and cuts forward/backward time by 26-28%, while mapped-weight output and gradient cosines remain above 0.999999. Keep the unweighted normalization contract without adding a learned scale. Signed-off-by: Sudhakar Singh --- .../attention/sparse_attention/dsv4_hybrid_attention.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py index 0d1df63353a..052f7e52b5b 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py @@ -187,9 +187,7 @@ def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = q_residual = self.q_a_norm(q_a) q = self.q_b_proj(q_residual).reshape(batch, seq, self.num_heads, self.head_dim) # Query-up norm is unweighted; TE RMSNorm would add a learned scale. - q = q * torch.rsqrt(q.float().square().mean(-1, keepdim=True) + self.rms_norm_eps).to( - q.dtype - ) + q = torch.nn.functional.rms_norm(q, (self.head_dim,), eps=self.rms_norm_eps) q = ( apply_rotary(q, *token_rope) .reshape(batch * seq, self.num_heads, self.head_dim) From 52bf9b781d3d83a0a9a8b4884456c96c177d0330 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Wed, 7 Oct 2026 13:51:52 -0700 Subject: [PATCH 09/37] Fix DSv4 indexer bound for partial windows cuDNN requires max_seqlen_q <= max_seqlen_k * ratio. Keep the compressed prefixes at the number of complete blocks, but pass a ceil-sized compile bound so non-divisible sequence lengths work. Signed-off-by: Sudhakar Singh --- .../pytorch/attention/sparse_attention/_dsv4_backend.py | 3 +++ .../pytorch/attention/sparse_attention/_dsv4_indexer.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py index f7df45ab518..a6bff8495d3 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py @@ -182,6 +182,9 @@ def select_blocks( scale and pass raw BF16 projected weights. Scaling in FP32 then rounding weights to BF16 can change close top-k boundaries. This helper does not implement the separate indexer auxiliary loss or autograd through IDs. + max_compressed_seqlen is a compile bound and must cover the partial final + ratio; use ceil(max_seqlen / ratio) even though prefixes count only complete + compressed blocks. """ if query.ndim != 3 or query.shape[1] not in (32, 64) or query.shape[2] != 128: raise ValueError("CSA index query must have shape [T,32 or 64,128].") diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py index fdc8e769d6b..93e94538d87 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py @@ -93,7 +93,9 @@ def forward( top_k=min(self.top_k, n_comp), ratio=self.ratio, max_seqlen=seq, - max_compressed_seqlen=n_comp, + # cuDNN's compile bound covers the partial trailing ratio even + # though cu_comp contains only complete compressed blocks. + max_compressed_seqlen=(seq + self.ratio - 1) // self.ratio, scale=(self.head_dim * self.n_heads) ** -0.5, ) return (indices, query, key, weights) if return_context else indices From 5e8ea3be3a8eef1b4d03b61dacf3db060ed33e37 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Wed, 7 Oct 2026 15:01:05 -0700 Subject: [PATCH 10/37] Refactor DSv4 sparse attention modules Name each module after the implementation it owns, and merge the parameter-free DSv4Attention core with the high-level attention layer. Remove the redundant dsv4 wrapper and legacy alias while preserving tensor contracts, state layout, and execution order. Signed-off-by: Sudhakar Singh --- tests/pytorch/test_dsv4_attention.py | 8 +- .../attention/sparse_attention/README.md | 37 +++++--- .../attention/sparse_attention/__init__.py | 14 ++- .../{_dsv4_compressor.py => compressor.py} | 2 +- ...{_dsv4_backend.py => dsa_cudnn_kernels.py} | 0 .../{_dsv4_rope.py => dsa_rope.py} | 0 .../attention/sparse_attention/dsv4.py | 91 ------------------- ..._hybrid_attention.py => dsv4_attention.py} | 81 +++++++++++++++-- .../{_dsv4_indexer.py => indexer.py} | 6 +- 9 files changed, 112 insertions(+), 127 deletions(-) rename transformer_engine/pytorch/attention/sparse_attention/{_dsv4_compressor.py => compressor.py} (98%) rename transformer_engine/pytorch/attention/sparse_attention/{_dsv4_backend.py => dsa_cudnn_kernels.py} (100%) rename transformer_engine/pytorch/attention/sparse_attention/{_dsv4_rope.py => dsa_rope.py} (100%) delete mode 100644 transformer_engine/pytorch/attention/sparse_attention/dsv4.py rename transformer_engine/pytorch/attention/sparse_attention/{dsv4_hybrid_attention.py => dsv4_attention.py} (77%) rename transformer_engine/pytorch/attention/sparse_attention/{_dsv4_indexer.py => indexer.py} (96%) diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index 9b189b67770..57a4ab76137 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -7,7 +7,7 @@ import pytest import torch -from transformer_engine.pytorch.attention.sparse_attention import dsv4 +from transformer_engine.pytorch.attention.sparse_attention import dsv4_attention @pytest.mark.parametrize("variant", ["hca", "csa"]) @@ -59,7 +59,7 @@ def test_dsv4_forward_backward(variant, head_dim): start += length comp_start += count - output = dsv4.DSv4Attention(window_size=32, ratio=ratio)( + output = dsv4_attention.DSv4Attention(window_size=32, ratio=ratio)( *leaves, cu, cu_comp, @@ -98,7 +98,7 @@ def test_dsv4_forward_backward(variant, head_dim): def test_dsv4_rejects_unsupported_metadata(): with pytest.raises(ValueError, match="positive"): - dsv4.DSv4Attention(window_size=0, ratio=4) - core = dsv4.DSv4Attention(window_size=32, ratio=4) + dsv4_attention.DSv4Attention(window_size=0, ratio=4) + core = dsv4_attention.DSv4Attention(window_size=32, ratio=4) with pytest.raises(ValueError, match="query"): core(torch.empty(1, 4, 512), None, None, None, None, None) diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index 1e38e17b0b6..3c3266d945a 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -37,7 +37,7 @@ indexer auxiliary loss and its attachment remain caller-owned; without one, the language-model loss does not train `indexer.q_proj` or the index-key/weight slices of `indexer.compressor.fused_proj`: top-k selection has no gradient. -`_dsv4_rope.py` caches FP32 token frequencies (prebuilt when `max_seqlen` is +`dsa_rope.py` caches FP32 token frequencies (prebuilt when `max_seqlen` is provided) and shares window-start slices with the CSA `_Indexer`. The indexer owns its projections, index-key compressor, and block selection; neither helper changes the cuDNN call contracts. @@ -68,14 +68,15 @@ window, all eligible compressed keys, and sink). Score helpers are not autograd operations; the caller owns the mean-reduced scalar KL, coefficient, and loss attachment. Per-token reduction is untested. -The lower-level `dsv4` calls expose three stages when a model owns its own -projections, RMSNorm, RoPE, sink, and output projection: +The lower-level `dsa_cudnn_kernels` calls expose compression and selection when +a model owns its own projections, RMSNorm, RoPE, sink, and output projection. +`DSv4Attention` owns the parameter-free attention call: ```text -model projected KV + gates ── dsv4.compress ── model norm/RoPE ── compressed KV -model index projections ───── dsv4.compress ── model norm/RoPE ── index key -model index query + key + weights ───────────── dsv4.select_blocks ── IDs (CSA) -model query + local KV + compressed KV + IDs ─ dsv4.DSv4Attention ── head output +model projected KV + gates ── dsa_cudnn_kernels.compress ── model norm/RoPE ── compressed KV +model index projections ───── dsa_cudnn_kernels.compress ── model norm/RoPE ── index key +model index query + key + weights ───────────── dsa_cudnn_kernels.select_blocks ── IDs (CSA) +model query + local KV + compressed KV + IDs ─ dsv4_attention.DSv4Attention ── head output ``` The model calls the same stages for HCA, omitting index compression and selection. @@ -85,27 +86,33 @@ projections. `select_blocks` returns global IDs into the packed **compressed** rows; the attention core maps them into its combined local/compressed KV space. ```python -from transformer_engine.pytorch.attention.sparse_attention import dsv4 +from transformer_engine.pytorch.attention.sparse_attention import ( + dsa_cudnn_kernels, + dsv4_attention, +) # Model code supplies BF16 projected tensors and applies its own norm/RoPE. -pooled = dsv4.compress(kv, gates, position_bias.float(), cu, cu_comp, - ratio=ratio, overlap=is_csa, total_comp=total_comp) +pooled = dsa_cudnn_kernels.compress( + kv, gates, position_bias.float(), cu, cu_comp, + ratio=ratio, overlap=is_csa, total_comp=total_comp, +) compressed_kv = model.finish_compressed(pooled) indices = None if is_csa: - index_pooled = dsv4.compress(index_kv, index_gates, - index_position_bias.float(), cu, cu_comp, - ratio=ratio, overlap=True, total_comp=total_comp) + index_pooled = dsa_cudnn_kernels.compress( + index_kv, index_gates, index_position_bias.float(), cu, cu_comp, + ratio=ratio, overlap=True, total_comp=total_comp, + ) index_key = model.finish_index_key(index_pooled) - indices = dsv4.select_blocks( + indices = dsa_cudnn_kernels.select_blocks( index_query, index_key, index_weights, cu, cu_comp, top_k=top_k, ratio=ratio, max_seqlen=max_seqlen, max_compressed_seqlen=max_compressed_seqlen, scale=(index_head_dim * index_n_heads) ** -0.5, ) -output = dsv4.DSv4Attention(window_size=window_size, ratio=ratio)( +output = dsv4_attention.DSv4Attention(window_size=window_size, ratio=ratio)( query, local_kv, compressed_kv, sink.float(), cu, cu_comp, indices=indices, max_compressed_seqlen=max_compressed_seqlen if indices is None else None, diff --git a/transformer_engine/pytorch/attention/sparse_attention/__init__.py b/transformer_engine/pytorch/attention/sparse_attention/__init__.py index a410f5e628e..62d3410e37d 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/__init__.py +++ b/transformer_engine/pytorch/attention/sparse_attention/__init__.py @@ -5,11 +5,15 @@ """Experimental sparse attention variants. Each variant owns its tensor/index contract and preparation helpers. Optional -kernels load on execution. Add future variants as siblings of ``dsv4``. +kernels load on execution. Add future variants as sibling modules. """ -from . import dsv4 -from .dsv4 import DSv4Attention -from .dsv4_hybrid_attention import DSv4HybridAttention +from . import dsa_cudnn_kernels, dsv4_attention +from .dsv4_attention import DSv4Attention, DSv4HybridAttention -__all__ = ["dsv4", "DSv4Attention", "DSv4HybridAttention"] +__all__ = [ + "dsa_cudnn_kernels", + "dsv4_attention", + "DSv4Attention", + "DSv4HybridAttention", +] diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_compressor.py b/transformer_engine/pytorch/attention/sparse_attention/compressor.py similarity index 98% rename from transformer_engine/pytorch/attention/sparse_attention/_dsv4_compressor.py rename to transformer_engine/pytorch/attention/sparse_attention/compressor.py index 2349965a733..b95e58b4481 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_compressor.py +++ b/transformer_engine/pytorch/attention/sparse_attention/compressor.py @@ -8,7 +8,7 @@ from transformer_engine.pytorch.module import Linear, RMSNorm -from .dsv4 import compress +from .dsa_cudnn_kernels import compress class _Compressor(torch.nn.Module): diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn_kernels.py similarity index 100% rename from transformer_engine/pytorch/attention/sparse_attention/_dsv4_backend.py rename to transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn_kernels.py diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_rope.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py similarity index 100% rename from transformer_engine/pytorch/attention/sparse_attention/_dsv4_rope.py rename to transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4.py deleted file mode 100644 index c656a0b72b1..00000000000 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# See LICENSE for license information. - -"""DSv4 core and preparation helpers, composed with model-owned projections.""" - -from typing import Optional - -import torch - -from ._dsv4_backend import attention as _attention -from ._dsv4_backend import ( - compress, - dense_indexer_loss_backward, - dense_indexer_loss_scores, - select_blocks, -) - -__all__ = [ - "DSv4Attention", - "compress", - "select_blocks", - "dense_indexer_loss_scores", - "dense_indexer_loss_backward", -] - - -class DSv4Attention(torch.nn.Module): - """Experimental causal joint local + compressed attention for DSv4. - - Parameter-free core: the caller owns projections, compression, normalization, - RoPE, and the learned sink. Use :func:`compress` for gated pooling and - :func:`select_blocks` for the CSA indexer. Pass selected indices for CSA; - omit them and supply ``max_compressed_seqlen`` for HCA. - - Inputs are contiguous CUDA BF16 query [T,64,D], local KV [T,D], compressed - KV [Tc,D], FP32 sink [64], and CUDA INT32 sequence prefixes [B+1]. D is 512 - or 576; values use the first 512 KV channels. Output is [T,64,512], retaining - the head axis for DSv4's output unrotation. Prefixes start at zero and end - at valid row counts; compressed lengths are floor(sequence length / ratio). - CSA indices are distinct global packed compressed-row IDs, or -1 for padding. - - Only full sequences with positions starting at zero are supported. Local - keys satisfy max(0,q-window_size+1) <= k <= q. Compressed block j becomes - visible when (j+1)*ratio <= q+1. All visible local and compressed entries - share one softmax with the sink; its value contribution is zero. - - Requires SM100 and cuDNN Frontend 1.29.0. Forward and first-order backward - use cuDNN's DSA namespace. No dropout, arbitrary masks, cache/decode, FP8, - distributed attention, or higher-order gradients. This module does not use - DPA backend-selection flags. cuDNN operations are loaded on execution. - """ - - def __init__(self, *, window_size: int, ratio: int): - super().__init__() - if window_size <= 0 or ratio <= 0: - raise ValueError("window_size and ratio must be positive.") - self.window_size = window_size - self.ratio = ratio - - def forward( - self, - query: torch.Tensor, - local_kv: torch.Tensor, - compressed_kv: torch.Tensor, - sink: torch.Tensor, - cu_seqlens: torch.Tensor, - cu_seqlens_comp: torch.Tensor, - *, - indices: Optional[torch.Tensor] = None, - scale: Optional[float] = None, - max_compressed_seqlen: Optional[int] = None, - ) -> torch.Tensor: - """Evaluate the joint attention, with scale defaulting to D**-0.5.""" - from transformer_engine.pytorch.quantization import FP8GlobalStateManager - - if FP8GlobalStateManager.is_fp8_enabled(): - raise NotImplementedError("DSv4Attention supports BF16 only; disable TE FP8 autocast.") - return _attention( - query, - local_kv, - compressed_kv, - sink, - cu_seqlens, - cu_seqlens_comp, - window_size=self.window_size, - ratio=self.ratio, - indices=indices, - scale=scale, - max_compressed_seqlen=max_compressed_seqlen, - ) diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py similarity index 77% rename from transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py rename to transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index 052f7e52b5b..cacd75c28aa 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_hybrid_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -2,7 +2,7 @@ # # See LICENSE for license information. -"""Unpadded DSv4 CSA/HCA attention.""" +"""DSv4 attention core and unpadded CSA/HCA attention layer.""" from typing import Optional @@ -10,10 +10,78 @@ from transformer_engine.pytorch.module import Linear, RMSNorm -from ._dsv4_compressor import _Compressor -from ._dsv4_indexer import _Indexer -from ._dsv4_rope import _DSv4RotaryEmbedding, apply_rotary -from .dsv4 import DSv4Attention +from .compressor import _Compressor +from .dsa_cudnn_kernels import attention as _attention +from .dsa_rope import _DSv4RotaryEmbedding, apply_rotary +from .indexer import _Indexer + +__all__ = ["DSv4Attention", "DSv4HybridAttention"] + + +class DSv4Attention(torch.nn.Module): + """Experimental causal joint local + compressed attention for DSv4. + + Parameter-free core: the caller owns projections, compression, normalization, + RoPE, and the learned sink. Use :mod:`dsa_cudnn_kernels` for gated pooling + and CSA selection. Pass selected indices for CSA; omit them and supply + ``max_compressed_seqlen`` for HCA. + + Inputs are contiguous CUDA BF16 query [T,64,D], local KV [T,D], compressed + KV [Tc,D], FP32 sink [64], and CUDA INT32 sequence prefixes [B+1]. D is 512 + or 576; values use the first 512 KV channels. Output is [T,64,512], retaining + the head axis for DSv4's output unrotation. Prefixes start at zero and end + at valid row counts; compressed lengths are floor(sequence length / ratio). + CSA indices are distinct global packed compressed-row IDs, or -1 for padding. + + Only full sequences with positions starting at zero are supported. Local + keys satisfy max(0,q-window_size+1) <= k <= q. Compressed block j becomes + visible when (j+1)*ratio <= q+1. All visible local and compressed entries + share one softmax with the sink; its value contribution is zero. + + Requires SM100 and cuDNN Frontend 1.29.0. Forward and first-order backward + use cuDNN's DSA namespace. No dropout, arbitrary masks, cache/decode, FP8, + distributed attention, or higher-order gradients. This module does not use + DPA backend-selection flags. cuDNN operations are loaded on execution. + """ + + def __init__(self, *, window_size: int, ratio: int): + super().__init__() + if window_size <= 0 or ratio <= 0: + raise ValueError("window_size and ratio must be positive.") + self.window_size = window_size + self.ratio = ratio + + def forward( + self, + query: torch.Tensor, + local_kv: torch.Tensor, + compressed_kv: torch.Tensor, + sink: torch.Tensor, + cu_seqlens: torch.Tensor, + cu_seqlens_comp: torch.Tensor, + *, + indices: Optional[torch.Tensor] = None, + scale: Optional[float] = None, + max_compressed_seqlen: Optional[int] = None, + ) -> torch.Tensor: + """Evaluate the joint attention, with scale defaulting to D**-0.5.""" + from transformer_engine.pytorch.quantization import FP8GlobalStateManager + + if FP8GlobalStateManager.is_fp8_enabled(): + raise NotImplementedError("DSv4Attention supports BF16 only; disable TE FP8 autocast.") + return _attention( + query, + local_kv, + compressed_kv, + sink, + cu_seqlens, + cu_seqlens_comp, + window_size=self.window_size, + ratio=self.ratio, + indices=indices, + scale=scale, + max_compressed_seqlen=max_compressed_seqlen, + ) class DSv4HybridAttention(torch.nn.Module): @@ -249,6 +317,3 @@ def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = "sink": self.sinks.float().detach(), } return result - - -__all__ = ["DSv4HybridAttention"] diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py b/transformer_engine/pytorch/attention/sparse_attention/indexer.py similarity index 96% rename from transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py rename to transformer_engine/pytorch/attention/sparse_attention/indexer.py index 93e94538d87..894ca67bf6a 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/_dsv4_indexer.py +++ b/transformer_engine/pytorch/attention/sparse_attention/indexer.py @@ -8,9 +8,9 @@ from transformer_engine.pytorch.module import Linear -from ._dsv4_compressor import _Compressor -from ._dsv4_rope import apply_rotary -from .dsv4 import select_blocks +from .compressor import _Compressor +from .dsa_cudnn_kernels import select_blocks +from .dsa_rope import apply_rotary class _Indexer(torch.nn.Module): From 6470ae17ef965f89de0695cdaa50705ce67df7cb Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Wed, 7 Oct 2026 15:22:50 -0700 Subject: [PATCH 11/37] Name the cuDNN DSA integration directly The module adapts PyTorch and autograd calls to cuDNN DSA APIs rather than implementing kernels, so dsa_cudnn is the shorter and more accurate boundary name. Signed-off-by: Sudhakar Singh --- .../pytorch/attention/sparse_attention/README.md | 16 ++++++++-------- .../attention/sparse_attention/__init__.py | 4 ++-- .../attention/sparse_attention/compressor.py | 2 +- .../{dsa_cudnn_kernels.py => dsa_cudnn.py} | 0 .../attention/sparse_attention/dsv4_attention.py | 4 ++-- .../attention/sparse_attention/indexer.py | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) rename transformer_engine/pytorch/attention/sparse_attention/{dsa_cudnn_kernels.py => dsa_cudnn.py} (100%) diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index 3c3266d945a..1173ba5f9f9 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -68,14 +68,14 @@ window, all eligible compressed keys, and sink). Score helpers are not autograd operations; the caller owns the mean-reduced scalar KL, coefficient, and loss attachment. Per-token reduction is untested. -The lower-level `dsa_cudnn_kernels` calls expose compression and selection when +The lower-level `dsa_cudnn` calls expose compression and selection when a model owns its own projections, RMSNorm, RoPE, sink, and output projection. `DSv4Attention` owns the parameter-free attention call: ```text -model projected KV + gates ── dsa_cudnn_kernels.compress ── model norm/RoPE ── compressed KV -model index projections ───── dsa_cudnn_kernels.compress ── model norm/RoPE ── index key -model index query + key + weights ───────────── dsa_cudnn_kernels.select_blocks ── IDs (CSA) +model projected KV + gates ── dsa_cudnn.compress ── model norm/RoPE ── compressed KV +model index projections ───── dsa_cudnn.compress ── model norm/RoPE ── index key +model index query + key + weights ───────────── dsa_cudnn.select_blocks ── IDs (CSA) model query + local KV + compressed KV + IDs ─ dsv4_attention.DSv4Attention ── head output ``` @@ -87,12 +87,12 @@ rows; the attention core maps them into its combined local/compressed KV space. ```python from transformer_engine.pytorch.attention.sparse_attention import ( - dsa_cudnn_kernels, + dsa_cudnn, dsv4_attention, ) # Model code supplies BF16 projected tensors and applies its own norm/RoPE. -pooled = dsa_cudnn_kernels.compress( +pooled = dsa_cudnn.compress( kv, gates, position_bias.float(), cu, cu_comp, ratio=ratio, overlap=is_csa, total_comp=total_comp, ) @@ -100,12 +100,12 @@ compressed_kv = model.finish_compressed(pooled) indices = None if is_csa: - index_pooled = dsa_cudnn_kernels.compress( + index_pooled = dsa_cudnn.compress( index_kv, index_gates, index_position_bias.float(), cu, cu_comp, ratio=ratio, overlap=True, total_comp=total_comp, ) index_key = model.finish_index_key(index_pooled) - indices = dsa_cudnn_kernels.select_blocks( + indices = dsa_cudnn.select_blocks( index_query, index_key, index_weights, cu, cu_comp, top_k=top_k, ratio=ratio, max_seqlen=max_seqlen, max_compressed_seqlen=max_compressed_seqlen, diff --git a/transformer_engine/pytorch/attention/sparse_attention/__init__.py b/transformer_engine/pytorch/attention/sparse_attention/__init__.py index 62d3410e37d..5cd7bb8cc53 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/__init__.py +++ b/transformer_engine/pytorch/attention/sparse_attention/__init__.py @@ -8,11 +8,11 @@ kernels load on execution. Add future variants as sibling modules. """ -from . import dsa_cudnn_kernels, dsv4_attention +from . import dsa_cudnn, dsv4_attention from .dsv4_attention import DSv4Attention, DSv4HybridAttention __all__ = [ - "dsa_cudnn_kernels", + "dsa_cudnn", "dsv4_attention", "DSv4Attention", "DSv4HybridAttention", diff --git a/transformer_engine/pytorch/attention/sparse_attention/compressor.py b/transformer_engine/pytorch/attention/sparse_attention/compressor.py index b95e58b4481..d5b1e0c99c2 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/compressor.py +++ b/transformer_engine/pytorch/attention/sparse_attention/compressor.py @@ -8,7 +8,7 @@ from transformer_engine.pytorch.module import Linear, RMSNorm -from .dsa_cudnn_kernels import compress +from .dsa_cudnn import compress class _Compressor(torch.nn.Module): diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn_kernels.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py similarity index 100% rename from transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn_kernels.py rename to transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index cacd75c28aa..53ec8ede0c4 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -11,7 +11,7 @@ from transformer_engine.pytorch.module import Linear, RMSNorm from .compressor import _Compressor -from .dsa_cudnn_kernels import attention as _attention +from .dsa_cudnn import attention as _attention from .dsa_rope import _DSv4RotaryEmbedding, apply_rotary from .indexer import _Indexer @@ -22,7 +22,7 @@ class DSv4Attention(torch.nn.Module): """Experimental causal joint local + compressed attention for DSv4. Parameter-free core: the caller owns projections, compression, normalization, - RoPE, and the learned sink. Use :mod:`dsa_cudnn_kernels` for gated pooling + RoPE, and the learned sink. Use :mod:`dsa_cudnn` for gated pooling and CSA selection. Pass selected indices for CSA; omit them and supply ``max_compressed_seqlen`` for HCA. diff --git a/transformer_engine/pytorch/attention/sparse_attention/indexer.py b/transformer_engine/pytorch/attention/sparse_attention/indexer.py index 894ca67bf6a..e8d762d8b90 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/indexer.py +++ b/transformer_engine/pytorch/attention/sparse_attention/indexer.py @@ -9,7 +9,7 @@ from transformer_engine.pytorch.module import Linear from .compressor import _Compressor -from .dsa_cudnn_kernels import select_blocks +from .dsa_cudnn import select_blocks from .dsa_rope import apply_rotary From 96f9f9cfa959a68d7d0fae8e209ac386d2ed0ae3 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Thu, 8 Oct 2026 10:48:33 -0700 Subject: [PATCH 12/37] Accelerate DSv4 RoPE through native CuTe dispatch Python-side CuTe launches dominate the otherwise fast rotary kernels. Register shape-specialized CuTe callables through the common TVM-FFI cache so steady-state SM100 execution dispatches from C++ while unsupported inputs keep the eager fallback. Compile the weightless query normalization and let its fresh output own the in-place query rotation. This removes one full-width query copy without changing the public attention or state-dict contract; other RoPE sites retain out-of-place semantics required by their autograd boundaries. Signed-off-by: Sudhakar Singh --- tests/pytorch/test_dsv4_attention.py | 56 +- .../common/fused_rope/fused_rope.cu | 97 ++++ .../include/transformer_engine/fused_rope.h | 9 + .../sparse_attention/_dsa_rope_cute.py | 510 ++++++++++++++++++ .../attention/sparse_attention/dsa_rope.py | 119 +++- .../sparse_attention/dsv4_attention.py | 17 +- .../attention/sparse_attention/indexer.py | 6 +- transformer_engine/pytorch/csrc/extensions.h | 3 + .../pytorch/csrc/extensions/apply_rope.cpp | 28 + .../pytorch/csrc/extensions/pybind.cpp | 2 + 10 files changed, 838 insertions(+), 9 deletions(-) create mode 100644 transformer_engine/pytorch/attention/sparse_attention/_dsa_rope_cute.py diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index 57a4ab76137..5014aeda356 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -4,10 +4,12 @@ """DSv4 core against a dense oracle, including packed sequence boundaries.""" +import os + import pytest import torch -from transformer_engine.pytorch.attention.sparse_attention import dsv4_attention +from transformer_engine.pytorch.attention.sparse_attention import dsa_rope, dsv4_attention @pytest.mark.parametrize("variant", ["hca", "csa"]) @@ -102,3 +104,55 @@ def test_dsv4_rejects_unsupported_metadata(): core = dsv4_attention.DSv4Attention(window_size=32, ratio=4) with pytest.raises(ValueError, match="query"): core(torch.empty(1, 4, 512), None, None, None, None, None) + + +def test_dsv4_cute_rope_batched_forward_backward(monkeypatch): + """The packed CuTe bridge must reset positions at each BSHD sequence.""" + if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 0): + pytest.skip("CuTe DSv4 RoPE requires SM100") + if dsa_rope._cute_rope_module() is None: + pytest.skip("CuTe DSL is unavailable") + if os.environ.get("NVTE_ENABLE_CUTEDSL_BACKEND", "0") == "0": + pytest.skip("native CuTe DSL dispatch is disabled") + + torch.manual_seed(23) + batch, seq, heads, dim, width = 2, 17, 64, 512, 64 + x = torch.randn(batch, seq, heads, dim, device="cuda", dtype=torch.bfloat16) + (cos, sin), _ = dsa_rope.rotary_embeddings(seq, 4, width, 160000.0, x.device) + cu = torch.arange(batch + 1, device=x.device, dtype=torch.int32) * seq + gradient = torch.randn_like(x) + + module = dsa_rope._cute_rope_module() + monkeypatch.setattr( + module, + "forward_inplace", + lambda *_: pytest.fail("native forward fell back to its Python launcher"), + ) + monkeypatch.setattr( + module, + "backward_inplace", + lambda *_: pytest.fail("native backward fell back to its Python launcher"), + ) + candidate_input = x.detach().requires_grad_() + candidate = dsa_rope.apply_rotary(candidate_input, cos, sin, cu) + candidate_grad = torch.autograd.grad(candidate, candidate_input, gradient)[0] + + reference_input = x.detach().requires_grad_() + reference = dsa_rope._apply_rotary_eager(reference_input, cos, sin) + reference_grad = torch.autograd.grad(reference, reference_input, gradient)[0] + + assert candidate.data_ptr() != candidate_input.data_ptr() + torch.testing.assert_close(candidate, reference, atol=2e-2, rtol=2e-2) + grad_delta = candidate_grad.float() - reference_grad.float() + assert grad_delta.abs().max() <= 0.03125 + assert grad_delta.norm() / reference_grad.float().norm() < 0.002 + + query_source = x.detach().requires_grad_() + query_input = query_source.clone() + query_ptr = query_input.data_ptr() + query = dsa_rope._apply_rotary_query(query_input, cos, sin, cu) + query_grad = torch.autograd.grad(query, query_source, gradient)[0] + + assert query.data_ptr() == query_ptr + torch.testing.assert_close(query, reference, atol=2e-2, rtol=2e-2) + torch.testing.assert_close(query_grad, reference_grad, atol=0.03125, rtol=0.002) diff --git a/transformer_engine/common/fused_rope/fused_rope.cu b/transformer_engine/common/fused_rope/fused_rope.cu index c7b762e9731..961f6aa29a2 100644 --- a/transformer_engine/common/fused_rope/fused_rope.cu +++ b/transformer_engine/common/fused_rope/fused_rope.cu @@ -9,11 +9,94 @@ #include #include "../common.h" +#ifdef NVTE_WITH_CUTEDSL +#include "../tvm_ffi_bridge.h" +#endif #include "../util/logging.h" #include "../utils.cuh" namespace transformer_engine { +#ifdef NVTE_WITH_CUTEDSL +namespace { + +struct DSv4RopeConfig { + static constexpr const char *kEntrypointName = "get_dsv4_rope_function"; + + uint32_t heads; + uint32_t head_dim; + uint32_t rope_dim; + uint32_t batch_size; + bool backward; + + uint32_t to_id() const { + NVTE_CHECK(heads < 256 && head_dim % 4 == 0 && head_dim / 4 < 256 && rope_dim % 4 == 0 && + rope_dim / 4 < 64 && batch_size < 256, + "DSv4 RoPE configuration is too large for its CuTe DSL cache key."); + return static_cast(backward) | (heads << 1) | ((head_dim / 4) << 9) | + ((rope_dim / 4) << 17) | (batch_size << 23); + } + + std::optional get_kernel() const { + static tvm_ffi_bridge::TVMFFIConfigCache &cache = + tvm_ffi_bridge::TVMFFIConfigCache::create(); + return cache.get_or_load(*this); + } + + std::string to_key() const { + return std::string("cutedsl_dsv4_rope_sm100_h") + std::to_string(heads) + "_d" + + std::to_string(head_dim) + "_r" + std::to_string(rope_dim) + "_b" + + std::to_string(batch_size) + (backward ? "_bwd" : "_fwd"); + } + + bool retrieve_func_from_python(const std::string &fn_name) const { + auto entrypoint = tvm::ffi::Function::GetGlobal(kEntrypointName); + if (!entrypoint.has_value()) return false; + tvm::ffi::Any result = + (*entrypoint)(tvm::ffi::String(fn_name), heads, head_dim, rope_dim, batch_size, backward); + return result.try_cast().value_or(false); + } +}; + +bool dsv4_rope_cutedsl(const Tensor &input, const Tensor &cos, const Tensor &sin, + const Tensor &cu_seqlens, const bool backward, cudaStream_t stream) { + if (!tvm_ffi_bridge::TVMFFICentral::getInstance().get_cutedsl_backend_enabled() || + cuda::sm_arch() != 100 || input.data.dtype != DType::kBFloat16 || + cos.data.dtype != DType::kFloat32 || sin.data.dtype != DType::kFloat32 || + cu_seqlens.data.dtype != DType::kInt32 || input.data.shape.size() != 3 || + cos.data.shape.size() != 2 || sin.data.shape != cos.data.shape || + cu_seqlens.data.shape.size() != 1 || input.data.shape[0] == 0 || + cu_seqlens.data.shape[0] < 2) { + return false; + } + + const DSv4RopeConfig config{/*heads=*/static_cast(input.data.shape[1]), + /*head_dim=*/static_cast(input.data.shape[2]), + /*rope_dim=*/static_cast(cos.data.shape[1]), + /*batch_size=*/static_cast(cu_seqlens.data.shape[0] - 1), + backward}; + if (config.rope_dim == 0 || config.rope_dim > config.head_dim || config.head_dim % 4 != 0 || + config.rope_dim % 4 != 0 || config.heads >= 256 || config.head_dim / 4 >= 256 || + config.rope_dim / 4 >= 64 || config.batch_size == 0 || config.batch_size >= 256) { + return false; + } + + checkCuDriverContext(stream); + auto kernel = config.get_kernel(); + if (!kernel.has_value()) return false; + + const int32_t device = cuda::current_device(); + tvm_ffi_bridge::DLTensorWrapper q(input.data, false, device); + tvm_ffi_bridge::DLTensorWrapper cos_table(cos.data, false, device); + tvm_ffi_bridge::DLTensorWrapper sin_table(sin.data, false, device); + tvm_ffi_bridge::DLTensorWrapper prefixes(cu_seqlens.data, false, device); + (*kernel)(&q, &cos_table, &sin_table, &prefixes, static_cast(stream)); + return true; +} + +} // namespace +#endif + // Returns the largest sequence index `b` such that // `cu_seqlens[b] / cp_size <= t_id`. Used by the linear-grid THD kernels to // locate the sequence span that owns local packed token `t_id`. Uses the same @@ -850,6 +933,20 @@ void nvte_fused_rope_backward(const NVTETensor output_grads, const NVTETensor cu cp_rank, s, b, h, d, d2, stride_s_or_t, stride_b, stride_h, stride_d, stream); } +bool nvte_dsv4_rope_cutedsl(NVTETensor input, const NVTETensor cos, const NVTETensor sin, + const NVTETensor cu_seqlens, const bool backward, + cudaStream_t stream) { + NVTE_API_CALL(nvte_dsv4_rope_cutedsl); +#ifdef NVTE_WITH_CUTEDSL + using namespace transformer_engine; + return dsv4_rope_cutedsl(*convertNVTETensorCheck(input), *convertNVTETensorCheck(cos), + *convertNVTETensorCheck(sin), *convertNVTETensorCheck(cu_seqlens), + backward, stream); +#else + return false; +#endif +} + void nvte_fused_qkv_rope_forward(const NVTETensor qkv_input, const NVTETensor q_freqs, const NVTETensor k_freqs, const NVTETensor start_positions, NVTETensor q_out, NVTETensor k_out, NVTETensor v_out, diff --git a/transformer_engine/common/include/transformer_engine/fused_rope.h b/transformer_engine/common/include/transformer_engine/fused_rope.h index aea5256a2cb..ce99baf43f9 100644 --- a/transformer_engine/common/include/transformer_engine/fused_rope.h +++ b/transformer_engine/common/include/transformer_engine/fused_rope.h @@ -76,6 +76,15 @@ void nvte_fused_rope_backward(const NVTETensor output_grads, const NVTETensor cu const int stride_s_or_t, const int stride_b, const int stride_h, const int stride_d, cudaStream_t stream); +/*! \brief Apply DSv4's trailing-channel RoPE in place with the optional CuTe DSL backend. + * + * Returns false when the backend or input configuration is unsupported so the + * caller can use its existing implementation. + */ +bool nvte_dsv4_rope_cutedsl(NVTETensor input, const NVTETensor cos, const NVTETensor sin, + const NVTETensor cu_seqlens, const bool backward, + cudaStream_t stream); + /*! \brief Apply rotary positional embedding to the combined QKV input tensor. * * \param[in] qkv_input Combined QKV input tensor for fused rope. diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsa_rope_cute.py b/transformer_engine/pytorch/attention/sparse_attention/_dsa_rope_cute.py new file mode 100644 index 00000000000..65df901ec9a --- /dev/null +++ b/transformer_engine/pytorch/attention/sparse_attention/_dsa_rope_cute.py @@ -0,0 +1,510 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""CuTe DSL kernels for DSv4 trailing-channel interleaved RoPE. + +The device bodies come from the retained DSv4 experiments: +forward SHA256 0355016c141365dfa436deda15a29cce49146a0815833ab7963d941ef67f7e5c +and backward SHA256 fdda9fb8bcf44e998d000bb8f78354fcf02f2c0b7cb4d9d3c9787a24abaaef82. +Only their host boundary is narrowed to TE's packed single-tensor contract. +""" + +import torch +import cutlass +import cutlass.cute as cute +import tvm_ffi +from cutlass.cute.runtime import from_dlpack +from cuda.bindings import driver as cuda + + +@cute.kernel +def _mla_rope_fwd_inplace_kernel( + q: cute.Tensor, + cos: cute.Tensor, + sin: cute.Tensor, + cu_seqlens, + positions, + nope_dim: cutlass.Constexpr, + emb_dim: cutlass.Constexpr, + batch_size: cutlass.Constexpr, + cp_rank: cutlass.Constexpr, + cp_size: cutlass.Constexpr, + inverse: cutlass.Constexpr, + remove_interleaving: cutlass.Constexpr, + packed_pairs: cutlass.Constexpr, + threads: cutlass.Constexpr, + heads_per_cta: cutlass.Constexpr, +): + tid, _, _ = cute.arch.thread_idx() + row, head_tile, _ = cute.arch.block_idx() + pos = cutlass.Int64(row // batch_size) + if cutlass.const_expr(positions is not None): + pos = positions[row].to(cutlass.Int64) + elif cutlass.const_expr(cu_seqlens is not None): + pos = cutlass.Int64(-1) + length = cutlass.Int64(0) + # Upper bound skips duplicate/empty ends and preserves CP floor division. + lo = cutlass.Int64(1) + hi = cutlass.Int64(cute.size(cu_seqlens)) + while lo < hi: + mid = (lo + hi) // 2 + end = cu_seqlens[mid].to(cutlass.Int64) // cp_size + if end > row: + hi = mid + else: + lo = mid + 1 + # The exclusive-bound sentinel denotes padding at/beyond the last end. + if lo < cute.size(cu_seqlens): + previous = cu_seqlens[lo - 1].to(cutlass.Int64) // cp_size + end = cu_seqlens[lo].to(cutlass.Int64) // cp_size + pos = row - previous + length = end - previous + if pos == -1: + pos = cutlass.Int64(0) + if cutlass.const_expr(cp_size > 1): + if pos < length // 2: + pos = pos + cp_rank * length // 2 + else: + pos = pos - length // 2 + (2 * cp_size - cp_rank - 1) * length // 2 + + if cutlass.const_expr(packed_pairs): + q_pairs = cute.recast_tensor(q, cutlass.Uint32) + half = emb_dim // 2 + count = cute.ceil_div(heads_per_cta * half, threads) + left = cute.make_rmem_tensor((count,), cutlass.Float32) + right = cute.make_rmem_tensor((count,), cutlass.Float32) + # Adding `threads` leaves `flat % half` unchanged only under this guard. + # Hoist the four same-address table loads once for each participating thread. + if cutlass.const_expr(threads % half == 0 and heads_per_cta * half >= threads): + table_pair = tid % half + cl_once = cos[pos, table_pair] + cr_once = cos[pos, half + table_pair] + sl_once = sin[pos, table_pair] + sr_once = sin[pos, half + table_pair] + for slot in cutlass.range_constexpr(count): + flat = tid + slot * threads + head = head_tile * heads_per_cta + flat // half + pair = flat % half + if flat < heads_per_cta * half and head < q.shape[1]: + if cutlass.const_expr(packed_pairs): + # Bit reinterpretation preserves the original BF16 operands. + values = cute.make_rmem_tensor((2,), q.element_type) + word = cute.recast_tensor(values, cutlass.Uint32) + word[0] = q_pairs[row, head, nope_dim // 2 + pair] + x, y = values[0], values[1] + else: + x = q[row, head, nope_dim + 2 * pair] + y = q[row, head, nope_dim + 2 * pair + 1] + if cutlass.const_expr(threads % half == 0 and heads_per_cta * half >= threads): + cl, cr, sl, sr = cl_once, cr_once, sl_once, sr_once + else: + cl = cos[pos, pair] + cr = cos[pos, half + pair] + sl = sin[pos, pair] + sr = sin[pos, half + pair] + if cutlass.const_expr(inverse): + sl, sr = -sl, -sr + # Floating promotion follows Triton: FP32 > FP16 > BF16. + left[slot] = (x * cl - y * sl).to(cutlass.Float32) + right[slot] = (y * cr + x * sr).to(cutlass.Float32) + + # Split-half stores alias other threads' interleaved inputs. Every warp + # must finish loading before any thread writes to those addresses. + if cutlass.const_expr(not remove_interleaving): + cute.arch.sync_threads() + for slot in cutlass.range_constexpr(count): + flat = tid + slot * threads + head = head_tile * heads_per_cta + flat // half + pair = flat % half + if flat < heads_per_cta * half and head < q.shape[1]: + if cutlass.const_expr(packed_pairs and remove_interleaving): + values = cute.make_rmem_tensor((2,), q.element_type) + values[0] = left[slot].to(q.element_type) + values[1] = right[slot].to(q.element_type) + word = cute.recast_tensor(values, cutlass.Uint32) + q_pairs[row, head, nope_dim // 2 + pair] = word[0] + else: + if cutlass.const_expr(remove_interleaving): + dst_l, dst_r = 2 * pair, 2 * pair + 1 + else: + dst_l, dst_r = pair, half + pair + q[row, head, nope_dim + dst_l] = left[slot].to(q.element_type) + q[row, head, nope_dim + dst_r] = right[slot].to(q.element_type) + + +@cute.jit +def _launch_forward( + q: cute.Tensor, + cos: cute.Tensor, + sin: cute.Tensor, + cu_seqlens, + positions, + nope_dim: cutlass.Constexpr, + emb_dim: cutlass.Constexpr, + batch_size: cutlass.Constexpr, + cp_rank: cutlass.Constexpr, + cp_size: cutlass.Constexpr, + inverse: cutlass.Constexpr, + remove_interleaving: cutlass.Constexpr, + packed_pairs: cutlass.Constexpr, + stream: cuda.CUstream, +): + threads = 128 + heads_per_cta = min(16, q.shape[1]) + _mla_rope_fwd_inplace_kernel( + q, + cos, + sin, + cu_seqlens, + positions, + nope_dim, + emb_dim, + batch_size, + cp_rank, + cp_size, + inverse, + remove_interleaving, + packed_pairs, + threads, + heads_per_cta, + ).launch( + grid=(q.shape[0], cute.ceil_div(q.shape[1], heads_per_cta), 1), + block=(threads, 1, 1), + stream=stream, + ) + + +@cute.kernel +def _mla_rope_bwd_inplace_kernel( + q: cute.Tensor, + cos: cute.Tensor, + sin: cute.Tensor, + cu_seqlens, + positions, + nope_dim: cutlass.Constexpr, + emb_dim: cutlass.Constexpr, + batch_size: cutlass.Constexpr, + cp_rank: cutlass.Constexpr, + cp_size: cutlass.Constexpr, + inverse: cutlass.Constexpr, + remove_interleaving: cutlass.Constexpr, + packed_pairs: cutlass.Constexpr, + threads: cutlass.Constexpr, + heads_per_cta: cutlass.Constexpr, +): + tid, _, _ = cute.arch.thread_idx() + row, head_tile, _ = cute.arch.block_idx() + pos = cutlass.Int64(row // batch_size) + if cutlass.const_expr(positions is not None): + pos = positions[row].to(cutlass.Int64) + elif cutlass.const_expr(cu_seqlens is not None): + pos = cutlass.Int64(-1) + length = cutlass.Int64(0) + previous = cu_seqlens[0].to(cutlass.Int64) // cp_size + for seq in range(cute.size(cu_seqlens) - 1): + end = cu_seqlens[seq + 1].to(cutlass.Int64) // cp_size + if pos == -1 and end > row: + pos = row - previous + length = end - previous + previous = end + if pos == -1: + pos = cutlass.Int64(0) + if cutlass.const_expr(cp_size > 1): + if pos < length // 2: + pos = pos + cp_rank * length // 2 + else: + pos = pos - length // 2 + (2 * cp_size - cp_rank - 1) * length // 2 + + if cutlass.const_expr(packed_pairs): + q_pairs = cute.recast_tensor(q, cutlass.Uint32) + half = emb_dim // 2 + count = cute.ceil_div(heads_per_cta * half, threads) + left = cute.make_rmem_tensor((count,), cutlass.Float32) + right = cute.make_rmem_tensor((count,), cutlass.Float32) + # Every slot has the same pair only when adding threads preserves remainder. + # Preserve table types and the original per-slot inverse/arithmetic below. + if cutlass.const_expr(threads % half == 0 and heads_per_cta * half >= threads): + table_pair = tid % half + cl_once = cos[pos, table_pair] + cr_once = cos[pos, half + table_pair] + sl_once = sin[pos, table_pair] + sr_once = sin[pos, half + table_pair] + for slot in cutlass.range_constexpr(count): + flat = tid + slot * threads + head = head_tile * heads_per_cta + flat // half + pair = flat % half + if flat < heads_per_cta * half and head < q.shape[1]: + if cutlass.const_expr(remove_interleaving): + src_l, src_r = 2 * pair, 2 * pair + 1 + else: + src_l, src_r = pair, half + pair + if cutlass.const_expr(packed_pairs and remove_interleaving): + # Adjacent inputs form one word; split inputs keep scalar loads. + values = cute.make_rmem_tensor((2,), q.element_type) + word = cute.recast_tensor(values, cutlass.Uint32) + word[0] = q_pairs[row, head, nope_dim // 2 + pair] + x, y = values[0], values[1] + else: + x = q[row, head, nope_dim + src_l] + y = q[row, head, nope_dim + src_r] + if cutlass.const_expr(threads % half == 0 and heads_per_cta * half >= threads): + cl, cr, sl, sr = cl_once, cr_once, sl_once, sr_once + else: + cl = cos[pos, pair] + cr = cos[pos, half + pair] + sl = sin[pos, pair] + sr = sin[pos, half + pair] + if cutlass.const_expr(inverse): + sl, sr = -sl, -sr + # CuTe and Triton both promote these floating operands to FP32 + # when a table is FP32, and otherwise preserve the narrower type. + left[slot] = (x * cl + y * sr).to(cutlass.Float32) + right[slot] = (-x * sl + y * cr).to(cutlass.Float32) + + # Split-input stores alias only within a head. When half divides 32, + # every head belongs to one warp; wider/non-divisor widths need the CTA. + if cutlass.const_expr(not remove_interleaving): + if cutlass.const_expr(half <= 32 and 32 % half == 0): + cute.arch.sync_warp() + else: + cute.arch.sync_threads() + for slot in cutlass.range_constexpr(count): + flat = tid + slot * threads + head = head_tile * heads_per_cta + flat // half + pair = flat % half + if flat < heads_per_cta * half and head < q.shape[1]: + if cutlass.const_expr(packed_pairs): + # Backward always writes adjacent pairs, after original casts. + values = cute.make_rmem_tensor((2,), q.element_type) + values[0] = left[slot].to(q.element_type) + values[1] = right[slot].to(q.element_type) + word = cute.recast_tensor(values, cutlass.Uint32) + q_pairs[row, head, nope_dim // 2 + pair] = word[0] + else: + q[row, head, nope_dim + 2 * pair] = left[slot].to(q.element_type) + q[row, head, nope_dim + 2 * pair + 1] = right[slot].to(q.element_type) + + +@cute.jit +def _launch_backward( + q: cute.Tensor, + cos: cute.Tensor, + sin: cute.Tensor, + cu_seqlens, + positions, + nope_dim: cutlass.Constexpr, + emb_dim: cutlass.Constexpr, + batch_size: cutlass.Constexpr, + cp_rank: cutlass.Constexpr, + cp_size: cutlass.Constexpr, + inverse: cutlass.Constexpr, + remove_interleaving: cutlass.Constexpr, + packed_pairs: cutlass.Constexpr, + stream: cuda.CUstream, +): + threads = 128 + heads_per_cta = min(16, q.shape[1]) + _mla_rope_bwd_inplace_kernel( + q, + cos, + sin, + cu_seqlens, + positions, + nope_dim, + emb_dim, + batch_size, + cp_rank, + cp_size, + inverse, + remove_interleaving, + packed_pairs, + threads, + heads_per_cta, + ).launch( + grid=(q.shape[0], cute.ceil_div(q.shape[1], heads_per_cta), 1), + block=(threads, 1, 1), + stream=stream, + ) + + +class _DSv4RopeEntry: + """TVM-FFI entrypoint with the kernel's static launch choices captured.""" + + def __init__(self, heads, head_dim, rope_dim, backward): + self.heads = heads + self.head_dim = head_dim + self.rope_dim = rope_dim + self.backward = backward + + @cute.jit + def __call__(self, q, cos, sin, cu_seqlens, stream): + if cutlass.const_expr(self.backward): + _launch_backward( + q, + cos, + sin, + cu_seqlens, + None, + self.head_dim - self.rope_dim, + self.rope_dim, + 1, + 0, + 1, + False, + True, + True, + stream, + ) + else: + _launch_forward( + q, + cos, + sin, + cu_seqlens, + None, + self.head_dim - self.rope_dim, + self.rope_dim, + 1, + 0, + 1, + False, + True, + True, + stream, + ) + + +def _compile_native(heads, head_dim, rope_dim, batch_size, backward): + """Compile one symbolic-token RoPE entrypoint for common C++ dispatch.""" + rows = cute.sym_int32() + positions = cute.sym_int32() + q = cute.runtime.make_fake_compact_tensor( + cutlass.BFloat16, + (rows, heads, head_dim), + stride_order=(2, 1, 0), + memspace=cute.AddressSpace.gmem, + assumed_align=4, + ) + table = lambda: cute.runtime.make_fake_compact_tensor( + cutlass.Float32, + (positions, rope_dim), + stride_order=(1, 0), + memspace=cute.AddressSpace.gmem, + assumed_align=4, + ) + cu_seqlens = cute.runtime.make_fake_compact_tensor( + cutlass.Int32, + (batch_size + 1,), + stride_order=(0,), + memspace=cute.AddressSpace.gmem, + assumed_align=4, + ) + return cute.compile( + _DSv4RopeEntry(heads, head_dim, rope_dim, backward), + q, + table(), + table(), + cu_seqlens, + cute.runtime.make_fake_stream(), + options="--enable-tvm-ffi", + ) + + +def get_dsv4_rope_function(fn_name, heads, head_dim, rope_dim, batch_size, backward): + """Compile and register the native callable requested by common C++.""" + if tvm_ffi.get_global_func(fn_name, allow_missing=True) is not None: + return True + if torch.cuda.get_device_capability() != (10, 0): + return False + compiled = _compile_native(heads, head_dim, rope_dim, batch_size, backward) + native = getattr(compiled, "__tvm_ffi_object__", lambda: None)() + tvm_ffi.register_global_func(fn_name, native if native is not None else compiled, override=True) + return True + + +tvm_ffi.register_global_func("get_dsv4_rope_function", get_dsv4_rope_function, override=True) + + +def _prepare(t, cos, sin, cu_seqlens): + """Validate and adapt TE's packed BSHD rows without changing device semantics.""" + emb_dim = cos.shape[-1] + nope_dim = t.shape[-1] - emb_dim + if t.ndim != 3 or t.stride(-1) != 1 or emb_dim <= 0 or emb_dim % 4 or nope_dim < 0: + raise ValueError("CuTe RoPE requires packed [T,H,D] with a divisible-by-4 rotary tail.") + if cos.ndim != 2 or sin.shape != cos.shape or cos.stride(-1) != 1 or sin.stride(-1) != 1: + raise ValueError("COS/SIN must be matching [P,R] tables with unit channel stride.") + if ( + cu_seqlens.ndim != 1 + or not cu_seqlens.is_contiguous() + or cu_seqlens.dtype not in (torch.int32, torch.int64) + ): + raise ValueError("Sequence prefixes must be a contiguous int32/int64 vector.") + for tensor in (t, cos, sin, cu_seqlens): + if not tensor.is_cuda or tensor.device != t.device: + raise ValueError("Input, tables, and sequence prefixes must share one CUDA device.") + if t.dtype not in (torch.float32, torch.float16, torch.bfloat16): + raise ValueError("CuTe RoPE supports FP32, FP16, and BF16 inputs.") + if cos.dtype not in (torch.float32, torch.float16, torch.bfloat16) or sin.dtype != cos.dtype: + raise ValueError("COS/SIN must share a supported floating type.") + packed_pairs = ( + t.dtype == torch.bfloat16 + and t.data_ptr() % 4 == 0 + and nope_dim % 2 == 0 + and all(stride % 2 == 0 for stride in t.stride()[:-1]) + ) + args = tuple( + from_dlpack(x, assumed_align=4 if packed_pairs and i == 0 else None) + for i, x in enumerate((t, cos, sin, cu_seqlens)) + ) + return args, nope_dim, emb_dim, packed_pairs + + +def forward_inplace(t, cos, sin, cu_seqlens): + """Apply the retained Q-style forward kernel to packed interleaved rows.""" + if t.numel() == 0: + return t + args, nope_dim, emb_dim, packed_pairs = _prepare(t, cos, sin, cu_seqlens) + q, cos_, sin_, cu_ = args + _launch_forward( + q, + cos_, + sin_, + cu_, + None, + nope_dim, + emb_dim, + 1, + 0, + 1, + False, + True, + packed_pairs, + cuda.CUstream(torch.cuda.current_stream(t.device).cuda_stream), + ) + return t + + +def backward_inplace(t, cos, sin, cu_seqlens): + """Apply the retained Q-style transpose kernel to packed interleaved gradients.""" + if t.numel() == 0: + return t + args, nope_dim, emb_dim, packed_pairs = _prepare(t, cos, sin, cu_seqlens) + q, cos_, sin_, cu_ = args + _launch_backward( + q, + cos_, + sin_, + cu_, + None, + nope_dim, + emb_dim, + 1, + 0, + 1, + False, + True, + packed_pairs, + cuda.CUstream(torch.cuda.current_stream(t.device).cuda_stream), + ) + return t diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py index 7165064322a..22aa5b0d68a 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py @@ -4,7 +4,10 @@ """DSv4's trailing-channel, interleaved RoPE for full-sequence attention.""" +from functools import lru_cache + import torch +import transformer_engine_torch as tex def rotary_embeddings(seq, ratio, width, theta, device): @@ -51,10 +54,124 @@ def forward(self, seq, device): ) -def apply_rotary(x, cos, sin): +def _apply_rotary_eager(x, cos, sin): """Rotate trailing interleaved pairs in FP32, then restore input dtype.""" width = cos.shape[-1] tail = x[..., -width:] pair = torch.stack((-tail[..., 1::2], tail[..., 0::2]), -1).flatten(-2) rotated = (tail.float() * cos + pair.float() * sin).to(x.dtype) return torch.cat((x[..., :-width], rotated), -1) + + +@lru_cache(maxsize=1) +def _cute_rope_module(): + """Load the optional CuTe DSL implementation only when DSv4 can use it.""" + try: + from . import _dsa_rope_cute + except ImportError: + return None + return _dsa_rope_cute + + +def _can_use_cute(x, cos, sin, cu_seqlens): + """Keep unsupported inputs on the behaviorally identical eager path.""" + return ( + cu_seqlens is not None + and x.is_cuda + and torch.cuda.get_device_capability(x.device) == (10, 0) + and x.ndim == 4 + and x.stride(-1) == 1 + and cos.shape == sin.shape + and cos.shape[-1] % 4 == 0 + and not cos.requires_grad + and not sin.requires_grad + and _cute_rope_module() is not None + ) + + +def _rope_tables(table): + """Convert TE's repeated pairs to the CuTe kernel's duplicated-half tables.""" + pairs = table[0, :, 0, ::2] + return torch.cat((pairs, pairs), dim=-1) + + +def _apply_rotary_cute(x, cos, sin, cu_seqlens): + """Apply CuTe RoPE out of place so callers retain the eager aliasing contract.""" + module = _cute_rope_module() + if module is None: + raise RuntimeError("CuTe DSL RoPE is unavailable.") + return _CuTeRotary.apply(x, _rope_tables(cos), _rope_tables(sin), cu_seqlens, module) + + +def _launch_cute(packed, cos, sin, cu_seqlens, module, backward): + native = ( + packed.dtype == torch.bfloat16 + and cos.dtype == torch.float32 + and cu_seqlens.dtype == torch.int32 + and tex.dsv4_rope_cutedsl_(packed, cos, sin, cu_seqlens, backward) + ) + if not native: + (module.backward_inplace if backward else module.forward_inplace)( + packed, cos, sin, cu_seqlens + ) + + +class _CuTeRotary(torch.autograd.Function): + """Pair the retained in-place CuTe kernels behind an out-of-place autograd API.""" + + @staticmethod + def forward(ctx, x, cos, sin, cu_seqlens, module): + # cuDNN attention saves its output for backward. Cloning also preserves + # apply_rotary's established out-of-place contract for every call site. + output = x.clone(memory_format=torch.contiguous_format) + packed = output.reshape(-1, x.shape[-2], x.shape[-1]) + _launch_cute(packed, cos, sin, cu_seqlens, module, False) + ctx.save_for_backward(cos, sin, cu_seqlens) + ctx.module = module + ctx.shape = x.shape + return output + + @staticmethod + def backward(ctx, grad_output): + cos, sin, cu_seqlens = ctx.saved_tensors + grad_input = grad_output.clone(memory_format=torch.contiguous_format) + packed = grad_input.reshape(-1, ctx.shape[-2], ctx.shape[-1]) + _launch_cute(packed, cos, sin, cu_seqlens, ctx.module, True) + return grad_input, None, None, None, None + + +class _CuTeQueryRotary(torch.autograd.Function): + """Rotate the fresh normalized query in place; keep backward out of place.""" + + @staticmethod + def forward(ctx, query, cos, sin, cu_seqlens, module): + packed = query.reshape(-1, query.shape[-2], query.shape[-1]) + _launch_cute(packed, cos, sin, cu_seqlens, module, False) + ctx.save_for_backward(cos, sin, cu_seqlens) + ctx.module = module + ctx.shape = query.shape + ctx.mark_dirty(query) + return query + + @staticmethod + def backward(ctx, grad_output): + cos, sin, cu_seqlens = ctx.saved_tensors + grad_input = grad_output.clone(memory_format=torch.contiguous_format) + packed = grad_input.reshape(-1, ctx.shape[-2], ctx.shape[-1]) + _launch_cute(packed, cos, sin, cu_seqlens, ctx.module, True) + return grad_input, None, None, None, None + + +def _apply_rotary_query(query, cos, sin, cu_seqlens): + """Use the fresh query tensor as CuTe output when its native path is available.""" + if not _can_use_cute(query, cos, sin, cu_seqlens): + return _apply_rotary_eager(query, cos, sin) + module = _cute_rope_module() + return _CuTeQueryRotary.apply(query, _rope_tables(cos), _rope_tables(sin), cu_seqlens, module) + + +def apply_rotary(x, cos, sin, cu_seqlens=None): + """Rotate trailing interleaved pairs, using CuTe when its contract is satisfied.""" + if _can_use_cute(x, cos, sin, cu_seqlens): + return _apply_rotary_cute(x, cos, sin, cu_seqlens) + return _apply_rotary_eager(x, cos, sin) diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index 53ec8ede0c4..aa25ef8dadd 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -12,12 +12,18 @@ from .compressor import _Compressor from .dsa_cudnn import attention as _attention -from .dsa_rope import _DSv4RotaryEmbedding, apply_rotary +from .dsa_rope import _apply_rotary_query, _DSv4RotaryEmbedding, apply_rotary from .indexer import _Indexer __all__ = ["DSv4Attention", "DSv4HybridAttention"] +@torch.compile +def _compiled_q_rms_norm(q: torch.Tensor, eps: float) -> torch.Tensor: + """Compile DSv4's weightless per-head query normalization.""" + return torch.nn.functional.rms_norm(q, (q.shape[-1],), eps=eps) + + class DSv4Attention(torch.nn.Module): """Experimental causal joint local + compressed attention for DSv4. @@ -255,17 +261,18 @@ def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = q_residual = self.q_a_norm(q_a) q = self.q_b_proj(q_residual).reshape(batch, seq, self.num_heads, self.head_dim) # Query-up norm is unweighted; TE RMSNorm would add a learned scale. - q = torch.nn.functional.rms_norm(q, (self.head_dim,), eps=self.rms_norm_eps) + q = _compiled_q_rms_norm(q, self.rms_norm_eps) q = ( - apply_rotary(q, *token_rope) + _apply_rotary_query(q, *token_rope, cu) .reshape(batch * seq, self.num_heads, self.head_dim) .contiguous() ) - local_kv = apply_rotary(self.kv_norm(local_kv_projected).unsqueeze(2), *token_rope) + local_kv = apply_rotary(self.kv_norm(local_kv_projected).unsqueeze(2), *token_rope, cu) local_kv = local_kv.reshape(batch * seq, self.head_dim).contiguous() compressed_kv = apply_rotary( self.compressor(hidden_states, cu, cu_comp).reshape(batch, n_comp, 1, self.head_dim), *compressed_rope, + cu_comp, ) compressed_kv = compressed_kv.reshape(batch * n_comp, self.head_dim).contiguous() @@ -299,7 +306,7 @@ def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = max_compressed_seqlen=None if self.is_csa else n_comp, ).reshape(batch, seq, self.num_heads, self.head_dim) cos, sin = token_rope - output = apply_rotary(output, cos, -sin).reshape(batch, seq, self.o_groups, -1) + output = apply_rotary(output, cos, -sin, cu).reshape(batch, seq, self.o_groups, -1) weight = self.o_a_proj.weight.reshape(self.o_groups, self.o_lora_rank, -1) grouped = torch.einsum("bsgd,grd->bsgr", output, weight).flatten(2) result = self.o_b_proj(grouped) diff --git a/transformer_engine/pytorch/attention/sparse_attention/indexer.py b/transformer_engine/pytorch/attention/sparse_attention/indexer.py index e8d762d8b90..1a7652675f3 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/indexer.py +++ b/transformer_engine/pytorch/attention/sparse_attention/indexer.py @@ -71,14 +71,16 @@ def forward( # The singleton axis is the shared index-key head, not a tunable head count. key = ( apply_rotary( - index_compressed.reshape(batch, n_comp, 1, self.head_dim), *compressed_rope + index_compressed.reshape(batch, n_comp, 1, self.head_dim), + *compressed_rope, + cu_comp, ) .reshape(batch * n_comp, self.head_dim) .contiguous() ) query = self.q_proj(q_residual).reshape(batch, seq, self.n_heads, self.head_dim) query = ( - apply_rotary(query, *token_rope) + apply_rotary(query, *token_rope, cu) .reshape(batch * seq, self.n_heads, self.head_dim) .contiguous() ) diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index 91db3a2a9ed..f83e9af2396 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -552,6 +552,9 @@ at::Tensor fused_rope_backward(const at::Tensor &output_grads, const at::Tensor const std::optional cu_seqlens, const int cp_size, const int cp_rank); +bool dsv4_rope_cutedsl_(at::Tensor input, const at::Tensor &cos, const at::Tensor &sin, + const at::Tensor &cu_seqlens, bool backward); + std::tuple fused_qkv_rope_forward( const at::Tensor &qkv_input, const at::Tensor &q_freqs, const at::Tensor &k_freqs, const std::optional start_positions, const std::vector &qkv_split_arg_list, diff --git a/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp b/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp index 4392fa4b433..c4f696da168 100644 --- a/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp +++ b/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp @@ -9,6 +9,34 @@ namespace transformer_engine::pytorch { +bool dsv4_rope_cutedsl_(at::Tensor input, const at::Tensor &cos, const at::Tensor &sin, + const at::Tensor &cu_seqlens, const bool backward) { + TORCH_CHECK(input.is_cuda() && cos.is_cuda() && sin.is_cuda() && cu_seqlens.is_cuda(), + "DSv4 CuTe RoPE tensors must be on CUDA."); + TORCH_CHECK(input.device() == cos.device() && input.device() == sin.device() && + input.device() == cu_seqlens.device(), + "DSv4 CuTe RoPE tensors must share one device."); + TORCH_CHECK(input.dim() == 3 && cos.dim() == 2 && sin.sizes() == cos.sizes() && + cu_seqlens.dim() == 1, + "Expected input [T,H,D], COS/SIN [P,R], and cu_seqlens [B+1]."); + TORCH_CHECK(input.is_contiguous() && cos.is_contiguous() && sin.is_contiguous() && + cu_seqlens.is_contiguous(), + "DSv4 CuTe RoPE tensors must be contiguous."); + TORCH_CHECK(input.scalar_type() == at::ScalarType::BFloat16 && + cos.scalar_type() == at::ScalarType::Float && + sin.scalar_type() == at::ScalarType::Float && + cu_seqlens.scalar_type() == at::ScalarType::Int, + "DSv4 CuTe RoPE expects BF16 input, FP32 tables, and int32 prefixes."); + + auto input_cu = makeTransformerEngineTensor(input); + auto cos_cu = makeTransformerEngineTensor(cos); + auto sin_cu = makeTransformerEngineTensor(sin); + auto cu_seqlens_cu = makeTransformerEngineTensor(cu_seqlens); + return nvte_dsv4_rope_cutedsl(input_cu.data(), cos_cu.data(), sin_cu.data(), + cu_seqlens_cu.data(), backward, + at::cuda::getCurrentCUDAStream()); +} + at::Tensor fused_rope_forward(const at::Tensor &input, const at::Tensor &freqs, const std::optional start_positions, const NVTE_QKV_Format qkv_format, const bool interleaved, diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index 00fdb9d15e7..bfa19834b54 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -577,6 +577,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { "Fused Apply RoPE FWD", py::call_guard()); m.def("fused_rope_backward", &transformer_engine::pytorch::fused_rope_backward, "Fused Apply RoPE BWD", py::call_guard()); + m.def("dsv4_rope_cutedsl_", &transformer_engine::pytorch::dsv4_rope_cutedsl_, + "In-place DSv4 CuTe DSL RoPE", py::call_guard()); m.def("fused_qkv_rope_forward", &transformer_engine::pytorch::fused_qkv_rope_forward, "Fused Apply QKV RoPE FWD", py::call_guard()); m.def("fused_qkv_rope_backward", &transformer_engine::pytorch::fused_qkv_rope_backward, From f6fc4f7eebb92a6dbd8ba279495b882533d806db Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:39:29 +0000 Subject: [PATCH 13/37] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- transformer_engine/common/fused_rope/fused_rope.cu | 6 ++---- .../common/include/transformer_engine/fused_rope.h | 3 +-- .../pytorch/csrc/extensions/apply_rope.cpp | 11 +++++------ 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/transformer_engine/common/fused_rope/fused_rope.cu b/transformer_engine/common/fused_rope/fused_rope.cu index 961f6aa29a2..ab33f057cc8 100644 --- a/transformer_engine/common/fused_rope/fused_rope.cu +++ b/transformer_engine/common/fused_rope/fused_rope.cu @@ -38,8 +38,7 @@ struct DSv4RopeConfig { } std::optional get_kernel() const { - static tvm_ffi_bridge::TVMFFIConfigCache &cache = - tvm_ffi_bridge::TVMFFIConfigCache::create(); + static tvm_ffi_bridge::TVMFFIConfigCache &cache = tvm_ffi_bridge::TVMFFIConfigCache::create(); return cache.get_or_load(*this); } @@ -934,8 +933,7 @@ void nvte_fused_rope_backward(const NVTETensor output_grads, const NVTETensor cu } bool nvte_dsv4_rope_cutedsl(NVTETensor input, const NVTETensor cos, const NVTETensor sin, - const NVTETensor cu_seqlens, const bool backward, - cudaStream_t stream) { + const NVTETensor cu_seqlens, const bool backward, cudaStream_t stream) { NVTE_API_CALL(nvte_dsv4_rope_cutedsl); #ifdef NVTE_WITH_CUTEDSL using namespace transformer_engine; diff --git a/transformer_engine/common/include/transformer_engine/fused_rope.h b/transformer_engine/common/include/transformer_engine/fused_rope.h index ce99baf43f9..13d999366a7 100644 --- a/transformer_engine/common/include/transformer_engine/fused_rope.h +++ b/transformer_engine/common/include/transformer_engine/fused_rope.h @@ -82,8 +82,7 @@ void nvte_fused_rope_backward(const NVTETensor output_grads, const NVTETensor cu * caller can use its existing implementation. */ bool nvte_dsv4_rope_cutedsl(NVTETensor input, const NVTETensor cos, const NVTETensor sin, - const NVTETensor cu_seqlens, const bool backward, - cudaStream_t stream); + const NVTETensor cu_seqlens, const bool backward, cudaStream_t stream); /*! \brief Apply rotary positional embedding to the combined QKV input tensor. * diff --git a/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp b/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp index c4f696da168..a028e02a90c 100644 --- a/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp +++ b/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp @@ -16,9 +16,9 @@ bool dsv4_rope_cutedsl_(at::Tensor input, const at::Tensor &cos, const at::Tenso TORCH_CHECK(input.device() == cos.device() && input.device() == sin.device() && input.device() == cu_seqlens.device(), "DSv4 CuTe RoPE tensors must share one device."); - TORCH_CHECK(input.dim() == 3 && cos.dim() == 2 && sin.sizes() == cos.sizes() && - cu_seqlens.dim() == 1, - "Expected input [T,H,D], COS/SIN [P,R], and cu_seqlens [B+1]."); + TORCH_CHECK( + input.dim() == 3 && cos.dim() == 2 && sin.sizes() == cos.sizes() && cu_seqlens.dim() == 1, + "Expected input [T,H,D], COS/SIN [P,R], and cu_seqlens [B+1]."); TORCH_CHECK(input.is_contiguous() && cos.is_contiguous() && sin.is_contiguous() && cu_seqlens.is_contiguous(), "DSv4 CuTe RoPE tensors must be contiguous."); @@ -32,9 +32,8 @@ bool dsv4_rope_cutedsl_(at::Tensor input, const at::Tensor &cos, const at::Tenso auto cos_cu = makeTransformerEngineTensor(cos); auto sin_cu = makeTransformerEngineTensor(sin); auto cu_seqlens_cu = makeTransformerEngineTensor(cu_seqlens); - return nvte_dsv4_rope_cutedsl(input_cu.data(), cos_cu.data(), sin_cu.data(), - cu_seqlens_cu.data(), backward, - at::cuda::getCurrentCUDAStream()); + return nvte_dsv4_rope_cutedsl(input_cu.data(), cos_cu.data(), sin_cu.data(), cu_seqlens_cu.data(), + backward, at::cuda::getCurrentCUDAStream()); } at::Tensor fused_rope_forward(const at::Tensor &input, const at::Tensor &freqs, From 865842e97d10bc6f5848cbfe99d9ed9279735ce0 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Thu, 8 Oct 2026 17:03:38 -0700 Subject: [PATCH 14/37] Add packed THD support to DSv4 attention Follow the existing TE packed-attention contract at the high-level layer so callers can pass [T,D] hidden states with CUDA cumulative lengths. Build compressed prefixes on device to avoid a per-step host synchronization, and clear unused capacity rows before normalization because cuDNN only guarantees rows below the final compressed prefix. Signed-off-by: Sudhakar Singh --- tests/pytorch/test_dsv4_attention.py | 160 +++++++++++++++++- .../attention/sparse_attention/compressor.py | 13 +- .../attention/sparse_attention/dsa_rope.py | 17 +- .../sparse_attention/dsv4_attention.py | 115 ++++++++++--- .../attention/sparse_attention/indexer.py | 24 +-- 5 files changed, 284 insertions(+), 45 deletions(-) diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index 5014aeda356..0e4b3bd01c0 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -9,7 +9,11 @@ import pytest import torch -from transformer_engine.pytorch.attention.sparse_attention import dsa_rope, dsv4_attention +from transformer_engine.pytorch.attention.sparse_attention import ( + compressor, + dsa_rope, + dsv4_attention, +) @pytest.mark.parametrize("variant", ["hca", "csa"]) @@ -156,3 +160,157 @@ def test_dsv4_cute_rope_batched_forward_backward(monkeypatch): assert query.data_ptr() == query_ptr torch.testing.assert_close(query, reference, atol=2e-2, rtol=2e-2) torch.testing.assert_close(query_grad, reference_grad, atol=0.03125, rtol=0.002) + + +def test_dsv4_thd_compressed_prefixes(): + """Floor each sequence independently, retaining a safe allocation capacity.""" + for lengths, capacity in (([129, 257], 96), ([131, 257], 97)): + x = torch.empty(sum(lengths), 8) + cu = torch.tensor([0, lengths[0], sum(lengths)], dtype=torch.int32) + cu_comp, total_comp, maximum = dsv4_attention._packed_prefixes(x, cu, 257, 4) + assert cu_comp.tolist() == [0, 32, 96] + assert total_comp == capacity + assert maximum == 257 + + +def test_dsv4_thd_compressor_clears_unused_capacity(monkeypatch): + """Unused cuDNN output must not contaminate normalization weight gradients.""" + layer = compressor._Compressor.__new__(compressor._Compressor) + torch.nn.Module.__init__(layer) + layer.kv_proj = layer.gate_proj = torch.nn.Identity() + layer.kv_norm = torch.nn.RMSNorm(4) + layer.position_bias = torch.nn.Parameter(torch.zeros(4, 4)) + layer.projected, layer.ratio, layer.overlap, layer.weight_width = 4, 4, False, 0 + monkeypatch.setattr( + compressor, + "compress", + lambda *args, **kwargs: torch.cat((torch.ones(1, 4), torch.full((1, 4), float("nan")))), + ) + cu = torch.tensor([0, 3, 8], dtype=torch.int32) + cu_comp = torch.tensor([0, 0, 1], dtype=torch.int32) + output = layer(torch.empty(8, 4), cu, cu_comp, total_comp=2) + output.sum().backward() + assert torch.isfinite(output).all() + assert torch.isfinite(layer.kv_norm.weight.grad).all() + assert torch.count_nonzero(output[1]) == 0 + + +@pytest.mark.parametrize( + "prefix,maximum", + [ + (None, 8), + ([0, 8], 8), + (torch.tensor([0, 8], dtype=torch.int64), 8), + (torch.tensor([[0, 8]], dtype=torch.int32), 8), + (torch.tensor([0], dtype=torch.int32), 8), + (torch.tensor([0, 0, 8, 8], dtype=torch.int32)[::2], 8), + (torch.tensor([0, 8], dtype=torch.int32), None), + (torch.tensor([0, 8], dtype=torch.int32), 0), + ], +) +def test_dsv4_thd_rejects_malformed_metadata(prefix, maximum): + with pytest.raises(ValueError, match="THD requires"): + dsv4_attention._packed_prefixes(torch.empty(8, 4), prefix, maximum, 4) + + +def test_dsv4_thd_rejects_hidden_batch_axis_and_multi_sequence_context(): + # These checks precede projections and do not require GPU-backed TE layers. + layer = dsv4_attention.DSv4HybridAttention.__new__(dsv4_attention.DSv4HybridAttention) + torch.nn.Module.__init__(layer) + layer.input_format, layer.hidden_size = "thd", 8 + layer.is_csa, layer.compression_ratio = True, 4 + with pytest.raises(ValueError, match="selected format"): + layer(torch.empty(16, 1, 8)) + with pytest.raises(ValueError, match="THD requires"): + layer(torch.empty(16, 8)) + with pytest.raises(ValueError, match="batch=1"): + layer( + torch.empty(16, 8), + cu_seqlens_q=torch.tensor([0, 8, 16], dtype=torch.int32), + max_seqlen_q=8, + return_indexer_context=True, + ) + + +def test_dsv4_packed_rope_eager_forward_backward(): + """Reset per-sequence positions and safely rotate unused compressed capacity.""" + torch.manual_seed(31) + cu = torch.tensor([0, 3, 8], dtype=torch.int32) + (cos, sin), _ = dsa_rope.rotary_embeddings(5, 1, 4, 12345.0, "cpu") + x = torch.randn(9, 2, 8, requires_grad=True) + reference_input = x.detach().clone().requires_grad_() + actual = dsa_rope.apply_rotary(x, cos, sin, cu) + expected = torch.cat( + [ + dsa_rope._apply_rotary_eager( + reference_input[start:end].unsqueeze(0), + cos[:, : end - start], + sin[:, : end - start], + ).squeeze(0) + for start, end in ((0, 3), (3, 8), (8, 9)) + ] + ) + gradient = torch.randn_like(actual) + actual.backward(gradient) + expected.backward(gradient) + torch.testing.assert_close(actual, expected) + torch.testing.assert_close(x.grad, reference_input.grad) + + +@pytest.mark.parametrize("variant,ratio", [("hca", 128), ("csa", 4)]) +@pytest.mark.parametrize("lengths", [[129, 257], [255, 383]]) +def test_dsv4_hybrid_thd_forward_backward(variant, ratio, lengths): + """Packed full-layer output and gradients match independent dense sequences.""" + if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 0): + pytest.skip("DSv4 requires SM100") + cudnn = pytest.importorskip("cudnn") + if not hasattr(getattr(cudnn, "DSA", None), "sparse_attention_forward_wrapper"): + pytest.skip("DSv4 requires cuDNN Frontend 1.29 sparse attention APIs") + torch.manual_seed(37) + layer = dsv4_attention.DSv4HybridAttention( + hidden_size=32, + q_lora_rank=16, + layer_type=( + "compressed_sparse_attention" if variant == "csa" else "heavily_compressed_attention" + ), + head_dim=512, + rope_head_dim=64, + sliding_window=32, + compression_ratio=ratio, + o_groups=8, + o_lora_rank=8, + index_topk=8, + rope=( + dsa_rope._DSv4RotaryEmbedding(ratio, 64, 12345.0, "cuda") if variant == "csa" else None + ), + params_dtype=torch.bfloat16, + input_format="thd", + ) + x = torch.randn(sum(lengths), 32, device="cuda", dtype=torch.bfloat16, requires_grad=True) + cu = torch.tensor([0, lengths[0], sum(lengths)], device=x.device, dtype=torch.int32) + actual = layer(x, cu_seqlens_q=cu, max_seqlen_q=max(lengths)) + assert actual.shape == x.shape + gradient = torch.randn_like(actual) + actual.backward(gradient) + packed_grads = { + name: p.grad.clone() for name, p in layer.named_parameters() if p.grad is not None + } + layer.zero_grad(set_to_none=True) + layer.input_format = "sbd" if variant == "hca" else "bsd" + batch_axis = 1 if layer.input_format == "sbd" else 0 + reference_input = x.detach().clone().requires_grad_() + expected = torch.cat( + [ + layer(part.unsqueeze(batch_axis)).squeeze(batch_axis) + for part in reference_input.split(lengths) + ] + ) + expected.backward(gradient) + torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2) + for packed_grad, reference_grad in [(x.grad, reference_input.grad)] + [ + (packed_grads[name], p.grad) for name, p in layer.named_parameters() if name in packed_grads + ]: + assert reference_grad is not None + assert torch.isfinite(packed_grad).all() and torch.isfinite(reference_grad).all() + error = (packed_grad.float() - reference_grad.float()).norm() + assert error / reference_grad.float().norm().clamp_min(1e-8) < 0.03 diff --git a/transformer_engine/pytorch/attention/sparse_attention/compressor.py b/transformer_engine/pytorch/attention/sparse_attention/compressor.py index d5b1e0c99c2..49832b1af51 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/compressor.py +++ b/transformer_engine/pytorch/attention/sparse_attention/compressor.py @@ -44,7 +44,7 @@ def __init__( self.ratio, self.overlap = ratio, overlap self.projected, self.weight_width = projected, weight_width - def forward(self, x, cu_seqlens, cu_seqlens_comp): + def forward(self, x, cu_seqlens, cu_seqlens_comp, *, total_comp): """Project and compress KV rows, returning optional fused index weights.""" if hasattr(self, "fused_proj"): projected = self.fused_proj(x) @@ -58,15 +58,20 @@ def forward(self, x, cu_seqlens, cu_seqlens_comp): else: kv, gate = self.kv_proj(x), self.gate_proj(x) pooled = compress( - kv.flatten(0, 1).contiguous(), - gate.flatten(0, 1).contiguous(), + kv.reshape(-1, self.projected).contiguous(), + gate.reshape(-1, self.projected).contiguous(), # cuDNN requires FP32 bias; the cast preserves gradients to the parameter. self.position_bias.float(), cu_seqlens, cu_seqlens_comp, ratio=self.ratio, overlap=self.overlap, - total_comp=x.shape[0] * (x.shape[1] // self.ratio), + total_comp=total_comp, ) + if x.ndim == 2: + # cuDNN only guarantees valid rows below the final prefix. Clear + # unused capacity before RMSNorm so it cannot poison weight gradients. + valid = torch.arange(total_comp, device=x.device) < cu_seqlens_comp[-1] + pooled = pooled.masked_fill(~valid[:, None], 0) compressed = self.kv_norm(pooled) return (compressed, parts[2]) if self.weight_width else compressed diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py index 22aa5b0d68a..6fe0ddf0378 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py @@ -54,8 +54,17 @@ def forward(self, seq, device): ) -def _apply_rotary_eager(x, cos, sin): +def _apply_rotary_eager(x, cos, sin, cu_seqlens=None): """Rotate trailing interleaved pairs in FP32, then restore input dtype.""" + if x.ndim == 3: + if cu_seqlens is None: + raise ValueError("Packed RoPE requires cu_seqlens.") + rows = torch.arange(x.shape[0], device=x.device) + sequences = torch.bucketize(rows, cu_seqlens[1:], right=True) + positions = rows - cu_seqlens[sequences] + # Compressed tensors may have unused capacity past the final prefix. + positions = torch.where(rows < cu_seqlens[-1], positions, 0) + cos, sin = cos[0, positions], sin[0, positions] width = cos.shape[-1] tail = x[..., -width:] pair = torch.stack((-tail[..., 1::2], tail[..., 0::2]), -1).flatten(-2) @@ -79,7 +88,7 @@ def _can_use_cute(x, cos, sin, cu_seqlens): cu_seqlens is not None and x.is_cuda and torch.cuda.get_device_capability(x.device) == (10, 0) - and x.ndim == 4 + and x.ndim in (3, 4) and x.stride(-1) == 1 and cos.shape == sin.shape and cos.shape[-1] % 4 == 0 @@ -165,7 +174,7 @@ def backward(ctx, grad_output): def _apply_rotary_query(query, cos, sin, cu_seqlens): """Use the fresh query tensor as CuTe output when its native path is available.""" if not _can_use_cute(query, cos, sin, cu_seqlens): - return _apply_rotary_eager(query, cos, sin) + return _apply_rotary_eager(query, cos, sin, cu_seqlens) module = _cute_rope_module() return _CuTeQueryRotary.apply(query, _rope_tables(cos), _rope_tables(sin), cu_seqlens, module) @@ -174,4 +183,4 @@ def apply_rotary(x, cos, sin, cu_seqlens=None): """Rotate trailing interleaved pairs, using CuTe when its contract is satisfied.""" if _can_use_cute(x, cos, sin, cu_seqlens): return _apply_rotary_cute(x, cos, sin, cu_seqlens) - return _apply_rotary_eager(x, cos, sin) + return _apply_rotary_eager(x, cos, sin, cu_seqlens) diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index aa25ef8dadd..3e9f71b87a1 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -24,6 +24,30 @@ def _compiled_q_rms_norm(q: torch.Tensor, eps: float) -> torch.Tensor: return torch.nn.functional.rms_norm(q, (q.shape[-1],), eps=eps) +def _packed_prefixes(hidden_states, cu_seqlens, max_seqlen, ratio): + """Lower packed metadata to complete-block prefixes and a safe row capacity.""" + if ( + not isinstance(cu_seqlens, torch.Tensor) + or cu_seqlens.dtype != torch.int32 + or cu_seqlens.ndim != 1 + or cu_seqlens.numel() < 2 + or not cu_seqlens.is_contiguous() + or cu_seqlens.device != hidden_states.device + ): + raise ValueError("THD requires contiguous INT32 cu_seqlens_q [B+1] on the input device.") + if not isinstance(max_seqlen, int) or max_seqlen < ratio: + raise ValueError("THD requires integer max_seqlen_q >= compression_ratio.") + # The sum of per-sequence floors can be smaller than floor(T / ratio). + # Allocate that safe capacity without reading CUDA prefix contents on host; + # cuDNN uses the true device prefix to exclude unused trailing rows. + lengths = cu_seqlens[1:] - cu_seqlens[:-1] + cu_comp = torch.cat((cu_seqlens.new_zeros(1), (lengths // ratio).cumsum(0, dtype=torch.int32))) + total_comp = hidden_states.shape[0] // ratio + if not total_comp: + raise ValueError("Sequences must contain at least one complete compression window.") + return cu_comp, total_comp, max_seqlen + + class DSv4Attention(torch.nn.Module): """Experimental causal joint local + compressed attention for DSv4. @@ -91,20 +115,21 @@ def forward( class DSv4HybridAttention(torch.nn.Module): - """DSv4 CSA/HCA attention from unpadded ``[S,B,D]`` to ``[S,B,D]``. + """DSv4 CSA/HCA attention in ``sbd``, ``bsd``, or packed ``thd`` format. This first-pass layer handles full-sequence BF16 attention forward/backward on SM100. By default, projections that share an input use one contiguous TE Linear weight and split its output before the separate DSv4 operations. It uses TE Linear/RMSNorm, the cuDNN DSv4 compressor and selector, and the existing parameter-free cuDNN attention core. Sequences in a batch have - the same length and start at position zero. The model has 64 heads of width + the same length in dense formats; THD accepts unequal lengths as ``[T,D]``. + Every sequence starts at position zero. The model has 64 heads of width 512; CSA index heads have width 128, matching the current cuDNN kernels. The low-level core's 576-wide Q/K mode returns only 512 value channels, which has no verified output-unrotation contract here. Cache/decode, - padding, TP/CP, FP8, sliding-only attention, and the indexer auxiliary - loss are unsupported. In CSA, top-k is discrete; the indexer projections - do not receive gradients from the returned hidden states alone. + physical inter-sequence padding, TP/SP/CP, FP8, sliding-only attention, and + the indexer auxiliary loss are unsupported. In CSA, top-k is discrete; + the indexer projections do not receive gradients from the returned hidden states alone. """ def __init__( @@ -136,8 +161,8 @@ def __init__( self.index_head_dim = 128 if layer_type not in ("compressed_sparse_attention", "heavily_compressed_attention"): raise ValueError("Only DSv4 CSA and HCA are supported.") - if input_format not in ("sbd", "bsd"): - raise ValueError("input_format must be 'sbd' or 'bsd'.") + if input_format not in ("sbd", "bsd", "thd"): + raise ValueError("input_format must be 'sbd', 'bsd', or 'thd'.") if head_dim != 512 or rope_head_dim < 2 or rope_head_dim % 2 or rope_head_dim > head_dim: raise ValueError("head_dim must be 512; rope_head_dim must be even and fit.") if any( @@ -230,9 +255,28 @@ def __init__( self.is_csa = is_csa self.input_format = input_format - def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = False): - """Run attention in the selected format and optionally expose CSA context.""" - if hidden_states.ndim != 3 or hidden_states.shape[-1] != self.hidden_size: + def forward( + self, + hidden_states: torch.Tensor, + *, + cu_seqlens_q: Optional[torch.Tensor] = None, + max_seqlen_q: Optional[int] = None, + return_indexer_context: bool = False, + ): + """Run full self-attention and optionally expose single-sequence CSA context. + + THD input/output is ``[T,D]`` and requires contiguous INT32 ``cu_seqlens_q`` + of shape ``[B+1]`` on the input device. Prefixes start at zero and end at + T, without physical padding or global offsets. ``max_seqlen_q`` is a + required upper bound covering every sequence. Prefix contents are a caller + contract and are not copied to the host for validation. Dense formats + do not accept these metadata arguments. + """ + packed = self.input_format == "thd" + if ( + hidden_states.ndim != (2 if packed else 3) + or hidden_states.shape[-1] != self.hidden_size + ): raise ValueError("hidden_states must have the selected format and hidden_size.") if return_indexer_context and not self.is_csa: raise ValueError("Only CSA has an indexer context.") @@ -240,14 +284,29 @@ def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = # cuDNN consumes batch-major packed rows. B=1 needs only a view; # larger batches require a physical repack with the current core. hidden_states = hidden_states.transpose(0, 1).contiguous() - batch, seq, _ = hidden_states.shape + if packed: + cu = cu_seqlens_q + cu_comp, total_comp, seq = _packed_prefixes( + hidden_states, cu, max_seqlen_q, self.compression_ratio + ) + batch = cu.numel() - 1 + comp_shape = (total_comp,) + else: + if cu_seqlens_q is not None or max_seqlen_q is not None: + raise ValueError("cu_seqlens_q and max_seqlen_q are only supported with THD input.") + batch, seq, _ = hidden_states.shape + n_comp = seq // self.compression_ratio + if not n_comp: + raise ValueError("Sequence must contain at least one complete compression window.") + cu = torch.arange(batch + 1, device=hidden_states.device, dtype=torch.int32) * seq + cu_comp = ( + torch.arange(batch + 1, device=hidden_states.device, dtype=torch.int32) * n_comp + ) + total_comp = batch * n_comp + comp_shape = (batch, n_comp) if return_indexer_context and batch != 1: raise ValueError("The CSA indexer context is currently supported for batch=1.") - n_comp = seq // self.compression_ratio - if not n_comp: - raise ValueError("Sequence must contain at least one complete compression window.") - cu = torch.arange(batch + 1, device=hidden_states.device, dtype=torch.int32) * seq - cu_comp = torch.arange(batch + 1, device=hidden_states.device, dtype=torch.int32) * n_comp + token_shape = hidden_states.shape[:-1] token_rope, compressed_rope = self.rope(seq, hidden_states.device) @@ -259,22 +318,24 @@ def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = q_a = self.q_a_proj(hidden_states) local_kv_projected = self.kv_proj(hidden_states) q_residual = self.q_a_norm(q_a) - q = self.q_b_proj(q_residual).reshape(batch, seq, self.num_heads, self.head_dim) + q = self.q_b_proj(q_residual).reshape(*token_shape, self.num_heads, self.head_dim) # Query-up norm is unweighted; TE RMSNorm would add a learned scale. q = _compiled_q_rms_norm(q, self.rms_norm_eps) q = ( _apply_rotary_query(q, *token_rope, cu) - .reshape(batch * seq, self.num_heads, self.head_dim) + .reshape(-1, self.num_heads, self.head_dim) .contiguous() ) - local_kv = apply_rotary(self.kv_norm(local_kv_projected).unsqueeze(2), *token_rope, cu) - local_kv = local_kv.reshape(batch * seq, self.head_dim).contiguous() + local_kv = apply_rotary(self.kv_norm(local_kv_projected).unsqueeze(-2), *token_rope, cu) + local_kv = local_kv.reshape(-1, self.head_dim).contiguous() compressed_kv = apply_rotary( - self.compressor(hidden_states, cu, cu_comp).reshape(batch, n_comp, 1, self.head_dim), + self.compressor(hidden_states, cu, cu_comp, total_comp=total_comp).reshape( + *comp_shape, 1, self.head_dim + ), *compressed_rope, cu_comp, ) - compressed_kv = compressed_kv.reshape(batch * n_comp, self.head_dim).contiguous() + compressed_kv = compressed_kv.reshape(total_comp, self.head_dim).contiguous() indices = None index_q = index_k = index_w = None @@ -288,6 +349,8 @@ def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = cu_comp, token_rope, compressed_rope, + total_comp=total_comp, + max_seqlen=seq, return_context=return_indexer_context, ) if return_indexer_context: @@ -303,12 +366,12 @@ def forward(self, hidden_states: torch.Tensor, *, return_indexer_context: bool = cu, cu_comp, indices=indices, - max_compressed_seqlen=None if self.is_csa else n_comp, - ).reshape(batch, seq, self.num_heads, self.head_dim) + max_compressed_seqlen=None if self.is_csa else seq // self.compression_ratio, + ).reshape(*token_shape, self.num_heads, self.head_dim) cos, sin = token_rope - output = apply_rotary(output, cos, -sin, cu).reshape(batch, seq, self.o_groups, -1) + output = apply_rotary(output, cos, -sin, cu).reshape(*token_shape, self.o_groups, -1) weight = self.o_a_proj.weight.reshape(self.o_groups, self.o_lora_rank, -1) - grouped = torch.einsum("bsgd,grd->bsgr", output, weight).flatten(2) + grouped = torch.einsum("...gd,grd->...gr", output, weight).flatten(-2) result = self.o_b_proj(grouped) if self.input_format == "sbd": result = result.transpose(0, 1).contiguous() diff --git a/transformer_engine/pytorch/attention/sparse_attention/indexer.py b/transformer_engine/pytorch/attention/sparse_attention/indexer.py index 1a7652675f3..38a699b8d52 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/indexer.py +++ b/transformer_engine/pytorch/attention/sparse_attention/indexer.py @@ -58,12 +58,14 @@ def forward( token_rope, compressed_rope, *, + total_comp, + max_seqlen, return_context=False, ): """Select compressed rows and optionally expose live tensors for the indexer loss.""" - batch, seq, _ = hidden_states.shape - n_comp = seq // self.ratio - index_compressed = self.compressor(hidden_states, cu, cu_comp) + n_comp = max_seqlen // self.ratio + comp_shape = (total_comp,) if hidden_states.ndim == 2 else (hidden_states.shape[0], n_comp) + index_compressed = self.compressor(hidden_states, cu, cu_comp, total_comp=total_comp) if self.fused: index_compressed, weights = index_compressed else: @@ -71,21 +73,23 @@ def forward( # The singleton axis is the shared index-key head, not a tunable head count. key = ( apply_rotary( - index_compressed.reshape(batch, n_comp, 1, self.head_dim), + index_compressed.reshape(*comp_shape, 1, self.head_dim), *compressed_rope, cu_comp, ) - .reshape(batch * n_comp, self.head_dim) + .reshape(total_comp, self.head_dim) .contiguous() ) - query = self.q_proj(q_residual).reshape(batch, seq, self.n_heads, self.head_dim) + query = self.q_proj(q_residual).reshape( + *hidden_states.shape[:-1], self.n_heads, self.head_dim + ) query = ( apply_rotary(query, *token_rope, cu) - .reshape(batch * seq, self.n_heads, self.head_dim) + .reshape(-1, self.n_heads, self.head_dim) .contiguous() ) # Top-k IDs are discrete; training index projections needs a separate loss. - weights = weights.reshape(batch * seq, self.n_heads).contiguous() + weights = weights.reshape(-1, self.n_heads).contiguous() indices = select_blocks( query, key, @@ -94,10 +98,10 @@ def forward( cu_comp, top_k=min(self.top_k, n_comp), ratio=self.ratio, - max_seqlen=seq, + max_seqlen=max_seqlen, # cuDNN's compile bound covers the partial trailing ratio even # though cu_comp contains only complete compressed blocks. - max_compressed_seqlen=(seq + self.ratio - 1) // self.ratio, + max_compressed_seqlen=(max_seqlen + self.ratio - 1) // self.ratio, scale=(self.head_dim * self.n_heads) ** -0.5, ) return (indices, query, key, weights) if return_context else indices From 3c7d4f216fb7f9b4c6607cfec0b68bf7e1188a21 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Fri, 9 Oct 2026 10:31:03 -0700 Subject: [PATCH 15/37] Replace DSv4 CuTe RoPE with Megatron Triton kernels Use the upstream forward/backward kernels from Megatron-LM c4129afff4867ca89b1fb7b4b911128424cdfbe8 while retaining TE tensor ownership and gradient semantics. Keeping the existing out-of-place KV/output and in-place fresh-query boundaries isolates the kernel-backend change and preserves autograd correctness. Remove the DSv4-specific CuTe implementation and native bridge so this branch has a single optimized RoPE backend. Signed-off-by: Sudhakar Singh --- tests/pytorch/test_dsv4_attention.py | 35 +- .../common/fused_rope/fused_rope.cu | 95 ---- .../include/transformer_engine/fused_rope.h | 8 - transformer_engine/common/triton/dsa_rope.py | 253 +++++++++ .../sparse_attention/_dsa_rope_cute.py | 510 ------------------ .../attention/sparse_attention/dsa_rope.py | 91 ++-- transformer_engine/pytorch/csrc/extensions.h | 3 - .../pytorch/csrc/extensions/apply_rope.cpp | 27 - .../pytorch/csrc/extensions/pybind.cpp | 2 - 9 files changed, 319 insertions(+), 705 deletions(-) create mode 100644 transformer_engine/common/triton/dsa_rope.py delete mode 100644 transformer_engine/pytorch/attention/sparse_attention/_dsa_rope_cute.py diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index 0e4b3bd01c0..aaf9a18f6d4 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -4,8 +4,6 @@ """DSv4 core against a dense oracle, including packed sequence boundaries.""" -import os - import pytest import torch @@ -110,39 +108,32 @@ def test_dsv4_rejects_unsupported_metadata(): core(torch.empty(1, 4, 512), None, None, None, None, None) -def test_dsv4_cute_rope_batched_forward_backward(monkeypatch): - """The packed CuTe bridge must reset positions at each BSHD sequence.""" - if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 0): - pytest.skip("CuTe DSv4 RoPE requires SM100") - if dsa_rope._cute_rope_module() is None: - pytest.skip("CuTe DSL is unavailable") - if os.environ.get("NVTE_ENABLE_CUTEDSL_BACKEND", "0") == "0": - pytest.skip("native CuTe DSL dispatch is disabled") - +@pytest.mark.parametrize("heads", [1, 64]) +def test_dsv4_triton_rope_batched_forward_backward(monkeypatch, heads): + """The packed Triton kernels must reset positions at each BSHD sequence.""" + if not torch.cuda.is_available(): + pytest.skip("Triton DSv4 RoPE requires CUDA") + if dsa_rope._triton_rope_module() is None: + pytest.skip("Triton DSL is unavailable") torch.manual_seed(23) - batch, seq, heads, dim, width = 2, 17, 64, 512, 64 + batch, seq, dim, width = 2, 17, 512, 64 x = torch.randn(batch, seq, heads, dim, device="cuda", dtype=torch.bfloat16) (cos, sin), _ = dsa_rope.rotary_embeddings(seq, 4, width, 160000.0, x.device) cu = torch.arange(batch + 1, device=x.device, dtype=torch.int32) * seq gradient = torch.randn_like(x) - module = dsa_rope._cute_rope_module() - monkeypatch.setattr( - module, - "forward_inplace", - lambda *_: pytest.fail("native forward fell back to its Python launcher"), - ) + eager = dsa_rope._apply_rotary_eager monkeypatch.setattr( - module, - "backward_inplace", - lambda *_: pytest.fail("native backward fell back to its Python launcher"), + dsa_rope, + "_apply_rotary_eager", + lambda *_: pytest.fail("Triton dispatch fell back to eager RoPE"), ) candidate_input = x.detach().requires_grad_() candidate = dsa_rope.apply_rotary(candidate_input, cos, sin, cu) candidate_grad = torch.autograd.grad(candidate, candidate_input, gradient)[0] reference_input = x.detach().requires_grad_() - reference = dsa_rope._apply_rotary_eager(reference_input, cos, sin) + reference = eager(reference_input, cos, sin) reference_grad = torch.autograd.grad(reference, reference_input, gradient)[0] assert candidate.data_ptr() != candidate_input.data_ptr() diff --git a/transformer_engine/common/fused_rope/fused_rope.cu b/transformer_engine/common/fused_rope/fused_rope.cu index ab33f057cc8..c7b762e9731 100644 --- a/transformer_engine/common/fused_rope/fused_rope.cu +++ b/transformer_engine/common/fused_rope/fused_rope.cu @@ -9,93 +9,11 @@ #include #include "../common.h" -#ifdef NVTE_WITH_CUTEDSL -#include "../tvm_ffi_bridge.h" -#endif #include "../util/logging.h" #include "../utils.cuh" namespace transformer_engine { -#ifdef NVTE_WITH_CUTEDSL -namespace { - -struct DSv4RopeConfig { - static constexpr const char *kEntrypointName = "get_dsv4_rope_function"; - - uint32_t heads; - uint32_t head_dim; - uint32_t rope_dim; - uint32_t batch_size; - bool backward; - - uint32_t to_id() const { - NVTE_CHECK(heads < 256 && head_dim % 4 == 0 && head_dim / 4 < 256 && rope_dim % 4 == 0 && - rope_dim / 4 < 64 && batch_size < 256, - "DSv4 RoPE configuration is too large for its CuTe DSL cache key."); - return static_cast(backward) | (heads << 1) | ((head_dim / 4) << 9) | - ((rope_dim / 4) << 17) | (batch_size << 23); - } - - std::optional get_kernel() const { - static tvm_ffi_bridge::TVMFFIConfigCache &cache = tvm_ffi_bridge::TVMFFIConfigCache::create(); - return cache.get_or_load(*this); - } - - std::string to_key() const { - return std::string("cutedsl_dsv4_rope_sm100_h") + std::to_string(heads) + "_d" + - std::to_string(head_dim) + "_r" + std::to_string(rope_dim) + "_b" + - std::to_string(batch_size) + (backward ? "_bwd" : "_fwd"); - } - - bool retrieve_func_from_python(const std::string &fn_name) const { - auto entrypoint = tvm::ffi::Function::GetGlobal(kEntrypointName); - if (!entrypoint.has_value()) return false; - tvm::ffi::Any result = - (*entrypoint)(tvm::ffi::String(fn_name), heads, head_dim, rope_dim, batch_size, backward); - return result.try_cast().value_or(false); - } -}; - -bool dsv4_rope_cutedsl(const Tensor &input, const Tensor &cos, const Tensor &sin, - const Tensor &cu_seqlens, const bool backward, cudaStream_t stream) { - if (!tvm_ffi_bridge::TVMFFICentral::getInstance().get_cutedsl_backend_enabled() || - cuda::sm_arch() != 100 || input.data.dtype != DType::kBFloat16 || - cos.data.dtype != DType::kFloat32 || sin.data.dtype != DType::kFloat32 || - cu_seqlens.data.dtype != DType::kInt32 || input.data.shape.size() != 3 || - cos.data.shape.size() != 2 || sin.data.shape != cos.data.shape || - cu_seqlens.data.shape.size() != 1 || input.data.shape[0] == 0 || - cu_seqlens.data.shape[0] < 2) { - return false; - } - - const DSv4RopeConfig config{/*heads=*/static_cast(input.data.shape[1]), - /*head_dim=*/static_cast(input.data.shape[2]), - /*rope_dim=*/static_cast(cos.data.shape[1]), - /*batch_size=*/static_cast(cu_seqlens.data.shape[0] - 1), - backward}; - if (config.rope_dim == 0 || config.rope_dim > config.head_dim || config.head_dim % 4 != 0 || - config.rope_dim % 4 != 0 || config.heads >= 256 || config.head_dim / 4 >= 256 || - config.rope_dim / 4 >= 64 || config.batch_size == 0 || config.batch_size >= 256) { - return false; - } - - checkCuDriverContext(stream); - auto kernel = config.get_kernel(); - if (!kernel.has_value()) return false; - - const int32_t device = cuda::current_device(); - tvm_ffi_bridge::DLTensorWrapper q(input.data, false, device); - tvm_ffi_bridge::DLTensorWrapper cos_table(cos.data, false, device); - tvm_ffi_bridge::DLTensorWrapper sin_table(sin.data, false, device); - tvm_ffi_bridge::DLTensorWrapper prefixes(cu_seqlens.data, false, device); - (*kernel)(&q, &cos_table, &sin_table, &prefixes, static_cast(stream)); - return true; -} - -} // namespace -#endif - // Returns the largest sequence index `b` such that // `cu_seqlens[b] / cp_size <= t_id`. Used by the linear-grid THD kernels to // locate the sequence span that owns local packed token `t_id`. Uses the same @@ -932,19 +850,6 @@ void nvte_fused_rope_backward(const NVTETensor output_grads, const NVTETensor cu cp_rank, s, b, h, d, d2, stride_s_or_t, stride_b, stride_h, stride_d, stream); } -bool nvte_dsv4_rope_cutedsl(NVTETensor input, const NVTETensor cos, const NVTETensor sin, - const NVTETensor cu_seqlens, const bool backward, cudaStream_t stream) { - NVTE_API_CALL(nvte_dsv4_rope_cutedsl); -#ifdef NVTE_WITH_CUTEDSL - using namespace transformer_engine; - return dsv4_rope_cutedsl(*convertNVTETensorCheck(input), *convertNVTETensorCheck(cos), - *convertNVTETensorCheck(sin), *convertNVTETensorCheck(cu_seqlens), - backward, stream); -#else - return false; -#endif -} - void nvte_fused_qkv_rope_forward(const NVTETensor qkv_input, const NVTETensor q_freqs, const NVTETensor k_freqs, const NVTETensor start_positions, NVTETensor q_out, NVTETensor k_out, NVTETensor v_out, diff --git a/transformer_engine/common/include/transformer_engine/fused_rope.h b/transformer_engine/common/include/transformer_engine/fused_rope.h index 13d999366a7..aea5256a2cb 100644 --- a/transformer_engine/common/include/transformer_engine/fused_rope.h +++ b/transformer_engine/common/include/transformer_engine/fused_rope.h @@ -76,14 +76,6 @@ void nvte_fused_rope_backward(const NVTETensor output_grads, const NVTETensor cu const int stride_s_or_t, const int stride_b, const int stride_h, const int stride_d, cudaStream_t stream); -/*! \brief Apply DSv4's trailing-channel RoPE in place with the optional CuTe DSL backend. - * - * Returns false when the backend or input configuration is unsupported so the - * caller can use its existing implementation. - */ -bool nvte_dsv4_rope_cutedsl(NVTETensor input, const NVTETensor cos, const NVTETensor sin, - const NVTETensor cu_seqlens, const bool backward, cudaStream_t stream); - /*! \brief Apply rotary positional embedding to the combined QKV input tensor. * * \param[in] qkv_input Combined QKV input tensor for fused rope. diff --git a/transformer_engine/common/triton/dsa_rope.py b/transformer_engine/common/triton/dsa_rope.py new file mode 100644 index 00000000000..5579c17084b --- /dev/null +++ b/transformer_engine/common/triton/dsa_rope.py @@ -0,0 +1,253 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""DSv4 RoPE kernels copied from NVIDIA/Megatron-LM. + +Source: megatron/core/fusions/fused_mla_yarn_rope_apply.py at +c4129afff4867ca89b1fb7b4b911128424cdfbe8. Kernel bodies and autotuning are +retained; TE owns tensor preparation and autograd. +""" + +import triton +import triton.language as tl + + +@triton.jit +def _get_thd_token_idx(cu_seqlens, pid_m, seq_num, cp_rank, cp_size): + # Cast ``pid_m`` and ``cu_seqlens`` loads to a single shared dtype so + # the loop-body reassignments don't surface as + # "initial value is int32 but redefined as int64" in newer Triton + # versions (which promote ``// Python_int`` to int64). + pid_m = pid_m.to(tl.int64) + token_idx = tl.full((), -1, dtype=tl.int64) + this_seq_len = tl.full((), 0, dtype=tl.int64) + seq_idx = 0 + last_cum_seqlen = tl.load(cu_seqlens).to(tl.int64) // cp_size + while seq_idx < seq_num: + cur_cum_seqlen = tl.load(cu_seqlens + seq_idx + 1).to(tl.int64) // cp_size + if token_idx == -1 and cur_cum_seqlen > pid_m: + token_idx = pid_m - last_cum_seqlen + this_seq_len = cur_cum_seqlen - last_cum_seqlen + last_cum_seqlen = cur_cum_seqlen + seq_idx += 1 + # Padding tokens beyond cu_seqlens[-1] (from THD CUDA-graph padding) + # never match any sequence, leaving token_idx == -1. Clamp to 0 so + # the cos/sin table loads stay in-bounds; the wrong RoPE result is + # harmless because padding positions are excluded by loss_mask. + if token_idx == -1: + token_idx = tl.full((), 0, dtype=tl.int64) + if cp_size > 1: + if token_idx < this_seq_len // 2: + token_idx = token_idx + cp_rank * this_seq_len // 2 + else: + token_idx = (token_idx - this_seq_len // 2) + ( + 2 * cp_size - cp_rank - 1 + ) * this_seq_len // 2 + return token_idx + + +@triton.jit +def _mla_rope_fwd_inplace_kernel( + Q, + COS, + SIN, + nope_dim, + emb_dim: tl.constexpr, + head_num: tl.constexpr, + batch_size, + seq_num, + cu_seqlens_q, + position_ids, + stride_x_seq, + stride_x_nheads, + stride_cos_seq, + stride_sin_seq, + cp_rank, + cp_size, + INVERSE: tl.constexpr, + REMOVE_INTERLEAVING: tl.constexpr, + BLOCK_H: tl.constexpr, +): + """ + Forward pass: apply RoPE inplace to the trailing emb_dim elements. + Reads from interleaved layout, writes back to interleaved layout. + + Input: + Q: [seq_len, batch_size, head_num, nope_dim + emb_dim] + or [total_seq_len, head_num, nope_dim + emb_dim] + COS/SIN: [max_seq_len, emb_dim] + + batch_size: batch size for sbhd format, not used for thd format + seq_num: number of sequences for thd format, not used for sbhd format + cu_seqlens_q: [seq_num + 1] accumulated sequence lengths for thd format + """ + pid_m = tl.program_id(axis=0) + pid_head = tl.program_id(axis=1) + + if position_ids is not None: + token_idx = tl.load(position_ids + pid_m) + elif cu_seqlens_q is None: + token_idx = pid_m // batch_size + else: + token_idx = _get_thd_token_idx(cu_seqlens_q, pid_m, seq_num, cp_rank, cp_size) + + cos_left = tl.load(COS + token_idx * stride_cos_seq + tl.arange(0, emb_dim // 2)) + sin_left = tl.load(SIN + token_idx * stride_sin_seq + tl.arange(0, emb_dim // 2)) + cos_right = tl.load( + COS + token_idx * stride_cos_seq + emb_dim // 2 + tl.arange(0, emb_dim // 2) + ) + sin_right = tl.load( + SIN + token_idx * stride_sin_seq + emb_dim // 2 + tl.arange(0, emb_dim // 2) + ) + if INVERSE: + sin_left = -sin_left + sin_right = -sin_right + cos_left = cos_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + sin_left = sin_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + cos_right = cos_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + sin_right = sin_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + + Q = Q + pid_m * stride_x_seq + pid_head * BLOCK_H * stride_x_nheads + + x_off = tl.arange(0, BLOCK_H)[:, None] * stride_x_nheads + nope_dim + mask = (pid_head * BLOCK_H + tl.arange(0, BLOCK_H))[:, None] < head_num + # x1 = t[..., 0::2], x2 = t[..., 1::2] + x_1_off = x_off + tl.arange(0, emb_dim // 2)[None, :] * 2 + x_2_off = x_1_off + 1 + x_1 = tl.load(Q + x_1_off, mask=mask) + x_2 = tl.load(Q + x_2_off, mask=mask) + + x_left = x_1 * cos_left - x_2 * sin_left + x_right = x_2 * cos_right + x_1 * sin_right + + if REMOVE_INTERLEAVING: + tl.store(Q + x_1_off, x_left, mask=mask) + tl.store(Q + x_2_off, x_right, mask=mask) + else: + # The interleaved input and split output layouts alias. Finish all loads + # before any warp stores to the overlapping destination addresses. + tl.debug_barrier() + x_left_off = x_off + tl.arange(0, emb_dim // 2)[None, :] + x_right_off = x_left_off + emb_dim // 2 + tl.store(Q + x_left_off, x_left, mask=mask) + tl.store(Q + x_right_off, x_right, mask=mask) + + +_autotuned_mla_rope_fwd_inplace_kernel = triton.autotune( + configs=[ + triton.Config({"BLOCK_H": 1}), + triton.Config({"BLOCK_H": 2}), + triton.Config({"BLOCK_H": 4}), + triton.Config({"BLOCK_H": 8}), + triton.Config({"BLOCK_H": 16}), + triton.Config({"BLOCK_H": 32}), + triton.Config({"BLOCK_H": 64}), + triton.Config({"BLOCK_H": 128}), + ], + key=["emb_dim", "head_num"], + restore_value=["Q"], +)(_mla_rope_fwd_inplace_kernel) + + +@triton.jit +def _mla_rope_bwd_inplace_kernel( + DO, + COS, + SIN, + nope_dim, + emb_dim: tl.constexpr, + head_num: tl.constexpr, + batch_size, + seq_num, + cu_seqlens_q, + position_ids, + stride_x_seq, + stride_x_nheads, + stride_cos_seq, + stride_sin_seq, + cp_rank, + cp_size, + INVERSE: tl.constexpr, + REMOVE_INTERLEAVING: tl.constexpr, + BLOCK_H: tl.constexpr, +): + """ + Backward pass: inverse RoPE inplace on the trailing emb_dim elements. + Reads from interleaved layout, writes to interleaved layout. + + Input: + DO: [seq_len, batch_size, head_num, nope_dim + emb_dim] + or [total_seq_len, head_num, nope_dim + emb_dim] + COS/SIN: [max_seq_len, emb_dim] + + batch_size, seq_num, and cu_seqlens_q are the same as in the forward pass + """ + pid_m = tl.program_id(axis=0) + pid_head = tl.program_id(axis=1) + + if position_ids is not None: + token_idx = tl.load(position_ids + pid_m) + elif cu_seqlens_q is None: + token_idx = pid_m // batch_size + else: + token_idx = _get_thd_token_idx(cu_seqlens_q, pid_m, seq_num, cp_rank, cp_size) + + cos_left = tl.load(COS + token_idx * stride_cos_seq + tl.arange(0, emb_dim // 2)) + sin_left = tl.load(SIN + token_idx * stride_sin_seq + tl.arange(0, emb_dim // 2)) + cos_right = tl.load( + COS + token_idx * stride_cos_seq + emb_dim // 2 + tl.arange(0, emb_dim // 2) + ) + sin_right = tl.load( + SIN + token_idx * stride_sin_seq + emb_dim // 2 + tl.arange(0, emb_dim // 2) + ) + if INVERSE: + sin_left = -sin_left + sin_right = -sin_right + cos_left = cos_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + sin_left = sin_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + cos_right = cos_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + sin_right = sin_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + + DO = DO + pid_m * stride_x_seq + pid_head * BLOCK_H * stride_x_nheads + + x_off = tl.arange(0, BLOCK_H)[:, None] * stride_x_nheads + nope_dim + mask = (pid_head * BLOCK_H + tl.arange(0, BLOCK_H))[:, None] < head_num + if REMOVE_INTERLEAVING: + x_1_off = x_off + tl.arange(0, emb_dim // 2)[None, :] * 2 + x_2_off = x_1_off + 1 + x_left = tl.load(DO + x_1_off, mask=mask) + x_right = tl.load(DO + x_2_off, mask=mask) + else: + x_left_off = x_off + tl.arange(0, emb_dim // 2)[None, :] + x_right_off = x_left_off + emb_dim // 2 + x_left = tl.load(DO + x_left_off, mask=mask) + x_right = tl.load(DO + x_right_off, mask=mask) + x_1_off = x_off + tl.arange(0, emb_dim // 2)[None, :] * 2 + x_2_off = x_1_off + 1 + + x_1 = x_left * cos_left + x_right * sin_right + x_2 = -x_left * sin_left + x_right * cos_right + + if not REMOVE_INTERLEAVING: + # The split input and interleaved output layouts alias. Finish all loads + # before any warp stores to the overlapping destination addresses. + tl.debug_barrier() + tl.store(DO + x_1_off, x_1, mask=mask) + tl.store(DO + x_2_off, x_2, mask=mask) + + +_autotuned_mla_rope_bwd_inplace_kernel = triton.autotune( + configs=[ + triton.Config({"BLOCK_H": 1}), + triton.Config({"BLOCK_H": 2}), + triton.Config({"BLOCK_H": 4}), + triton.Config({"BLOCK_H": 8}), + triton.Config({"BLOCK_H": 16}), + triton.Config({"BLOCK_H": 32}), + triton.Config({"BLOCK_H": 64}), + triton.Config({"BLOCK_H": 128}), + ], + key=["emb_dim", "head_num"], + restore_value=["DO"], +)(_mla_rope_bwd_inplace_kernel) diff --git a/transformer_engine/pytorch/attention/sparse_attention/_dsa_rope_cute.py b/transformer_engine/pytorch/attention/sparse_attention/_dsa_rope_cute.py deleted file mode 100644 index 65df901ec9a..00000000000 --- a/transformer_engine/pytorch/attention/sparse_attention/_dsa_rope_cute.py +++ /dev/null @@ -1,510 +0,0 @@ -# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# See LICENSE for license information. - -"""CuTe DSL kernels for DSv4 trailing-channel interleaved RoPE. - -The device bodies come from the retained DSv4 experiments: -forward SHA256 0355016c141365dfa436deda15a29cce49146a0815833ab7963d941ef67f7e5c -and backward SHA256 fdda9fb8bcf44e998d000bb8f78354fcf02f2c0b7cb4d9d3c9787a24abaaef82. -Only their host boundary is narrowed to TE's packed single-tensor contract. -""" - -import torch -import cutlass -import cutlass.cute as cute -import tvm_ffi -from cutlass.cute.runtime import from_dlpack -from cuda.bindings import driver as cuda - - -@cute.kernel -def _mla_rope_fwd_inplace_kernel( - q: cute.Tensor, - cos: cute.Tensor, - sin: cute.Tensor, - cu_seqlens, - positions, - nope_dim: cutlass.Constexpr, - emb_dim: cutlass.Constexpr, - batch_size: cutlass.Constexpr, - cp_rank: cutlass.Constexpr, - cp_size: cutlass.Constexpr, - inverse: cutlass.Constexpr, - remove_interleaving: cutlass.Constexpr, - packed_pairs: cutlass.Constexpr, - threads: cutlass.Constexpr, - heads_per_cta: cutlass.Constexpr, -): - tid, _, _ = cute.arch.thread_idx() - row, head_tile, _ = cute.arch.block_idx() - pos = cutlass.Int64(row // batch_size) - if cutlass.const_expr(positions is not None): - pos = positions[row].to(cutlass.Int64) - elif cutlass.const_expr(cu_seqlens is not None): - pos = cutlass.Int64(-1) - length = cutlass.Int64(0) - # Upper bound skips duplicate/empty ends and preserves CP floor division. - lo = cutlass.Int64(1) - hi = cutlass.Int64(cute.size(cu_seqlens)) - while lo < hi: - mid = (lo + hi) // 2 - end = cu_seqlens[mid].to(cutlass.Int64) // cp_size - if end > row: - hi = mid - else: - lo = mid + 1 - # The exclusive-bound sentinel denotes padding at/beyond the last end. - if lo < cute.size(cu_seqlens): - previous = cu_seqlens[lo - 1].to(cutlass.Int64) // cp_size - end = cu_seqlens[lo].to(cutlass.Int64) // cp_size - pos = row - previous - length = end - previous - if pos == -1: - pos = cutlass.Int64(0) - if cutlass.const_expr(cp_size > 1): - if pos < length // 2: - pos = pos + cp_rank * length // 2 - else: - pos = pos - length // 2 + (2 * cp_size - cp_rank - 1) * length // 2 - - if cutlass.const_expr(packed_pairs): - q_pairs = cute.recast_tensor(q, cutlass.Uint32) - half = emb_dim // 2 - count = cute.ceil_div(heads_per_cta * half, threads) - left = cute.make_rmem_tensor((count,), cutlass.Float32) - right = cute.make_rmem_tensor((count,), cutlass.Float32) - # Adding `threads` leaves `flat % half` unchanged only under this guard. - # Hoist the four same-address table loads once for each participating thread. - if cutlass.const_expr(threads % half == 0 and heads_per_cta * half >= threads): - table_pair = tid % half - cl_once = cos[pos, table_pair] - cr_once = cos[pos, half + table_pair] - sl_once = sin[pos, table_pair] - sr_once = sin[pos, half + table_pair] - for slot in cutlass.range_constexpr(count): - flat = tid + slot * threads - head = head_tile * heads_per_cta + flat // half - pair = flat % half - if flat < heads_per_cta * half and head < q.shape[1]: - if cutlass.const_expr(packed_pairs): - # Bit reinterpretation preserves the original BF16 operands. - values = cute.make_rmem_tensor((2,), q.element_type) - word = cute.recast_tensor(values, cutlass.Uint32) - word[0] = q_pairs[row, head, nope_dim // 2 + pair] - x, y = values[0], values[1] - else: - x = q[row, head, nope_dim + 2 * pair] - y = q[row, head, nope_dim + 2 * pair + 1] - if cutlass.const_expr(threads % half == 0 and heads_per_cta * half >= threads): - cl, cr, sl, sr = cl_once, cr_once, sl_once, sr_once - else: - cl = cos[pos, pair] - cr = cos[pos, half + pair] - sl = sin[pos, pair] - sr = sin[pos, half + pair] - if cutlass.const_expr(inverse): - sl, sr = -sl, -sr - # Floating promotion follows Triton: FP32 > FP16 > BF16. - left[slot] = (x * cl - y * sl).to(cutlass.Float32) - right[slot] = (y * cr + x * sr).to(cutlass.Float32) - - # Split-half stores alias other threads' interleaved inputs. Every warp - # must finish loading before any thread writes to those addresses. - if cutlass.const_expr(not remove_interleaving): - cute.arch.sync_threads() - for slot in cutlass.range_constexpr(count): - flat = tid + slot * threads - head = head_tile * heads_per_cta + flat // half - pair = flat % half - if flat < heads_per_cta * half and head < q.shape[1]: - if cutlass.const_expr(packed_pairs and remove_interleaving): - values = cute.make_rmem_tensor((2,), q.element_type) - values[0] = left[slot].to(q.element_type) - values[1] = right[slot].to(q.element_type) - word = cute.recast_tensor(values, cutlass.Uint32) - q_pairs[row, head, nope_dim // 2 + pair] = word[0] - else: - if cutlass.const_expr(remove_interleaving): - dst_l, dst_r = 2 * pair, 2 * pair + 1 - else: - dst_l, dst_r = pair, half + pair - q[row, head, nope_dim + dst_l] = left[slot].to(q.element_type) - q[row, head, nope_dim + dst_r] = right[slot].to(q.element_type) - - -@cute.jit -def _launch_forward( - q: cute.Tensor, - cos: cute.Tensor, - sin: cute.Tensor, - cu_seqlens, - positions, - nope_dim: cutlass.Constexpr, - emb_dim: cutlass.Constexpr, - batch_size: cutlass.Constexpr, - cp_rank: cutlass.Constexpr, - cp_size: cutlass.Constexpr, - inverse: cutlass.Constexpr, - remove_interleaving: cutlass.Constexpr, - packed_pairs: cutlass.Constexpr, - stream: cuda.CUstream, -): - threads = 128 - heads_per_cta = min(16, q.shape[1]) - _mla_rope_fwd_inplace_kernel( - q, - cos, - sin, - cu_seqlens, - positions, - nope_dim, - emb_dim, - batch_size, - cp_rank, - cp_size, - inverse, - remove_interleaving, - packed_pairs, - threads, - heads_per_cta, - ).launch( - grid=(q.shape[0], cute.ceil_div(q.shape[1], heads_per_cta), 1), - block=(threads, 1, 1), - stream=stream, - ) - - -@cute.kernel -def _mla_rope_bwd_inplace_kernel( - q: cute.Tensor, - cos: cute.Tensor, - sin: cute.Tensor, - cu_seqlens, - positions, - nope_dim: cutlass.Constexpr, - emb_dim: cutlass.Constexpr, - batch_size: cutlass.Constexpr, - cp_rank: cutlass.Constexpr, - cp_size: cutlass.Constexpr, - inverse: cutlass.Constexpr, - remove_interleaving: cutlass.Constexpr, - packed_pairs: cutlass.Constexpr, - threads: cutlass.Constexpr, - heads_per_cta: cutlass.Constexpr, -): - tid, _, _ = cute.arch.thread_idx() - row, head_tile, _ = cute.arch.block_idx() - pos = cutlass.Int64(row // batch_size) - if cutlass.const_expr(positions is not None): - pos = positions[row].to(cutlass.Int64) - elif cutlass.const_expr(cu_seqlens is not None): - pos = cutlass.Int64(-1) - length = cutlass.Int64(0) - previous = cu_seqlens[0].to(cutlass.Int64) // cp_size - for seq in range(cute.size(cu_seqlens) - 1): - end = cu_seqlens[seq + 1].to(cutlass.Int64) // cp_size - if pos == -1 and end > row: - pos = row - previous - length = end - previous - previous = end - if pos == -1: - pos = cutlass.Int64(0) - if cutlass.const_expr(cp_size > 1): - if pos < length // 2: - pos = pos + cp_rank * length // 2 - else: - pos = pos - length // 2 + (2 * cp_size - cp_rank - 1) * length // 2 - - if cutlass.const_expr(packed_pairs): - q_pairs = cute.recast_tensor(q, cutlass.Uint32) - half = emb_dim // 2 - count = cute.ceil_div(heads_per_cta * half, threads) - left = cute.make_rmem_tensor((count,), cutlass.Float32) - right = cute.make_rmem_tensor((count,), cutlass.Float32) - # Every slot has the same pair only when adding threads preserves remainder. - # Preserve table types and the original per-slot inverse/arithmetic below. - if cutlass.const_expr(threads % half == 0 and heads_per_cta * half >= threads): - table_pair = tid % half - cl_once = cos[pos, table_pair] - cr_once = cos[pos, half + table_pair] - sl_once = sin[pos, table_pair] - sr_once = sin[pos, half + table_pair] - for slot in cutlass.range_constexpr(count): - flat = tid + slot * threads - head = head_tile * heads_per_cta + flat // half - pair = flat % half - if flat < heads_per_cta * half and head < q.shape[1]: - if cutlass.const_expr(remove_interleaving): - src_l, src_r = 2 * pair, 2 * pair + 1 - else: - src_l, src_r = pair, half + pair - if cutlass.const_expr(packed_pairs and remove_interleaving): - # Adjacent inputs form one word; split inputs keep scalar loads. - values = cute.make_rmem_tensor((2,), q.element_type) - word = cute.recast_tensor(values, cutlass.Uint32) - word[0] = q_pairs[row, head, nope_dim // 2 + pair] - x, y = values[0], values[1] - else: - x = q[row, head, nope_dim + src_l] - y = q[row, head, nope_dim + src_r] - if cutlass.const_expr(threads % half == 0 and heads_per_cta * half >= threads): - cl, cr, sl, sr = cl_once, cr_once, sl_once, sr_once - else: - cl = cos[pos, pair] - cr = cos[pos, half + pair] - sl = sin[pos, pair] - sr = sin[pos, half + pair] - if cutlass.const_expr(inverse): - sl, sr = -sl, -sr - # CuTe and Triton both promote these floating operands to FP32 - # when a table is FP32, and otherwise preserve the narrower type. - left[slot] = (x * cl + y * sr).to(cutlass.Float32) - right[slot] = (-x * sl + y * cr).to(cutlass.Float32) - - # Split-input stores alias only within a head. When half divides 32, - # every head belongs to one warp; wider/non-divisor widths need the CTA. - if cutlass.const_expr(not remove_interleaving): - if cutlass.const_expr(half <= 32 and 32 % half == 0): - cute.arch.sync_warp() - else: - cute.arch.sync_threads() - for slot in cutlass.range_constexpr(count): - flat = tid + slot * threads - head = head_tile * heads_per_cta + flat // half - pair = flat % half - if flat < heads_per_cta * half and head < q.shape[1]: - if cutlass.const_expr(packed_pairs): - # Backward always writes adjacent pairs, after original casts. - values = cute.make_rmem_tensor((2,), q.element_type) - values[0] = left[slot].to(q.element_type) - values[1] = right[slot].to(q.element_type) - word = cute.recast_tensor(values, cutlass.Uint32) - q_pairs[row, head, nope_dim // 2 + pair] = word[0] - else: - q[row, head, nope_dim + 2 * pair] = left[slot].to(q.element_type) - q[row, head, nope_dim + 2 * pair + 1] = right[slot].to(q.element_type) - - -@cute.jit -def _launch_backward( - q: cute.Tensor, - cos: cute.Tensor, - sin: cute.Tensor, - cu_seqlens, - positions, - nope_dim: cutlass.Constexpr, - emb_dim: cutlass.Constexpr, - batch_size: cutlass.Constexpr, - cp_rank: cutlass.Constexpr, - cp_size: cutlass.Constexpr, - inverse: cutlass.Constexpr, - remove_interleaving: cutlass.Constexpr, - packed_pairs: cutlass.Constexpr, - stream: cuda.CUstream, -): - threads = 128 - heads_per_cta = min(16, q.shape[1]) - _mla_rope_bwd_inplace_kernel( - q, - cos, - sin, - cu_seqlens, - positions, - nope_dim, - emb_dim, - batch_size, - cp_rank, - cp_size, - inverse, - remove_interleaving, - packed_pairs, - threads, - heads_per_cta, - ).launch( - grid=(q.shape[0], cute.ceil_div(q.shape[1], heads_per_cta), 1), - block=(threads, 1, 1), - stream=stream, - ) - - -class _DSv4RopeEntry: - """TVM-FFI entrypoint with the kernel's static launch choices captured.""" - - def __init__(self, heads, head_dim, rope_dim, backward): - self.heads = heads - self.head_dim = head_dim - self.rope_dim = rope_dim - self.backward = backward - - @cute.jit - def __call__(self, q, cos, sin, cu_seqlens, stream): - if cutlass.const_expr(self.backward): - _launch_backward( - q, - cos, - sin, - cu_seqlens, - None, - self.head_dim - self.rope_dim, - self.rope_dim, - 1, - 0, - 1, - False, - True, - True, - stream, - ) - else: - _launch_forward( - q, - cos, - sin, - cu_seqlens, - None, - self.head_dim - self.rope_dim, - self.rope_dim, - 1, - 0, - 1, - False, - True, - True, - stream, - ) - - -def _compile_native(heads, head_dim, rope_dim, batch_size, backward): - """Compile one symbolic-token RoPE entrypoint for common C++ dispatch.""" - rows = cute.sym_int32() - positions = cute.sym_int32() - q = cute.runtime.make_fake_compact_tensor( - cutlass.BFloat16, - (rows, heads, head_dim), - stride_order=(2, 1, 0), - memspace=cute.AddressSpace.gmem, - assumed_align=4, - ) - table = lambda: cute.runtime.make_fake_compact_tensor( - cutlass.Float32, - (positions, rope_dim), - stride_order=(1, 0), - memspace=cute.AddressSpace.gmem, - assumed_align=4, - ) - cu_seqlens = cute.runtime.make_fake_compact_tensor( - cutlass.Int32, - (batch_size + 1,), - stride_order=(0,), - memspace=cute.AddressSpace.gmem, - assumed_align=4, - ) - return cute.compile( - _DSv4RopeEntry(heads, head_dim, rope_dim, backward), - q, - table(), - table(), - cu_seqlens, - cute.runtime.make_fake_stream(), - options="--enable-tvm-ffi", - ) - - -def get_dsv4_rope_function(fn_name, heads, head_dim, rope_dim, batch_size, backward): - """Compile and register the native callable requested by common C++.""" - if tvm_ffi.get_global_func(fn_name, allow_missing=True) is not None: - return True - if torch.cuda.get_device_capability() != (10, 0): - return False - compiled = _compile_native(heads, head_dim, rope_dim, batch_size, backward) - native = getattr(compiled, "__tvm_ffi_object__", lambda: None)() - tvm_ffi.register_global_func(fn_name, native if native is not None else compiled, override=True) - return True - - -tvm_ffi.register_global_func("get_dsv4_rope_function", get_dsv4_rope_function, override=True) - - -def _prepare(t, cos, sin, cu_seqlens): - """Validate and adapt TE's packed BSHD rows without changing device semantics.""" - emb_dim = cos.shape[-1] - nope_dim = t.shape[-1] - emb_dim - if t.ndim != 3 or t.stride(-1) != 1 or emb_dim <= 0 or emb_dim % 4 or nope_dim < 0: - raise ValueError("CuTe RoPE requires packed [T,H,D] with a divisible-by-4 rotary tail.") - if cos.ndim != 2 or sin.shape != cos.shape or cos.stride(-1) != 1 or sin.stride(-1) != 1: - raise ValueError("COS/SIN must be matching [P,R] tables with unit channel stride.") - if ( - cu_seqlens.ndim != 1 - or not cu_seqlens.is_contiguous() - or cu_seqlens.dtype not in (torch.int32, torch.int64) - ): - raise ValueError("Sequence prefixes must be a contiguous int32/int64 vector.") - for tensor in (t, cos, sin, cu_seqlens): - if not tensor.is_cuda or tensor.device != t.device: - raise ValueError("Input, tables, and sequence prefixes must share one CUDA device.") - if t.dtype not in (torch.float32, torch.float16, torch.bfloat16): - raise ValueError("CuTe RoPE supports FP32, FP16, and BF16 inputs.") - if cos.dtype not in (torch.float32, torch.float16, torch.bfloat16) or sin.dtype != cos.dtype: - raise ValueError("COS/SIN must share a supported floating type.") - packed_pairs = ( - t.dtype == torch.bfloat16 - and t.data_ptr() % 4 == 0 - and nope_dim % 2 == 0 - and all(stride % 2 == 0 for stride in t.stride()[:-1]) - ) - args = tuple( - from_dlpack(x, assumed_align=4 if packed_pairs and i == 0 else None) - for i, x in enumerate((t, cos, sin, cu_seqlens)) - ) - return args, nope_dim, emb_dim, packed_pairs - - -def forward_inplace(t, cos, sin, cu_seqlens): - """Apply the retained Q-style forward kernel to packed interleaved rows.""" - if t.numel() == 0: - return t - args, nope_dim, emb_dim, packed_pairs = _prepare(t, cos, sin, cu_seqlens) - q, cos_, sin_, cu_ = args - _launch_forward( - q, - cos_, - sin_, - cu_, - None, - nope_dim, - emb_dim, - 1, - 0, - 1, - False, - True, - packed_pairs, - cuda.CUstream(torch.cuda.current_stream(t.device).cuda_stream), - ) - return t - - -def backward_inplace(t, cos, sin, cu_seqlens): - """Apply the retained Q-style transpose kernel to packed interleaved gradients.""" - if t.numel() == 0: - return t - args, nope_dim, emb_dim, packed_pairs = _prepare(t, cos, sin, cu_seqlens) - q, cos_, sin_, cu_ = args - _launch_backward( - q, - cos_, - sin_, - cu_, - None, - nope_dim, - emb_dim, - 1, - 0, - 1, - False, - True, - packed_pairs, - cuda.CUstream(torch.cuda.current_stream(t.device).cuda_stream), - ) - return t diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py index 6fe0ddf0378..e42882b7019 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py @@ -7,7 +7,6 @@ from functools import lru_cache import torch -import transformer_engine_torch as tex def rotary_embeddings(seq, ratio, width, theta, device): @@ -73,60 +72,76 @@ def _apply_rotary_eager(x, cos, sin, cu_seqlens=None): @lru_cache(maxsize=1) -def _cute_rope_module(): - """Load the optional CuTe DSL implementation only when DSv4 can use it.""" +def _triton_rope_module(): + """Load the optional Triton implementation only when DSv4 can use it.""" try: - from . import _dsa_rope_cute + from transformer_engine.common.triton import dsa_rope except ImportError: return None - return _dsa_rope_cute + return dsa_rope -def _can_use_cute(x, cos, sin, cu_seqlens): +def _can_use_triton(x, cos, sin, cu_seqlens): """Keep unsupported inputs on the behaviorally identical eager path.""" return ( cu_seqlens is not None and x.is_cuda - and torch.cuda.get_device_capability(x.device) == (10, 0) and x.ndim in (3, 4) and x.stride(-1) == 1 and cos.shape == sin.shape and cos.shape[-1] % 4 == 0 and not cos.requires_grad and not sin.requires_grad - and _cute_rope_module() is not None + and _triton_rope_module() is not None ) def _rope_tables(table): - """Convert TE's repeated pairs to the CuTe kernel's duplicated-half tables.""" + """Convert TE's repeated pairs to the Triton kernel's duplicated-half tables.""" pairs = table[0, :, 0, ::2] return torch.cat((pairs, pairs), dim=-1) -def _apply_rotary_cute(x, cos, sin, cu_seqlens): - """Apply CuTe RoPE out of place so callers retain the eager aliasing contract.""" - module = _cute_rope_module() +def _apply_rotary_triton(x, cos, sin, cu_seqlens): + """Apply Triton RoPE out of place so callers retain the eager aliasing contract.""" + module = _triton_rope_module() if module is None: - raise RuntimeError("CuTe DSL RoPE is unavailable.") - return _CuTeRotary.apply(x, _rope_tables(cos), _rope_tables(sin), cu_seqlens, module) + raise RuntimeError("Triton RoPE is unavailable.") + return _TritonRotary.apply(x, _rope_tables(cos), _rope_tables(sin), cu_seqlens, module) -def _launch_cute(packed, cos, sin, cu_seqlens, module, backward): - native = ( - packed.dtype == torch.bfloat16 - and cos.dtype == torch.float32 - and cu_seqlens.dtype == torch.int32 - and tex.dsv4_rope_cutedsl_(packed, cos, sin, cu_seqlens, backward) +def _launch_triton(packed, cos, sin, cu_seqlens, module, backward): + kernel = ( + module._autotuned_mla_rope_bwd_inplace_kernel + if backward + else module._autotuned_mla_rope_fwd_inplace_kernel + ) + heads, dim = packed.shape[-2:] + grid = lambda meta: (packed.shape[0], module.triton.cdiv(heads, meta["BLOCK_H"])) + kernel[grid]( + packed, + cos, + sin, + dim - cos.shape[-1], + cos.shape[-1], + heads, + 1, + cu_seqlens.numel() - 1, + cu_seqlens, + None, + packed.stride(0), + packed.stride(1), + cos.stride(0), + sin.stride(0), + 0, + 1, + INVERSE=False, + REMOVE_INTERLEAVING=True, ) - if not native: - (module.backward_inplace if backward else module.forward_inplace)( - packed, cos, sin, cu_seqlens - ) -class _CuTeRotary(torch.autograd.Function): - """Pair the retained in-place CuTe kernels behind an out-of-place autograd API.""" +class _TritonRotary(torch.autograd.Function): + """Pair the retained in-place Triton kernels behind an out-of-place autograd API.""" @staticmethod def forward(ctx, x, cos, sin, cu_seqlens, module): @@ -134,7 +149,7 @@ def forward(ctx, x, cos, sin, cu_seqlens, module): # apply_rotary's established out-of-place contract for every call site. output = x.clone(memory_format=torch.contiguous_format) packed = output.reshape(-1, x.shape[-2], x.shape[-1]) - _launch_cute(packed, cos, sin, cu_seqlens, module, False) + _launch_triton(packed, cos, sin, cu_seqlens, module, False) ctx.save_for_backward(cos, sin, cu_seqlens) ctx.module = module ctx.shape = x.shape @@ -145,17 +160,17 @@ def backward(ctx, grad_output): cos, sin, cu_seqlens = ctx.saved_tensors grad_input = grad_output.clone(memory_format=torch.contiguous_format) packed = grad_input.reshape(-1, ctx.shape[-2], ctx.shape[-1]) - _launch_cute(packed, cos, sin, cu_seqlens, ctx.module, True) + _launch_triton(packed, cos, sin, cu_seqlens, ctx.module, True) return grad_input, None, None, None, None -class _CuTeQueryRotary(torch.autograd.Function): +class _TritonQueryRotary(torch.autograd.Function): """Rotate the fresh normalized query in place; keep backward out of place.""" @staticmethod def forward(ctx, query, cos, sin, cu_seqlens, module): packed = query.reshape(-1, query.shape[-2], query.shape[-1]) - _launch_cute(packed, cos, sin, cu_seqlens, module, False) + _launch_triton(packed, cos, sin, cu_seqlens, module, False) ctx.save_for_backward(cos, sin, cu_seqlens) ctx.module = module ctx.shape = query.shape @@ -167,20 +182,20 @@ def backward(ctx, grad_output): cos, sin, cu_seqlens = ctx.saved_tensors grad_input = grad_output.clone(memory_format=torch.contiguous_format) packed = grad_input.reshape(-1, ctx.shape[-2], ctx.shape[-1]) - _launch_cute(packed, cos, sin, cu_seqlens, ctx.module, True) + _launch_triton(packed, cos, sin, cu_seqlens, ctx.module, True) return grad_input, None, None, None, None def _apply_rotary_query(query, cos, sin, cu_seqlens): - """Use the fresh query tensor as CuTe output when its native path is available.""" - if not _can_use_cute(query, cos, sin, cu_seqlens): + """Use the fresh query tensor as Triton output when its native path is available.""" + if not _can_use_triton(query, cos, sin, cu_seqlens): return _apply_rotary_eager(query, cos, sin, cu_seqlens) - module = _cute_rope_module() - return _CuTeQueryRotary.apply(query, _rope_tables(cos), _rope_tables(sin), cu_seqlens, module) + module = _triton_rope_module() + return _TritonQueryRotary.apply(query, _rope_tables(cos), _rope_tables(sin), cu_seqlens, module) def apply_rotary(x, cos, sin, cu_seqlens=None): - """Rotate trailing interleaved pairs, using CuTe when its contract is satisfied.""" - if _can_use_cute(x, cos, sin, cu_seqlens): - return _apply_rotary_cute(x, cos, sin, cu_seqlens) + """Rotate trailing interleaved pairs, using Triton when its contract is satisfied.""" + if _can_use_triton(x, cos, sin, cu_seqlens): + return _apply_rotary_triton(x, cos, sin, cu_seqlens) return _apply_rotary_eager(x, cos, sin, cu_seqlens) diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index f83e9af2396..91db3a2a9ed 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -552,9 +552,6 @@ at::Tensor fused_rope_backward(const at::Tensor &output_grads, const at::Tensor const std::optional cu_seqlens, const int cp_size, const int cp_rank); -bool dsv4_rope_cutedsl_(at::Tensor input, const at::Tensor &cos, const at::Tensor &sin, - const at::Tensor &cu_seqlens, bool backward); - std::tuple fused_qkv_rope_forward( const at::Tensor &qkv_input, const at::Tensor &q_freqs, const at::Tensor &k_freqs, const std::optional start_positions, const std::vector &qkv_split_arg_list, diff --git a/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp b/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp index a028e02a90c..4392fa4b433 100644 --- a/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp +++ b/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp @@ -9,33 +9,6 @@ namespace transformer_engine::pytorch { -bool dsv4_rope_cutedsl_(at::Tensor input, const at::Tensor &cos, const at::Tensor &sin, - const at::Tensor &cu_seqlens, const bool backward) { - TORCH_CHECK(input.is_cuda() && cos.is_cuda() && sin.is_cuda() && cu_seqlens.is_cuda(), - "DSv4 CuTe RoPE tensors must be on CUDA."); - TORCH_CHECK(input.device() == cos.device() && input.device() == sin.device() && - input.device() == cu_seqlens.device(), - "DSv4 CuTe RoPE tensors must share one device."); - TORCH_CHECK( - input.dim() == 3 && cos.dim() == 2 && sin.sizes() == cos.sizes() && cu_seqlens.dim() == 1, - "Expected input [T,H,D], COS/SIN [P,R], and cu_seqlens [B+1]."); - TORCH_CHECK(input.is_contiguous() && cos.is_contiguous() && sin.is_contiguous() && - cu_seqlens.is_contiguous(), - "DSv4 CuTe RoPE tensors must be contiguous."); - TORCH_CHECK(input.scalar_type() == at::ScalarType::BFloat16 && - cos.scalar_type() == at::ScalarType::Float && - sin.scalar_type() == at::ScalarType::Float && - cu_seqlens.scalar_type() == at::ScalarType::Int, - "DSv4 CuTe RoPE expects BF16 input, FP32 tables, and int32 prefixes."); - - auto input_cu = makeTransformerEngineTensor(input); - auto cos_cu = makeTransformerEngineTensor(cos); - auto sin_cu = makeTransformerEngineTensor(sin); - auto cu_seqlens_cu = makeTransformerEngineTensor(cu_seqlens); - return nvte_dsv4_rope_cutedsl(input_cu.data(), cos_cu.data(), sin_cu.data(), cu_seqlens_cu.data(), - backward, at::cuda::getCurrentCUDAStream()); -} - at::Tensor fused_rope_forward(const at::Tensor &input, const at::Tensor &freqs, const std::optional start_positions, const NVTE_QKV_Format qkv_format, const bool interleaved, diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index bfa19834b54..00fdb9d15e7 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -577,8 +577,6 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { "Fused Apply RoPE FWD", py::call_guard()); m.def("fused_rope_backward", &transformer_engine::pytorch::fused_rope_backward, "Fused Apply RoPE BWD", py::call_guard()); - m.def("dsv4_rope_cutedsl_", &transformer_engine::pytorch::dsv4_rope_cutedsl_, - "In-place DSv4 CuTe DSL RoPE", py::call_guard()); m.def("fused_qkv_rope_forward", &transformer_engine::pytorch::fused_qkv_rope_forward, "Fused Apply QKV RoPE FWD", py::call_guard()); m.def("fused_qkv_rope_backward", &transformer_engine::pytorch::fused_qkv_rope_backward, From 31fc926a28f1ac0aafbfb5aa329d16d6c11335b1 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Mon, 12 Oct 2026 14:59:30 -0700 Subject: [PATCH 16/37] Make DSv4Attention the hidden-state attention layer Remove the parameter-free module that only stored window/ratio and delegated to the backend function. Call that function directly from the high-level layer and retain its FP8 rejection, giving DSv4Attention one clear hidden-state contract without an additional wrapper or compatibility alias. Prepared-tensor callers continue to use dsa_cudnn.attention. Signed-off-by: Sudhakar Singh --- tests/pytorch/test_dsv4_attention.py | 43 ++++++++-- transformer_engine/pytorch/__init__.py | 1 - .../pytorch/attention/__init__.py | 3 +- .../attention/sparse_attention/README.md | 15 ++-- .../attention/sparse_attention/__init__.py | 3 +- .../sparse_attention/dsv4_attention.py | 80 +++---------------- 6 files changed, 56 insertions(+), 89 deletions(-) diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index aaf9a18f6d4..2f65778e7d6 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -9,6 +9,7 @@ from transformer_engine.pytorch.attention.sparse_attention import ( compressor, + dsa_cudnn, dsa_rope, dsv4_attention, ) @@ -63,10 +64,12 @@ def test_dsv4_forward_backward(variant, head_dim): start += length comp_start += count - output = dsv4_attention.DSv4Attention(window_size=32, ratio=ratio)( + output = dsa_cudnn.attention( *leaves, cu, cu_comp, + window_size=32, + ratio=ratio, indices=indices, max_compressed_seqlen=max(comp_lengths) if indices is None else None, ) @@ -102,10 +105,38 @@ def test_dsv4_forward_backward(variant, head_dim): def test_dsv4_rejects_unsupported_metadata(): with pytest.raises(ValueError, match="positive"): - dsv4_attention.DSv4Attention(window_size=0, ratio=4) - core = dsv4_attention.DSv4Attention(window_size=32, ratio=4) + dsa_cudnn.attention( + torch.empty(1, 64, 512), + torch.empty(1, 512), + torch.empty(1, 512), + torch.empty(64), + None, + None, + window_size=0, + ratio=4, + ) with pytest.raises(ValueError, match="query"): - core(torch.empty(1, 4, 512), None, None, None, None, None) + dsa_cudnn.attention( + torch.empty(1, 4, 512), + None, + None, + None, + None, + None, + window_size=32, + ratio=4, + ) + + +def test_dsv4_attention_rejects_fp8(monkeypatch): + from transformer_engine.pytorch.quantization import FP8GlobalStateManager + + monkeypatch.setattr(FP8GlobalStateManager, "is_fp8_enabled", lambda: True) + layer = dsv4_attention.DSv4Attention.__new__(dsv4_attention.DSv4Attention) + torch.nn.Module.__init__(layer) + layer.input_format, layer.hidden_size = "sbd", 8 + with pytest.raises(NotImplementedError, match="BF16 only"): + layer(torch.empty(4, 1, 8)) @pytest.mark.parametrize("heads", [1, 64]) @@ -206,7 +237,7 @@ def test_dsv4_thd_rejects_malformed_metadata(prefix, maximum): def test_dsv4_thd_rejects_hidden_batch_axis_and_multi_sequence_context(): # These checks precede projections and do not require GPU-backed TE layers. - layer = dsv4_attention.DSv4HybridAttention.__new__(dsv4_attention.DSv4HybridAttention) + layer = dsv4_attention.DSv4Attention.__new__(dsv4_attention.DSv4Attention) torch.nn.Module.__init__(layer) layer.input_format, layer.hidden_size = "thd", 8 layer.is_csa, layer.compression_ratio = True, 4 @@ -258,7 +289,7 @@ def test_dsv4_hybrid_thd_forward_backward(variant, ratio, lengths): if not hasattr(getattr(cudnn, "DSA", None), "sparse_attention_forward_wrapper"): pytest.skip("DSv4 requires cuDNN Frontend 1.29 sparse attention APIs") torch.manual_seed(37) - layer = dsv4_attention.DSv4HybridAttention( + layer = dsv4_attention.DSv4Attention( hidden_size=32, q_lora_rank=16, layer_type=( diff --git a/transformer_engine/pytorch/__init__.py b/transformer_engine/pytorch/__init__.py index 36e3d46f774..9601a617a67 100644 --- a/transformer_engine/pytorch/__init__.py +++ b/transformer_engine/pytorch/__init__.py @@ -31,7 +31,6 @@ from transformer_engine.pytorch.attention import DotProductAttention from transformer_engine.pytorch.attention import ( DSv4Attention, - DSv4HybridAttention, sparse_attention, ) from transformer_engine.pytorch.attention import GatedDeltaNetAttention diff --git a/transformer_engine/pytorch/attention/__init__.py b/transformer_engine/pytorch/attention/__init__.py index db2d9b4be54..2b8ef866340 100644 --- a/transformer_engine/pytorch/attention/__init__.py +++ b/transformer_engine/pytorch/attention/__init__.py @@ -6,7 +6,7 @@ from .dot_product_attention import DotProductAttention from . import sparse_attention -from .sparse_attention import DSv4Attention, DSv4HybridAttention +from .sparse_attention import DSv4Attention from .linear_attention import ( GatedDeltaNetAttention, GatedDeltaNet2Attention, @@ -20,7 +20,6 @@ __all__ = [ "DotProductAttention", "DSv4Attention", - "DSv4HybridAttention", "sparse_attention", "GatedDeltaNetAttention", "GatedDeltaNet2Attention", diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index 1173ba5f9f9..0818451ddee 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -1,14 +1,14 @@ # DSv4 model integration -For stateless, unpadded BF16 CSA/HCA layers, `DSv4HybridAttention` owns the +For stateless, unpadded BF16 CSA/HCA layers, `DSv4Attention` owns the DSv4 projections, norms, RoPE, compressors, indexer, sink, and grouped output projection. Its default input and output are both `[sequence, batch, hidden_size]`: ```python import torch -from transformer_engine.pytorch.attention.sparse_attention import DSv4HybridAttention +from transformer_engine.pytorch.attention.sparse_attention import DSv4Attention -attention = DSv4HybridAttention( +attention = DSv4Attention( hidden_size=4096, q_lora_rank=1536, layer_type="compressed_sparse_attention", head_dim=512, rope_head_dim=64, sliding_window=128, compression_ratio=4, o_groups=8, o_lora_rank=1024, @@ -70,13 +70,13 @@ attachment. Per-token reduction is untested. The lower-level `dsa_cudnn` calls expose compression and selection when a model owns its own projections, RMSNorm, RoPE, sink, and output projection. -`DSv4Attention` owns the parameter-free attention call: +`dsa_cudnn.attention` provides the parameter-free prepared-tensor attention call: ```text model projected KV + gates ── dsa_cudnn.compress ── model norm/RoPE ── compressed KV model index projections ───── dsa_cudnn.compress ── model norm/RoPE ── index key model index query + key + weights ───────────── dsa_cudnn.select_blocks ── IDs (CSA) -model query + local KV + compressed KV + IDs ─ dsv4_attention.DSv4Attention ── head output +model query + local KV + compressed KV + IDs ─ dsa_cudnn.attention ── head output ``` The model calls the same stages for HCA, omitting index compression and selection. @@ -88,7 +88,6 @@ rows; the attention core maps them into its combined local/compressed KV space. ```python from transformer_engine.pytorch.attention.sparse_attention import ( dsa_cudnn, - dsv4_attention, ) # Model code supplies BF16 projected tensors and applies its own norm/RoPE. @@ -112,9 +111,9 @@ if is_csa: scale=(index_head_dim * index_n_heads) ** -0.5, ) -output = dsv4_attention.DSv4Attention(window_size=window_size, ratio=ratio)( +output = dsa_cudnn.attention( query, local_kv, compressed_kv, sink.float(), cu, cu_comp, - indices=indices, + window_size=window_size, ratio=ratio, indices=indices, max_compressed_seqlen=max_compressed_seqlen if indices is None else None, ) output = model.finish_output(output) diff --git a/transformer_engine/pytorch/attention/sparse_attention/__init__.py b/transformer_engine/pytorch/attention/sparse_attention/__init__.py index 5cd7bb8cc53..1bc31b27637 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/__init__.py +++ b/transformer_engine/pytorch/attention/sparse_attention/__init__.py @@ -9,11 +9,10 @@ """ from . import dsa_cudnn, dsv4_attention -from .dsv4_attention import DSv4Attention, DSv4HybridAttention +from .dsv4_attention import DSv4Attention __all__ = [ "dsa_cudnn", "dsv4_attention", "DSv4Attention", - "DSv4HybridAttention", ] diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index 3e9f71b87a1..cf420c5356b 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -2,7 +2,7 @@ # # See LICENSE for license information. -"""DSv4 attention core and unpadded CSA/HCA attention layer.""" +"""DSv4 CSA/HCA hidden-state attention layer.""" from typing import Optional @@ -15,7 +15,7 @@ from .dsa_rope import _apply_rotary_query, _DSv4RotaryEmbedding, apply_rotary from .indexer import _Indexer -__all__ = ["DSv4Attention", "DSv4HybridAttention"] +__all__ = ["DSv4Attention"] @torch.compile @@ -49,72 +49,6 @@ def _packed_prefixes(hidden_states, cu_seqlens, max_seqlen, ratio): class DSv4Attention(torch.nn.Module): - """Experimental causal joint local + compressed attention for DSv4. - - Parameter-free core: the caller owns projections, compression, normalization, - RoPE, and the learned sink. Use :mod:`dsa_cudnn` for gated pooling - and CSA selection. Pass selected indices for CSA; omit them and supply - ``max_compressed_seqlen`` for HCA. - - Inputs are contiguous CUDA BF16 query [T,64,D], local KV [T,D], compressed - KV [Tc,D], FP32 sink [64], and CUDA INT32 sequence prefixes [B+1]. D is 512 - or 576; values use the first 512 KV channels. Output is [T,64,512], retaining - the head axis for DSv4's output unrotation. Prefixes start at zero and end - at valid row counts; compressed lengths are floor(sequence length / ratio). - CSA indices are distinct global packed compressed-row IDs, or -1 for padding. - - Only full sequences with positions starting at zero are supported. Local - keys satisfy max(0,q-window_size+1) <= k <= q. Compressed block j becomes - visible when (j+1)*ratio <= q+1. All visible local and compressed entries - share one softmax with the sink; its value contribution is zero. - - Requires SM100 and cuDNN Frontend 1.29.0. Forward and first-order backward - use cuDNN's DSA namespace. No dropout, arbitrary masks, cache/decode, FP8, - distributed attention, or higher-order gradients. This module does not use - DPA backend-selection flags. cuDNN operations are loaded on execution. - """ - - def __init__(self, *, window_size: int, ratio: int): - super().__init__() - if window_size <= 0 or ratio <= 0: - raise ValueError("window_size and ratio must be positive.") - self.window_size = window_size - self.ratio = ratio - - def forward( - self, - query: torch.Tensor, - local_kv: torch.Tensor, - compressed_kv: torch.Tensor, - sink: torch.Tensor, - cu_seqlens: torch.Tensor, - cu_seqlens_comp: torch.Tensor, - *, - indices: Optional[torch.Tensor] = None, - scale: Optional[float] = None, - max_compressed_seqlen: Optional[int] = None, - ) -> torch.Tensor: - """Evaluate the joint attention, with scale defaulting to D**-0.5.""" - from transformer_engine.pytorch.quantization import FP8GlobalStateManager - - if FP8GlobalStateManager.is_fp8_enabled(): - raise NotImplementedError("DSv4Attention supports BF16 only; disable TE FP8 autocast.") - return _attention( - query, - local_kv, - compressed_kv, - sink, - cu_seqlens, - cu_seqlens_comp, - window_size=self.window_size, - ratio=self.ratio, - indices=indices, - scale=scale, - max_compressed_seqlen=max_compressed_seqlen, - ) - - -class DSv4HybridAttention(torch.nn.Module): """DSv4 CSA/HCA attention in ``sbd``, ``bsd``, or packed ``thd`` format. This first-pass layer handles full-sequence BF16 attention forward/backward @@ -234,7 +168,7 @@ def __init__( self.sinks = torch.nn.Parameter( torch.zeros(self.num_heads, device=device, dtype=params_dtype) ) - self.core_attention = DSv4Attention(window_size=sliding_window, ratio=compression_ratio) + self.sliding_window = sliding_window # A model can supply its RoPE frequencies (for example YaRN) without # duplicating that policy here. It must return the same token/compressed # (cos, sin) pairs as _DSv4RotaryEmbedding.forward. @@ -278,6 +212,10 @@ def forward( or hidden_states.shape[-1] != self.hidden_size ): raise ValueError("hidden_states must have the selected format and hidden_size.") + from transformer_engine.pytorch.quantization import FP8GlobalStateManager + + if FP8GlobalStateManager.is_fp8_enabled(): + raise NotImplementedError("DSv4Attention supports BF16 only; disable TE FP8 autocast.") if return_indexer_context and not self.is_csa: raise ValueError("Only CSA has an indexer context.") if self.input_format == "sbd": @@ -358,13 +296,15 @@ def forward( else: indices = index_result - output = self.core_attention( + output = _attention( q, local_kv, compressed_kv, self.sinks.float(), cu, cu_comp, + window_size=self.sliding_window, + ratio=self.compression_ratio, indices=indices, max_compressed_seqlen=None if self.is_csa else seq // self.compression_ratio, ).reshape(*token_shape, self.num_heads, self.head_dim) From 17e4b0de81fa99015ba938942b9b8beaa5b76f98 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Mon, 12 Oct 2026 15:28:44 -0700 Subject: [PATCH 17/37] Align DSv4 layout naming with TE attention Use qkv_format with sbhd, bshd, and thd as in MultiheadAttention and DotProductAttention. The names describe the attention layout while high-level hidden states omit the head axis. Preserve the same dense and packed computation without aliases for the retired spelling. Signed-off-by: Sudhakar Singh --- tests/pytorch/test_dsv4_attention.py | 10 +++---- .../attention/sparse_attention/README.md | 29 +++++++++++++++---- .../sparse_attention/dsv4_attention.py | 16 +++++----- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index 2f65778e7d6..2383c6de901 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -134,7 +134,7 @@ def test_dsv4_attention_rejects_fp8(monkeypatch): monkeypatch.setattr(FP8GlobalStateManager, "is_fp8_enabled", lambda: True) layer = dsv4_attention.DSv4Attention.__new__(dsv4_attention.DSv4Attention) torch.nn.Module.__init__(layer) - layer.input_format, layer.hidden_size = "sbd", 8 + layer.qkv_format, layer.hidden_size = "sbhd", 8 with pytest.raises(NotImplementedError, match="BF16 only"): layer(torch.empty(4, 1, 8)) @@ -239,7 +239,7 @@ def test_dsv4_thd_rejects_hidden_batch_axis_and_multi_sequence_context(): # These checks precede projections and do not require GPU-backed TE layers. layer = dsv4_attention.DSv4Attention.__new__(dsv4_attention.DSv4Attention) torch.nn.Module.__init__(layer) - layer.input_format, layer.hidden_size = "thd", 8 + layer.qkv_format, layer.hidden_size = "thd", 8 layer.is_csa, layer.compression_ratio = True, 4 with pytest.raises(ValueError, match="selected format"): layer(torch.empty(16, 1, 8)) @@ -306,7 +306,7 @@ def test_dsv4_hybrid_thd_forward_backward(variant, ratio, lengths): dsa_rope._DSv4RotaryEmbedding(ratio, 64, 12345.0, "cuda") if variant == "csa" else None ), params_dtype=torch.bfloat16, - input_format="thd", + qkv_format="thd", ) x = torch.randn(sum(lengths), 32, device="cuda", dtype=torch.bfloat16, requires_grad=True) cu = torch.tensor([0, lengths[0], sum(lengths)], device=x.device, dtype=torch.int32) @@ -318,8 +318,8 @@ def test_dsv4_hybrid_thd_forward_backward(variant, ratio, lengths): name: p.grad.clone() for name, p in layer.named_parameters() if p.grad is not None } layer.zero_grad(set_to_none=True) - layer.input_format = "sbd" if variant == "hca" else "bsd" - batch_axis = 1 if layer.input_format == "sbd" else 0 + layer.qkv_format = "sbhd" if variant == "hca" else "bshd" + batch_axis = 1 if layer.qkv_format == "sbhd" else 0 reference_input = x.detach().clone().requires_grad_() expected = torch.cat( [ diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index 0818451ddee..334ca943f4a 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -19,19 +19,36 @@ output = attention(hidden_states) # [S, B, 4096] -> [S, B, 4096] ``` ```text -SBD ── TE query/local-KV projections + norms ───────────────┐ - ├── TE compressor projections/norm ── cuDNN compression ──┼── cuDNN attention ── TE grouped output ── SBD +SBHD ── TE query/local-KV projections + norms ───────────────┐ + ├── TE compressor projections/norm ── cuDNN compression ──┼── cuDNN attention ── TE grouped output ── SBHD └── (CSA) TE index projections/norm ── cuDNN compression ── cuDNN selection ──┘ ``` The high-level layer uses 64 heads of width 512 and defaults to plain interleaved partial RoPE with `rope_theta`; callers may inject a `rope` module returning token and compressed `(cos, sin)` pairs. -Use `input_format="bsd"` for batch-major callers. The SBD boundary conversion +The layout names follow TE MHA/DPA conventions; hidden states omit the head axis: + +| `qkv_format` | Hidden input/output | Sequence lengths | +| --- | --- | --- | +| `sbhd` (default) | `[S,B,D]` | Equal within a dense batch | +| `bshd` | `[B,S,D]` | Equal within a dense batch | +| `thd` | `[T,D]` | May differ between packed sequences | + +For packed sequences, pass CUDA INT32 prefixes and a maximum sequence length: + +```python +attention = DSv4Attention(..., qkv_format="thd") +output = attention(hidden_states, cu_seqlens_q=cu, max_seqlen_q=max_length) +# hidden_states/output: [T,D]; cu: [0, length_0, length_0 + length_1, ...] +``` + +Use `qkv_format="bshd"` for batch-major callers. The SBHD boundary conversion is a view when batch size is one; larger batches require a repack for the current cuDNN packed-row contract. -It supports full sequences starting at position zero, equal unpadded lengths within -a batch, and at least one complete compression window. Sliding-only layers, +It supports full sequences starting at position zero and at least one complete +compression window. Dense layouts require equal unpadded lengths within a batch; +THD supports unequal packed lengths. Sliding-only layers, cache/decode, padding, TP/CP, and FP8 remain outside this first pass. The indexer auxiliary loss and its attachment remain caller-owned; without one, the language-model loss does not train `indexer.q_proj` or the @@ -57,7 +74,7 @@ The cuDNN calls used by this layer are: | Fused indexer score + top-k | — | Once | | Sparse attention forward/backward | Once | Once | -For CSA, `return_indexer_context=True` returns `(SBD_output, context)` by default; HCA rejects +For CSA, `return_indexer_context=True` returns `(SBHD_output, context)` by default; HCA rejects it. The live BF16 index Q/K/W have shapes `[B*S,H,128]`, `[B*S/ratio,128]`, and `[B*S,H]`; selected IDs are `[B*S,topk]` global compressed-row IDs. Detached attention Q/local KV/compressed KV/sink have shapes `[B*S,64,512]`, diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index cf420c5356b..114bce21611 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -49,7 +49,7 @@ def _packed_prefixes(hidden_states, cu_seqlens, max_seqlen, ratio): class DSv4Attention(torch.nn.Module): - """DSv4 CSA/HCA attention in ``sbd``, ``bsd``, or packed ``thd`` format. + """DSv4 CSA/HCA attention in ``sbhd``, ``bshd``, or packed ``thd`` format. This first-pass layer handles full-sequence BF16 attention forward/backward on SM100. By default, projections that share an input use one contiguous @@ -86,7 +86,7 @@ def __init__( max_seqlen: Optional[int] = None, device: str = "cuda", params_dtype: torch.dtype = torch.float32, - input_format: str = "sbd", + qkv_format: str = "sbhd", _fuse_projections: bool = True, ): super().__init__() @@ -95,8 +95,8 @@ def __init__( self.index_head_dim = 128 if layer_type not in ("compressed_sparse_attention", "heavily_compressed_attention"): raise ValueError("Only DSv4 CSA and HCA are supported.") - if input_format not in ("sbd", "bsd", "thd"): - raise ValueError("input_format must be 'sbd', 'bsd', or 'thd'.") + if qkv_format not in ("sbhd", "bshd", "thd"): + raise ValueError("qkv_format must be 'sbhd', 'bshd', or 'thd'.") if head_dim != 512 or rope_head_dim < 2 or rope_head_dim % 2 or rope_head_dim > head_dim: raise ValueError("head_dim must be 512; rope_head_dim must be even and fit.") if any( @@ -187,7 +187,7 @@ def __init__( self.compression_ratio = compression_ratio self.o_groups, self.o_lora_rank = o_groups, o_lora_rank self.is_csa = is_csa - self.input_format = input_format + self.qkv_format = qkv_format def forward( self, @@ -206,7 +206,7 @@ def forward( contract and are not copied to the host for validation. Dense formats do not accept these metadata arguments. """ - packed = self.input_format == "thd" + packed = self.qkv_format == "thd" if ( hidden_states.ndim != (2 if packed else 3) or hidden_states.shape[-1] != self.hidden_size @@ -218,7 +218,7 @@ def forward( raise NotImplementedError("DSv4Attention supports BF16 only; disable TE FP8 autocast.") if return_indexer_context and not self.is_csa: raise ValueError("Only CSA has an indexer context.") - if self.input_format == "sbd": + if self.qkv_format == "sbhd": # cuDNN consumes batch-major packed rows. B=1 needs only a view; # larger batches require a physical repack with the current core. hidden_states = hidden_states.transpose(0, 1).contiguous() @@ -313,7 +313,7 @@ def forward( weight = self.o_a_proj.weight.reshape(self.o_groups, self.o_lora_rank, -1) grouped = torch.einsum("...gd,grd->...gr", output, weight).flatten(-2) result = self.o_b_proj(grouped) - if self.input_format == "sbd": + if self.qkv_format == "sbhd": result = result.transpose(0, 1).contiguous() if return_indexer_context: return result, { From 69661dbe80bfb5353a944db14279ec08939b0883 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Tue, 13 Oct 2026 11:45:25 -0700 Subject: [PATCH 18/37] Clarify the DSv4 API stack and sparse attention entry point Name the prepared-tensor operation sparse_attention to distinguish it from the hidden-state layer. Update callers and tests together, and document the module hierarchy and CSA/HCA usage without repeating implementation detail. Signed-off-by: Sudhakar Singh --- tests/pytorch/test_dsv4_attention.py | 6 +- .../attention/sparse_attention/README.md | 201 ++++++++---------- .../attention/sparse_attention/dsa_cudnn.py | 4 +- .../sparse_attention/dsv4_attention.py | 4 +- 4 files changed, 90 insertions(+), 125 deletions(-) diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index 2383c6de901..89ccd40eaf3 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -64,7 +64,7 @@ def test_dsv4_forward_backward(variant, head_dim): start += length comp_start += count - output = dsa_cudnn.attention( + output = dsa_cudnn.sparse_attention( *leaves, cu, cu_comp, @@ -105,7 +105,7 @@ def test_dsv4_forward_backward(variant, head_dim): def test_dsv4_rejects_unsupported_metadata(): with pytest.raises(ValueError, match="positive"): - dsa_cudnn.attention( + dsa_cudnn.sparse_attention( torch.empty(1, 64, 512), torch.empty(1, 512), torch.empty(1, 512), @@ -116,7 +116,7 @@ def test_dsv4_rejects_unsupported_metadata(): ratio=4, ) with pytest.raises(ValueError, match="query"): - dsa_cudnn.attention( + dsa_cudnn.sparse_attention( torch.empty(1, 4, 512), None, None, diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index 334ca943f4a..2e348eded0f 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -1,118 +1,48 @@ -# DSv4 model integration +# DSv4 attention API -For stateless, unpadded BF16 CSA/HCA layers, `DSv4Attention` owns the -DSv4 projections, norms, RoPE, compressors, indexer, sink, and grouped output -projection. Its default input and output are both `[sequence, batch, hidden_size]`: +`DSv4Attention` is an experimental API in TE that composes cuDNN compression, +top-k indexing, and sparse attention kernels with Megatron's DSA RoPE kernels +through helper submodules such as Compressor (`_Compressor`) and Indexer +(`_Indexer`). cuDNN supplies compression and sparse attention backward kernels +and separate indexer-loss score/backward primitives; top-k selection itself is +discrete. The layer maps hidden states to hidden states with the same shape. +The helper submodules are internal; `dsa_cudnn` exposes the underlying +operations for models that supply their own prepared tensors. -```python -import torch -from transformer_engine.pytorch.attention.sparse_attention import DSv4Attention - -attention = DSv4Attention( - hidden_size=4096, q_lora_rank=1536, - layer_type="compressed_sparse_attention", head_dim=512, rope_head_dim=64, - sliding_window=128, compression_ratio=4, o_groups=8, o_lora_rank=1024, - index_n_heads=32, index_topk=32, max_seqlen=2048, - params_dtype=torch.bfloat16, -) -output = attention(hidden_states) # [S, B, 4096] -> [S, B, 4096] -``` +## Module hierarchy ```text -SBHD ── TE query/local-KV projections + norms ───────────────┐ - ├── TE compressor projections/norm ── cuDNN compression ──┼── cuDNN attention ── TE grouped output ── SBHD - └── (CSA) TE index projections/norm ── cuDNN compression ── cuDNN selection ──┘ +DSv4Attention +├── TE query/local-KV projections + norms +├── _Compressor +│ ├── TE KV/gate projections + learned position bias +│ ├── dsa_cudnn.compress -> cuDNN compressor +│ └── TE RMSNorm +├── _Indexer (CSA only) +│ ├── TE index-query/head-weight projections +│ ├── own _Compressor (index keys) +│ ├── dsa_rope.apply_rotary (query + index keys) +│ └── dsa_cudnn.select_blocks -> cuDNN score + top-k +├── dsa_rope embeddings + rotary application +├── dsa_cudnn.sparse_attention -> cuDNN sparse attention +└── TE grouped output projection ``` -The high-level layer uses 64 heads of width 512 and defaults to plain interleaved -partial RoPE with `rope_theta`; callers may inject a `rope` module returning token -and compressed `(cos, sin)` pairs. -The layout names follow TE MHA/DPA conventions; hidden states omit the head axis: +## Inside DSv4Attention -| `qkv_format` | Hidden input/output | Sequence lengths | -| --- | --- | --- | -| `sbhd` (default) | `[S,B,D]` | Equal within a dense batch | -| `bshd` | `[B,S,D]` | Equal within a dense batch | -| `thd` | `[T,D]` | May differ between packed sequences | - -For packed sequences, pass CUDA INT32 prefixes and a maximum sequence length: - -```python -attention = DSv4Attention(..., qkv_format="thd") -output = attention(hidden_states, cu_seqlens_q=cu, max_seqlen_q=max_length) -# hidden_states/output: [T,D]; cu: [0, length_0, length_0 + length_1, ...] -``` - -Use `qkv_format="bshd"` for batch-major callers. The SBHD boundary conversion -is a view when batch size is one; larger batches require a repack for the current -cuDNN packed-row contract. -It supports full sequences starting at position zero and at least one complete -compression window. Dense layouts require equal unpadded lengths within a batch; -THD supports unequal packed lengths. Sliding-only layers, -cache/decode, padding, TP/CP, and FP8 remain outside this first pass. The -indexer auxiliary loss and its attachment remain caller-owned; without one, the -language-model loss does not train `indexer.q_proj` or the -index-key/weight slices of `indexer.compressor.fused_proj`: top-k selection has no gradient. - -`dsa_rope.py` caches FP32 token frequencies (prebuilt when `max_seqlen` is -provided) and shares window-start slices with the CSA `_Indexer`. The indexer -owns its projections, index-key compressor, and block selection; neither helper -changes the cuDNN call contracts. - -The default projection layout groups operations that read the same input: one -TE `Linear` for query-down plus local KV, one for compressor KV plus gates, and -in CSA one for index-compressor KV plus gates plus per-head weights. Each weight -is contiguous; output slices feed the existing norms and cuDNN calls. The -current wrapper validates 64 attention heads. `_fuse_projections=False` retains -separate Linear calls as a comparison path. - -The cuDNN calls used by this layer are: - -| Stage | HCA | CSA | -| --- | --- | --- | -| Compressor forward/backward | Once | Twice: attention KV and index keys | -| Fused indexer score + top-k | — | Once | -| Sparse attention forward/backward | Once | Once | - -For CSA, `return_indexer_context=True` returns `(SBHD_output, context)` by default; HCA rejects -it. The live BF16 index Q/K/W have shapes `[B*S,H,128]`, `[B*S/ratio,128]`, and -`[B*S,H]`; selected IDs are `[B*S,topk]` global compressed-row IDs. Detached -attention Q/local KV/compressed KV/sink have shapes `[B*S,64,512]`, -`[B*S,512]`, `[B*S/ratio,512]`, and `[64]`. The opt-in context and dense -helpers currently support B1 BF16; the helpers take BSHD views plus a -caller-supplied FP32 full LSE `[1,S,64]` (local -window, all eligible compressed keys, and sink). Score helpers are not autograd -operations; the caller owns the mean-reduced scalar KL, coefficient, and loss -attachment. Per-token reduction is untested. - -The lower-level `dsa_cudnn` calls expose compression and selection when -a model owns its own projections, RMSNorm, RoPE, sink, and output projection. -`dsa_cudnn.attention` provides the parameter-free prepared-tensor attention call: - -```text -model projected KV + gates ── dsa_cudnn.compress ── model norm/RoPE ── compressed KV -model index projections ───── dsa_cudnn.compress ── model norm/RoPE ── index key -model index query + key + weights ───────────── dsa_cudnn.select_blocks ── IDs (CSA) -model query + local KV + compressed KV + IDs ─ dsa_cudnn.attention ── head output -``` - -The model calls the same stages for HCA, omitting index compression and selection. -It passes `indices=None` and `max_compressed_seqlen` to attention. CSA calls -`compress` twice because its attention memory and index keys use separate -projections. `select_blocks` returns global IDs into the packed **compressed** -rows; the attention core maps them into its combined local/compressed KV space. +The following sketch expands the cuDNN calls underneath Compressor, Indexer, and attention. +Query and local KV are already projected, normalized, and rotated; tensors are contiguous CUDA BF16, with FP32 position bias/sink and CUDA INT32 sequence prefixes. ```python from transformer_engine.pytorch.attention.sparse_attention import ( dsa_cudnn, ) -# Model code supplies BF16 projected tensors and applies its own norm/RoPE. pooled = dsa_cudnn.compress( kv, gates, position_bias.float(), cu, cu_comp, ratio=ratio, overlap=is_csa, total_comp=total_comp, ) -compressed_kv = model.finish_compressed(pooled) +compressed_kv = ... # Apply RMSNorm and RoPE to pooled rows. indices = None if is_csa: @@ -128,29 +58,64 @@ if is_csa: scale=(index_head_dim * index_n_heads) ** -0.5, ) -output = dsa_cudnn.attention( +output = dsa_cudnn.sparse_attention( query, local_kv, compressed_kv, sink.float(), cu, cu_comp, window_size=window_size, ratio=ratio, indices=indices, max_compressed_seqlen=max_compressed_seqlen if indices is None else None, ) -output = model.finish_output(output) +output = ... # Apply inverse RoPE and grouped output projection. +``` + +`dsa_rope` supplies token/compressed-position embeddings and rotary application. +A caller may override the default embeddings with a `rope` module returning +token and compressed `(cos, sin)` pairs. + +## Usage + +```python +import torch +from transformer_engine.pytorch.attention.sparse_attention import DSv4Attention + +attention = DSv4Attention( + hidden_size=4096, q_lora_rank=1536, + layer_type="compressed_sparse_attention", head_dim=512, rope_head_dim=64, + sliding_window=128, compression_ratio=4, o_groups=8, o_lora_rank=1024, + index_n_heads=32, index_topk=32, max_seqlen=2048, + params_dtype=torch.bfloat16, +) +output = attention(hidden_states) # [S, B, 4096] -> [S, B, 4096] +``` + +HCA uses non-overlapping compression and omits the Indexer: + +```python +hca = DSv4Attention( + hidden_size=4096, q_lora_rank=1536, + layer_type="heavily_compressed_attention", head_dim=512, rope_head_dim=64, + sliding_window=128, compression_ratio=128, o_groups=8, o_lora_rank=1024, + max_seqlen=2048, params_dtype=torch.bfloat16, +) +output = hca(hidden_states) # [S, B, 4096] -> [S, B, 4096] ``` -This is a wiring sketch: `model.finish_*` are model-owned transforms, not TE APIs. -The model must apply its own normalization and RoPE to `query`, `local_kv`, -`compressed_kv`, `index_query`, and `index_key` before their TE calls. `cu` and -`cu_comp` are CUDA INT32 packed-sequence prefixes; `total_comp` is the valid -number of compressed rows. All Q/KV tensors passed to these stages are -contiguous CUDA BF16. For CSA, pass the raw BF16 `index_weights` projection and -put both positive index scaling factors in `scale` to preserve the FP32 score -scale without an extra BF16 weight rounding. The cuDNN compressor pools with -FP32 intermediates and one BF16 output cast, so keys can differ slightly from -an eager model that rounds its softmax weights or products to BF16. - -The current core supports 64 query heads and head width 512 or 576 on SM100; -the selector supports 32 or 64 index heads of width 128. It handles -full-sequence prefill starting at position zero, with exact packed row counts -in the prefix arrays. A model using segment-local compressed IDs must convert -them to the global packed IDs expected here. Cache/decode, padded-capacity -attention rows, context parallelism, and the separate indexer auxiliary loss -are outside this API. +Supported input tensor shapes are "batch-first" (shape `BSD`), "sequence-first" (shape `SBD`), +and packed (shape `TD`). Packed calls require contiguous CUDA INT32 +`cu_seqlens_q [B+1]` (from zero to `T`) and `max_seqlen_q`: + +```python +# Construct attention with qkv_format="thd". +output = attention(hidden_states, cu_seqlens_q=cu, max_seqlen_q=max_length) +``` + +## Note on support + +The current layer supports full-sequence BF16 forward/backward on SM100, +with 64 heads of width 512 and at least one complete compression window +per sequence. Sequences start at position zero; cache/decode, padding, +TP/SP/CP, FP8, and sliding-only layers are unsupported. + +Compression and attention support autograd; top-k selection is discrete. +CSA's `return_indexer_context=True` returns `(output, context)` for a +single-sequence, caller-owned indexer loss. Without that objective, the +indexer is not trained by the attention output alone. Pre-attention +normalization, residuals, and loss calculation remain model-owned. diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py index a6bff8495d3..e50b3df7d24 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py @@ -18,7 +18,7 @@ __all__ = [ "compress", "select_blocks", - "attention", + "sparse_attention", "dense_indexer_loss_scores", "dense_indexer_loss_backward", ] @@ -402,7 +402,7 @@ def backward(ctx, grad_out): return result["dq"], result["dkv"], None, result["d_sink"], None -def attention( +def sparse_attention( query: torch.Tensor, local_kv: torch.Tensor, compressed_kv: torch.Tensor, diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index 114bce21611..1b49acb43b3 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -11,7 +11,7 @@ from transformer_engine.pytorch.module import Linear, RMSNorm from .compressor import _Compressor -from .dsa_cudnn import attention as _attention +from .dsa_cudnn import sparse_attention as _sparse_attention from .dsa_rope import _apply_rotary_query, _DSv4RotaryEmbedding, apply_rotary from .indexer import _Indexer @@ -296,7 +296,7 @@ def forward( else: indices = index_result - output = _attention( + output = _sparse_attention( q, local_kv, compressed_kv, From 06013cea132d54bc7f331ed1334c02b4b7ae4513 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Tue, 13 Oct 2026 11:48:42 -0700 Subject: [PATCH 19/37] Resolve DSv4 Python lint findings Use a named launch-grid function and document the rotary autograd methods. Split packed-prefix type validation from tensor checks to satisfy the Boolean-expression limit while preserving the same validation and error message. Signed-off-by: Sudhakar Singh --- .../pytorch/attention/sparse_attention/dsa_rope.py | 9 ++++++++- .../pytorch/attention/sparse_attention/dsv4_attention.py | 5 +++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py index e42882b7019..2f8059f7454 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py @@ -117,7 +117,10 @@ def _launch_triton(packed, cos, sin, cu_seqlens, module, backward): else module._autotuned_mla_rope_fwd_inplace_kernel ) heads, dim = packed.shape[-2:] - grid = lambda meta: (packed.shape[0], module.triton.cdiv(heads, meta["BLOCK_H"])) + + def grid(meta): + return packed.shape[0], module.triton.cdiv(heads, meta["BLOCK_H"]) + kernel[grid]( packed, cos, @@ -145,6 +148,7 @@ class _TritonRotary(torch.autograd.Function): @staticmethod def forward(ctx, x, cos, sin, cu_seqlens, module): + """Rotate a copy and save the frequency tables for backward.""" # cuDNN attention saves its output for backward. Cloning also preserves # apply_rotary's established out-of-place contract for every call site. output = x.clone(memory_format=torch.contiguous_format) @@ -157,6 +161,7 @@ def forward(ctx, x, cos, sin, cu_seqlens, module): @staticmethod def backward(ctx, grad_output): + """Apply the backward rotation to a copy of the output gradient.""" cos, sin, cu_seqlens = ctx.saved_tensors grad_input = grad_output.clone(memory_format=torch.contiguous_format) packed = grad_input.reshape(-1, ctx.shape[-2], ctx.shape[-1]) @@ -169,6 +174,7 @@ class _TritonQueryRotary(torch.autograd.Function): @staticmethod def forward(ctx, query, cos, sin, cu_seqlens, module): + """Rotate the fresh query and record its in-place mutation.""" packed = query.reshape(-1, query.shape[-2], query.shape[-1]) _launch_triton(packed, cos, sin, cu_seqlens, module, False) ctx.save_for_backward(cos, sin, cu_seqlens) @@ -179,6 +185,7 @@ def forward(ctx, query, cos, sin, cu_seqlens, module): @staticmethod def backward(ctx, grad_output): + """Apply the backward rotation without mutating the output gradient.""" cos, sin, cu_seqlens = ctx.saved_tensors grad_input = grad_output.clone(memory_format=torch.contiguous_format) packed = grad_input.reshape(-1, ctx.shape[-2], ctx.shape[-1]) diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index 1b49acb43b3..0f3daa85ea2 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -26,9 +26,10 @@ def _compiled_q_rms_norm(q: torch.Tensor, eps: float) -> torch.Tensor: def _packed_prefixes(hidden_states, cu_seqlens, max_seqlen, ratio): """Lower packed metadata to complete-block prefixes and a safe row capacity.""" + if not isinstance(cu_seqlens, torch.Tensor): + raise ValueError("THD requires contiguous INT32 cu_seqlens_q [B+1] on the input device.") if ( - not isinstance(cu_seqlens, torch.Tensor) - or cu_seqlens.dtype != torch.int32 + cu_seqlens.dtype != torch.int32 or cu_seqlens.ndim != 1 or cu_seqlens.numel() < 2 or not cu_seqlens.is_contiguous() From e02f6c101d8848cd567cb050f7c34e0a9f58e4d7 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Tue, 13 Oct 2026 15:37:07 -0700 Subject: [PATCH 20/37] Expose DSA building blocks and validate indexer gradients Compressor and Indexer are reusable attention building blocks, so expose them directly without changing computation or parameter layout. Test the standalone indexer score/backward primitives against a dense PyTorch KL reference and keep full-layer context checks in the DSv4 wiring test. Restrict Triton RoPE dispatch to power-of-two widths so other accepted widths retain the eager fallback. Cover width 96 forward/backward and both 32/64 index heads; all 27 focused GPU tests passed on B200. Signed-off-by: Sudhakar Singh --- tests/pytorch/test_dsv4_attention.py | 184 ++++++++++++++++-- .../attention/sparse_attention/README.md | 14 +- .../attention/sparse_attention/__init__.py | 4 + .../attention/sparse_attention/compressor.py | 11 +- .../attention/sparse_attention/dsa_rope.py | 4 +- .../sparse_attention/dsv4_attention.py | 8 +- .../attention/sparse_attention/indexer.py | 15 +- 7 files changed, 208 insertions(+), 32 deletions(-) diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index 89ccd40eaf3..8298a1a89d4 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -2,12 +2,14 @@ # # See LICENSE for license information. -"""DSv4 core against a dense oracle, including packed sequence boundaries.""" +"""DSA primitives and DSv4 layer wiring, including packed sequence boundaries.""" import pytest import torch from transformer_engine.pytorch.attention.sparse_attention import ( + Compressor, + Indexer, compressor, dsa_cudnn, dsa_rope, @@ -17,7 +19,7 @@ @pytest.mark.parametrize("variant", ["hca", "csa"]) @pytest.mark.parametrize("head_dim", [512, 576]) -def test_dsv4_forward_backward(variant, head_dim): +def test_dsa_sparse_attention_forward_backward(variant, head_dim): if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 0): pytest.skip("DSv4 requires SM100") cudnn = pytest.importorskip("cudnn") @@ -140,7 +142,7 @@ def test_dsv4_attention_rejects_fp8(monkeypatch): @pytest.mark.parametrize("heads", [1, 64]) -def test_dsv4_triton_rope_batched_forward_backward(monkeypatch, heads): +def test_dsa_triton_rope_batched_forward_backward(monkeypatch, heads): """The packed Triton kernels must reset positions at each BSHD sequence.""" if not torch.cuda.is_available(): pytest.skip("Triton DSv4 RoPE requires CUDA") @@ -184,6 +186,157 @@ def test_dsv4_triton_rope_batched_forward_backward(monkeypatch, heads): torch.testing.assert_close(query_grad, reference_grad, atol=0.03125, rtol=0.002) +def test_dsa_rope_non_power_of_two_forward_backward(monkeypatch): + """An accepted width of 96 must fall back for both rotary entry points.""" + if not torch.cuda.is_available(): + pytest.skip("DSv4 rotary dispatch requires CUDA") + if dsa_rope._triton_rope_module() is None: + pytest.skip("Triton DSL is unavailable") + torch.manual_seed(41) + x = torch.randn(17, 2, 128, device="cuda", dtype=torch.bfloat16) + (cos, sin), _ = dsa_rope.rotary_embeddings(17, 4, 96, 160000.0, x.device) + cu = torch.tensor([0, 17], device=x.device, dtype=torch.int32) + monkeypatch.setattr( + dsa_rope, "_launch_triton", lambda *_: pytest.fail("Width 96 reached Triton") + ) + assert not dsa_rope._can_use_triton(x, cos, sin, cu) + gradient = torch.randn_like(x) + reference_input = x.detach().requires_grad_() + reference = dsa_rope._apply_rotary_eager(reference_input, cos, sin, cu) + reference_grad = torch.autograd.grad(reference, reference_input, gradient)[0] + for apply in (dsa_rope.apply_rotary, dsa_rope._apply_rotary_query): + candidate_input = x.detach().requires_grad_() + candidate = apply(candidate_input, cos, sin, cu) + candidate_grad = torch.autograd.grad(candidate, candidate_input, gradient)[0] + torch.testing.assert_close(candidate, reference) + torch.testing.assert_close(candidate_grad, reference_grad) + + +@pytest.mark.parametrize("index_heads", [32, 64]) +def test_dsa_indexer_backward(index_heads): + """Train the standalone Indexer and its Compressor against a dense KL reference.""" + if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 0): + pytest.skip("DSv4 requires SM100") + cudnn = pytest.importorskip("cudnn") + if not all( + hasattr(getattr(cudnn, "DSA", None), op) + for op in ( + "dense_indexer_score_recompute_wrapper", + "dense_attn_score_recompute_wrapper", + "dense_indexer_backward_wrapper", + ) + ): + pytest.skip("DSv4 requires cuDNN dense indexer training APIs") + torch.manual_seed(43) + seq, ratio, window = 256, 4, 32 + layer = Indexer(32, 16, 128, index_heads, 8, ratio, 1e-6, "cuda", torch.bfloat16, fused=True) + x = torch.randn(1, seq, 32, device="cuda", dtype=torch.bfloat16) + q_residual = torch.randn(1, seq, 16, device="cuda", dtype=torch.bfloat16) + cu = torch.tensor([0, seq], device=x.device, dtype=torch.int32) + cu_comp = torch.tensor([0, seq // ratio], device=x.device, dtype=torch.int32) + token_rope, compressed_rope = dsa_rope.rotary_embeddings(seq, ratio, 64, 160000.0, x.device) + ids, query, key, weights = layer( + x, + q_residual, + cu, + cu_comp, + token_rope, + compressed_rope, + total_comp=seq // ratio, + max_seqlen=seq, + return_context=True, + ) + assert ids.shape == (seq, 8) + qi, ki, wi = (t.unsqueeze(0) for t in (query, key, weights)) + qa = (torch.randn(1, seq, 64, 512, device=x.device, dtype=torch.bfloat16) * 0.1).float() + kc = (torch.randn(1, seq // ratio, 512, device=x.device, dtype=torch.bfloat16) * 0.1).float() + local_kv = (torch.randn(seq, 512, device=x.device, dtype=torch.bfloat16) * 0.1).float() + sink = torch.randn(64, device=x.device) + pos = torch.arange(seq, device=x.device) + valid = (torch.arange(seq // ratio, device=x.device) + 1) * ratio <= pos[:, None] + 1 + local_mask = (pos[None, :] <= pos[:, None]) & (pos[None, :] > pos[:, None] - window) + scale_i, scale_a = (index_heads * 128) ** -0.5, 512**-0.5 + comp_logits = (torch.einsum("bshd,bkd->bshk", qa, kc) * scale_a).masked_fill( + ~valid[None, :, None, :], -torch.inf + ) + local_logits = (torch.einsum("bshd,kd->bshk", qa, local_kv) * scale_a).masked_fill( + ~local_mask[None, :, None, :], -torch.inf + ) + # The dense teacher denominator includes local keys, ALL compressed keys, and sink. + full_lse = torch.logaddexp( + torch.logaddexp(local_logits.logsumexp(-1), comp_logits.logsumexp(-1)), + sink[None, None, :], + ).contiguous() + target = (comp_logits - full_lse[..., None]).exp().sum(2) + target = target / target.sum(-1, keepdim=True).clamp_min(1e-30) + index_score, attn_score = dsa_cudnn.dense_indexer_loss_scores( + qi, + ki, + wi, + qa.to(torch.bfloat16), + kc.to(torch.bfloat16), + full_lse, + ratio=ratio, + index_scale=scale_i, + attention_scale=scale_a, + ) + actual_target = attn_score["out"] / attn_score["denom"].unsqueeze(-1).clamp_min(1e-30) + # Masked score-buffer entries are not valid teacher probabilities. + actual_target = actual_target.masked_fill(~valid[None], 0) + torch.testing.assert_close(actual_target, target, atol=2e-4, rtol=2e-2) + + qr, kr, wr = (t.detach().float().requires_grad_() for t in (qi, ki, wi)) + logits = (torch.einsum("bshd,bkd->bshk", qr, kr).relu() * wr[..., None]).sum(2) * scale_i + logits = logits.masked_fill(~valid[None], -torch.inf) + logits = torch.where(valid.any(-1)[None, :, None], logits, 0) + log_prob = logits.log_softmax(-1) + actual_log_prob = index_score["out"] - index_score["denom"].unsqueeze(-1) + torch.testing.assert_close(actual_log_prob[0][valid], log_prob[0][valid], atol=2e-3, rtol=2e-2) + coefficient, upstream = 0.01, 0.7 + loss = ( + coefficient * upstream * (target * (target.clamp_min(1e-30).log() - log_prob)).sum() / seq + ) + reference_grads = torch.autograd.grad(loss, (qr, kr, wr)) + actual_grads = dsa_cudnn.dense_indexer_loss_backward( + qi, + ki, + wi, + index_score, + attn_score, + torch.tensor(upstream, device=x.device), + ratio=ratio, + index_scale=scale_i, + loss_coeff=coefficient, + ) + for actual, expected in zip(actual_grads, reference_grads): + assert actual.shape == expected.shape and torch.isfinite(actual).all() + error = (actual.float() - expected).norm() / expected.norm().clamp_min(1e-8) + assert error < 0.03, error.item() + parameters = tuple(layer.parameters()) + candidate = torch.autograd.grad((qi, ki, wi), parameters, actual_grads) + # TE Linear releases its backward workspace; compare using a fresh forward. + _, ref_q, ref_k, ref_w = layer( + x, + q_residual, + cu, + cu_comp, + token_rope, + compressed_rope, + total_comp=seq // ratio, + max_seqlen=seq, + return_context=True, + ) + reference = torch.autograd.grad( + (ref_q, ref_k, ref_w), + parameters, + tuple(g[0].to(t.dtype) for g, t in zip(reference_grads, (ref_q, ref_k, ref_w))), + ) + for actual, expected in zip(candidate, reference): + assert torch.isfinite(actual).all() and actual.float().norm() > 0 + error = (actual.float() - expected.float()).norm() / expected.float().norm().clamp_min(1e-8) + assert error < 0.03, error.item() + + def test_dsv4_thd_compressed_prefixes(): """Floor each sequence independently, retaining a safe allocation capacity.""" for lengths, capacity in (([129, 257], 96), ([131, 257], 97)): @@ -195,9 +348,9 @@ def test_dsv4_thd_compressed_prefixes(): assert maximum == 257 -def test_dsv4_thd_compressor_clears_unused_capacity(monkeypatch): +def test_dsa_compressor_clears_unused_capacity(monkeypatch): """Unused cuDNN output must not contaminate normalization weight gradients.""" - layer = compressor._Compressor.__new__(compressor._Compressor) + layer = Compressor.__new__(Compressor) torch.nn.Module.__init__(layer) layer.kv_proj = layer.gate_proj = torch.nn.Identity() layer.kv_norm = torch.nn.RMSNorm(4) @@ -254,7 +407,7 @@ def test_dsv4_thd_rejects_hidden_batch_axis_and_multi_sequence_context(): ) -def test_dsv4_packed_rope_eager_forward_backward(): +def test_dsa_packed_rope_eager_forward_backward(): """Reset per-sequence positions and safely rotate unused compressed capacity.""" torch.manual_seed(31) cu = torch.tensor([0, 3, 8], dtype=torch.int32) @@ -321,12 +474,19 @@ def test_dsv4_hybrid_thd_forward_backward(variant, ratio, lengths): layer.qkv_format = "sbhd" if variant == "hca" else "bshd" batch_axis = 1 if layer.qkv_format == "sbhd" else 0 reference_input = x.detach().clone().requires_grad_() - expected = torch.cat( - [ - layer(part.unsqueeze(batch_axis)).squeeze(batch_axis) - for part in reference_input.split(lengths) - ] - ) + parts = [] + for part in reference_input.split(lengths): + if variant == "csa": + output, context = layer(part.unsqueeze(batch_axis), return_indexer_context=True) + assert all(context[name].requires_grad for name in ("index_q", "index_k", "index_w")) + assert not any( + context[name].requires_grad + for name in ("attn_q", "local_kv", "compressed_kv", "sink") + ) + else: + output = layer(part.unsqueeze(batch_axis)) + parts.append(output.squeeze(batch_axis)) + expected = torch.cat(parts) expected.backward(gradient) torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2) for packed_grad, reference_grad in [(x.grad, reference_input.grad)] + [ diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index 2e348eded0f..0af9bae9fbf 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -2,25 +2,25 @@ `DSv4Attention` is an experimental API in TE that composes cuDNN compression, top-k indexing, and sparse attention kernels with Megatron's DSA RoPE kernels -through helper submodules such as Compressor (`_Compressor`) and Indexer -(`_Indexer`). cuDNN supplies compression and sparse attention backward kernels +through reusable building blocks such as `Compressor` and `Indexer`. cuDNN +supplies compression and sparse attention backward kernels and separate indexer-loss score/backward primitives; top-k selection itself is discrete. The layer maps hidden states to hidden states with the same shape. -The helper submodules are internal; `dsa_cudnn` exposes the underlying -operations for models that supply their own prepared tensors. +`Compressor` and `Indexer` can also be composed into other attention layers; +`dsa_cudnn` exposes the underlying operations for prepared tensors. ## Module hierarchy ```text DSv4Attention ├── TE query/local-KV projections + norms -├── _Compressor +├── Compressor │ ├── TE KV/gate projections + learned position bias │ ├── dsa_cudnn.compress -> cuDNN compressor │ └── TE RMSNorm -├── _Indexer (CSA only) +├── Indexer (CSA only) │ ├── TE index-query/head-weight projections -│ ├── own _Compressor (index keys) +│ ├── own Compressor (index keys) │ ├── dsa_rope.apply_rotary (query + index keys) │ └── dsa_cudnn.select_blocks -> cuDNN score + top-k ├── dsa_rope embeddings + rotary application diff --git a/transformer_engine/pytorch/attention/sparse_attention/__init__.py b/transformer_engine/pytorch/attention/sparse_attention/__init__.py index 1bc31b27637..21c762380e8 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/__init__.py +++ b/transformer_engine/pytorch/attention/sparse_attention/__init__.py @@ -9,10 +9,14 @@ """ from . import dsa_cudnn, dsv4_attention +from .compressor import Compressor from .dsv4_attention import DSv4Attention +from .indexer import Indexer __all__ = [ "dsa_cudnn", "dsv4_attention", "DSv4Attention", + "Compressor", + "Indexer", ] diff --git a/transformer_engine/pytorch/attention/sparse_attention/compressor.py b/transformer_engine/pytorch/attention/sparse_attention/compressor.py index 49832b1af51..923abbcebd9 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/compressor.py +++ b/transformer_engine/pytorch/attention/sparse_attention/compressor.py @@ -2,7 +2,7 @@ # # See LICENSE for license information. -"""Learned DSv4 projections and norm around cuDNN compression.""" +"""Learned projections and normalization around cuDNN compression.""" import torch @@ -11,8 +11,13 @@ from .dsa_cudnn import compress -class _Compressor(torch.nn.Module): - """Project KV/gates and normalize the cuDNN-pooled rows.""" +class Compressor(torch.nn.Module): + """Project KV/gates and normalize the cuDNN-pooled rows. + + Accept batch-major ``[B,S,D]`` or packed ``[T,D]`` hidden states with + token/compressed prefixes. Return ``[total_comp,width]`` rows before RoPE, + plus projected head weights when ``weight_width`` is nonzero in fused mode. + """ def __init__( self, diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py index 2f8059f7454..c6c2d75d9c2 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py @@ -89,7 +89,9 @@ def _can_use_triton(x, cos, sin, cu_seqlens): and x.ndim in (3, 4) and x.stride(-1) == 1 and cos.shape == sin.shape - and cos.shape[-1] % 4 == 0 + and cos.shape[-1] >= 4 + # The Triton kernels use tl.arange over half the rotary width. + and cos.shape[-1] & (cos.shape[-1] - 1) == 0 and not cos.requires_grad and not sin.requires_grad and _triton_rope_module() is not None diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index 0f3daa85ea2..78829a8a914 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -10,10 +10,10 @@ from transformer_engine.pytorch.module import Linear, RMSNorm -from .compressor import _Compressor +from .compressor import Compressor from .dsa_cudnn import sparse_attention as _sparse_attention from .dsa_rope import _apply_rotary_query, _DSv4RotaryEmbedding, apply_rotary -from .indexer import _Indexer +from .indexer import Indexer __all__ = ["DSv4Attention"] @@ -139,7 +139,7 @@ def __init__( if not _fuse_projections: self.kv_proj = Linear(hidden_size, head_dim, **kw) self.kv_norm = RMSNorm(head_dim, eps=rms_norm_eps, device=device, dtype=params_dtype) - self.compressor = _Compressor( + self.compressor = Compressor( hidden_size, head_dim, compression_ratio, @@ -150,7 +150,7 @@ def __init__( fused=_fuse_projections, ) if is_csa: - self.indexer = _Indexer( + self.indexer = Indexer( hidden_size, q_lora_rank, self.index_head_dim, diff --git a/transformer_engine/pytorch/attention/sparse_attention/indexer.py b/transformer_engine/pytorch/attention/sparse_attention/indexer.py index 38a699b8d52..c0b8b09243e 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/indexer.py +++ b/transformer_engine/pytorch/attention/sparse_attention/indexer.py @@ -2,19 +2,24 @@ # # See LICENSE for license information. -"""CSA index projections, compression, and cuDNN block selection.""" +"""Index projections, compression, and cuDNN block selection.""" import torch from transformer_engine.pytorch.module import Linear -from .compressor import _Compressor +from .compressor import Compressor from .dsa_cudnn import select_blocks from .dsa_rope import apply_rotary -class _Indexer(torch.nn.Module): - """Select compressed rows using the indexer's own Q/K/weight projections.""" +class Indexer(torch.nn.Module): + """Select compressed rows using the indexer's own Q/K/weight projections. + + Accept batch-major or packed hidden states and query residuals, sequence + prefixes, and caller-provided RoPE tables. Return global compressed-row IDs; + ``return_context=True`` also returns live Q/K/head weights for a separate loss. + """ def __init__( self, @@ -31,7 +36,7 @@ def __init__( fused, ): super().__init__() - self.compressor = _Compressor( + self.compressor = Compressor( hidden_size, head_dim, ratio, From a5d597f45b7e3d7711ae396127736b10d365e367 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Thu, 15 Oct 2026 07:07:37 -0700 Subject: [PATCH 21/37] Clarify sparse attention contracts and naming Use generic cuDNN terminology for reusable primitives and name selection/preparation helpers after the indices they return. Trim repeated documentation while retaining kernel constraints and the distinct rotary mutation contracts. Group unfused input projections under one branch. This changes their seeded initialization order while preserving parameter names, shapes, and matched-weight computation. Local source tests and copied-weight projection checks passed. Signed-off-by: Sudhakar Singh --- tests/pytorch/test_dsv4_attention.py | 12 +++---- transformer_engine/common/triton/dsa_rope.py | 2 +- .../attention/sparse_attention/README.md | 4 +-- .../attention/sparse_attention/dsa_cudnn.py | 29 +++++++-------- .../attention/sparse_attention/dsa_rope.py | 10 ++++-- .../sparse_attention/dsv4_attention.py | 35 +++++++------------ .../attention/sparse_attention/indexer.py | 4 +-- 7 files changed, 46 insertions(+), 50 deletions(-) diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index 8298a1a89d4..7a82dc42ec3 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -21,7 +21,7 @@ @pytest.mark.parametrize("head_dim", [512, 576]) def test_dsa_sparse_attention_forward_backward(variant, head_dim): if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 0): - pytest.skip("DSv4 requires SM100") + pytest.skip("cuDNN sparse attention requires SM100") cudnn = pytest.importorskip("cudnn") dsa = getattr(cudnn, "DSA", None) if not all( @@ -31,7 +31,7 @@ def test_dsa_sparse_attention_forward_backward(variant, head_dim): "sparse_attention_backward_wrapper", ) ): - pytest.skip("DSv4 requires cuDNN Frontend 1.29 sparse attention APIs") + pytest.skip("cuDNN sparse attention APIs are unavailable") torch.manual_seed(17) ratio = 128 if variant == "hca" else 4 lengths = [256, 128] @@ -145,7 +145,7 @@ def test_dsv4_attention_rejects_fp8(monkeypatch): def test_dsa_triton_rope_batched_forward_backward(monkeypatch, heads): """The packed Triton kernels must reset positions at each BSHD sequence.""" if not torch.cuda.is_available(): - pytest.skip("Triton DSv4 RoPE requires CUDA") + pytest.skip("Triton DSA RoPE requires CUDA") if dsa_rope._triton_rope_module() is None: pytest.skip("Triton DSL is unavailable") torch.manual_seed(23) @@ -189,7 +189,7 @@ def test_dsa_triton_rope_batched_forward_backward(monkeypatch, heads): def test_dsa_rope_non_power_of_two_forward_backward(monkeypatch): """An accepted width of 96 must fall back for both rotary entry points.""" if not torch.cuda.is_available(): - pytest.skip("DSv4 rotary dispatch requires CUDA") + pytest.skip("DSA rotary dispatch requires CUDA") if dsa_rope._triton_rope_module() is None: pytest.skip("Triton DSL is unavailable") torch.manual_seed(41) @@ -216,7 +216,7 @@ def test_dsa_rope_non_power_of_two_forward_backward(monkeypatch): def test_dsa_indexer_backward(index_heads): """Train the standalone Indexer and its Compressor against a dense KL reference.""" if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 0): - pytest.skip("DSv4 requires SM100") + pytest.skip("cuDNN indexer training requires SM100") cudnn = pytest.importorskip("cudnn") if not all( hasattr(getattr(cudnn, "DSA", None), op) @@ -226,7 +226,7 @@ def test_dsa_indexer_backward(index_heads): "dense_indexer_backward_wrapper", ) ): - pytest.skip("DSv4 requires cuDNN dense indexer training APIs") + pytest.skip("cuDNN dense indexer training APIs are unavailable") torch.manual_seed(43) seq, ratio, window = 256, 4, 32 layer = Indexer(32, 16, 128, index_heads, 8, ratio, 1e-6, "cuda", torch.bfloat16, fused=True) diff --git a/transformer_engine/common/triton/dsa_rope.py b/transformer_engine/common/triton/dsa_rope.py index 5579c17084b..b7876e8220e 100644 --- a/transformer_engine/common/triton/dsa_rope.py +++ b/transformer_engine/common/triton/dsa_rope.py @@ -2,7 +2,7 @@ # # See LICENSE for license information. -"""DSv4 RoPE kernels copied from NVIDIA/Megatron-LM. +"""DSA RoPE kernels copied from NVIDIA/Megatron-LM. Source: megatron/core/fusions/fused_mla_yarn_rope_apply.py at c4129afff4867ca89b1fb7b4b911128424cdfbe8. Kernel bodies and autotuning are diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index 0af9bae9fbf..a8417d8aa91 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -22,7 +22,7 @@ DSv4Attention │ ├── TE index-query/head-weight projections │ ├── own Compressor (index keys) │ ├── dsa_rope.apply_rotary (query + index keys) -│ └── dsa_cudnn.select_blocks -> cuDNN score + top-k +│ └── dsa_cudnn.select_indices -> cuDNN score + top-k ├── dsa_rope embeddings + rotary application ├── dsa_cudnn.sparse_attention -> cuDNN sparse attention └── TE grouped output projection @@ -51,7 +51,7 @@ if is_csa: ratio=ratio, overlap=True, total_comp=total_comp, ) index_key = model.finish_index_key(index_pooled) - indices = dsa_cudnn.select_blocks( + indices = dsa_cudnn.select_indices( index_query, index_key, index_weights, cu, cu_comp, top_k=top_k, ratio=ratio, max_seqlen=max_seqlen, max_compressed_seqlen=max_compressed_seqlen, diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py index e50b3df7d24..76a95ac0b72 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py @@ -2,7 +2,7 @@ # # See LICENSE for license information. -"""Experimental PyTorch bindings for cuDNN Frontend 1.29.0 DSv4 primitives. +"""Experimental PyTorch bindings for cuDNN sparse attention primitives. Compression, selection, sparse attention forward, and backward call cuDNN Frontend APIs. There are no numerical fallbacks. Inputs use packed sequences; @@ -17,7 +17,7 @@ __all__ = [ "compress", - "select_blocks", + "select_indices", "sparse_attention", "dense_indexer_loss_scores", "dense_indexer_loss_backward", @@ -29,7 +29,8 @@ def _namespace(name): return getattr(import_module("cudnn"), name) except (ImportError, AttributeError) as exc: raise ImportError( - f"DSv4 requires nvidia-cudnn-frontend>=1.29.0 with the cudnn.{name} namespace." + f"Sparse attention requires nvidia-cudnn-frontend>=1.29.0 with the cudnn.{name}" + " namespace." ) from exc @@ -38,7 +39,7 @@ def _dsa_op(name): return getattr(_namespace("DSA"), name) except AttributeError as exc: raise ImportError( - f"DSv4 requires cudnn.DSA.{name} from nvidia-cudnn-frontend>=1.29.0." + f"Sparse attention requires cudnn.DSA.{name} from nvidia-cudnn-frontend>=1.29.0." ) from exc @@ -52,14 +53,14 @@ def _validate_tensors(reference, bf16, fp32, cu_seqlens, cu_seqlens_comp): 10, 0, ): - raise ValueError("This DSv4 implementation requires an SM100 GPU.") + raise ValueError("These cuDNN sparse attention kernels require an SM100 GPU.") for tensor in (reference, *bf16, *fp32, cu_seqlens, cu_seqlens_comp): if tensor.device != reference.device or not tensor.is_contiguous(): - raise ValueError("DSv4 inputs must be contiguous on the same CUDA device.") + raise ValueError("Inputs must be contiguous on the same CUDA device.") if any(t.dtype != torch.bfloat16 for t in (reference, *bf16)): - raise TypeError("DSv4 activations must be BF16.") + raise TypeError("Activations must be BF16.") if any(t.dtype != torch.float32 for t in fp32): - raise TypeError("DSv4 position biases and sinks must be FP32.") + raise TypeError("Position biases and sinks must be FP32.") for offsets in (cu_seqlens, cu_seqlens_comp): if offsets.dtype != torch.int32 or offsets.ndim != 1 or offsets.numel() < 2: raise ValueError("Sequence prefixes must be INT32 [batch+1].") @@ -156,7 +157,7 @@ def compress( @torch.no_grad() -def select_blocks( +def select_indices( query: torch.Tensor, compressed_key: torch.Tensor, weights: torch.Tensor, @@ -171,7 +172,7 @@ def select_blocks( ) -> torch.Tensor: """CSA selection using cuDNN's SM100 combined score + Top-K operation. - This is the indexer, with its own projected Q/K/weights, not attention Q/KV. + Inputs are the indexer's projected Q/K/head weights. BF16 query [T,H,128], compressed_key [Tc,128], weights [T,H]; H=32 or 64. Full packed-sequence prefill only: causal timeline offsets start at zero. Returns INT32 [T,top_k] indices into compressed_key's global packed rows; @@ -309,7 +310,7 @@ def dense_indexer_loss_backward( ) -def _attention_indices( +def _prepare_sparse_indices( query: torch.Tensor, cu_seqlens: torch.Tensor, cu_seqlens_comp: torch.Tensor, @@ -416,7 +417,7 @@ def sparse_attention( scale: Optional[float] = None, max_compressed_seqlen: Optional[int] = None, ) -> torch.Tensor: - """Joint local + compressed DSv4 attention with first-order autograd. + """Joint local + compressed sparse attention with first-order autograd. Full packed-sequence prefill: query [T,H,D], local_kv [T,D], compressed_kv [Tc,D], sink [H]. Q/KV have already had the model's norm/RoPE applied. @@ -437,7 +438,7 @@ def sparse_attention( reading a CUDA prefix on host. """ if query.ndim != 3 or query.shape[1] != 64: - raise ValueError("DSv4 attention requires query [T,64,D].") + raise ValueError("cuDNN sparse attention requires query [T,64,D].") if query.shape[-1] not in (512, 576): raise ValueError("attention head dimension must be 512 or 576.") if local_kv.ndim != 2 or compressed_kv.ndim != 2: @@ -458,7 +459,7 @@ def sparse_attention( raise ValueError("CSA indices must be CUDA INT32 on the query device.") if scale is None: scale = query.shape[-1] ** -0.5 - selected = _attention_indices( + selected = _prepare_sparse_indices( query, cu_seqlens, cu_seqlens_comp, diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py index c6c2d75d9c2..26cf10b1fc6 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py @@ -2,7 +2,7 @@ # # See LICENSE for license information. -"""DSv4's trailing-channel, interleaved RoPE for full-sequence attention.""" +"""Trailing-channel, interleaved DSA RoPE for full-sequence attention.""" from functools import lru_cache @@ -73,7 +73,7 @@ def _apply_rotary_eager(x, cos, sin, cu_seqlens=None): @lru_cache(maxsize=1) def _triton_rope_module(): - """Load the optional Triton implementation only when DSv4 can use it.""" + """Load the optional Triton DSA RoPE implementation.""" try: from transformer_engine.common.triton import dsa_rope except ImportError: @@ -196,7 +196,11 @@ def backward(ctx, grad_output): def _apply_rotary_query(query, cos, sin, cu_seqlens): - """Use the fresh query tensor as Triton output when its native path is available.""" + """Rotate fresh normalized queries in place with Triton, avoiding a copy. + + Unlike apply_rotary, this private path requires an input safe to mutate. + Its backward and eager fallback remain out of place. + """ if not _can_use_triton(query, cos, sin, cu_seqlens): return _apply_rotary_eager(query, cos, sin, cu_seqlens) module = _triton_rope_module() diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index 78829a8a914..a8f6b31332b 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -25,7 +25,7 @@ def _compiled_q_rms_norm(q: torch.Tensor, eps: float) -> torch.Tensor: def _packed_prefixes(hidden_states, cu_seqlens, max_seqlen, ratio): - """Lower packed metadata to complete-block prefixes and a safe row capacity.""" + """Build compressed sequence prefixes and output capacity.""" if not isinstance(cu_seqlens, torch.Tensor): raise ValueError("THD requires contiguous INT32 cu_seqlens_q [B+1] on the input device.") if ( @@ -52,19 +52,13 @@ def _packed_prefixes(hidden_states, cu_seqlens, max_seqlen, ratio): class DSv4Attention(torch.nn.Module): """DSv4 CSA/HCA attention in ``sbhd``, ``bshd``, or packed ``thd`` format. - This first-pass layer handles full-sequence BF16 attention forward/backward - on SM100. By default, projections that share an input use one contiguous - TE Linear weight and split its output before the separate DSv4 operations. - It uses TE Linear/RMSNorm, the cuDNN DSv4 compressor and selector, and the - existing parameter-free cuDNN attention core. Sequences in a batch have - the same length in dense formats; THD accepts unequal lengths as ``[T,D]``. - Every sequence starts at position zero. The model has 64 heads of width - 512; CSA index heads have width 128, matching the current cuDNN kernels. - The low-level core's 576-wide Q/K mode returns only 512 value channels, - which has no verified output-unrotation contract here. Cache/decode, - physical inter-sequence padding, TP/SP/CP, FP8, sliding-only attention, and - the indexer auxiliary loss are unsupported. In CSA, top-k is discrete; - the indexer projections do not receive gradients from the returned hidden states alone. + Composes TE projections/norms, Compressor, Indexer, and RoPE with cuDNN + sparse attention. Supports full-sequence BF16 forward/backward on SM100 + with 64 heads of width 512 and CSA index heads of width 128. Shared-input + projections are fused by default. Each sequence starts at position zero. + Cache/decode, physical inter-sequence padding, TP/SP/CP, FP8, and sliding-only + attention are unsupported. The model owns the indexer auxiliary loss; + discrete top-k selection does not train the indexer from attention output alone. """ def __init__( @@ -133,11 +127,10 @@ def __init__( ) else: self.q_a_proj = Linear(hidden_size, q_lora_rank, **kw) + self.kv_proj = Linear(hidden_size, head_dim, **kw) # LayerNormLinear would normalize before projection; DSv4 normalizes after it. self.q_a_norm = RMSNorm(q_lora_rank, eps=rms_norm_eps, device=device, dtype=params_dtype) self.q_b_proj = Linear(q_lora_rank, self.num_heads * head_dim, **kw) - if not _fuse_projections: - self.kv_proj = Linear(hidden_size, head_dim, **kw) self.kv_norm = RMSNorm(head_dim, eps=rms_norm_eps, device=device, dtype=params_dtype) self.compressor = Compressor( hidden_size, @@ -200,12 +193,10 @@ def forward( ): """Run full self-attention and optionally expose single-sequence CSA context. - THD input/output is ``[T,D]`` and requires contiguous INT32 ``cu_seqlens_q`` - of shape ``[B+1]`` on the input device. Prefixes start at zero and end at - T, without physical padding or global offsets. ``max_seqlen_q`` is a - required upper bound covering every sequence. Prefix contents are a caller - contract and are not copied to the host for validation. Dense formats - do not accept these metadata arguments. + Packed calls require contiguous INT32 ``cu_seqlens_q`` on the input + device and ``max_seqlen_q``. Prefixes have no padding/global offsets; + their contents are caller-owned and are not checked on the host. + These metadata arguments are accepted only for ``thd``. """ packed = self.qkv_format == "thd" if ( diff --git a/transformer_engine/pytorch/attention/sparse_attention/indexer.py b/transformer_engine/pytorch/attention/sparse_attention/indexer.py index c0b8b09243e..6b163ad4908 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/indexer.py +++ b/transformer_engine/pytorch/attention/sparse_attention/indexer.py @@ -9,7 +9,7 @@ from transformer_engine.pytorch.module import Linear from .compressor import Compressor -from .dsa_cudnn import select_blocks +from .dsa_cudnn import select_indices from .dsa_rope import apply_rotary @@ -95,7 +95,7 @@ def forward( ) # Top-k IDs are discrete; training index projections needs a separate loss. weights = weights.reshape(-1, self.n_heads).contiguous() - indices = select_blocks( + indices = select_indices( query, key, weights, From 0003c129cd34880bf421cb6037d09b901b5d5b04 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Thu, 15 Oct 2026 13:12:36 -0700 Subject: [PATCH 22/37] Clarify sparse attention building blocks and examples Expose Compressor and Indexer at the standard PyTorch import levels so callers can compose attention variants. Explain overlapping compression, first-window masking and incomplete tails to make the source-token contract explicit. Simplify documentation and make the CSA, packed-input and hardware support examples consistent without changing numerical behavior. Preserve the compressed output-capacity contract while removing implementation rationale. Validation: explicit-path pre-commit, sparse-attention pylint, focused export identity checks, README constructor signature checks, and git diff --check passed. Signed-off-by: Sudhakar Singh --- transformer_engine/pytorch/__init__.py | 2 ++ .../pytorch/attention/__init__.py | 4 +++- .../attention/sparse_attention/README.md | 24 +++++++++++++------ .../attention/sparse_attention/__init__.py | 6 +---- .../attention/sparse_attention/compressor.py | 21 +++++++++++++++- .../attention/sparse_attention/dsa_cudnn.py | 18 +++++--------- .../sparse_attention/dsv4_attention.py | 7 +++--- 7 files changed, 53 insertions(+), 29 deletions(-) diff --git a/transformer_engine/pytorch/__init__.py b/transformer_engine/pytorch/__init__.py index 9601a617a67..2ea8384c2df 100644 --- a/transformer_engine/pytorch/__init__.py +++ b/transformer_engine/pytorch/__init__.py @@ -30,7 +30,9 @@ from transformer_engine.pytorch.module import UserBufferQuantizationMode from transformer_engine.pytorch.attention import DotProductAttention from transformer_engine.pytorch.attention import ( + Compressor, DSv4Attention, + Indexer, sparse_attention, ) from transformer_engine.pytorch.attention import GatedDeltaNetAttention diff --git a/transformer_engine/pytorch/attention/__init__.py b/transformer_engine/pytorch/attention/__init__.py index 2b8ef866340..4b145b28578 100644 --- a/transformer_engine/pytorch/attention/__init__.py +++ b/transformer_engine/pytorch/attention/__init__.py @@ -6,7 +6,7 @@ from .dot_product_attention import DotProductAttention from . import sparse_attention -from .sparse_attention import DSv4Attention +from .sparse_attention import Compressor, DSv4Attention, Indexer from .linear_attention import ( GatedDeltaNetAttention, GatedDeltaNet2Attention, @@ -20,6 +20,8 @@ __all__ = [ "DotProductAttention", "DSv4Attention", + "Compressor", + "Indexer", "sparse_attention", "GatedDeltaNetAttention", "GatedDeltaNet2Attention", diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index a8417d8aa91..39532b9da66 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -42,7 +42,7 @@ pooled = dsa_cudnn.compress( kv, gates, position_bias.float(), cu, cu_comp, ratio=ratio, overlap=is_csa, total_comp=total_comp, ) -compressed_kv = ... # Apply RMSNorm and RoPE to pooled rows. +compressed_kv = ... # Apply RMSNorm and RoPE to compressed KV. indices = None if is_csa: @@ -72,18 +72,21 @@ token and compressed `(cos, sin)` pairs. ## Usage +CSA uses overlapping compression (see `Compressor`) and an Indexer to select +compressed KV entries. + ```python import torch from transformer_engine.pytorch.attention.sparse_attention import DSv4Attention -attention = DSv4Attention( +csa = DSv4Attention( hidden_size=4096, q_lora_rank=1536, layer_type="compressed_sparse_attention", head_dim=512, rope_head_dim=64, sliding_window=128, compression_ratio=4, o_groups=8, o_lora_rank=1024, index_n_heads=32, index_topk=32, max_seqlen=2048, params_dtype=torch.bfloat16, ) -output = attention(hidden_states) # [S, B, 4096] -> [S, B, 4096] +output = csa(hidden_states) # [S, B, 4096] -> [S, B, 4096] ``` HCA uses non-overlapping compression and omits the Indexer: @@ -103,16 +106,23 @@ and packed (shape `TD`). Packed calls require contiguous CUDA INT32 `cu_seqlens_q [B+1]` (from zero to `T`) and `max_seqlen_q`: ```python -# Construct attention with qkv_format="thd". -output = attention(hidden_states, cu_seqlens_q=cu, max_seqlen_q=max_length) +packed_csa = DSv4Attention( + hidden_size=4096, q_lora_rank=1536, + layer_type="compressed_sparse_attention", head_dim=512, rope_head_dim=64, + sliding_window=128, compression_ratio=4, o_groups=8, o_lora_rank=1024, + index_n_heads=32, index_topk=32, max_seqlen=2048, + params_dtype=torch.bfloat16, qkv_format="thd", +) +output = packed_csa(hidden_states, cu_seqlens_q=cu, max_seqlen_q=max_length) ``` ## Note on support -The current layer supports full-sequence BF16 forward/backward on SM100, +The current layer supports full-sequence BF16 forward/backward on SM100 only, with 64 heads of width 512 and at least one complete compression window per sequence. Sequences start at position zero; cache/decode, padding, -TP/SP/CP, FP8, and sliding-only layers are unsupported. +TP/SP/CP, and FP8 are unsupported. CSA and HCA include sliding-window attention; +standalone sliding-window-only layers are unsupported. Compression and attention support autograd; top-k selection is discrete. CSA's `return_indexer_context=True` returns `(output, context)` for a diff --git a/transformer_engine/pytorch/attention/sparse_attention/__init__.py b/transformer_engine/pytorch/attention/sparse_attention/__init__.py index 21c762380e8..f6cc5415a12 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/__init__.py +++ b/transformer_engine/pytorch/attention/sparse_attention/__init__.py @@ -2,11 +2,7 @@ # # See LICENSE for license information. -"""Experimental sparse attention variants. - -Each variant owns its tensor/index contract and preparation helpers. Optional -kernels load on execution. Add future variants as sibling modules. -""" +"""Sparse attention. This feature is experimental and subject to change.""" from . import dsa_cudnn, dsv4_attention from .compressor import Compressor diff --git a/transformer_engine/pytorch/attention/sparse_attention/compressor.py b/transformer_engine/pytorch/attention/sparse_attention/compressor.py index 923abbcebd9..edf2b73638d 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/compressor.py +++ b/transformer_engine/pytorch/attention/sparse_attention/compressor.py @@ -12,7 +12,26 @@ class Compressor(torch.nn.Module): - """Project KV/gates and normalize the cuDNN-pooled rows. + """Project KV/gates, compress token windows with cuDNN, and apply RMSNorm. + + Non-overlapping compression combines each block of ``ratio`` tokens into + one compressed KV entry. Overlapping compression mixes the current block + with the entire preceding block to create one compressed KV entry, using + separate learned KV/gate projections for the current and preceding blocks. + Neighboring compressed KV entries therefore share an entire source block, + but use different projections of its tokens. + Weights are computed per KV channel as ``softmax(gates + position_bias)``. + + For ``ratio=2`` and ``[a b c d e f g h i j k]``: + + - Non-overlapping windows: ``[a b]``, ``[c d]``, ``[e f]``, ``[g h]``, ``[i j]``. + - Overlapping windows: ``[a b]``, ``[a b c d]``, ``[c d e f]``, + ``[e f g h]``, ``[g h i j]``. + + The first overlapping entry masks the missing previous block. Both modes emit + ``floor(length / ratio)`` entries per sequence; incomplete trailing blocks + (``k`` here) are excluded from compression. In DSv4Attention, these tokens + remain available through local sliding-window attention. Accept batch-major ``[B,S,D]`` or packed ``[T,D]`` hidden states with token/compressed prefixes. Return ``[total_comp,width]`` rows before RoPE, diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py index 76a95ac0b72..22256dac1cd 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py @@ -2,12 +2,7 @@ # # See LICENSE for license information. -"""Experimental PyTorch bindings for cuDNN sparse attention primitives. - -Compression, selection, sparse attention forward, and backward call cuDNN -Frontend APIs. There are no numerical fallbacks. Inputs use packed sequences; -model projections/norm/RoPE are external. -""" +"""PyTorch wrappers for cuDNN DSA/CSA primitives.""" from importlib import import_module from typing import Optional @@ -128,16 +123,15 @@ def compress( overlap: bool, total_comp: int, ) -> torch.Tensor: - """Gated pooling, with cuDNN forward and first-order backward. + """Compress token windows with cuDNN forward and backward. kv/score: contiguous BF16 [T, coff*D]; ape: FP32 [ratio, coff*D]. - coff is 2 for overlapping pooling, 1 otherwise. Sequence prefixes are + coff is 2 for overlapping compression, 1 otherwise. Sequence prefixes are CUDA INT32 [B+1]; compressed segment lengths are floor(length / ratio). Returns BF16 [total_comp, D]. Per-segment tails are dropped by cuDNN. - Pass total_comp explicitly because inferring it from a CUDA prefix would - synchronize. It may be a capacity; only rows below cu_seqlens_comp[-1] - are valid. Remaining kernel-specific constraints are checked by cuDNN. + total_comp specifies output capacity; only entries before + cu_seqlens_comp[-1] are valid. """ if ratio <= 0 or total_comp < 0: raise ValueError("ratio must be positive and total_comp nonnegative.") @@ -417,7 +411,7 @@ def sparse_attention( scale: Optional[float] = None, max_compressed_seqlen: Optional[int] = None, ) -> torch.Tensor: - """Joint local + compressed sparse attention with first-order autograd. + """Joint local + compressed sparse attention with autograd. Full packed-sequence prefill: query [T,H,D], local_kv [T,D], compressed_kv [Tc,D], sink [H]. Q/KV have already had the model's norm/RoPE applied. diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index a8f6b31332b..0b7b1dd4848 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -53,11 +53,12 @@ class DSv4Attention(torch.nn.Module): """DSv4 CSA/HCA attention in ``sbhd``, ``bshd``, or packed ``thd`` format. Composes TE projections/norms, Compressor, Indexer, and RoPE with cuDNN - sparse attention. Supports full-sequence BF16 forward/backward on SM100 + sparse attention. Supports full-sequence BF16 forward/backward on SM100 only, with 64 heads of width 512 and CSA index heads of width 128. Shared-input projections are fused by default. Each sequence starts at position zero. - Cache/decode, physical inter-sequence padding, TP/SP/CP, FP8, and sliding-only - attention are unsupported. The model owns the indexer auxiliary loss; + CSA and HCA include sliding-window attention; standalone sliding-window-only + layers are unsupported. Cache/decode, physical inter-sequence padding, + TP/SP/CP, and FP8 are unsupported. The model owns the indexer auxiliary loss; discrete top-k selection does not train the indexer from attention output alone. """ From 77f68aa79dd5bf0717d8e91dd4911ff88f997862 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Fri, 16 Oct 2026 11:31:40 -0700 Subject: [PATCH 23/37] Reject unsupported Compressor weight projection settings Optional head weights are produced only by the fused projection. Reject nonzero weight_width without fusion at construction instead of reaching an undefined local in forward. Cover the rejection in the existing metadata test to keep the test count unchanged. Clarify generic indexer-loss reduction and hidden-state layout contracts, and align remaining compression terminology and test names. Supported numerical paths and rotary ownership are unchanged. Validation: two focused tests passed; explicit-path pre-commit, sparse-attention pylint, and git diff --check passed. Signed-off-by: Sudhakar Singh --- tests/pytorch/test_dsv4_attention.py | 6 ++++-- .../pytorch/attention/sparse_attention/README.md | 11 ++++++----- .../pytorch/attention/sparse_attention/compressor.py | 10 ++++++---- .../pytorch/attention/sparse_attention/dsa_cudnn.py | 5 ++--- .../pytorch/attention/sparse_attention/indexer.py | 6 +++--- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index 7a82dc42ec3..c09ad805b89 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -105,7 +105,9 @@ def test_dsa_sparse_attention_forward_backward(variant, head_dim): assert relative_error < 0.02, relative_error.item() -def test_dsv4_rejects_unsupported_metadata(): +def test_dsa_rejects_unsupported_metadata(): + with pytest.raises(ValueError, match="weight_width requires fused=True"): + Compressor(8, 4, 2, False, 1e-6, "cpu", torch.float32, weight_width=1) with pytest.raises(ValueError, match="positive"): dsa_cudnn.sparse_attention( torch.empty(1, 64, 512), @@ -434,7 +436,7 @@ def test_dsa_packed_rope_eager_forward_backward(): @pytest.mark.parametrize("variant,ratio", [("hca", 128), ("csa", 4)]) @pytest.mark.parametrize("lengths", [[129, 257], [255, 383]]) -def test_dsv4_hybrid_thd_forward_backward(variant, ratio, lengths): +def test_dsv4_attention_thd_forward_backward(variant, ratio, lengths): """Packed full-layer output and gradients match independent dense sequences.""" if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 0): pytest.skip("DSv4 requires SM100") diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index 39532b9da66..cbbc7a07764 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -38,7 +38,7 @@ from transformer_engine.pytorch.attention.sparse_attention import ( dsa_cudnn, ) -pooled = dsa_cudnn.compress( +compressed_kv = dsa_cudnn.compress( kv, gates, position_bias.float(), cu, cu_comp, ratio=ratio, overlap=is_csa, total_comp=total_comp, ) @@ -46,11 +46,11 @@ compressed_kv = ... # Apply RMSNorm and RoPE to compressed KV. indices = None if is_csa: - index_pooled = dsa_cudnn.compress( + index_key = dsa_cudnn.compress( index_kv, index_gates, index_position_bias.float(), cu, cu_comp, ratio=ratio, overlap=True, total_comp=total_comp, ) - index_key = model.finish_index_key(index_pooled) + index_key = ... # Apply RMSNorm and RoPE to compressed index keys. indices = dsa_cudnn.select_indices( index_query, index_key, index_weights, cu, cu_comp, top_k=top_k, ratio=ratio, max_seqlen=max_seqlen, @@ -101,8 +101,9 @@ hca = DSv4Attention( output = hca(hidden_states) # [S, B, 4096] -> [S, B, 4096] ``` -Supported input tensor shapes are "batch-first" (shape `BSD`), "sequence-first" (shape `SBD`), -and packed (shape `TD`). Packed calls require contiguous CUDA INT32 +`hidden_states` has shape `[S,B,hidden_size]` for `qkv_format="sbhd"` (default), +`[B,S,hidden_size]` for `"bshd"`, or `[T,hidden_size]` for `"thd"`. +Packed calls require contiguous CUDA INT32 `cu_seqlens_q [B+1]` (from zero to `T`) and `max_seqlen_q`: ```python diff --git a/transformer_engine/pytorch/attention/sparse_attention/compressor.py b/transformer_engine/pytorch/attention/sparse_attention/compressor.py index edf2b73638d..af3124849b6 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/compressor.py +++ b/transformer_engine/pytorch/attention/sparse_attention/compressor.py @@ -52,6 +52,8 @@ def __init__( weight_width=0, ): super().__init__() + if weight_width and not fused: + raise ValueError("weight_width requires fused=True.") projected = width * (2 if overlap else 1) kw = {"bias": False, "device": device, "params_dtype": params_dtype} if fused: @@ -69,7 +71,7 @@ def __init__( self.projected, self.weight_width = projected, weight_width def forward(self, x, cu_seqlens, cu_seqlens_comp, *, total_comp): - """Project and compress KV rows, returning optional fused index weights.""" + """Project and compress KV entries, returning optional fused index weights.""" if hasattr(self, "fused_proj"): projected = self.fused_proj(x) widths = ( @@ -81,7 +83,7 @@ def forward(self, x, cu_seqlens, cu_seqlens_comp, *, total_comp): kv, gate = parts[:2] else: kv, gate = self.kv_proj(x), self.gate_proj(x) - pooled = compress( + compressed_kv = compress( kv.reshape(-1, self.projected).contiguous(), gate.reshape(-1, self.projected).contiguous(), # cuDNN requires FP32 bias; the cast preserves gradients to the parameter. @@ -96,6 +98,6 @@ def forward(self, x, cu_seqlens, cu_seqlens_comp, *, total_comp): # cuDNN only guarantees valid rows below the final prefix. Clear # unused capacity before RMSNorm so it cannot poison weight gradients. valid = torch.arange(total_comp, device=x.device) < cu_seqlens_comp[-1] - pooled = pooled.masked_fill(~valid[:, None], 0) - compressed = self.kv_norm(pooled) + compressed_kv = compressed_kv.masked_fill(~valid[:, None], 0) + compressed = self.kv_norm(compressed_kv) return (compressed, parts[2]) if self.weight_width else compressed diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py index 22256dac1cd..22e46e3a576 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py @@ -277,10 +277,9 @@ def dense_indexer_loss_backward( index_scale: float, loss_coeff: float, ): - """Return dQ/dK/dW for B1 BF16 mean-reduced KL, padding H32 at cuDNN. + """Return dQ/dK/dW for B1 BF16 KL averaged over query tokens, padding H32 at cuDNN. - The tested scalar grad_loss/loss_coeff contract is Megatron's default - calculate_per_token_loss=False reduction; per-token reduction is untested. + grad_loss is the scalar upstream gradient; loss_coeff scales the loss. """ padded_q, padded_w = _pad_index_heads(index_q, index_w) result = _dsa_op("dense_indexer_backward_wrapper")( diff --git a/transformer_engine/pytorch/attention/sparse_attention/indexer.py b/transformer_engine/pytorch/attention/sparse_attention/indexer.py index 6b163ad4908..dcd056abac0 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/indexer.py +++ b/transformer_engine/pytorch/attention/sparse_attention/indexer.py @@ -2,7 +2,7 @@ # # See LICENSE for license information. -"""Index projections, compression, and cuDNN block selection.""" +"""Index projections, compression, and cuDNN compressed KV selection.""" import torch @@ -14,7 +14,7 @@ class Indexer(torch.nn.Module): - """Select compressed rows using the indexer's own Q/K/weight projections. + """Select compressed KV entries using the indexer's own Q/K/weight projections. Accept batch-major or packed hidden states and query residuals, sequence prefixes, and caller-provided RoPE tables. Return global compressed-row IDs; @@ -67,7 +67,7 @@ def forward( max_seqlen, return_context=False, ): - """Select compressed rows and optionally expose live tensors for the indexer loss.""" + """Select compressed KV entries and optionally expose live tensors for the indexer loss.""" n_comp = max_seqlen // self.ratio comp_shape = (total_comp,) if hidden_states.ndim == 2 else (hidden_states.shape[0], n_comp) index_compressed = self.compressor(hidden_states, cu, cu_comp, total_comp=total_comp) From 3fe1c73b2e4547f0b6bae00083e809319b024eb8 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Tue, 10 Nov 2026 17:39:56 -0800 Subject: [PATCH 24/37] Share RoPE kernels across sparse attention and MLA Remove duplicated GPU computation while preserving distinct DSA interleaved and MLA split-half/KV adapters, inverse rotation and packed-position behavior. Keep framework dependencies out of the common layer and add independent layout, context-position and gradient regressions. Validation: 35 DSv4 and 21 focused MLA cases passed on each of SM103 and SM107, including the shared-kernel regressions. Signed-off-by: Sudhakar Singh --- tests/pytorch/attention/mla_rope_utils.py | 328 +----------------- tests/pytorch/attention/test_fused_mla.py | 38 ++ tests/pytorch/test_dsv4_attention.py | 70 ++++ transformer_engine/common/triton/dsa_rope.py | 180 +++++++++- .../pytorch/attention/fused_mla_q_uproj.py | 115 +----- 5 files changed, 307 insertions(+), 424 deletions(-) diff --git a/tests/pytorch/attention/mla_rope_utils.py b/tests/pytorch/attention/mla_rope_utils.py index 90eebfc66ae..0aedc863a05 100644 --- a/tests/pytorch/attention/mla_rope_utils.py +++ b/tests/pytorch/attention/mla_rope_utils.py @@ -17,7 +17,12 @@ try: import triton - import triton.language as tl + from transformer_engine.common.triton.dsa_rope import ( + _autotuned_mla_rope_fwd_inplace_kernel as rotary_fwd_q_kernel, + _autotuned_mla_rope_bwd_inplace_kernel as rotary_bwd_q_kernel, + rotary_fwd_kv_kernel, + rotary_bwd_kv_kernel, + ) HAVE_TRITON = True except ImportError: @@ -46,317 +51,6 @@ def build_rope_tables( if HAVE_TRITON: - # Not used for non-packed batches; kept for THD compatibility. - @triton.jit - def _get_thd_token_idx(cu_seqlens, pid_m, seq_num, cp_rank, cp_size): - token_idx = -1 - this_seq_len = 0 - seq_idx = 0 - last_cum_seqlen = tl.load(cu_seqlens) // cp_size - while seq_idx < seq_num: - cur_cum_seqlen = tl.load(cu_seqlens + seq_idx + 1) // cp_size - if token_idx == -1 and cur_cum_seqlen > pid_m: - token_idx = pid_m - last_cum_seqlen - this_seq_len = cur_cum_seqlen - last_cum_seqlen - last_cum_seqlen = cur_cum_seqlen - seq_idx += 1 - if cp_size > 1: - if token_idx < this_seq_len // 2: - token_idx = token_idx + cp_rank * this_seq_len // 2 - else: - token_idx = (token_idx - this_seq_len // 2) + ( - 2 * cp_size - cp_rank - 1 - ) * this_seq_len // 2 - return token_idx - - @triton.autotune( - configs=[ - triton.Config({"BLOCK_H": 1}), - triton.Config({"BLOCK_H": 2}), - triton.Config({"BLOCK_H": 4}), - triton.Config({"BLOCK_H": 8}), - triton.Config({"BLOCK_H": 16}), - triton.Config({"BLOCK_H": 32}), - triton.Config({"BLOCK_H": 64}), - triton.Config({"BLOCK_H": 128}), - ], - key=["emb_dim", "head_num"], - restore_value=["Q"], - ) - @triton.jit - def rotary_fwd_q_kernel( - Q, - COS, - SIN, - qk_head_dim, - emb_dim: tl.constexpr, - head_num: tl.constexpr, - batch_size, - seq_num, - cu_seqlens_q, - stride_x_seq, - stride_x_nheads, - cp_rank, - cp_size, - BLOCK_H: tl.constexpr, - ): - pid_m = tl.program_id(axis=0) - pid_head = tl.program_id(axis=1) - if cu_seqlens_q is None: - token_idx = pid_m // batch_size - else: - token_idx = _get_thd_token_idx(cu_seqlens_q, pid_m, seq_num, cp_rank, cp_size) - cos_left = tl.load(COS + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - sin_left = tl.load(SIN + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - cos_right = tl.load(COS + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) - sin_right = tl.load(SIN + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) - cos_left = cos_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - sin_left = sin_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - cos_right = cos_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - sin_right = sin_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - head_offsets = pid_head * BLOCK_H + tl.arange(0, BLOCK_H) - Q = Q + pid_m * stride_x_seq - x_off = head_offsets[:, None] * stride_x_nheads + qk_head_dim - mask = head_offsets[:, None] < head_num - x_1_off = x_off + tl.arange(0, emb_dim // 2)[None, :] * 2 - x_2_off = x_1_off + 1 - x_1 = tl.load(Q + x_1_off, mask=mask) - x_2 = tl.load(Q + x_2_off, mask=mask) - x_left = x_1 * cos_left - x_2 * sin_left - x_right = x_2 * cos_right + x_1 * sin_right - x_left_off = x_off + tl.arange(0, emb_dim // 2)[None, :] - x_right_off = x_left_off + emb_dim // 2 - tl.store(Q + x_left_off, x_left, mask=mask) - tl.store(Q + x_right_off, x_right, mask=mask) - - @triton.autotune( - configs=[ - triton.Config({"BLOCK_H": 1}), - triton.Config({"BLOCK_H": 2}), - triton.Config({"BLOCK_H": 4}), - triton.Config({"BLOCK_H": 8}), - triton.Config({"BLOCK_H": 16}), - triton.Config({"BLOCK_H": 32}), - triton.Config({"BLOCK_H": 64}), - triton.Config({"BLOCK_H": 128}), - ], - key=["emb_dim", "head_num"], - restore_value=["DO"], - ) - @triton.jit - def rotary_bwd_q_kernel( - DO, - COS, - SIN, - qk_head_dim, - emb_dim: tl.constexpr, - head_num: tl.constexpr, - batch_size, - seq_num, - cu_seqlens_q, - stride_x_seq, - stride_x_nheads, - cp_rank, - cp_size, - BLOCK_H: tl.constexpr, - ): - pid_m = tl.program_id(axis=0) - pid_head = tl.program_id(axis=1) - if cu_seqlens_q is None: - token_idx = pid_m // batch_size - else: - token_idx = _get_thd_token_idx(cu_seqlens_q, pid_m, seq_num, cp_rank, cp_size) - cos_left = tl.load(COS + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - sin_left = tl.load(SIN + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - cos_right = tl.load(COS + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) - sin_right = tl.load(SIN + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) - cos_left = cos_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - sin_left = sin_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - cos_right = cos_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - sin_right = sin_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - head_offsets = pid_head * BLOCK_H + tl.arange(0, BLOCK_H) - DO = DO + pid_m * stride_x_seq - x_off = head_offsets[:, None] * stride_x_nheads + qk_head_dim - mask = head_offsets[:, None] < head_num - x_left_off = x_off + tl.arange(0, emb_dim // 2)[None, :] - x_right_off = x_left_off + emb_dim // 2 - x_left = tl.load(DO + x_left_off, mask=mask) - x_right = tl.load(DO + x_right_off, mask=mask) - x_1 = x_left * cos_left + x_right * sin_right - x_2 = -x_left * sin_left + x_right * cos_right - x_1_off = x_off + tl.arange(0, emb_dim // 2)[None, :] * 2 - x_2_off = x_1_off + 1 - tl.store(DO + x_1_off, x_1, mask=mask) - tl.store(DO + x_2_off, x_2, mask=mask) - - @triton.autotune( - configs=[ - triton.Config({"BLOCK_H": 1}), - triton.Config({"BLOCK_H": 2}), - triton.Config({"BLOCK_H": 4}), - triton.Config({"BLOCK_H": 8}), - triton.Config({"BLOCK_H": 16}), - triton.Config({"BLOCK_H": 32}), - triton.Config({"BLOCK_H": 64}), - triton.Config({"BLOCK_H": 128}), - ], - key=["emb_dim", "k_dim", "v_dim", "head_num"], - ) - @triton.jit - def rotary_fwd_kv_kernel( - KV, - K_POS_EMB, - O_KEY, - O_VALUE, - COS, - SIN, - emb_dim: tl.constexpr, - k_dim: tl.constexpr, - v_dim: tl.constexpr, - head_num: tl.constexpr, - batch_size, - seq_num, - cu_seqlens_kv, - stride_kv_seq, - stride_kv_nheads, - stride_emb_seq, - stride_k_seq, - stride_k_nheads, - stride_v_seq, - stride_v_nheads, - cp_rank, - cp_size, - BLOCK_H: tl.constexpr, - ): - pid_m = tl.program_id(axis=0) - pid_head = tl.program_id(axis=1) - if cu_seqlens_kv is None: - token_idx = pid_m // batch_size - else: - token_idx = _get_thd_token_idx(cu_seqlens_kv, pid_m, seq_num, cp_rank, cp_size) - cos_left = tl.load(COS + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - sin_left = tl.load(SIN + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - cos_right = tl.load(COS + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) - sin_right = tl.load(SIN + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) - head_offsets = pid_head * BLOCK_H + tl.arange(0, BLOCK_H) - KV_ptr = KV + pid_m * stride_kv_seq - kv_off = head_offsets[:, None] * stride_kv_nheads - mask = head_offsets[:, None] < head_num - k_in_off = kv_off + tl.arange(0, k_dim)[None, :] - v_in_off = kv_off + k_dim + tl.arange(0, v_dim)[None, :] - k = tl.load(KV_ptr + k_in_off, mask=mask) - v = tl.load(KV_ptr + v_in_off, mask=mask) - K_ptr = O_KEY + pid_m * stride_k_seq + pid_head * BLOCK_H * stride_k_nheads - V_ptr = O_VALUE + pid_m * stride_v_seq + pid_head * BLOCK_H * stride_v_nheads - k_out_off = tl.arange(0, BLOCK_H)[:, None] * stride_k_nheads + tl.arange(0, k_dim)[None, :] - v_out_off = tl.arange(0, BLOCK_H)[:, None] * stride_v_nheads + tl.arange(0, v_dim)[None, :] - tl.store(K_ptr + k_out_off, k, mask=mask) - tl.store(V_ptr + v_out_off, v, mask=mask) - EMB = K_POS_EMB + pid_m * stride_emb_seq - x_1 = tl.load(EMB + tl.arange(0, emb_dim // 2) * 2) - x_2 = tl.load(EMB + tl.arange(0, emb_dim // 2) * 2 + 1) - x_left = x_1 * cos_left - x_2 * sin_left - x_right = x_2 * cos_right + x_1 * sin_right - x_left = x_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - x_right = x_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - x_left_off = ( - tl.arange(0, BLOCK_H)[:, None] * stride_k_nheads - + k_dim - + tl.arange(0, emb_dim // 2)[None, :] - ) - x_right_off = x_left_off + emb_dim // 2 - tl.store(K_ptr + x_left_off, x_left, mask=mask) - tl.store(K_ptr + x_right_off, x_right, mask=mask) - - @triton.autotune( - configs=[ - triton.Config({"BLOCK_H": 1}), - triton.Config({"BLOCK_H": 2}), - triton.Config({"BLOCK_H": 4}), - triton.Config({"BLOCK_H": 8}), - triton.Config({"BLOCK_H": 16}), - triton.Config({"BLOCK_H": 32}), - triton.Config({"BLOCK_H": 64}), - triton.Config({"BLOCK_H": 128}), - ], - key=["emb_dim", "k_dim", "v_dim", "head_num"], - ) - @triton.jit - def rotary_bwd_kv_kernel( - dK, - dV, - dKV, - dEMB, - COS, - SIN, - emb_dim: tl.constexpr, - k_dim: tl.constexpr, - v_dim: tl.constexpr, - head_num: tl.constexpr, - batch_size, - seq_num, - cu_seqlens_kv, - stride_dk_seq, - stride_dk_nheads, - stride_dv_seq, - stride_dv_nheads, - stride_dkv_seq, - stride_dkv_nheads, - stride_demb_seq, - cp_rank, - cp_size, - BLOCK_H: tl.constexpr, - ): - pid_m = tl.program_id(axis=0) - pid_head = tl.program_id(axis=1) - if cu_seqlens_kv is None: - token_idx = pid_m // batch_size - else: - token_idx = _get_thd_token_idx(cu_seqlens_kv, pid_m, seq_num, cp_rank, cp_size) - head_offsets = pid_head * BLOCK_H + tl.arange(0, BLOCK_H) - dKV_ptr = dKV + pid_m * stride_dkv_seq - dkv_off = head_offsets[:, None] * stride_dkv_nheads - mask = head_offsets[:, None] < head_num - dk_out_off = dkv_off + tl.arange(0, k_dim)[None, :] - dv_out_off = dkv_off + k_dim + tl.arange(0, v_dim)[None, :] - dK_ptr = dK + pid_m * stride_dk_seq + pid_head * BLOCK_H * stride_dk_nheads - dV_ptr = dV + pid_m * stride_dv_seq + pid_head * BLOCK_H * stride_dv_nheads - dk_in_off = tl.arange(0, BLOCK_H)[:, None] * stride_dk_nheads + tl.arange(0, k_dim)[None, :] - dv_in_off = tl.arange(0, BLOCK_H)[:, None] * stride_dv_nheads + tl.arange(0, v_dim)[None, :] - dk = tl.load(dK_ptr + dk_in_off, mask=mask) - dv = tl.load(dV_ptr + dv_in_off, mask=mask) - tl.store(dKV_ptr + dk_out_off, dk, mask=mask) - tl.store(dKV_ptr + dv_out_off, dv, mask=mask) - if pid_head == 0: - x_left_accum = tl.zeros((BLOCK_H, emb_dim // 2), dtype=tl.float32) - x_right_accum = tl.zeros((BLOCK_H, emb_dim // 2), dtype=tl.float32) - for i in tl.static_range(triton.cdiv(head_num, BLOCK_H)): - head_offsets_i = i * BLOCK_H + tl.arange(0, BLOCK_H) - dK_ptr_i = dK + pid_m * stride_dk_seq - x_off = head_offsets_i[:, None] * stride_dk_nheads + k_dim - mask_i = head_offsets_i[:, None] < head_num - x_left_off = x_off + tl.arange(0, emb_dim // 2)[None, :] - x_right_off = x_left_off + emb_dim // 2 - x_left_accum += tl.load(dK_ptr_i + x_left_off, mask=mask_i) - x_right_accum += tl.load(dK_ptr_i + x_right_off, mask=mask_i) - x_left_accum = tl.sum(x_left_accum, axis=0) - x_right_accum = tl.sum(x_right_accum, axis=0) - x_left_accum = x_left_accum.to(dEMB.dtype.element_ty) - x_right_accum = x_right_accum.to(dEMB.dtype.element_ty) - cos_left = tl.load(COS + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - sin_left = tl.load(SIN + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - cos_right = tl.load( - COS + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2) - ) - sin_right = tl.load( - SIN + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2) - ) - x_1 = x_left_accum * cos_left + x_right_accum * sin_right - x_2 = -x_left_accum * sin_left + x_right_accum * cos_right - dEMB_ptr = dEMB + pid_m * stride_demb_seq - tl.store(dEMB_ptr + tl.arange(0, emb_dim // 2) * 2, x_1) - tl.store(dEMB_ptr + tl.arange(0, emb_dim // 2) * 2 + 1, x_2) - def _flattened_token_stride(tensor: torch.Tensor) -> int: if tensor.dim() == 4: return tensor.stride(1) @@ -379,10 +73,15 @@ def forward(ctx, q, cos, sin, head_dim_nope, head_dim_rope): b, None, None, + None, _flattened_token_stride(q), q.stride(2), + cos.stride(0), + sin.stride(0), 0, 1, + INVERSE=False, + REMOVE_INTERLEAVING=False, ) ctx.save_for_backward(cos, sin) @@ -410,10 +109,15 @@ def backward(ctx, dq): b, None, None, + None, _flattened_token_stride(dq), dq.stride(2), + cos.stride(0), + sin.stride(0), 0, 1, + INVERSE=False, + REMOVE_INTERLEAVING=False, ) return dq, None, None, None, None diff --git a/tests/pytorch/attention/test_fused_mla.py b/tests/pytorch/attention/test_fused_mla.py index 1fcd9a8e218..4a5e1fc6a3a 100644 --- a/tests/pytorch/attention/test_fused_mla.py +++ b/tests/pytorch/attention/test_fused_mla.py @@ -289,10 +289,15 @@ def test_fused_mla_q_uproj_autograd() -> None: 1, None, None, + None, dq3.stride(0), dq3.stride(1), + cos_flat.stride(0), + sin_flat.stride(0), 0, 1, + INVERSE=False, + REMOVE_INTERLEAVING=False, ) dq2d = dq3.reshape(tokens, PROJ_DIM).contiguous() gy_quantizer = MXFP8Quantizer(fp8_dtype=tex.DType.kFloat8E4M3, rowwise=True, columnwise=True) @@ -462,3 +467,36 @@ def test_inputs_on_non_current_device(): _assert_identical(value, ref_value, "value") with pytest.raises(RuntimeError, match="same device"): fused_mla_kv_rope_mxfp8(kv, k_rope.to("cuda:0"), cos, sin, HEAD_DIM_V) + + +@pytest.mark.skipif(not HAVE_TRITON, reason="MLA RoPE requires Triton") +def test_mla_rope_kv_equations_backward(): + """Separate MLA K/V assembly and positional-gradient reduction survive reuse.""" + if not torch.cuda.is_available(): + pytest.skip("MLA RoPE requires CUDA") + torch.manual_seed(59) + kv = torch.randn(7, 2, 3, 32, device="cuda", dtype=torch.bfloat16, requires_grad=True) + positional = torch.randn(7, 2, 1, 64, device="cuda", dtype=torch.bfloat16, requires_grad=True) + cos, sin = build_rope_tables(7, emb_dim=64, device=kv.device) + key, value = apply_mla_rope_kv( + kv, + positional, + cos_table=cos, + sin_table=sin, + head_dim_nope=16, + head_dim_v=16, + ) + ref_kv = kv.detach().requires_grad_() + ref_pos = positional.detach().requires_grad_() + c, s = cos[:, None, None], sin[:, None, None] + left = ref_pos[..., 0::2].float() * c[..., :32] - ref_pos[..., 1::2].float() * s[..., :32] + right = ref_pos[..., 1::2].float() * c[..., 32:] + ref_pos[..., 0::2].float() * s[..., 32:] + ref_key = torch.cat( + (ref_kv[..., :16], torch.cat((left, right), -1).to(kv.dtype).expand(-1, -1, 3, -1)), -1 + ) + ref_value = ref_kv[..., 16:] + gradients = (torch.randn_like(key), torch.randn_like(value)) + actual_grads = torch.autograd.grad((key, value), (kv, positional), gradients) + ref_grads = torch.autograd.grad((ref_key, ref_value), (ref_kv, ref_pos), gradients) + for actual, expected in zip((key, value, *actual_grads), (ref_key, ref_value, *ref_grads)): + torch.testing.assert_close(actual, expected, atol=0.03125, rtol=0.02) diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index c09ad805b89..edf63b4e7a7 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -498,3 +498,73 @@ def test_dsv4_attention_thd_forward_backward(variant, ratio, lengths): assert torch.isfinite(packed_grad).all() and torch.isfinite(reference_grad).all() error = (packed_grad.float() - reference_grad.float()).norm() assert error / reference_grad.float().norm().clamp_min(1e-8) < 0.03 + + +@pytest.mark.parametrize("interleaved", [False, True]) +@pytest.mark.parametrize("inverse", [False, True]) +@pytest.mark.parametrize("cp_rank", [0, 1]) +def test_common_rope_layout_inverse_packed(interleaved, inverse, cp_rank): + """Independent pair equations cover shared lookup, both layouts and transpose.""" + if not torch.cuda.is_available(): + pytest.skip("Triton RoPE requires CUDA") + kernels = dsa_rope._triton_rope_module() + if kernels is None: + pytest.skip("Triton DSL is unavailable") + import triton + + torch.manual_seed(53) + # CP2 local lengths are 4 and 6, plus one padded row. Each rank owns + # the first/last chunks prescribed by the zig-zag CP mapping. + positions = ( + [0, 1, 6, 7, 0, 1, 2, 9, 10, 11, 0] if cp_rank == 0 else [2, 3, 4, 5, 3, 4, 5, 6, 7, 8, 0] + ) + cu = torch.tensor([0, 8, 20], device="cuda", dtype=torch.int32) + x = torch.randn(11, 3, 80, device="cuda", dtype=torch.bfloat16) + # Noncontiguous rows ensure callers preserve the explicit table strides. + cos = torch.randn(12, 128, device="cuda")[:, ::2] + sin = torch.randn(12, 128, device="cuda")[:, ::2] + reference_input = x.detach().requires_grad_() + tail = reference_input[..., 16:].float() + c, s = cos[positions, None], sin[positions, None] + if inverse: + s = -s + left = tail[..., 0::2] * c[..., :32] - tail[..., 1::2] * s[..., :32] + right = tail[..., 1::2] * c[..., 32:] + tail[..., 0::2] * s[..., 32:] + rotated = ( + torch.stack((left, right), -1).flatten(-2) if interleaved else torch.cat((left, right), -1) + ) + reference = torch.cat((reference_input[..., :16], rotated.to(x.dtype)), -1) + gradient = torch.randn_like(x) + expected_grad = torch.autograd.grad(reference, reference_input, gradient)[0] + candidate, actual_grad = x.clone(), gradient.clone() + grid = lambda meta: (11, triton.cdiv(3, meta["BLOCK_H"])) + # The tables have strided columns; materialize those columns while retaining + # a larger row stride, which is the kernel's supported table contract. + cos = torch.stack((cos, cos), 1)[:, 0, :] + sin = torch.stack((sin, sin), 1)[:, 0, :] + for kernel, tensor in ( + (kernels._autotuned_mla_rope_fwd_inplace_kernel, candidate), + (kernels._autotuned_mla_rope_bwd_inplace_kernel, actual_grad), + ): + kernel[grid]( + tensor, + cos, + sin, + 16, + 64, + 3, + 1, + 2, + cu, + None, + tensor.stride(0), + tensor.stride(1), + cos.stride(0), + sin.stride(0), + cp_rank, + 2, + INVERSE=inverse, + REMOVE_INTERLEAVING=interleaved, + ) + torch.testing.assert_close(candidate, reference, atol=0.03125, rtol=0.002) + torch.testing.assert_close(actual_grad, expected_grad, atol=0.03125, rtol=0.002) diff --git a/transformer_engine/common/triton/dsa_rope.py b/transformer_engine/common/triton/dsa_rope.py index b7876e8220e..39e9a67a4ea 100644 --- a/transformer_engine/common/triton/dsa_rope.py +++ b/transformer_engine/common/triton/dsa_rope.py @@ -2,11 +2,11 @@ # # See LICENSE for license information. -"""DSA RoPE kernels copied from NVIDIA/Megatron-LM. +"""Shared DSA and MLA RoPE kernels from NVIDIA/Megatron-LM. -Source: megatron/core/fusions/fused_mla_yarn_rope_apply.py at -c4129afff4867ca89b1fb7b4b911128424cdfbe8. Kernel bodies and autotuning are -retained; TE owns tensor preparation and autograd. +Query kernels: megatron/core/fusions/fused_mla_yarn_rope_apply.py at +c4129afff4867ca89b1fb7b4b911128424cdfbe8. Separate MLA KV kernels were moved +from TE's mla_rope_utils.py; framework adapters own tensor preparation/autograd. """ import triton @@ -71,7 +71,7 @@ def _mla_rope_fwd_inplace_kernel( ): """ Forward pass: apply RoPE inplace to the trailing emb_dim elements. - Reads from interleaved layout, writes back to interleaved layout. + Reads interleaved input; writes interleaved or split halves per REMOVE_INTERLEAVING. Input: Q: [seq_len, batch_size, head_num, nope_dim + emb_dim] @@ -174,7 +174,7 @@ def _mla_rope_bwd_inplace_kernel( ): """ Backward pass: inverse RoPE inplace on the trailing emb_dim elements. - Reads from interleaved layout, writes to interleaved layout. + Reads interleaved or split halves per REMOVE_INTERLEAVING; writes interleaved gradients. Input: DO: [seq_len, batch_size, head_num, nope_dim + emb_dim] @@ -251,3 +251,171 @@ def _mla_rope_bwd_inplace_kernel( key=["emb_dim", "head_num"], restore_value=["DO"], )(_mla_rope_bwd_inplace_kernel) + + +@triton.autotune( + configs=[ + triton.Config({"BLOCK_H": 1}), + triton.Config({"BLOCK_H": 2}), + triton.Config({"BLOCK_H": 4}), + triton.Config({"BLOCK_H": 8}), + triton.Config({"BLOCK_H": 16}), + triton.Config({"BLOCK_H": 32}), + triton.Config({"BLOCK_H": 64}), + triton.Config({"BLOCK_H": 128}), + ], + key=["emb_dim", "k_dim", "v_dim", "head_num"], +) +@triton.jit +def rotary_fwd_kv_kernel( + KV, + K_POS_EMB, + O_KEY, + O_VALUE, + COS, + SIN, + emb_dim: tl.constexpr, + k_dim: tl.constexpr, + v_dim: tl.constexpr, + head_num: tl.constexpr, + batch_size, + seq_num, + cu_seqlens_kv, + stride_kv_seq, + stride_kv_nheads, + stride_emb_seq, + stride_k_seq, + stride_k_nheads, + stride_v_seq, + stride_v_nheads, + cp_rank, + cp_size, + BLOCK_H: tl.constexpr, +): + """Rotate positional keys and assemble separate MLA key/value tensors.""" + pid_m = tl.program_id(axis=0) + pid_head = tl.program_id(axis=1) + if cu_seqlens_kv is None: + token_idx = pid_m // batch_size + else: + token_idx = _get_thd_token_idx(cu_seqlens_kv, pid_m, seq_num, cp_rank, cp_size) + cos_left = tl.load(COS + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) + sin_left = tl.load(SIN + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) + cos_right = tl.load(COS + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) + sin_right = tl.load(SIN + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) + head_offsets = pid_head * BLOCK_H + tl.arange(0, BLOCK_H) + KV_ptr = KV + pid_m * stride_kv_seq + kv_off = head_offsets[:, None] * stride_kv_nheads + mask = head_offsets[:, None] < head_num + k_in_off = kv_off + tl.arange(0, k_dim)[None, :] + v_in_off = kv_off + k_dim + tl.arange(0, v_dim)[None, :] + k = tl.load(KV_ptr + k_in_off, mask=mask) + v = tl.load(KV_ptr + v_in_off, mask=mask) + K_ptr = O_KEY + pid_m * stride_k_seq + pid_head * BLOCK_H * stride_k_nheads + V_ptr = O_VALUE + pid_m * stride_v_seq + pid_head * BLOCK_H * stride_v_nheads + k_out_off = tl.arange(0, BLOCK_H)[:, None] * stride_k_nheads + tl.arange(0, k_dim)[None, :] + v_out_off = tl.arange(0, BLOCK_H)[:, None] * stride_v_nheads + tl.arange(0, v_dim)[None, :] + tl.store(K_ptr + k_out_off, k, mask=mask) + tl.store(V_ptr + v_out_off, v, mask=mask) + EMB = K_POS_EMB + pid_m * stride_emb_seq + x_1 = tl.load(EMB + tl.arange(0, emb_dim // 2) * 2) + x_2 = tl.load(EMB + tl.arange(0, emb_dim // 2) * 2 + 1) + x_left = x_1 * cos_left - x_2 * sin_left + x_right = x_2 * cos_right + x_1 * sin_right + x_left = x_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + x_right = x_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + x_left_off = ( + tl.arange(0, BLOCK_H)[:, None] * stride_k_nheads + + k_dim + + tl.arange(0, emb_dim // 2)[None, :] + ) + x_right_off = x_left_off + emb_dim // 2 + tl.store(K_ptr + x_left_off, x_left, mask=mask) + tl.store(K_ptr + x_right_off, x_right, mask=mask) + + +@triton.autotune( + configs=[ + triton.Config({"BLOCK_H": 1}), + triton.Config({"BLOCK_H": 2}), + triton.Config({"BLOCK_H": 4}), + triton.Config({"BLOCK_H": 8}), + triton.Config({"BLOCK_H": 16}), + triton.Config({"BLOCK_H": 32}), + triton.Config({"BLOCK_H": 64}), + triton.Config({"BLOCK_H": 128}), + ], + key=["emb_dim", "k_dim", "v_dim", "head_num"], +) +@triton.jit +def rotary_bwd_kv_kernel( + dK, + dV, + dKV, + dEMB, + COS, + SIN, + emb_dim: tl.constexpr, + k_dim: tl.constexpr, + v_dim: tl.constexpr, + head_num: tl.constexpr, + batch_size, + seq_num, + cu_seqlens_kv, + stride_dk_seq, + stride_dk_nheads, + stride_dv_seq, + stride_dv_nheads, + stride_dkv_seq, + stride_dkv_nheads, + stride_demb_seq, + cp_rank, + cp_size, + BLOCK_H: tl.constexpr, +): + """Propagate MLA K/V gradients and reduce shared positional-key gradients.""" + pid_m = tl.program_id(axis=0) + pid_head = tl.program_id(axis=1) + if cu_seqlens_kv is None: + token_idx = pid_m // batch_size + else: + token_idx = _get_thd_token_idx(cu_seqlens_kv, pid_m, seq_num, cp_rank, cp_size) + head_offsets = pid_head * BLOCK_H + tl.arange(0, BLOCK_H) + dKV_ptr = dKV + pid_m * stride_dkv_seq + dkv_off = head_offsets[:, None] * stride_dkv_nheads + mask = head_offsets[:, None] < head_num + dk_out_off = dkv_off + tl.arange(0, k_dim)[None, :] + dv_out_off = dkv_off + k_dim + tl.arange(0, v_dim)[None, :] + dK_ptr = dK + pid_m * stride_dk_seq + pid_head * BLOCK_H * stride_dk_nheads + dV_ptr = dV + pid_m * stride_dv_seq + pid_head * BLOCK_H * stride_dv_nheads + dk_in_off = tl.arange(0, BLOCK_H)[:, None] * stride_dk_nheads + tl.arange(0, k_dim)[None, :] + dv_in_off = tl.arange(0, BLOCK_H)[:, None] * stride_dv_nheads + tl.arange(0, v_dim)[None, :] + dk = tl.load(dK_ptr + dk_in_off, mask=mask) + dv = tl.load(dV_ptr + dv_in_off, mask=mask) + tl.store(dKV_ptr + dk_out_off, dk, mask=mask) + tl.store(dKV_ptr + dv_out_off, dv, mask=mask) + if pid_head == 0: + x_left_accum = tl.zeros((BLOCK_H, emb_dim // 2), dtype=tl.float32) + x_right_accum = tl.zeros((BLOCK_H, emb_dim // 2), dtype=tl.float32) + for i in tl.static_range(triton.cdiv(head_num, BLOCK_H)): + head_offsets_i = i * BLOCK_H + tl.arange(0, BLOCK_H) + dK_ptr_i = dK + pid_m * stride_dk_seq + x_off = head_offsets_i[:, None] * stride_dk_nheads + k_dim + mask_i = head_offsets_i[:, None] < head_num + x_left_off = x_off + tl.arange(0, emb_dim // 2)[None, :] + x_right_off = x_left_off + emb_dim // 2 + x_left_accum += tl.load(dK_ptr_i + x_left_off, mask=mask_i) + x_right_accum += tl.load(dK_ptr_i + x_right_off, mask=mask_i) + x_left_accum = tl.sum(x_left_accum, axis=0) + x_right_accum = tl.sum(x_right_accum, axis=0) + x_left_accum = x_left_accum.to(dEMB.dtype.element_ty) + x_right_accum = x_right_accum.to(dEMB.dtype.element_ty) + cos_left = tl.load(COS + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) + sin_left = tl.load(SIN + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) + cos_right = tl.load(COS + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) + sin_right = tl.load(SIN + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) + x_1 = x_left_accum * cos_left + x_right_accum * sin_right + x_2 = -x_left_accum * sin_left + x_right_accum * cos_right + dEMB_ptr = dEMB + pid_m * stride_demb_seq + tl.store(dEMB_ptr + tl.arange(0, emb_dim // 2) * 2, x_1) + tl.store(dEMB_ptr + tl.arange(0, emb_dim // 2) * 2 + 1, x_2) diff --git a/transformer_engine/pytorch/attention/fused_mla_q_uproj.py b/transformer_engine/pytorch/attention/fused_mla_q_uproj.py index b15e53e6ee3..643d4287ba7 100644 --- a/transformer_engine/pytorch/attention/fused_mla_q_uproj.py +++ b/transformer_engine/pytorch/attention/fused_mla_q_uproj.py @@ -20,10 +20,12 @@ try: import triton - import triton.language as tl + from transformer_engine.common.triton.dsa_rope import ( + _autotuned_mla_rope_bwd_inplace_kernel as rotary_bwd_q_kernel, + ) except ImportError: triton = None - tl = None + rotary_bwd_q_kernel = None from ..constants import MXFP8_BLOCK_SCALING_SIZE from ..distributed import get_distributed_world_size @@ -44,110 +46,6 @@ def _cudnn_frontend_version_supported() -> bool: return False -if triton is not None: - - @triton.jit - def _get_thd_token_idx(cu_seqlens, pid_m, seq_num, cp_rank, cp_size): - token_idx = -1 - this_seq_len = 0 - seq_idx = 0 - last_cum_seqlen = tl.load(cu_seqlens) // cp_size - while seq_idx < seq_num: - cur_cum_seqlen = tl.load(cu_seqlens + seq_idx + 1) // cp_size - if token_idx == -1 and cur_cum_seqlen > pid_m: - token_idx = pid_m - last_cum_seqlen - this_seq_len = cur_cum_seqlen - last_cum_seqlen - last_cum_seqlen = cur_cum_seqlen - seq_idx += 1 - if cp_size > 1: - if token_idx < this_seq_len // 2: - token_idx = token_idx + cp_rank * this_seq_len // 2 - else: - token_idx = (token_idx - this_seq_len // 2) + ( - 2 * cp_size - cp_rank - 1 - ) * this_seq_len // 2 - return token_idx - - @triton.autotune( - configs=[ - triton.Config({"BLOCK_H": 1}), - triton.Config({"BLOCK_H": 2}), - triton.Config({"BLOCK_H": 4}), - triton.Config({"BLOCK_H": 8}), - triton.Config({"BLOCK_H": 16}), - triton.Config({"BLOCK_H": 32}), - triton.Config({"BLOCK_H": 64}), - triton.Config({"BLOCK_H": 128}), - ], - key=["emb_dim", "head_num"], - restore_value=["DO"], - ) - @triton.jit - def rotary_bwd_q_kernel( - DO, - COS, - SIN, - qk_head_dim, - emb_dim: tl.constexpr, - head_num: tl.constexpr, - batch_size, - seq_num, - cu_seqlens_q, - stride_x_seq, - stride_x_nheads, - cp_rank, - cp_size, - BLOCK_H: tl.constexpr, - ): - """ - Triton kernel of the backward pass for applying YARN RoPE to MLA's query. - This kernel inplace modifies the input tensor DO. - - Input: - DO: [seq_len, batch_size, head_num, qk_head_dim + emb_dim] - or [total_seq_len, head_num, qk_head_dim + emb_dim] - COS/SIN: [max_seq_len, emb_dim] - - batch_size, seq_num, and cu_seqlens_q are the same as in the forward pass - """ - pid_m = tl.program_id(axis=0) - pid_head = tl.program_id(axis=1) - - if cu_seqlens_q is None: - token_idx = pid_m // batch_size - else: - token_idx = _get_thd_token_idx(cu_seqlens_q, pid_m, seq_num, cp_rank, cp_size) - - cos_left = tl.load(COS + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - sin_left = tl.load(SIN + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - cos_right = tl.load(COS + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) - sin_right = tl.load(SIN + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) - cos_left = cos_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - sin_left = sin_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - cos_right = cos_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - sin_right = sin_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - - DO = DO + pid_m * stride_x_seq + pid_head * BLOCK_H * stride_x_nheads - - x_off = tl.arange(0, BLOCK_H)[:, None] * stride_x_nheads + qk_head_dim - mask = x_off < head_num * stride_x_nheads - x_left_off = x_off + tl.arange(0, emb_dim // 2)[None, :] - x_right_off = x_left_off + emb_dim // 2 - x_left = tl.load(DO + x_left_off, mask=mask) - x_right = tl.load(DO + x_right_off, mask=mask) - - x_1 = x_left * cos_left + x_right * sin_right - x_2 = -x_left * sin_left + x_right * cos_right - - x_1_off = x_off + tl.arange(0, emb_dim // 2)[None, :] * 2 - x_2_off = x_1_off + 1 - tl.store(DO + x_1_off, x_1, mask=mask) - tl.store(DO + x_2_off, x_2, mask=mask) - -else: - rotary_bwd_q_kernel = None - - class FusedMLAQUpProjRopeQuant: """Wrapper for the cuDNN fused MLA Q up-proj + per-head RoPE + MXFP8 quantize kernel. @@ -473,10 +371,15 @@ def grid(META): 1, None, None, + None, dq3.stride(0), dq3.stride(1), + cos.shape[-1], + sin.shape[-1], 0, 1, + INVERSE=False, + REMOVE_INTERLEAVING=False, ) # grad w.r.t. the (pre-RoPE) up-proj GEMM output; bf16. dq2d = dq3.reshape(tokens, nh * q_head_dim).contiguous() From 3e4865e2d42fc2dab206814a3f168b65ab9ec72d Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Tue, 10 Nov 2026 17:39:56 -0800 Subject: [PATCH 25/37] Validate sparse attention on additional Blackwell GPUs Admit SM103 and SM107 after exercising compression, sparse attention and indexer training on the actual architectures. Retain BF16 activation checks because the compression and indexer dependencies do not support FP16. Centralize support and qualification requirements instead of repeating hardware restrictions throughout operation docstrings. Validation: 35 DSv4 and 21 focused MLA cases passed on each architecture; outputs, applicable gradients and checkpoint loading match the original implementation with only its architecture guard relaxed. Signed-off-by: Sudhakar Singh --- tests/pytorch/test_dsv4_attention.py | 24 ++++++++++++----- .../attention/sparse_attention/README.md | 19 ++++++++----- .../attention/sparse_attention/dsa_cudnn.py | 27 ++++++++++--------- 3 files changed, 46 insertions(+), 24 deletions(-) diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index edf63b4e7a7..4568d283bc7 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -20,8 +20,12 @@ @pytest.mark.parametrize("variant", ["hca", "csa"]) @pytest.mark.parametrize("head_dim", [512, 576]) def test_dsa_sparse_attention_forward_backward(variant, head_dim): - if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 0): - pytest.skip("cuDNN sparse attention requires SM100") + if not torch.cuda.is_available() or torch.cuda.get_device_capability() not in ( + (10, 0), + (10, 3), + (10, 7), + ): + pytest.skip("cuDNN sparse attention requires SM100, SM103, or SM107") cudnn = pytest.importorskip("cudnn") dsa = getattr(cudnn, "DSA", None) if not all( @@ -217,8 +221,12 @@ def test_dsa_rope_non_power_of_two_forward_backward(monkeypatch): @pytest.mark.parametrize("index_heads", [32, 64]) def test_dsa_indexer_backward(index_heads): """Train the standalone Indexer and its Compressor against a dense KL reference.""" - if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 0): - pytest.skip("cuDNN indexer training requires SM100") + if not torch.cuda.is_available() or torch.cuda.get_device_capability() not in ( + (10, 0), + (10, 3), + (10, 7), + ): + pytest.skip("cuDNN indexer training requires SM100, SM103, or SM107") cudnn = pytest.importorskip("cudnn") if not all( hasattr(getattr(cudnn, "DSA", None), op) @@ -438,8 +446,12 @@ def test_dsa_packed_rope_eager_forward_backward(): @pytest.mark.parametrize("lengths", [[129, 257], [255, 383]]) def test_dsv4_attention_thd_forward_backward(variant, ratio, lengths): """Packed full-layer output and gradients match independent dense sequences.""" - if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (10, 0): - pytest.skip("DSv4 requires SM100") + if not torch.cuda.is_available() or torch.cuda.get_device_capability() not in ( + (10, 0), + (10, 3), + (10, 7), + ): + pytest.skip("DSv4 requires SM100, SM103, or SM107") cudnn = pytest.importorskip("cudnn") if not hasattr(getattr(cudnn, "DSA", None), "sparse_attention_forward_wrapper"): pytest.skip("DSv4 requires cuDNN Frontend 1.29 sparse attention APIs") diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index cbbc7a07764..08e1be6e792 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -31,7 +31,7 @@ DSv4Attention ## Inside DSv4Attention The following sketch expands the cuDNN calls underneath Compressor, Indexer, and attention. -Query and local KV are already projected, normalized, and rotated; tensors are contiguous CUDA BF16, with FP32 position bias/sink and CUDA INT32 sequence prefixes. +Query and local KV are already projected, normalized, and rotated; tensors are contiguous CUDA, with FP32 position bias/sink and CUDA INT32 sequence prefixes. ```python from transformer_engine.pytorch.attention.sparse_attention import ( @@ -119,11 +119,18 @@ output = packed_csa(hidden_states, cu_seqlens_q=cu, max_seqlen_q=max_length) ## Note on support -The current layer supports full-sequence BF16 forward/backward on SM100 only, -with 64 heads of width 512 and at least one complete compression window -per sequence. Sequences start at position zero; cache/decode, padding, -TP/SP/CP, and FP8 are unsupported. CSA and HCA include sliding-window attention; -standalone sliding-window-only layers are unsupported. +The layer admits SM100, SM103, and SM107 GPUs for full-sequence +forward/backward and requires BF16 activations. FP16 is unsupported by the +compression and indexer training dependencies. The operations require cuDNN +Frontend 1.29 or newer with its CSA/DSA APIs and are delivered as CuTe kernels. +Hardware validation includes prior SM100 runs and SM103/SM107 runs with +cuDNN Frontend 1.30.0, Cutlass DSL 4.8.0, CUDA 13.4, and cuDNN 9.27. + +The layer uses 64 heads of width 512 and requires at least one complete +compression window per sequence. Sequences start at position zero; +cache/decode, physical inter-sequence padding, TP/SP/CP, and FP8 are unsupported. +CSA and HCA include sliding-window attention; standalone sliding-window-only +layers are unsupported. Compression and attention support autograd; top-k selection is discrete. CSA's `return_indexer_context=True` returns `(output, context)` for a diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py index 22e46e3a576..49945fe06d2 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py @@ -44,11 +44,14 @@ def _validate_tensors(reference, bf16, fp32, cu_seqlens, cu_seqlens_comp): Callers own prefix contents: start at zero, monotonic, final offsets equal valid row counts, and compressed lengths floor(length / ratio). """ - if not reference.is_cuda or torch.cuda.get_device_capability(reference.device) != ( - 10, - 0, + if not reference.is_cuda or torch.cuda.get_device_capability(reference.device) not in ( + (10, 0), + (10, 3), + (10, 7), ): - raise ValueError("These cuDNN sparse attention kernels require an SM100 GPU.") + raise ValueError( + "These cuDNN sparse attention kernels require an SM100, SM103, or SM107 GPU." + ) for tensor in (reference, *bf16, *fp32, cu_seqlens, cu_seqlens_comp): if tensor.device != reference.device or not tensor.is_contiguous(): raise ValueError("Inputs must be contiguous on the same CUDA device.") @@ -125,10 +128,10 @@ def compress( ) -> torch.Tensor: """Compress token windows with cuDNN forward and backward. - kv/score: contiguous BF16 [T, coff*D]; ape: FP32 [ratio, coff*D]. + kv/score: contiguous [T, coff*D]; ape: FP32 [ratio, coff*D]. coff is 2 for overlapping compression, 1 otherwise. Sequence prefixes are CUDA INT32 [B+1]; compressed segment lengths are floor(length / ratio). - Returns BF16 [total_comp, D]. Per-segment tails are dropped by cuDNN. + Returns [total_comp, D]. Per-segment tails are dropped by cuDNN. total_comp specifies output capacity; only entries before cu_seqlens_comp[-1] are valid. @@ -164,18 +167,18 @@ def select_indices( max_compressed_seqlen: int, scale: Optional[float] = None, ) -> torch.Tensor: - """CSA selection using cuDNN's SM100 combined score + Top-K operation. + """CSA selection using cuDNN's combined score + Top-K operation. Inputs are the indexer's projected Q/K/head weights. - BF16 query [T,H,128], compressed_key [Tc,128], weights [T,H]; H=32 or 64. + query [T,H,128], compressed_key [Tc,128], weights [T,H]; H=32 or 64. Full packed-sequence prefill only: causal timeline offsets start at zero. Returns INT32 [T,top_k] indices into compressed_key's global packed rows; -1 denotes an invalid slot. These are NOT IDs into local+compressed KV. Selection is discrete. Scale defaults to 1/sqrt(index head dim). If the model also scales by 1/sqrt(index heads), fold that positive factor into - scale and pass raw BF16 projected weights. Scaling in FP32 then rounding - weights to BF16 can change close top-k boundaries. This helper does not + scale and pass raw projected weights. Scaling in FP32 then rounding + weights back to activation dtype can change close top-k boundaries. This helper does not implement the separate indexer auxiliary loss or autograd through IDs. max_compressed_seqlen is a compile bound and must cover the partial final ratio; use ceil(max_seqlen / ratio) even though prefixes count only complete @@ -237,7 +240,7 @@ def dense_indexer_loss_scores( index_scale: float, attention_scale: float, ): - """Return cuDNN dense index/teacher score dictionaries for B1 BF16 prefill. + """Return cuDNN dense index/teacher score dictionaries for B1 prefill. full_lse is FP32 [1,S,64] and must include the local window, every eligible compressed key, and the sink. The sparse-attention saved LSE covers only @@ -277,7 +280,7 @@ def dense_indexer_loss_backward( index_scale: float, loss_coeff: float, ): - """Return dQ/dK/dW for B1 BF16 KL averaged over query tokens, padding H32 at cuDNN. + """Return dQ/dK/dW for B1 KL averaged over query tokens, padding H32 at cuDNN. grad_loss is the scalar upstream gradient; loss_coeff scales the loss. """ From e35f1b067000cece9f74ff2da76aa715180e04e6 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Tue, 10 Nov 2026 17:39:56 -0800 Subject: [PATCH 26/37] Keep one fused projection path in DSv4Attention Remove the private high-level unfused comparison branch to avoid maintaining two checkpoint layouts. Preserve default fused parameter names and shapes, and keep standalone Compressor and Indexer projection options available. Validation: strict checkpoint loading, outputs and applicable gradients match the original fused path on SM103 and SM107. Signed-off-by: Sudhakar Singh --- .../attention/sparse_attention/README.md | 7 +++ .../sparse_attention/dsv4_attention.py | 43 ++++++------------- 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index 08e1be6e792..2fe0d5ab801 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -137,3 +137,10 @@ CSA's `return_indexer_context=True` returns `(output, context)` for a single-sequence, caller-owned indexer loss. Without that objective, the indexer is not trained by the attention output alone. Pre-attention normalization, residuals, and loss calculation remain model-owned. + +Shared-input projections in `DSv4Attention` are fused: `q_a_kv_proj.weight` +stores query-down then local-KV rows; `compressor.fused_proj.weight` stores +KV then gate rows; the CSA `indexer.compressor.fused_proj.weight` additionally +stores head-weight rows last. These default checkpoint keys and shapes are +preserved. `Compressor` and `Indexer` retain their independent projection +options. Separate-weight model imports must concatenate those rows explicitly. diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index 0b7b1dd4848..09e6c12b649 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -53,13 +53,8 @@ class DSv4Attention(torch.nn.Module): """DSv4 CSA/HCA attention in ``sbhd``, ``bshd``, or packed ``thd`` format. Composes TE projections/norms, Compressor, Indexer, and RoPE with cuDNN - sparse attention. Supports full-sequence BF16 forward/backward on SM100 only, - with 64 heads of width 512 and CSA index heads of width 128. Shared-input - projections are fused by default. Each sequence starts at position zero. - CSA and HCA include sliding-window attention; standalone sliding-window-only - layers are unsupported. Cache/decode, physical inter-sequence padding, - TP/SP/CP, and FP8 are unsupported. The model owns the indexer auxiliary loss; - discrete top-k selection does not train the indexer from attention output alone. + sparse attention. See the sparse_attention README for support and training + requirements. Shared-input projections are fused. """ def __init__( @@ -83,7 +78,6 @@ def __init__( device: str = "cuda", params_dtype: torch.dtype = torch.float32, qkv_format: str = "sbhd", - _fuse_projections: bool = True, ): super().__init__() # Keep the current kernel geometry explicit without exposing untested sizes. @@ -118,17 +112,13 @@ def __init__( ) kw = {"bias": False, "device": device, "params_dtype": params_dtype} - if _fuse_projections: - # Query-down and local-KV projections read the same hidden states. - # Keep slices explicit below while one contiguous weight feeds the GEMM. - self.q_a_kv_proj = Linear( - hidden_size, - q_lora_rank + head_dim, - **kw, - ) - else: - self.q_a_proj = Linear(hidden_size, q_lora_rank, **kw) - self.kv_proj = Linear(hidden_size, head_dim, **kw) + # Query-down and local-KV projections read the same hidden states. + # Keep slices explicit below while one contiguous weight feeds the GEMM. + self.q_a_kv_proj = Linear( + hidden_size, + q_lora_rank + head_dim, + **kw, + ) # LayerNormLinear would normalize before projection; DSv4 normalizes after it. self.q_a_norm = RMSNorm(q_lora_rank, eps=rms_norm_eps, device=device, dtype=params_dtype) self.q_b_proj = Linear(q_lora_rank, self.num_heads * head_dim, **kw) @@ -141,7 +131,7 @@ def __init__( rms_norm_eps, device, params_dtype, - fused=_fuse_projections, + fused=True, ) if is_csa: self.indexer = Indexer( @@ -154,7 +144,7 @@ def __init__( rms_norm_eps, device, params_dtype, - fused=_fuse_projections, + fused=True, ) # HF/Megatron use one independent block per output group. A normal TE # Linear forward mixes groups, so use its weight in the block multiply. @@ -175,7 +165,6 @@ def __init__( ) ) self.hidden_size, self.q_lora_rank = hidden_size, q_lora_rank - self.fused_projections = _fuse_projections self.head_dim, self.rope_head_dim = head_dim, rope_head_dim self.rope_theta = rope_theta if rope is None else None self.rms_norm_eps = rms_norm_eps @@ -241,13 +230,9 @@ def forward( token_rope, compressed_rope = self.rope(seq, hidden_states.device) - if self.fused_projections: - q_a, local_kv_projected = self.q_a_kv_proj(hidden_states).split( - (self.q_lora_rank, self.head_dim), dim=-1 - ) - else: - q_a = self.q_a_proj(hidden_states) - local_kv_projected = self.kv_proj(hidden_states) + q_a, local_kv_projected = self.q_a_kv_proj(hidden_states).split( + (self.q_lora_rank, self.head_dim), dim=-1 + ) q_residual = self.q_a_norm(q_a) q = self.q_b_proj(q_residual).reshape(*token_shape, self.num_heads, self.head_dim) # Query-up norm is unweighted; TE RMSNorm would add a learned scale. From ef2106a2dd3e0493879d0792d7214931064bfd35 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Tue, 10 Nov 2026 17:39:56 -0800 Subject: [PATCH 27/37] Clarify sparse attention building-block contracts Make the reusable classes discoverable without advertising implementation modules as package exports. Explain packed addressing, exact dense capacity, position bias and caller-owned indexer gradients so users can compose the operations without assuming that discrete selection trains the indexer. Rename the Compressor input to hidden_states without changing computation. Validation: syntax, formatter, changed-production pylint and whitespace checks pass; independent review checked related callers and contracts. Signed-off-by: Sudhakar Singh --- docs/api/pytorch.rst | 9 ++++ transformer_engine/pytorch/__init__.py | 3 +- .../pytorch/attention/__init__.py | 4 +- .../attention/sparse_attention/README.md | 52 ++++++++++++++++--- .../attention/sparse_attention/__init__.py | 5 +- .../attention/sparse_attention/compressor.py | 35 ++++++++++--- .../attention/sparse_attention/dsa_cudnn.py | 33 ++++++++++-- .../sparse_attention/dsv4_attention.py | 28 ++++++++-- .../attention/sparse_attention/indexer.py | 31 ++++++++++- 9 files changed, 167 insertions(+), 33 deletions(-) diff --git a/docs/api/pytorch.rst b/docs/api/pytorch.rst index 31d469cb77f..bdd14514df7 100644 --- a/docs/api/pytorch.rst +++ b/docs/api/pytorch.rst @@ -25,6 +25,15 @@ PyTorch .. autoapiclass:: transformer_engine.pytorch.DotProductAttention(num_attention_heads, kv_channels, **kwargs) :members: forward, set_context_parallel_group +.. autoapiclass:: transformer_engine.pytorch.DSv4Attention(hidden_size, q_lora_rank, **kwargs) + :members: forward + +.. autoapiclass:: transformer_engine.pytorch.Compressor(hidden_size, width, ratio, overlap, eps, device, params_dtype, **kwargs) + :members: forward + +.. autoapiclass:: transformer_engine.pytorch.Indexer(hidden_size, q_lora_rank, head_dim, n_heads, top_k, ratio, eps, device, params_dtype, **kwargs) + :members: forward + .. autoapiclass:: transformer_engine.pytorch.GatedDeltaNetAttention(num_attention_heads, kv_channels, **kwargs) :members: forward diff --git a/transformer_engine/pytorch/__init__.py b/transformer_engine/pytorch/__init__.py index c6395cb8468..601dfe43e65 100644 --- a/transformer_engine/pytorch/__init__.py +++ b/transformer_engine/pytorch/__init__.py @@ -30,10 +30,9 @@ from transformer_engine.pytorch.module import UserBufferQuantizationMode from transformer_engine.pytorch.attention import DotProductAttention from transformer_engine.pytorch.attention import ( - Compressor, DSv4Attention, + Compressor, Indexer, - sparse_attention, ) from transformer_engine.pytorch.attention import GatedDeltaNetAttention from transformer_engine.pytorch.attention import GatedDeltaNet2Attention diff --git a/transformer_engine/pytorch/attention/__init__.py b/transformer_engine/pytorch/attention/__init__.py index e79ec28047c..aa8f21b2b36 100644 --- a/transformer_engine/pytorch/attention/__init__.py +++ b/transformer_engine/pytorch/attention/__init__.py @@ -5,8 +5,7 @@ """Python interface for attention""" from .dot_product_attention import DotProductAttention -from . import sparse_attention -from .sparse_attention import Compressor, DSv4Attention, Indexer +from .sparse_attention import DSv4Attention, Compressor, Indexer from .linear_attention import ( GatedDeltaNetAttention, GatedDeltaNet2Attention, @@ -23,7 +22,6 @@ "DSv4Attention", "Compressor", "Indexer", - "sparse_attention", "GatedDeltaNetAttention", "GatedDeltaNet2Attention", "GatedDeltaProductAttention", diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index 2fe0d5ab801..e5f7fa63a9e 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -3,7 +3,7 @@ `DSv4Attention` is an experimental API in TE that composes cuDNN compression, top-k indexing, and sparse attention kernels with Megatron's DSA RoPE kernels through reusable building blocks such as `Compressor` and `Indexer`. cuDNN -supplies compression and sparse attention backward kernels +supplies compression and sparse attention forward and backward kernels and separate indexer-loss score/backward primitives; top-k selection itself is discrete. The layer maps hidden states to hidden states with the same shape. `Compressor` and `Indexer` can also be composed into other attention layers; @@ -14,7 +14,7 @@ discrete. The layer maps hidden states to hidden states with the same shape. ```text DSv4Attention ├── TE query/local-KV projections + norms -├── Compressor +├── Compressor (HCA or CSA) │ ├── TE KV/gate projections + learned position bias │ ├── dsa_cudnn.compress -> cuDNN compressor │ └── TE RMSNorm @@ -34,9 +34,7 @@ The following sketch expands the cuDNN calls underneath Compressor, Indexer, and Query and local KV are already projected, normalized, and rotated; tensors are contiguous CUDA, with FP32 position bias/sink and CUDA INT32 sequence prefixes. ```python -from transformer_engine.pytorch.attention.sparse_attention import ( - dsa_cudnn, -) +import transformer_engine.pytorch.attention.sparse_attention.dsa_cudnn as dsa_cudnn compressed_kv = dsa_cudnn.compress( kv, gates, position_bias.float(), cu, cu_comp, @@ -66,6 +64,14 @@ output = dsa_cudnn.sparse_attention( output = ... # Apply inverse RoPE and grouped output projection. ``` +Dense inputs are repacked to batch-major rows before these calls; packed THD +inputs already have that layout. Attention uses the original query +`[T,64,D]` and concatenated local/compressed KV `[T+Tc,D]`; there is no +compressed query. `_prepare_sparse_indices` creates one row per query containing +causal local IDs and compressed IDs offset by `T`, with invalid slots set to +`-1`. This address preparation differs from `select_indices`, which scores +the indexer's projected Q/K/head weights and returns compressed-row IDs. + `dsa_rope` supplies token/compressed-position embeddings and rotary application. A caller may override the default embeddings with a `rope` module returning token and compressed `(cos, sin)` pairs. @@ -130,7 +136,7 @@ The layer uses 64 heads of width 512 and requires at least one complete compression window per sequence. Sequences start at position zero; cache/decode, physical inter-sequence padding, TP/SP/CP, and FP8 are unsupported. CSA and HCA include sliding-window attention; standalone sliding-window-only -layers are unsupported. +layers should use `DotProductAttention` with `window_size` instead. Compression and attention support autograd; top-k selection is discrete. CSA's `return_indexer_context=True` returns `(output, context)` for a @@ -138,6 +144,40 @@ single-sequence, caller-owned indexer loss. Without that objective, the indexer is not trained by the attention output alone. Pre-attention normalization, residuals, and loss calculation remain model-owned. +For B=1, the caller can train the indexer with the dense helper primitives: + +```python +output, context = csa(hidden_states, return_indexer_context=True) +qi, ki, wi = (context[name].unsqueeze(0) + for name in ("index_q", "index_k", "index_w")) +qa, kc = (context[name].unsqueeze(0) + for name in ("attn_q", "compressed_kv")) +# Caller computes detached FP32 full_lse [1,T,64] over the local window, +# ALL eligible compressed keys, and sink, using context's teacher tensors. +# The selected sparse-attention LSE excludes sink and is not this denominator. +full_lse = ... +index_scale = (qi.shape[2] * qi.shape[3]) ** -0.5 +index_scores, teacher_scores = dsa_cudnn.dense_indexer_loss_scores( + qi, ki, wi, qa, kc, full_lse, + ratio=csa.compression_ratio, index_scale=index_scale, + attention_scale=csa.head_dim ** -0.5, +) +index_grads = dsa_cudnn.dense_indexer_loss_backward( + qi, ki, wi, index_scores, teacher_scores, + torch.ones((), device=qi.device), + ratio=csa.compression_ratio, index_scale=index_scale, + loss_coeff=loss_coefficient, +) +torch.autograd.backward((qi, ki, wi), index_grads) +``` + +These helpers return score buffers and explicit gradients, not an autograd +loss. The model owns its loss coefficient and propagates its main objective +through `output` separately. Auxiliary gradients reach the live indexer +projections and compressor; they do not reach the shared hidden/query +projections or detached attention teacher. The dense-reference example in +`tests/pytorch/test_dsv4_attention.py` shows full teacher-LSE construction. + Shared-input projections in `DSv4Attention` are fused: `q_a_kv_proj.weight` stores query-down then local-KV rows; `compressor.fused_proj.weight` stores KV then gate rows; the CSA `indexer.compressor.fused_proj.weight` additionally diff --git a/transformer_engine/pytorch/attention/sparse_attention/__init__.py b/transformer_engine/pytorch/attention/sparse_attention/__init__.py index f6cc5415a12..93d93f14f7f 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/__init__.py +++ b/transformer_engine/pytorch/attention/sparse_attention/__init__.py @@ -4,14 +4,11 @@ """Sparse attention. This feature is experimental and subject to change.""" -from . import dsa_cudnn, dsv4_attention -from .compressor import Compressor from .dsv4_attention import DSv4Attention +from .compressor import Compressor from .indexer import Indexer __all__ = [ - "dsa_cudnn", - "dsv4_attention", "DSv4Attention", "Compressor", "Indexer", diff --git a/transformer_engine/pytorch/attention/sparse_attention/compressor.py b/transformer_engine/pytorch/attention/sparse_attention/compressor.py index af3124849b6..a0d4f9bda5f 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/compressor.py +++ b/transformer_engine/pytorch/attention/sparse_attention/compressor.py @@ -12,7 +12,7 @@ class Compressor(torch.nn.Module): - """Project KV/gates, compress token windows with cuDNN, and apply RMSNorm. + """Project KV/gates for HCA or CSA, compress with cuDNN, and apply RMSNorm. Non-overlapping compression combines each block of ``ratio`` tokens into one compressed KV entry. Overlapping compression mixes the current block @@ -57,7 +57,7 @@ def __init__( projected = width * (2 if overlap else 1) kw = {"bias": False, "device": device, "params_dtype": params_dtype} if fused: - # Fuse the KV, gate, and optional index-weight projections of x. + # Fuse the KV, gate, and optional index-weight projections of hidden states. # Keep one contiguous weight for the shared projection call. self.fused_proj = Linear(hidden_size, 2 * projected + weight_width, **kw) else: @@ -70,10 +70,29 @@ def __init__( self.ratio, self.overlap = ratio, overlap self.projected, self.weight_width = projected, weight_width - def forward(self, x, cu_seqlens, cu_seqlens_comp, *, total_comp): - """Project and compress KV entries, returning optional fused index weights.""" + def forward(self, hidden_states, cu_seqlens, cu_seqlens_comp, *, total_comp): + """Project and compress KV entries before RoPE. + + Parameters + ---------- + hidden_states : torch.Tensor + Batch-major ``[B,S,hidden_size]`` or packed ``[T,hidden_size]`` input. + cu_seqlens, cu_seqlens_comp : torch.Tensor + Contiguous CUDA INT32 ``[B+1]`` token/compressed prefixes. Compressed + segment lengths are ``floor(length / ratio)``. + total_comp : int + Output row capacity. For dense input, use exactly ``B * (S // ratio)``; + packed input may reserve extra rows beyond ``cu_seqlens_comp[-1]``. + + Returns + ------- + torch.Tensor or tuple of torch.Tensor + Normalized ``[total_comp,width]`` KV rows. When ``weight_width > 0``, + also return head weights with shape ``hidden_states.shape[:-1] + + (weight_width,)``. Only rows below the final compressed prefix are valid. + """ if hasattr(self, "fused_proj"): - projected = self.fused_proj(x) + projected = self.fused_proj(hidden_states) widths = ( (self.projected, self.projected, self.weight_width) if self.weight_width @@ -82,7 +101,7 @@ def forward(self, x, cu_seqlens, cu_seqlens_comp, *, total_comp): parts = projected.split(widths, dim=-1) kv, gate = parts[:2] else: - kv, gate = self.kv_proj(x), self.gate_proj(x) + kv, gate = self.kv_proj(hidden_states), self.gate_proj(hidden_states) compressed_kv = compress( kv.reshape(-1, self.projected).contiguous(), gate.reshape(-1, self.projected).contiguous(), @@ -94,10 +113,10 @@ def forward(self, x, cu_seqlens, cu_seqlens_comp, *, total_comp): overlap=self.overlap, total_comp=total_comp, ) - if x.ndim == 2: + if hidden_states.ndim == 2: # cuDNN only guarantees valid rows below the final prefix. Clear # unused capacity before RMSNorm so it cannot poison weight gradients. - valid = torch.arange(total_comp, device=x.device) < cu_seqlens_comp[-1] + valid = torch.arange(total_comp, device=hidden_states.device) < cu_seqlens_comp[-1] compressed_kv = compressed_kv.masked_fill(~valid[:, None], 0) compressed = self.kv_norm(compressed_kv) return (compressed, parts[2]) if self.weight_width else compressed diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py index 49945fe06d2..05340d1c41b 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py @@ -128,7 +128,9 @@ def compress( ) -> torch.Tensor: """Compress token windows with cuDNN forward and backward. - kv/score: contiguous [T, coff*D]; ape: FP32 [ratio, coff*D]. + kv/score: contiguous [T, coff*D] projected KV/gate scores. + ape: FP32 [ratio, coff*D] additive position embedding, a learned per-channel + bias added to gate scores before the compression softmax. coff is 2 for overlapping compression, 1 otherwise. Sequence prefixes are CUDA INT32 [B+1]; compressed segment lengths are floor(length / ratio). Returns [total_comp, D]. Per-segment tails are dropped by cuDNN. @@ -167,10 +169,12 @@ def select_indices( max_compressed_seqlen: int, scale: Optional[float] = None, ) -> torch.Tensor: - """CSA selection using cuDNN's combined score + Top-K operation. + """Select compressed rows with cuDNN's combined index-score + Top-K operation. - Inputs are the indexer's projected Q/K/head weights. + Inputs are the indexer's projected Q/K/head weights, after compression/RoPE. + Unlike the Indexer module, this operation does not project or compress inputs. query [T,H,128], compressed_key [Tc,128], weights [T,H]; H=32 or 64. + cu_seqlens/cu_seqlens_comp are CUDA INT32 [B+1] token/compressed prefixes. Full packed-sequence prefill only: causal timeline offsets start at zero. Returns INT32 [T,top_k] indices into compressed_key's global packed rows; -1 denotes an invalid slot. These are NOT IDs into local+compressed KV. @@ -242,9 +246,13 @@ def dense_indexer_loss_scores( ): """Return cuDNN dense index/teacher score dictionaries for B1 prefill. + index_q/index_k/index_w are [1,S,H,128], [1,Sc,128], [1,S,H], + with H=32 or 64. attn_q/compressed_kv are [1,S,64,D], [1,Sc,D]. full_lse is FP32 [1,S,64] and must include the local window, every eligible compressed key, and the sink. The sparse-attention saved LSE covers only selected keys and excludes the sink, so it cannot serve this dense teacher. + Returned score/denominator dictionaries feed dense_indexer_loss_backward; + these calls do not attach an autograd loss to the indexer tensors. """ if any(t.dtype != torch.bfloat16 for t in (index_k, attn_q, compressed_kv)): raise TypeError("Dense indexer keys and attention Q/K must be BF16.") @@ -282,6 +290,9 @@ def dense_indexer_loss_backward( ): """Return dQ/dK/dW for B1 KL averaged over query tokens, padding H32 at cuDNN. + index_q/index_k/index_w and score dictionaries are the inputs/outputs of + dense_indexer_loss_scores. Returned gradients match those input shapes and + must be propagated through the live indexer tensors by the caller. grad_loss is the scalar upstream gradient; loss_coeff scales the loss. """ padded_q, padded_w = _pad_index_heads(index_q, index_w) @@ -315,7 +326,13 @@ def _prepare_sparse_indices( indices: Optional[torch.Tensor], max_compressed_seqlen: Optional[int], ) -> torch.Tensor: - """Map local and compressed entries into one concatenated KV address space.""" + """Build one KV-index row per original query token, using its shape/device. + + Query values are not read. Concatenate causal local-row IDs and compressed + IDs offset by T into [T, window_size + compressed_slots]; invalid slots + are -1. This prepares attention addresses, whereas select_indices chooses + compressed entries from indexer scores. + """ total_q = query.shape[0] device = query.device lengths = cu_seqlens[1:] - cu_seqlens[:-1] @@ -366,7 +383,11 @@ def _prepare_sparse_indices( class _SparseAttention(torch.autograd.Function): @staticmethod def forward(ctx, query, kv, indices, sink, scale): - """Run cuDNN sparse prefill and save its backward inputs.""" + """Attend query [T,64,D] to concatenated KV [T+Tc,D] using INT32 rows. + + indices [T,window_size+compressed_slots] addresses that KV tensor, with + -1 for invalid slots. No compressed query is constructed. + """ result = _dsa_op("sparse_attention_forward_wrapper")( query, kv, @@ -417,6 +438,8 @@ def sparse_attention( Full packed-sequence prefill: query [T,H,D], local_kv [T,D], compressed_kv [Tc,D], sink [H]. Q/KV have already had the model's norm/RoPE applied. + The high-level layer repacks dense SBHD/BSHD inputs to these batch-major + packed rows; THD inputs already use packed rows. There is no compressed Q. cu_seqlens describes both query and local KV; cu_seqlens_comp describes valid compressed rows, excluding capacity padding. Values are the first 512 channels of each shared KV row (Q/K width may also be 576). diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index 09e6c12b649..1a69c7b94bd 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -183,10 +183,30 @@ def forward( ): """Run full self-attention and optionally expose single-sequence CSA context. - Packed calls require contiguous INT32 ``cu_seqlens_q`` on the input - device and ``max_seqlen_q``. Prefixes have no padding/global offsets; - their contents are caller-owned and are not checked on the host. - These metadata arguments are accepted only for ``thd``. + Parameters + ---------- + hidden_states : torch.Tensor + Input ``[S,B,hidden_size]`` for ``sbhd``, ``[B,S,hidden_size]`` for + ``bshd``, or ``[T,hidden_size]`` for packed ``thd``. + cu_seqlens_q : torch.Tensor, optional + Contiguous CUDA INT32 ``[B+1]`` prefixes, required only for ``thd``. + Prefixes start at zero, end at T, and contain no padding/global offsets; + their contents are caller-owned and are not checked on the host. + max_seqlen_q : int, optional + Maximum sequence length, required only for ``thd``. + return_indexer_context : bool, default = False + Return context for a caller-owned CSA auxiliary loss; requires B=1. + + Returns + ------- + torch.Tensor or tuple + Output with the input shape, or ``(output, context)``. Context contains + live ``index_q[T,H,128]``, ``index_k[Tc,128]``, ``index_w[T,H]`` and + discrete compressed-row ``indices[T,K]``; H is ``index_n_heads``. + Teacher ``attn_q[T,64,512]``, ``local_kv[T,512]``, + ``compressed_kv[Tc,512]`` and ``sink[64]`` are detached. Indexer inputs + are detached from the shared hidden/query projections, so auxiliary + gradients train only the indexer tower. See the README loss example. """ packed = self.qkv_format == "thd" if ( diff --git a/transformer_engine/pytorch/attention/sparse_attention/indexer.py b/transformer_engine/pytorch/attention/sparse_attention/indexer.py index dcd056abac0..0c7d3ba9839 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/indexer.py +++ b/transformer_engine/pytorch/attention/sparse_attention/indexer.py @@ -67,7 +67,36 @@ def forward( max_seqlen, return_context=False, ): - """Select compressed KV entries and optionally expose live tensors for the indexer loss.""" + """Select compressed KV entries and optionally expose live indexer tensors. + + Parameters + ---------- + hidden_states : torch.Tensor + Batch-major ``[B,S,hidden_size]`` or packed ``[T,hidden_size]`` input. + q_residual : torch.Tensor + Query-down residual with matching leading axes and width ``q_lora_rank``. + cu, cu_comp : torch.Tensor + CUDA INT32 ``[B+1]`` token/compressed prefixes. + token_rope, compressed_rope : tuple of torch.Tensor + FP32 cosine/sine pairs ``[1,max_positions,1,rotary_width]`` for + token and compressed-window positions, respectively. + total_comp : int + Compressed output capacity: exactly ``B * (S // ratio)`` for dense + input; packed input may reserve extra rows beyond ``cu_comp[-1]``. + max_seqlen : int + Maximum token sequence length used to bound selection. For dense + input this must equal S; for packed input use the maximum segment length. + return_context : bool, default = False + Also return differentiable Q/K/head weights for a caller-owned loss. + + Returns + ------- + torch.Tensor or tuple of torch.Tensor + INT32 ``[T,min(top_k,max_seqlen // ratio)]`` global compressed-row + IDs, with invalid slots set to -1. With context, return + ``(indices, query[T,n_heads,head_dim], key[total_comp,head_dim], + weights[T,n_heads])``. Selection IDs do not carry gradients. + """ n_comp = max_seqlen // self.ratio comp_shape = (total_comp,) if hidden_states.ndim == 2 else (hidden_states.shape[0], n_comp) index_compressed = self.compressor(hidden_states, cu, cu_comp, total_comp=total_comp) From 47d7fe537b652cd4a1702570bdb06dd4f1dca581 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Tue, 10 Nov 2026 17:39:57 -0800 Subject: [PATCH 28/37] Register and organize sparse attention regression tests Include the sparse tests in the standard PyTorch launcher with its existing JUnit and failure handling. Group cases by component and remove redundant forward_backward suffixes while retaining the packed-layout distinction. Validation: AST comparison preserves all 13 test functions, assertions and 35 parametrized cases from the GPU-validated candidate. Shell syntax and pre-commit checks pass; this editorial change does not add a GPU run. Signed-off-by: Sudhakar Singh --- qa/L0_pytorch_unittest/test.sh | 1 + tests/pytorch/test_dsv4_attention.py | 275 ++++++++++++++------------- 2 files changed, 146 insertions(+), 130 deletions(-) diff --git a/qa/L0_pytorch_unittest/test.sh b/qa/L0_pytorch_unittest/test.sh index f45e9369d10..a1884c3f74b 100644 --- a/qa/L0_pytorch_unittest/test.sh +++ b/qa/L0_pytorch_unittest/test.sh @@ -65,6 +65,7 @@ python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_identity_quantiz NVTE_ALLOW_UNSAFE_PICKLE_EXTRA_STATE=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_attention.xml $TE_PATH/tests/pytorch/attention/test_attention.py || test_fail "test_attention.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_attention_backend_selection.xml $TE_PATH/tests/pytorch/attention/test_attention_backend_selection.py || test_fail "test_attention_backend_selection.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_flex_attention.xml $TE_PATH/tests/pytorch/attention/test_flex_attention.py || test_fail "test_flex_attention.py" +python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_dsv4_attention.xml $TE_PATH/tests/pytorch/test_dsv4_attention.py || test_fail "test_dsv4_attention.py" NVTE_GDN_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_gdn_attention.xml $TE_PATH/tests/pytorch/attention/test_gdn_attention.py || test_fail "test_gdn_attention.py" NVTE_GDN2_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_gdn2_attention.xml $TE_PATH/tests/pytorch/attention/test_gdn2_attention.py || test_fail "test_gdn2_attention.py" NVTE_GDP_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_gdp_attention.xml $TE_PATH/tests/pytorch/attention/test_gdp_attention.py || test_fail "test_gdp_attention.py" diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index 4568d283bc7..c886726f3a2 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -17,9 +17,12 @@ ) +# Sparse attention + + @pytest.mark.parametrize("variant", ["hca", "csa"]) @pytest.mark.parametrize("head_dim", [512, 576]) -def test_dsa_sparse_attention_forward_backward(variant, head_dim): +def test_dsa_sparse_attention(variant, head_dim): if not torch.cuda.is_available() or torch.cuda.get_device_capability() not in ( (10, 0), (10, 3), @@ -136,19 +139,11 @@ def test_dsa_rejects_unsupported_metadata(): ) -def test_dsv4_attention_rejects_fp8(monkeypatch): - from transformer_engine.pytorch.quantization import FP8GlobalStateManager - - monkeypatch.setattr(FP8GlobalStateManager, "is_fp8_enabled", lambda: True) - layer = dsv4_attention.DSv4Attention.__new__(dsv4_attention.DSv4Attention) - torch.nn.Module.__init__(layer) - layer.qkv_format, layer.hidden_size = "sbhd", 8 - with pytest.raises(NotImplementedError, match="BF16 only"): - layer(torch.empty(4, 1, 8)) +# RoPE @pytest.mark.parametrize("heads", [1, 64]) -def test_dsa_triton_rope_batched_forward_backward(monkeypatch, heads): +def test_dsa_triton_rope_batched(monkeypatch, heads): """The packed Triton kernels must reset positions at each BSHD sequence.""" if not torch.cuda.is_available(): pytest.skip("Triton DSA RoPE requires CUDA") @@ -192,7 +187,7 @@ def test_dsa_triton_rope_batched_forward_backward(monkeypatch, heads): torch.testing.assert_close(query_grad, reference_grad, atol=0.03125, rtol=0.002) -def test_dsa_rope_non_power_of_two_forward_backward(monkeypatch): +def test_dsa_rope_non_power_of_two(monkeypatch): """An accepted width of 96 must fall back for both rotary entry points.""" if not torch.cuda.is_available(): pytest.skip("DSA rotary dispatch requires CUDA") @@ -218,6 +213,129 @@ def test_dsa_rope_non_power_of_two_forward_backward(monkeypatch): torch.testing.assert_close(candidate_grad, reference_grad) +def test_dsa_packed_rope_eager(): + """Reset per-sequence positions and safely rotate unused compressed capacity.""" + torch.manual_seed(31) + cu = torch.tensor([0, 3, 8], dtype=torch.int32) + (cos, sin), _ = dsa_rope.rotary_embeddings(5, 1, 4, 12345.0, "cpu") + x = torch.randn(9, 2, 8, requires_grad=True) + reference_input = x.detach().clone().requires_grad_() + actual = dsa_rope.apply_rotary(x, cos, sin, cu) + expected = torch.cat( + [ + dsa_rope._apply_rotary_eager( + reference_input[start:end].unsqueeze(0), + cos[:, : end - start], + sin[:, : end - start], + ).squeeze(0) + for start, end in ((0, 3), (3, 8), (8, 9)) + ] + ) + gradient = torch.randn_like(actual) + actual.backward(gradient) + expected.backward(gradient) + torch.testing.assert_close(actual, expected) + torch.testing.assert_close(x.grad, reference_input.grad) + + +@pytest.mark.parametrize("interleaved", [False, True]) +@pytest.mark.parametrize("inverse", [False, True]) +@pytest.mark.parametrize("cp_rank", [0, 1]) +def test_common_rope_layout_inverse_packed(interleaved, inverse, cp_rank): + """Independent pair equations cover shared lookup, both layouts and transpose.""" + if not torch.cuda.is_available(): + pytest.skip("Triton RoPE requires CUDA") + kernels = dsa_rope._triton_rope_module() + if kernels is None: + pytest.skip("Triton DSL is unavailable") + import triton + + torch.manual_seed(53) + # CP2 local lengths are 4 and 6, plus one padded row. Each rank owns + # the first/last chunks prescribed by the zig-zag CP mapping. + positions = ( + [0, 1, 6, 7, 0, 1, 2, 9, 10, 11, 0] if cp_rank == 0 else [2, 3, 4, 5, 3, 4, 5, 6, 7, 8, 0] + ) + cu = torch.tensor([0, 8, 20], device="cuda", dtype=torch.int32) + x = torch.randn(11, 3, 80, device="cuda", dtype=torch.bfloat16) + # Noncontiguous rows ensure callers preserve the explicit table strides. + cos = torch.randn(12, 128, device="cuda")[:, ::2] + sin = torch.randn(12, 128, device="cuda")[:, ::2] + reference_input = x.detach().requires_grad_() + tail = reference_input[..., 16:].float() + c, s = cos[positions, None], sin[positions, None] + if inverse: + s = -s + left = tail[..., 0::2] * c[..., :32] - tail[..., 1::2] * s[..., :32] + right = tail[..., 1::2] * c[..., 32:] + tail[..., 0::2] * s[..., 32:] + rotated = ( + torch.stack((left, right), -1).flatten(-2) if interleaved else torch.cat((left, right), -1) + ) + reference = torch.cat((reference_input[..., :16], rotated.to(x.dtype)), -1) + gradient = torch.randn_like(x) + expected_grad = torch.autograd.grad(reference, reference_input, gradient)[0] + candidate, actual_grad = x.clone(), gradient.clone() + grid = lambda meta: (11, triton.cdiv(3, meta["BLOCK_H"])) + # The tables have strided columns; materialize those columns while retaining + # a larger row stride, which is the kernel's supported table contract. + cos = torch.stack((cos, cos), 1)[:, 0, :] + sin = torch.stack((sin, sin), 1)[:, 0, :] + for kernel, tensor in ( + (kernels._autotuned_mla_rope_fwd_inplace_kernel, candidate), + (kernels._autotuned_mla_rope_bwd_inplace_kernel, actual_grad), + ): + kernel[grid]( + tensor, + cos, + sin, + 16, + 64, + 3, + 1, + 2, + cu, + None, + tensor.stride(0), + tensor.stride(1), + cos.stride(0), + sin.stride(0), + cp_rank, + 2, + INVERSE=inverse, + REMOVE_INTERLEAVING=interleaved, + ) + torch.testing.assert_close(candidate, reference, atol=0.03125, rtol=0.002) + torch.testing.assert_close(actual_grad, expected_grad, atol=0.03125, rtol=0.002) + + +# Compressor + + +def test_dsa_compressor_clears_unused_capacity(monkeypatch): + """Unused cuDNN output must not contaminate normalization weight gradients.""" + layer = Compressor.__new__(Compressor) + torch.nn.Module.__init__(layer) + layer.kv_proj = layer.gate_proj = torch.nn.Identity() + layer.kv_norm = torch.nn.RMSNorm(4) + layer.position_bias = torch.nn.Parameter(torch.zeros(4, 4)) + layer.projected, layer.ratio, layer.overlap, layer.weight_width = 4, 4, False, 0 + monkeypatch.setattr( + compressor, + "compress", + lambda *args, **kwargs: torch.cat((torch.ones(1, 4), torch.full((1, 4), float("nan")))), + ) + cu = torch.tensor([0, 3, 8], dtype=torch.int32) + cu_comp = torch.tensor([0, 0, 1], dtype=torch.int32) + output = layer(torch.empty(8, 4), cu, cu_comp, total_comp=2) + output.sum().backward() + assert torch.isfinite(output).all() + assert torch.isfinite(layer.kv_norm.weight.grad).all() + assert torch.count_nonzero(output[1]) == 0 + + +# Indexer + + @pytest.mark.parametrize("index_heads", [32, 64]) def test_dsa_indexer_backward(index_heads): """Train the standalone Indexer and its Compressor against a dense KL reference.""" @@ -347,6 +465,20 @@ def test_dsa_indexer_backward(index_heads): assert error < 0.03, error.item() +# DSv4 layer wiring + + +def test_dsv4_attention_rejects_fp8(monkeypatch): + from transformer_engine.pytorch.quantization import FP8GlobalStateManager + + monkeypatch.setattr(FP8GlobalStateManager, "is_fp8_enabled", lambda: True) + layer = dsv4_attention.DSv4Attention.__new__(dsv4_attention.DSv4Attention) + torch.nn.Module.__init__(layer) + layer.qkv_format, layer.hidden_size = "sbhd", 8 + with pytest.raises(NotImplementedError, match="BF16 only"): + layer(torch.empty(4, 1, 8)) + + def test_dsv4_thd_compressed_prefixes(): """Floor each sequence independently, retaining a safe allocation capacity.""" for lengths, capacity in (([129, 257], 96), ([131, 257], 97)): @@ -358,28 +490,6 @@ def test_dsv4_thd_compressed_prefixes(): assert maximum == 257 -def test_dsa_compressor_clears_unused_capacity(monkeypatch): - """Unused cuDNN output must not contaminate normalization weight gradients.""" - layer = Compressor.__new__(Compressor) - torch.nn.Module.__init__(layer) - layer.kv_proj = layer.gate_proj = torch.nn.Identity() - layer.kv_norm = torch.nn.RMSNorm(4) - layer.position_bias = torch.nn.Parameter(torch.zeros(4, 4)) - layer.projected, layer.ratio, layer.overlap, layer.weight_width = 4, 4, False, 0 - monkeypatch.setattr( - compressor, - "compress", - lambda *args, **kwargs: torch.cat((torch.ones(1, 4), torch.full((1, 4), float("nan")))), - ) - cu = torch.tensor([0, 3, 8], dtype=torch.int32) - cu_comp = torch.tensor([0, 0, 1], dtype=torch.int32) - output = layer(torch.empty(8, 4), cu, cu_comp, total_comp=2) - output.sum().backward() - assert torch.isfinite(output).all() - assert torch.isfinite(layer.kv_norm.weight.grad).all() - assert torch.count_nonzero(output[1]) == 0 - - @pytest.mark.parametrize( "prefix,maximum", [ @@ -417,34 +527,9 @@ def test_dsv4_thd_rejects_hidden_batch_axis_and_multi_sequence_context(): ) -def test_dsa_packed_rope_eager_forward_backward(): - """Reset per-sequence positions and safely rotate unused compressed capacity.""" - torch.manual_seed(31) - cu = torch.tensor([0, 3, 8], dtype=torch.int32) - (cos, sin), _ = dsa_rope.rotary_embeddings(5, 1, 4, 12345.0, "cpu") - x = torch.randn(9, 2, 8, requires_grad=True) - reference_input = x.detach().clone().requires_grad_() - actual = dsa_rope.apply_rotary(x, cos, sin, cu) - expected = torch.cat( - [ - dsa_rope._apply_rotary_eager( - reference_input[start:end].unsqueeze(0), - cos[:, : end - start], - sin[:, : end - start], - ).squeeze(0) - for start, end in ((0, 3), (3, 8), (8, 9)) - ] - ) - gradient = torch.randn_like(actual) - actual.backward(gradient) - expected.backward(gradient) - torch.testing.assert_close(actual, expected) - torch.testing.assert_close(x.grad, reference_input.grad) - - @pytest.mark.parametrize("variant,ratio", [("hca", 128), ("csa", 4)]) @pytest.mark.parametrize("lengths", [[129, 257], [255, 383]]) -def test_dsv4_attention_thd_forward_backward(variant, ratio, lengths): +def test_dsv4_attention_thd(variant, ratio, lengths): """Packed full-layer output and gradients match independent dense sequences.""" if not torch.cuda.is_available() or torch.cuda.get_device_capability() not in ( (10, 0), @@ -510,73 +595,3 @@ def test_dsv4_attention_thd_forward_backward(variant, ratio, lengths): assert torch.isfinite(packed_grad).all() and torch.isfinite(reference_grad).all() error = (packed_grad.float() - reference_grad.float()).norm() assert error / reference_grad.float().norm().clamp_min(1e-8) < 0.03 - - -@pytest.mark.parametrize("interleaved", [False, True]) -@pytest.mark.parametrize("inverse", [False, True]) -@pytest.mark.parametrize("cp_rank", [0, 1]) -def test_common_rope_layout_inverse_packed(interleaved, inverse, cp_rank): - """Independent pair equations cover shared lookup, both layouts and transpose.""" - if not torch.cuda.is_available(): - pytest.skip("Triton RoPE requires CUDA") - kernels = dsa_rope._triton_rope_module() - if kernels is None: - pytest.skip("Triton DSL is unavailable") - import triton - - torch.manual_seed(53) - # CP2 local lengths are 4 and 6, plus one padded row. Each rank owns - # the first/last chunks prescribed by the zig-zag CP mapping. - positions = ( - [0, 1, 6, 7, 0, 1, 2, 9, 10, 11, 0] if cp_rank == 0 else [2, 3, 4, 5, 3, 4, 5, 6, 7, 8, 0] - ) - cu = torch.tensor([0, 8, 20], device="cuda", dtype=torch.int32) - x = torch.randn(11, 3, 80, device="cuda", dtype=torch.bfloat16) - # Noncontiguous rows ensure callers preserve the explicit table strides. - cos = torch.randn(12, 128, device="cuda")[:, ::2] - sin = torch.randn(12, 128, device="cuda")[:, ::2] - reference_input = x.detach().requires_grad_() - tail = reference_input[..., 16:].float() - c, s = cos[positions, None], sin[positions, None] - if inverse: - s = -s - left = tail[..., 0::2] * c[..., :32] - tail[..., 1::2] * s[..., :32] - right = tail[..., 1::2] * c[..., 32:] + tail[..., 0::2] * s[..., 32:] - rotated = ( - torch.stack((left, right), -1).flatten(-2) if interleaved else torch.cat((left, right), -1) - ) - reference = torch.cat((reference_input[..., :16], rotated.to(x.dtype)), -1) - gradient = torch.randn_like(x) - expected_grad = torch.autograd.grad(reference, reference_input, gradient)[0] - candidate, actual_grad = x.clone(), gradient.clone() - grid = lambda meta: (11, triton.cdiv(3, meta["BLOCK_H"])) - # The tables have strided columns; materialize those columns while retaining - # a larger row stride, which is the kernel's supported table contract. - cos = torch.stack((cos, cos), 1)[:, 0, :] - sin = torch.stack((sin, sin), 1)[:, 0, :] - for kernel, tensor in ( - (kernels._autotuned_mla_rope_fwd_inplace_kernel, candidate), - (kernels._autotuned_mla_rope_bwd_inplace_kernel, actual_grad), - ): - kernel[grid]( - tensor, - cos, - sin, - 16, - 64, - 3, - 1, - 2, - cu, - None, - tensor.stride(0), - tensor.stride(1), - cos.stride(0), - sin.stride(0), - cp_rank, - 2, - INVERSE=inverse, - REMOVE_INTERLEAVING=interleaved, - ) - torch.testing.assert_close(candidate, reference, atol=0.03125, rtol=0.002) - torch.testing.assert_close(actual_grad, expected_grad, atol=0.03125, rtol=0.002) From 3521f19a4a28dfc72eb25a6e20bd485099047249 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Tue, 10 Nov 2026 18:05:31 -0800 Subject: [PATCH 29/37] Remove redundant fusion options from sparse attention helpers Apply fused projection composition consistently to Compressor and Indexer instead of retaining unused unfused branches and passing a constant flag. Remove the obsolete mode-specific rejection and update the capacity fixture to exercise the fused split. The prior standalone unfused parameter layout is intentionally retired; existing fused and high-level DSv4 parameter keys and row order are preserved. Validation: the existing Compressor CPU output/gradient regression passes with its fused fixture; pre-commit and changed-production pylint pass. A separate review checked related source, callers, documentation and tests. No fresh GPU integration run was performed for this branch simplification. Signed-off-by: Sudhakar Singh --- tests/pytorch/test_dsv4_attention.py | 8 ++--- .../attention/sparse_attention/README.md | 3 +- .../attention/sparse_attention/compressor.py | 34 +++++++------------ .../sparse_attention/dsv4_attention.py | 2 -- .../attention/sparse_attention/indexer.py | 17 +++------- 5 files changed, 21 insertions(+), 43 deletions(-) diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index c886726f3a2..c829e0e84f2 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -113,8 +113,6 @@ def test_dsa_sparse_attention(variant, head_dim): def test_dsa_rejects_unsupported_metadata(): - with pytest.raises(ValueError, match="weight_width requires fused=True"): - Compressor(8, 4, 2, False, 1e-6, "cpu", torch.float32, weight_width=1) with pytest.raises(ValueError, match="positive"): dsa_cudnn.sparse_attention( torch.empty(1, 64, 512), @@ -315,7 +313,7 @@ def test_dsa_compressor_clears_unused_capacity(monkeypatch): """Unused cuDNN output must not contaminate normalization weight gradients.""" layer = Compressor.__new__(Compressor) torch.nn.Module.__init__(layer) - layer.kv_proj = layer.gate_proj = torch.nn.Identity() + layer.fused_proj = torch.nn.Identity() layer.kv_norm = torch.nn.RMSNorm(4) layer.position_bias = torch.nn.Parameter(torch.zeros(4, 4)) layer.projected, layer.ratio, layer.overlap, layer.weight_width = 4, 4, False, 0 @@ -326,7 +324,7 @@ def test_dsa_compressor_clears_unused_capacity(monkeypatch): ) cu = torch.tensor([0, 3, 8], dtype=torch.int32) cu_comp = torch.tensor([0, 0, 1], dtype=torch.int32) - output = layer(torch.empty(8, 4), cu, cu_comp, total_comp=2) + output = layer(torch.empty(8, 8), cu, cu_comp, total_comp=2) output.sum().backward() assert torch.isfinite(output).all() assert torch.isfinite(layer.kv_norm.weight.grad).all() @@ -357,7 +355,7 @@ def test_dsa_indexer_backward(index_heads): pytest.skip("cuDNN dense indexer training APIs are unavailable") torch.manual_seed(43) seq, ratio, window = 256, 4, 32 - layer = Indexer(32, 16, 128, index_heads, 8, ratio, 1e-6, "cuda", torch.bfloat16, fused=True) + layer = Indexer(32, 16, 128, index_heads, 8, ratio, 1e-6, "cuda", torch.bfloat16) x = torch.randn(1, seq, 32, device="cuda", dtype=torch.bfloat16) q_residual = torch.randn(1, seq, 16, device="cuda", dtype=torch.bfloat16) cu = torch.tensor([0, seq], device=x.device, dtype=torch.int32) diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index e5f7fa63a9e..ce3d98d5fb0 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -182,5 +182,4 @@ Shared-input projections in `DSv4Attention` are fused: `q_a_kv_proj.weight` stores query-down then local-KV rows; `compressor.fused_proj.weight` stores KV then gate rows; the CSA `indexer.compressor.fused_proj.weight` additionally stores head-weight rows last. These default checkpoint keys and shapes are -preserved. `Compressor` and `Indexer` retain their independent projection -options. Separate-weight model imports must concatenate those rows explicitly. +preserved. Separate-weight model imports must concatenate those rows explicitly. diff --git a/transformer_engine/pytorch/attention/sparse_attention/compressor.py b/transformer_engine/pytorch/attention/sparse_attention/compressor.py index a0d4f9bda5f..fe111b8d423 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/compressor.py +++ b/transformer_engine/pytorch/attention/sparse_attention/compressor.py @@ -35,7 +35,7 @@ class Compressor(torch.nn.Module): Accept batch-major ``[B,S,D]`` or packed ``[T,D]`` hidden states with token/compressed prefixes. Return ``[total_comp,width]`` rows before RoPE, - plus projected head weights when ``weight_width`` is nonzero in fused mode. + plus projected head weights when ``weight_width`` is nonzero. """ def __init__( @@ -48,21 +48,14 @@ def __init__( device, params_dtype, *, - fused=False, weight_width=0, ): super().__init__() - if weight_width and not fused: - raise ValueError("weight_width requires fused=True.") projected = width * (2 if overlap else 1) kw = {"bias": False, "device": device, "params_dtype": params_dtype} - if fused: - # Fuse the KV, gate, and optional index-weight projections of hidden states. - # Keep one contiguous weight for the shared projection call. - self.fused_proj = Linear(hidden_size, 2 * projected + weight_width, **kw) - else: - self.kv_proj = Linear(hidden_size, projected, **kw) - self.gate_proj = Linear(hidden_size, projected, **kw) + # Fuse the KV, gate, and optional index-weight projections of hidden states. + # Keep one contiguous weight for the shared projection call. + self.fused_proj = Linear(hidden_size, 2 * projected + weight_width, **kw) self.position_bias = torch.nn.Parameter( torch.zeros(ratio, projected, device=device, dtype=params_dtype) ) @@ -91,17 +84,14 @@ def forward(self, hidden_states, cu_seqlens, cu_seqlens_comp, *, total_comp): also return head weights with shape ``hidden_states.shape[:-1] + (weight_width,)``. Only rows below the final compressed prefix are valid. """ - if hasattr(self, "fused_proj"): - projected = self.fused_proj(hidden_states) - widths = ( - (self.projected, self.projected, self.weight_width) - if self.weight_width - else (self.projected, self.projected) - ) - parts = projected.split(widths, dim=-1) - kv, gate = parts[:2] - else: - kv, gate = self.kv_proj(hidden_states), self.gate_proj(hidden_states) + projected = self.fused_proj(hidden_states) + widths = ( + (self.projected, self.projected, self.weight_width) + if self.weight_width + else (self.projected, self.projected) + ) + parts = projected.split(widths, dim=-1) + kv, gate = parts[:2] compressed_kv = compress( kv.reshape(-1, self.projected).contiguous(), gate.reshape(-1, self.projected).contiguous(), diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index 1a69c7b94bd..f9e497dae65 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -131,7 +131,6 @@ def __init__( rms_norm_eps, device, params_dtype, - fused=True, ) if is_csa: self.indexer = Indexer( @@ -144,7 +143,6 @@ def __init__( rms_norm_eps, device, params_dtype, - fused=True, ) # HF/Megatron use one independent block per output group. A normal TE # Linear forward mixes groups, so use its weight in the block multiply. diff --git a/transformer_engine/pytorch/attention/sparse_attention/indexer.py b/transformer_engine/pytorch/attention/sparse_attention/indexer.py index 0c7d3ba9839..1774aa86d26 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/indexer.py +++ b/transformer_engine/pytorch/attention/sparse_attention/indexer.py @@ -32,8 +32,6 @@ def __init__( eps, device, params_dtype, - *, - fused, ): super().__init__() self.compressor = Compressor( @@ -44,15 +42,12 @@ def __init__( eps, device, params_dtype, - fused=fused, - weight_width=n_heads if fused else 0, + weight_width=n_heads, ) kw = {"bias": False, "device": device, "params_dtype": params_dtype} self.q_proj = Linear(q_lora_rank, n_heads * head_dim, **kw) - if not fused: - self.weights_proj = Linear(hidden_size, n_heads, **kw) self.head_dim, self.n_heads = head_dim, n_heads - self.top_k, self.ratio, self.fused = top_k, ratio, fused + self.top_k, self.ratio = top_k, ratio def forward( self, @@ -99,11 +94,9 @@ def forward( """ n_comp = max_seqlen // self.ratio comp_shape = (total_comp,) if hidden_states.ndim == 2 else (hidden_states.shape[0], n_comp) - index_compressed = self.compressor(hidden_states, cu, cu_comp, total_comp=total_comp) - if self.fused: - index_compressed, weights = index_compressed - else: - weights = self.weights_proj(hidden_states) + index_compressed, weights = self.compressor( + hidden_states, cu, cu_comp, total_comp=total_comp + ) # The singleton axis is the shared index-key head, not a tunable head count. key = ( apply_rotary( From 7aeb880e167fff5fb6763b56294a1bea4ec6a00b Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Mon, 16 Nov 2026 14:28:27 -0800 Subject: [PATCH 30/37] Use token terminology consistently in sparse attention Describe query, local KV and compressed KV token indices explicitly so selection indices cannot be confused with indices into concatenated KV. Distinguish incomplete input blocks from packed output padding and retain the normalization cleanup rationale. Preserve physical weight/table rows. Only docstrings, comments, one error message and a private local variable name change. Executable AST equivalence, pre-commit, changed-production pylint and whitespace checks pass; no numerical behavior or public call contract changes. Signed-off-by: Sudhakar Singh --- tests/pytorch/test_dsv4_attention.py | 6 +- .../attention/sparse_attention/README.md | 14 +++-- .../attention/sparse_attention/compressor.py | 28 ++++++---- .../attention/sparse_attention/dsa_cudnn.py | 55 +++++++++++-------- .../attention/sparse_attention/dsa_rope.py | 8 +-- .../sparse_attention/dsv4_attention.py | 8 ++- .../attention/sparse_attention/indexer.py | 17 +++--- 7 files changed, 78 insertions(+), 58 deletions(-) diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index c829e0e84f2..193e9493af5 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -60,8 +60,8 @@ def test_dsa_sparse_attention(variant, head_dim): reference_leaves = tuple(x.detach().float().requires_grad_() for x in leaves) indices = None if variant == "csa": - # Select alternating compressed entries; distinct global IDs expose - # mistakes at the second sequence's compressed offset. + # Select alternating compressed KV tokens; global compressed KV token indices + # expose mistakes at the second sequence's compressed token offset. indices = torch.full((total, 32), -1, device="cuda", dtype=torch.int32) start = comp_start = 0 for length, count in zip(lengths, comp_lengths): @@ -249,7 +249,7 @@ def test_common_rope_layout_inverse_packed(interleaved, inverse, cp_rank): import triton torch.manual_seed(53) - # CP2 local lengths are 4 and 6, plus one padded row. Each rank owns + # CP2 local lengths are 4 and 6, plus one padded token position. Each rank owns # the first/last chunks prescribed by the zig-zag CP mapping. positions = ( [0, 1, 6, 7, 0, 1, 2, 9, 10, 11, 0] if cp_rank == 0 else [2, 3, 4, 5, 3, 4, 5, 6, 7, 8, 0] diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index ce3d98d5fb0..3f8f6196840 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -64,13 +64,15 @@ output = dsa_cudnn.sparse_attention( output = ... # Apply inverse RoPE and grouped output projection. ``` -Dense inputs are repacked to batch-major rows before these calls; packed THD -inputs already have that layout. Attention uses the original query +Dense inputs are repacked into tokens in batch-major order before these calls; +packed THD inputs already have that layout. Attention uses the original query `[T,64,D]` and concatenated local/compressed KV `[T+Tc,D]`; there is no -compressed query. `_prepare_sparse_indices` creates one row per query containing -causal local IDs and compressed IDs offset by `T`, with invalid slots set to -`-1`. This address preparation differs from `select_indices`, which scores -the indexer's projected Q/K/head weights and returns compressed-row IDs. +compressed query. `T` is the total query/local KV token count. +`_prepare_sparse_indices` creates a list of KV token indices for each query token, +combining causal local KV token indices with compressed KV token indices offset +by `T`; invalid entries are `-1`. This differs from `select_indices`, which scores +the indexer's projected Q/K/head weights and returns global compressed KV token +indices addressing only packed compressed KV, before the offset is applied. `dsa_rope` supplies token/compressed-position embeddings and rotary application. A caller may override the default embeddings with a `rope` module returning diff --git a/transformer_engine/pytorch/attention/sparse_attention/compressor.py b/transformer_engine/pytorch/attention/sparse_attention/compressor.py index fe111b8d423..510705012b8 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/compressor.py +++ b/transformer_engine/pytorch/attention/sparse_attention/compressor.py @@ -29,13 +29,13 @@ class Compressor(torch.nn.Module): ``[e f g h]``, ``[g h i j]``. The first overlapping entry masks the missing previous block. Both modes emit - ``floor(length / ratio)`` entries per sequence; incomplete trailing blocks - (``k`` here) are excluded from compression. In DSv4Attention, these tokens - remain available through local sliding-window attention. + ``floor(length / ratio)`` compressed KV tokens per sequence. Trailing tokens + that do not fill a complete block (``k`` here) are excluded from compression. + In DSv4Attention, they remain available through local sliding-window attention. Accept batch-major ``[B,S,D]`` or packed ``[T,D]`` hidden states with - token/compressed prefixes. Return ``[total_comp,width]`` rows before RoPE, - plus projected head weights when ``weight_width`` is nonzero. + token/compressed prefixes. Return ``[total_comp,width]`` compressed KV representations + before RoPE, plus projected head weights when ``weight_width`` is nonzero. """ def __init__( @@ -74,15 +74,18 @@ def forward(self, hidden_states, cu_seqlens, cu_seqlens_comp, *, total_comp): Contiguous CUDA INT32 ``[B+1]`` token/compressed prefixes. Compressed segment lengths are ``floor(length / ratio)``. total_comp : int - Output row capacity. For dense input, use exactly ``B * (S // ratio)``; - packed input may reserve extra rows beyond ``cu_seqlens_comp[-1]``. + Allocated compressed output length. For dense input, use exactly + ``B * (S // ratio)``; packed output may include padding after the valid + compressed KV tokens. Returns ------- torch.Tensor or tuple of torch.Tensor - Normalized ``[total_comp,width]`` KV rows. When ``weight_width > 0``, - also return head weights with shape ``hidden_states.shape[:-1] + - (weight_width,)``. Only rows below the final compressed prefix are valid. + Normalized ``[total_comp,width]`` compressed KV representations. + When ``weight_width > 0``, also return head weights with shape + ``hidden_states.shape[:-1] + + (weight_width,)``. Only outputs for compressed KV token indices below + ``cu_seqlens_comp[-1]`` are valid; the remaining output is padding. """ projected = self.fused_proj(hidden_states) widths = ( @@ -104,8 +107,9 @@ def forward(self, hidden_states, cu_seqlens, cu_seqlens_comp, *, total_comp): total_comp=total_comp, ) if hidden_states.ndim == 2: - # cuDNN only guarantees valid rows below the final prefix. Clear - # unused capacity before RMSNorm so it cannot poison weight gradients. + # cuDNN guarantees valid output only before the final compressed prefix. + # Zero output from cu_seqlens_comp[-1] onward before RMSNorm so + # uninitialized values cannot affect weight gradients. valid = torch.arange(total_comp, device=hidden_states.device) < cu_seqlens_comp[-1] compressed_kv = compressed_kv.masked_fill(~valid[:, None], 0) compressed = self.kv_norm(compressed_kv) diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py index 05340d1c41b..37015960cd2 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py @@ -42,7 +42,8 @@ def _validate_tensors(reference, bf16, fp32, cu_seqlens, cu_seqlens_comp): """Check metadata without copying packed sequence offsets to the CPU. Callers own prefix contents: start at zero, monotonic, final offsets equal - valid row counts, and compressed lengths floor(length / ratio). + valid query/local KV and compressed KV token counts, respectively, and + compressed lengths floor(length / ratio). """ if not reference.is_cuda or torch.cuda.get_device_capability(reference.device) not in ( (10, 0), @@ -133,10 +134,11 @@ def compress( bias added to gate scores before the compression softmax. coff is 2 for overlapping compression, 1 otherwise. Sequence prefixes are CUDA INT32 [B+1]; compressed segment lengths are floor(length / ratio). - Returns [total_comp, D]. Per-segment tails are dropped by cuDNN. + Returns [total_comp, D]. Trailing input tokens that do not fill a complete + block are excluded from compression. - total_comp specifies output capacity; only entries before - cu_seqlens_comp[-1] are valid. + total_comp specifies output capacity. Only outputs for compressed KV token + indices below cu_seqlens_comp[-1] are valid; the remaining output is padding. """ if ratio <= 0 or total_comp < 0: raise ValueError("ratio must be positive and total_comp nonnegative.") @@ -169,21 +171,22 @@ def select_indices( max_compressed_seqlen: int, scale: Optional[float] = None, ) -> torch.Tensor: - """Select compressed rows with cuDNN's combined index-score + Top-K operation. + """Select compressed KV tokens with cuDNN's combined index-score + Top-K operation. Inputs are the indexer's projected Q/K/head weights, after compression/RoPE. Unlike the Indexer module, this operation does not project or compress inputs. query [T,H,128], compressed_key [Tc,128], weights [T,H]; H=32 or 64. cu_seqlens/cu_seqlens_comp are CUDA INT32 [B+1] token/compressed prefixes. Full packed-sequence prefill only: causal timeline offsets start at zero. - Returns INT32 [T,top_k] indices into compressed_key's global packed rows; - -1 denotes an invalid slot. These are NOT IDs into local+compressed KV. + Returns INT32 [T,top_k] global compressed KV token indices into packed + compressed_key, with one list per query token; -1 denotes an invalid entry. + These indices address compressed KV only, not concatenated local + compressed KV. Selection is discrete. Scale defaults to 1/sqrt(index head dim). If the model also scales by 1/sqrt(index heads), fold that positive factor into scale and pass raw projected weights. Scaling in FP32 then rounding weights back to activation dtype can change close top-k boundaries. This helper does not - implement the separate indexer auxiliary loss or autograd through IDs. + implement the separate indexer auxiliary loss or autograd through token indices. max_compressed_seqlen is a compile bound and must cover the partial final ratio; use ceil(max_seqlen / ratio) even though prefixes count only complete compressed blocks. @@ -326,12 +329,16 @@ def _prepare_sparse_indices( indices: Optional[torch.Tensor], max_compressed_seqlen: Optional[int], ) -> torch.Tensor: - """Build one KV-index row per original query token, using its shape/device. + """Build a list of KV token indices for each query token. - Query values are not read. Concatenate causal local-row IDs and compressed - IDs offset by T into [T, window_size + compressed_slots]; invalid slots - are -1. This prepares attention addresses, whereas select_indices chooses - compressed entries from indexer scores. + Use query only for its token count T and device; query values are not read. + Return INT32 [T, window_size + compressed_slots] indices into concatenated + [local_kv; compressed_kv], where local_kv also contains T tokens. + + Combine causal local KV token indices with eligible compressed KV token + indices. Compressed indices are global within packed compressed KV, not + relative to each sequence, and are offset by T here. Invalid entries are -1. + select_indices chooses compressed KV token indices before this offset. """ total_q = query.shape[0] device = query.device @@ -383,10 +390,11 @@ def _prepare_sparse_indices( class _SparseAttention(torch.autograd.Function): @staticmethod def forward(ctx, query, kv, indices, sink, scale): - """Attend query [T,64,D] to concatenated KV [T+Tc,D] using INT32 rows. + """Attend query [T,64,D] to concatenated KV [T+Tc,D] using INT32 KV token indices. - indices [T,window_size+compressed_slots] addresses that KV tensor, with - -1 for invalid slots. No compressed query is constructed. + indices [T,window_size+compressed_slots] contains one list per query token + addressing concatenated local + compressed KV, with -1 for invalid entries. + No compressed query is constructed. """ result = _dsa_op("sparse_attention_forward_wrapper")( query, @@ -438,16 +446,17 @@ def sparse_attention( Full packed-sequence prefill: query [T,H,D], local_kv [T,D], compressed_kv [Tc,D], sink [H]. Q/KV have already had the model's norm/RoPE applied. - The high-level layer repacks dense SBHD/BSHD inputs to these batch-major - packed rows; THD inputs already use packed rows. There is no compressed Q. + The high-level layer repacks dense SBHD/BSHD inputs into tokens packed in + batch-major order; THD inputs already use this token order. There is no compressed Q. cu_seqlens describes both query and local KV; cu_seqlens_comp describes - valid compressed rows, excluding capacity padding. Values are the first - 512 channels of each shared KV row (Q/K width may also be 576). + valid compressed KV tokens, excluding output padding. Values are the first + 512 channels of each shared KV token representation (Q/K width may also be 576). Local positions satisfy max(0,q-window_size+1) <= k <= q per sequence. Compressed block j is eligible when (j+1)*ratio <= q+1. With indices (CSA), - use its selected, valid, causal compressed rows; without indices (HCA), - use every eligible compressed row. IDs refer to compressed_kv only. + use its selected, valid, causal compressed KV tokens; without indices (HCA), + use every eligible compressed KV token. Input indices address packed compressed_kv + only, before adding the local KV offset. One cuDNN sparse-attention call normalizes the combined visible entries with the sink and provides the KV-only LSE consumed by cuDNN backward. Output is @@ -465,7 +474,7 @@ def sparse_attention( if local_kv.shape[-1] != query.shape[-1] or compressed_kv.shape[-1] != query.shape[-1]: raise ValueError("Q and both KV tensors must have the same QK dimension.") if local_kv.shape[0] != query.shape[0]: - raise ValueError("Full-sequence attention requires one local KV row per query.") + raise ValueError("Full-sequence attention requires one local KV token per query token.") if sink.shape != (query.shape[1],): raise ValueError("sink must have shape [64].") if window_size <= 0 or ratio <= 0: diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py index 26cf10b1fc6..800d6ff42e8 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py @@ -58,11 +58,11 @@ def _apply_rotary_eager(x, cos, sin, cu_seqlens=None): if x.ndim == 3: if cu_seqlens is None: raise ValueError("Packed RoPE requires cu_seqlens.") - rows = torch.arange(x.shape[0], device=x.device) - sequences = torch.bucketize(rows, cu_seqlens[1:], right=True) - positions = rows - cu_seqlens[sequences] + token_indices = torch.arange(x.shape[0], device=x.device) + sequences = torch.bucketize(token_indices, cu_seqlens[1:], right=True) + positions = token_indices - cu_seqlens[sequences] # Compressed tensors may have unused capacity past the final prefix. - positions = torch.where(rows < cu_seqlens[-1], positions, 0) + positions = torch.where(token_indices < cu_seqlens[-1], positions, 0) cos, sin = cos[0, positions], sin[0, positions] width = cos.shape[-1] tail = x[..., -width:] diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index f9e497dae65..24921dc0647 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -40,7 +40,8 @@ def _packed_prefixes(hidden_states, cu_seqlens, max_seqlen, ratio): raise ValueError("THD requires integer max_seqlen_q >= compression_ratio.") # The sum of per-sequence floors can be smaller than floor(T / ratio). # Allocate that safe capacity without reading CUDA prefix contents on host; - # cuDNN uses the true device prefix to exclude unused trailing rows. + # cuDNN uses the compressed prefix to exclude output padding beyond the valid + # compressed KV tokens. lengths = cu_seqlens[1:] - cu_seqlens[:-1] cu_comp = torch.cat((cu_seqlens.new_zeros(1), (lengths // ratio).cumsum(0, dtype=torch.int32))) total_comp = hidden_states.shape[0] // ratio @@ -200,7 +201,8 @@ def forward( torch.Tensor or tuple Output with the input shape, or ``(output, context)``. Context contains live ``index_q[T,H,128]``, ``index_k[Tc,128]``, ``index_w[T,H]`` and - discrete compressed-row ``indices[T,K]``; H is ``index_n_heads``. + discrete compressed KV token indices ``indices[T,K]``, before adding + the local KV offset; H is ``index_n_heads``. Teacher ``attn_q[T,64,512]``, ``local_kv[T,512]``, ``compressed_kv[Tc,512]`` and ``sink[64]`` are detached. Indexer inputs are detached from the shared hidden/query projections, so auxiliary @@ -219,7 +221,7 @@ def forward( if return_indexer_context and not self.is_csa: raise ValueError("Only CSA has an indexer context.") if self.qkv_format == "sbhd": - # cuDNN consumes batch-major packed rows. B=1 needs only a view; + # cuDNN consumes tokens packed in batch-major order. B=1 needs only a view; # larger batches require a physical repack with the current core. hidden_states = hidden_states.transpose(0, 1).contiguous() if packed: diff --git a/transformer_engine/pytorch/attention/sparse_attention/indexer.py b/transformer_engine/pytorch/attention/sparse_attention/indexer.py index 1774aa86d26..ba496ee11d6 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/indexer.py +++ b/transformer_engine/pytorch/attention/sparse_attention/indexer.py @@ -17,7 +17,8 @@ class Indexer(torch.nn.Module): """Select compressed KV entries using the indexer's own Q/K/weight projections. Accept batch-major or packed hidden states and query residuals, sequence - prefixes, and caller-provided RoPE tables. Return global compressed-row IDs; + prefixes, and caller-provided RoPE tables. Return global compressed KV token + indices addressing compressed KV only; ``return_context=True`` also returns live Q/K/head weights for a separate loss. """ @@ -76,8 +77,9 @@ def forward( FP32 cosine/sine pairs ``[1,max_positions,1,rotary_width]`` for token and compressed-window positions, respectively. total_comp : int - Compressed output capacity: exactly ``B * (S // ratio)`` for dense - input; packed input may reserve extra rows beyond ``cu_comp[-1]``. + Allocated compressed output length: exactly ``B * (S // ratio)`` for + dense input; packed output may include padding after the valid + compressed KV tokens. max_seqlen : int Maximum token sequence length used to bound selection. For dense input this must equal S; for packed input use the maximum segment length. @@ -87,10 +89,11 @@ def forward( Returns ------- torch.Tensor or tuple of torch.Tensor - INT32 ``[T,min(top_k,max_seqlen // ratio)]`` global compressed-row - IDs, with invalid slots set to -1. With context, return + INT32 ``[T,min(top_k,max_seqlen // ratio)]`` global compressed KV token + indices addressing compressed KV only, with invalid entries set to -1. + With context, return ``(indices, query[T,n_heads,head_dim], key[total_comp,head_dim], - weights[T,n_heads])``. Selection IDs do not carry gradients. + weights[T,n_heads])``. Selected token indices do not carry gradients. """ n_comp = max_seqlen // self.ratio comp_shape = (total_comp,) if hidden_states.ndim == 2 else (hidden_states.shape[0], n_comp) @@ -115,7 +118,7 @@ def forward( .reshape(-1, self.n_heads, self.head_dim) .contiguous() ) - # Top-k IDs are discrete; training index projections needs a separate loss. + # Top-k token indices are discrete; training index projections needs a separate loss. weights = weights.reshape(-1, self.n_heads).contiguous() indices = select_indices( query, From 02630edf6e0550013c0a2ef8f90583a93c8041fe Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Mon, 16 Nov 2026 23:37:09 -0800 Subject: [PATCH 31/37] Clarify sparse attention APIs and worked index examples Document existing operation and constructor contracts consistently, using explicit batched/packed shapes and cumulative token counts. Explain the TE output allocation separately from valid compressed tokens and illustrate local/compressed index conversion beside the actual implementation. Preserve user prose while resolving inline notes. Changes are documentation, comments and the clarified full-sequence shape-error text; numerical code and public call contracts are unchanged. Validation: pre-commit, pylint and executable AST checks pass. Twelve CUDA checks of actual source helpers and instrumented packing/padding/test boundaries match expected results. Numerical backends were intercepted; this is not a new native cuDNN attention validation. Signed-off-by: Sudhakar Singh --- tests/pytorch/test_dsv4_attention.py | 6 +- .../attention/sparse_attention/README.md | 20 +- .../attention/sparse_attention/compressor.py | 58 +++- .../attention/sparse_attention/dsa_cudnn.py | 283 ++++++++++++------ .../attention/sparse_attention/dsa_rope.py | 2 +- .../sparse_attention/dsv4_attention.py | 75 ++++- .../attention/sparse_attention/indexer.py | 49 ++- 7 files changed, 361 insertions(+), 132 deletions(-) diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index 193e9493af5..2c08c47b1f9 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -16,7 +16,6 @@ dsv4_attention, ) - # Sparse attention @@ -60,8 +59,9 @@ def test_dsa_sparse_attention(variant, head_dim): reference_leaves = tuple(x.detach().float().requires_grad_() for x in leaves) indices = None if variant == "csa": - # Select alternating compressed KV tokens; global compressed KV token indices - # expose mistakes at the second sequence's compressed token offset. + # With ratio=4, sequence one has 64 compressed tokens. Sequence two + # therefore selects packed indices 64, 66, ... rather than 0, 2, ... . + # Comparing with its separate reference catches a missing sequence offset. indices = torch.full((total, 32), -1, device="cuda", dtype=torch.int32) start = comp_start = 0 for length, count in zip(lengths, comp_lengths): diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index 3f8f6196840..bb7cc586114 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -64,15 +64,23 @@ output = dsa_cudnn.sparse_attention( output = ... # Apply inverse RoPE and grouped output projection. ``` -Dense inputs are repacked into tokens in batch-major order before these calls; -packed THD inputs already have that layout. Attention uses the original query +Batched inputs contain B sequences of the same length S. `DSv4Attention` accepts +`[S,B,hidden_size]` (sbhd) or `[B,S,hidden_size]` (bshd), and packs tokens in +batch-major order before these calls. Packed `[T,hidden_size]` (thd) inputs +concatenate sequences, with cumulative token counts identifying their boundaries. +Standalone `Compressor` and `Indexer` accept only batched `[B,S,D]` or packed +`thd` `[T,D]` input; callers must transpose sequence-major inputs first. Hidden +states have no explicit head axis; prepared queries have shape `[T,H,D]`. +Attention uses the original query `[T,64,D]` and concatenated local/compressed KV `[T+Tc,D]`; there is no compressed query. `T` is the total query/local KV token count. -`_prepare_sparse_indices` creates a list of KV token indices for each query token, +`_prepare_sparse_kv_indices` creates a list of KV token indices for each query token, combining causal local KV token indices with compressed KV token indices offset -by `T`; invalid entries are `-1`. This differs from `select_indices`, which scores -the indexer's projected Q/K/head weights and returns global compressed KV token -indices addressing only packed compressed KV, before the offset is applied. +by `T`; invalid entries are `-1`. CSA supplies global compressed KV token indices +from `select_indices`; HCA passes `indices=None` and includes all eligible +compressed tokens. `_prepare_sparse_kv_indices` documents a small example of these +index spaces. Selection indices address only packed compressed KV before the +local KV offset is applied. `dsa_rope` supplies token/compressed-position embeddings and rotary application. A caller may override the default embeddings with a `rope` module returning diff --git a/transformer_engine/pytorch/attention/sparse_attention/compressor.py b/transformer_engine/pytorch/attention/sparse_attention/compressor.py index 510705012b8..b5a3b770802 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/compressor.py +++ b/transformer_engine/pytorch/attention/sparse_attention/compressor.py @@ -33,9 +33,34 @@ class Compressor(torch.nn.Module): that do not fill a complete block (``k`` here) are excluded from compression. In DSv4Attention, they remain available through local sliding-window attention. - Accept batch-major ``[B,S,D]`` or packed ``[T,D]`` hidden states with - token/compressed prefixes. Return ``[total_comp,width]`` compressed KV representations + Accept batched ``[B,S,D]`` or packed ``thd`` hidden states ``[T,D]`` with + cumulative sequence token counts. Standalone calls do not interpret + sequence-major ``[S,B,D]`` input; DSv4Attention performs that conversion. + Return ``[total_comp,width]`` compressed KV representations before RoPE, plus projected head weights when ``weight_width`` is nonzero. + + See the sparse_attention README for support requirements. + + Parameters + ---------- + hidden_size : int + Input hidden-state width. + width : int + Width of each compressed KV representation. + ratio : int + Number of input tokens per compression block. + overlap : bool + Whether each output also uses the preceding complete block. + eps : float + Epsilon for RMSNorm after compression. + device : torch.device or str + Device on which to allocate parameters. + params_dtype : torch.dtype + Dtype used to initialize projection, position-bias and normalization parameters. + weight_width : int, default = 0 + Additional per-input-token head-weight projection width. Indexer sets this + to n_heads for its scoring weights; DSv4Attention's main KV compressor + leaves it at zero. """ def __init__( @@ -64,28 +89,31 @@ def __init__( self.projected, self.weight_width = projected, weight_width def forward(self, hidden_states, cu_seqlens, cu_seqlens_comp, *, total_comp): - """Project and compress KV entries before RoPE. + """Project and compress KV blocks before RoPE. Parameters ---------- hidden_states : torch.Tensor - Batch-major ``[B,S,hidden_size]`` or packed ``[T,hidden_size]`` input. + Batched ``[B,S,hidden_size]`` with equal sequence length S, or packed ``thd`` + ``[T,hidden_size]`` with sequences concatenated along the token dimension. cu_seqlens, cu_seqlens_comp : torch.Tensor - Contiguous CUDA INT32 ``[B+1]`` token/compressed prefixes. Compressed - segment lengths are ``floor(length / ratio)``. + Contiguous CUDA INT32 ``[B+1]`` cumulative input/compressed token counts, + starting at zero. Each compressed count increment is the corresponding + input sequence length divided by ratio, rounded down. total_comp : int - Allocated compressed output length. For dense input, use exactly - ``B * (S // ratio)``; packed output may include padding after the valid - compressed KV tokens. + Allocated compressed output length. For ``[B,S,hidden_size]`` input, use + exactly ``B * (S // ratio)``; DSv4Attention allocates ``T // ratio`` for ``thd`` + input, which may include output padding after the + valid compressed tokens counted by ``cu_seqlens_comp[-1]``. Returns ------- torch.Tensor or tuple of torch.Tensor - Normalized ``[total_comp,width]`` compressed KV representations. - When ``weight_width > 0``, also return head weights with shape - ``hidden_states.shape[:-1] + - (weight_width,)``. Only outputs for compressed KV token indices below - ``cu_seqlens_comp[-1]`` are valid; the remaining output is padding. + Normalized compressed KV ``[total_comp,width]``. If ``weight_width > 0``, + return ``(compressed_kv, weights)`` with per-input-token head weights of + shape ``hidden_states.shape[:-1] + (weight_width,)``. Indexer uses this + projection for scoring; the main KV compressor does not. Only compressed + token indices below ``cu_seqlens_comp[-1]`` are valid; the rest is padding. """ projected = self.fused_proj(hidden_states) widths = ( @@ -107,7 +135,7 @@ def forward(self, hidden_states, cu_seqlens, cu_seqlens_comp, *, total_comp): total_comp=total_comp, ) if hidden_states.ndim == 2: - # cuDNN guarantees valid output only before the final compressed prefix. + # cuDNN guarantees valid output only for the cu_seqlens_comp[-1] compressed tokens. # Zero output from cu_seqlens_comp[-1] onward before RMSNorm so # uninitialized values cannot affect weight gradients. valid = torch.arange(total_comp, device=hidden_states.device) < cu_seqlens_comp[-1] diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py index 37015960cd2..e4d1989eb01 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py @@ -39,12 +39,7 @@ def _dsa_op(name): def _validate_tensors(reference, bf16, fp32, cu_seqlens, cu_seqlens_comp): - """Check metadata without copying packed sequence offsets to the CPU. - - Callers own prefix contents: start at zero, monotonic, final offsets equal - valid query/local KV and compressed KV token counts, respectively, and - compressed lengths floor(length / ratio). - """ + """Check tensor metadata without copying sequence boundaries to the CPU.""" if not reference.is_cuda or torch.cuda.get_device_capability(reference.device) not in ( (10, 0), (10, 3), @@ -127,18 +122,39 @@ def compress( overlap: bool, total_comp: int, ) -> torch.Tensor: - """Compress token windows with cuDNN forward and backward. - - kv/score: contiguous [T, coff*D] projected KV/gate scores. - ape: FP32 [ratio, coff*D] additive position embedding, a learned per-channel - bias added to gate scores before the compression softmax. - coff is 2 for overlapping compression, 1 otherwise. Sequence prefixes are - CUDA INT32 [B+1]; compressed segment lengths are floor(length / ratio). - Returns [total_comp, D]. Trailing input tokens that do not fill a complete - block are excluded from compression. - - total_comp specifies output capacity. Only outputs for compressed KV token - indices below cu_seqlens_comp[-1] are valid; the remaining output is padding. + """Compress token blocks with cuDNN forward and backward. + + Overlapping compression combines the current complete block with the previous + block; otherwise each block is compressed independently. Trailing input tokens + that do not fill a complete block are excluded from compression. This operation + accepts projected tensors; it does not apply the module's projections, RMSNorm + or RoPE. See the sparse_attention README for supported dtypes and devices. + + Parameters + ---------- + kv, score : torch.Tensor + Contiguous projected KV and gate scores of shape ``[T,coff*D]`` on the + same device. ``coff`` is 2 when ``overlap=True``, otherwise 1. + ape : torch.Tensor + FP32 learned position bias ``[ratio,coff*D]``, added to gate scores + before softmax over each compression block. + cu_seqlens, cu_seqlens_comp : torch.Tensor + Contiguous CUDA INT32 ``[B+1]`` cumulative input/compressed token counts. + Start at zero; each increment is the corresponding sequence's token + count. Compressed counts are ``floor(sequence_length / ratio)``. + ratio : int + Positive number of input tokens per block producing one compressed token. + overlap : bool + Whether to include the previous block using its separate projections. + total_comp : int + Allocated compressed output length, at least ``cu_seqlens_comp[-1]``. + + Returns + ------- + torch.Tensor + Compressed KV ``[total_comp,D]`` before RMSNorm and RoPE. Only indices + below ``cu_seqlens_comp[-1]`` contain valid compressed tokens; remaining + output is padding. Autograd propagates gradients to kv, score and ape. """ if ratio <= 0 or total_comp < 0: raise ValueError("ratio must be positive and total_comp nonnegative.") @@ -171,25 +187,47 @@ def select_indices( max_compressed_seqlen: int, scale: Optional[float] = None, ) -> torch.Tensor: - """Select compressed KV tokens with cuDNN's combined index-score + Top-K operation. - - Inputs are the indexer's projected Q/K/head weights, after compression/RoPE. - Unlike the Indexer module, this operation does not project or compress inputs. - query [T,H,128], compressed_key [Tc,128], weights [T,H]; H=32 or 64. - cu_seqlens/cu_seqlens_comp are CUDA INT32 [B+1] token/compressed prefixes. - Full packed-sequence prefill only: causal timeline offsets start at zero. - Returns INT32 [T,top_k] global compressed KV token indices into packed - compressed_key, with one list per query token; -1 denotes an invalid entry. - These indices address compressed KV only, not concatenated local + compressed KV. - - Selection is discrete. Scale defaults to 1/sqrt(index head dim). If the - model also scales by 1/sqrt(index heads), fold that positive factor into - scale and pass raw projected weights. Scaling in FP32 then rounding - weights back to activation dtype can change close top-k boundaries. This helper does not - implement the separate indexer auxiliary loss or autograd through token indices. - max_compressed_seqlen is a compile bound and must cover the partial final - ratio; use ceil(max_seqlen / ratio) even though prefixes count only complete - compressed blocks. + """Select compressed KV tokens using cuDNN's index scores and top-k operation. + + Inputs are the indexer's projected Q/K/head weights after compression/RoPE; + this operation does not project or compress inputs. It supports complete + sequences starting at position zero. See the sparse_attention README for + supported dtypes and devices. + + Parameters + ---------- + query : torch.Tensor + Contiguous index queries ``[T,H,128]``, with H=32 or 64. + compressed_key : torch.Tensor + Contiguous packed compressed index keys ``[Tc,128]``. + weights : torch.Tensor + Contiguous projected per-token head weights ``[T,H]``. + cu_seqlens, cu_seqlens_comp : torch.Tensor + Contiguous CUDA INT32 ``[B+1]`` cumulative query/compressed token counts, + starting at zero. Adjacent differences give each sequence's token count. + top_k : int + Positive maximum number of compressed KV token indices per query token. + ratio : int + Positive number of input tokens per compressed block. + max_seqlen : int + Maximum input sequence length, passed to cuDNN as ``max_seqlen_q``. + max_compressed_seqlen : int + Size passed to cuDNN as ``max_seqlen_k``. Use + ``(max_seqlen + ratio - 1) // ratio``, as Indexer does, even though + ``cu_seqlens_comp`` counts only complete compressed blocks. + scale : float, optional + Score multiplier. None uses ``query.shape[-1] ** -0.5``. Indexer includes + the positive head-normalization factor using + ``(head_dim * n_heads) ** -0.5``. Pass raw projected weights; scaling then + rounding weights back to their activation dtype can change close top-k results. + + Returns + ------- + torch.Tensor + CUDA INT32 ``[T,top_k]`` global indices into packed compressed_key, one + list per query token; -1 marks invalid entries. Indices address compressed + KV only, before adding the local KV offset. Selection has no autograd; + train the indexer with a separate caller-owned auxiliary loss. """ if query.ndim != 3 or query.shape[1] not in (32, 64) or query.shape[2] != 128: raise ValueError("CSA index query must have shape [T,32 or 64,128].") @@ -247,15 +285,35 @@ def dense_indexer_loss_scores( index_scale: float, attention_scale: float, ): - """Return cuDNN dense index/teacher score dictionaries for B1 prefill. - - index_q/index_k/index_w are [1,S,H,128], [1,Sc,128], [1,S,H], - with H=32 or 64. attn_q/compressed_kv are [1,S,64,D], [1,Sc,D]. - full_lse is FP32 [1,S,64] and must include the local window, every eligible - compressed key, and the sink. The sparse-attention saved LSE covers only - selected keys and excludes the sink, so it cannot serve this dense teacher. - Returned score/denominator dictionaries feed dense_indexer_loss_backward; - these calls do not attach an autograd loss to the indexer tensors. + """Recompute indexer and teacher scores for a single-sequence auxiliary loss. + + See the sparse_attention README for support requirements and the loss example. + These primitives do not attach an autograd loss to the indexer tensors. + + Parameters + ---------- + index_q, index_k, index_w : torch.Tensor + Indexer query/key/head weights of shapes ``[1,S,H,128]``, ``[1,Sc,128]`` + and ``[1,S,H]``, with H=32 or 64. + attn_q, compressed_kv : torch.Tensor + Teacher attention queries ``[1,S,64,D]`` and compressed KV ``[1,Sc,D]``. + Use detached teacher tensors so the auxiliary loss trains only the indexer. + full_lse : torch.Tensor + Detached FP32 ``[1,S,64]`` log-sum-exp of teacher logits over the local + window, every eligible compressed KV token, and the sink. The saved sparse + attention LSE excludes the sink and covers only selected compressed tokens, + so it cannot be used as this dense teacher denominator. + ratio : int + Number of input tokens per compressed block. + index_scale, attention_scale : float + Score multipliers for the indexer and teacher attention, respectively. + + Returns + ------- + tuple of dict + ``(index_score, attn_score)`` cuDNN dictionaries with ``out`` and ``denom`` + tensors, passed to ``dense_indexer_loss_backward``. The caller owns the + auxiliary objective and propagation through its live indexer tensors. """ if any(t.dtype != torch.bfloat16 for t in (index_k, attn_q, compressed_kv)): raise TypeError("Dense indexer keys and attention Q/K must be BF16.") @@ -291,12 +349,33 @@ def dense_indexer_loss_backward( index_scale: float, loss_coeff: float, ): - """Return dQ/dK/dW for B1 KL averaged over query tokens, padding H32 at cuDNN. - - index_q/index_k/index_w and score dictionaries are the inputs/outputs of - dense_indexer_loss_scores. Returned gradients match those input shapes and - must be propagated through the live indexer tensors by the caller. - grad_loss is the scalar upstream gradient; loss_coeff scales the loss. + """Compute indexer gradients for KL loss averaged over query tokens, with B=1. + + Use the same tensors, compression ratio and index scale as in + ``dense_indexer_loss_scores``. See the sparse_attention README loss example. + + Parameters + ---------- + index_q, index_k, index_w : torch.Tensor + Live indexer query/key/head weights ``[1,S,H,128]``, ``[1,Sc,128]`` and + ``[1,S,H]``, with H=32 or 64. H=32 is padded internally for cuDNN. + index_score, attn_score : dict + Indexer and teacher dictionaries returned by ``dense_indexer_loss_scores``. + grad_loss : torch.Tensor + Scalar upstream gradient for the auxiliary loss. + ratio : int + Number of input tokens per compressed block. + index_scale : float + Index score multiplier used to compute index_score. + loss_coeff : float + Multiplier for the auxiliary loss and its gradients. + + Returns + ------- + tuple of torch.Tensor + ``(d_query, d_key, d_weights)`` matching the input tensor shapes. The caller + must propagate these gradients through the live indexer tensors, for example + with ``torch.autograd.backward``; this helper does not do that propagation. """ padded_q, padded_w = _pad_index_heads(index_q, index_w) result = _dsa_op("dense_indexer_backward_wrapper")( @@ -320,7 +399,7 @@ def dense_indexer_loss_backward( ) -def _prepare_sparse_indices( +def _prepare_sparse_kv_indices( query: torch.Tensor, cu_seqlens: torch.Tensor, cu_seqlens_comp: torch.Tensor, @@ -331,14 +410,15 @@ def _prepare_sparse_indices( ) -> torch.Tensor: """Build a list of KV token indices for each query token. - Use query only for its token count T and device; query values are not read. - Return INT32 [T, window_size + compressed_slots] indices into concatenated - [local_kv; compressed_kv], where local_kv also contains T tokens. + Query is already in packed thd format [T,H,D]; only its token count and device are used. + This helper does not flatten it. Return INT32 [T,window_size+compressed_slots] + indices into torch.cat((local_kv, compressed_kv), dim=0). - Combine causal local KV token indices with eligible compressed KV token - indices. Compressed indices are global within packed compressed KV, not - relative to each sequence, and are offset by T here. Invalid entries are -1. - select_indices chooses compressed KV token indices before this offset. + Local KV has T tokens. Compressed indices count from the start of the entire + packed compressed tensor, including preceding sequences. Add T to address + compressed KV after local KV in the concatenated tensor. Invalid entries are -1. + CSA supplies top-k indices from select_indices; HCA supplies indices=None and + this helper includes every eligible compressed token. """ total_q = query.shape[0] device = query.device @@ -381,6 +461,14 @@ def _prepare_sparse_indices( & (compressed < comp_end[:, None]) & ((comp_position + 1) * ratio <= position[:, None] + 1) ) + # Example: lengths=[4,11,17], T=32; query 7 is position 3 of sequence two. + # With indices=None and max_compressed_seqlen=8 and 5, respectively: + # window=ratio=2: cu_comp=[0,2,7,15]; local=[6,7]; compressed=[2,3]. + # Combined KV indices: [6,7,34,35,-1,-1,-1,-1,-1,-1]. + # window=ratio=3: cu_comp=[0,1,4,9]; local=[5,6,7]; compressed=[1]. + # Combined KV indices: [5,6,7,33,-1,-1,-1,-1]. + # Ratio 3 illustrates only this helper's index arithmetic, not cuDNN compression support. + # Only compressed indices receive offset T; -1 marks invalid entries. compressed = torch.where(valid, compressed + query.shape[0], -1) # cuDNN accepts an arbitrary logical width and pads its internal kernel # layout, so preserve only the model-visible local + compressed slots. @@ -442,28 +530,53 @@ def sparse_attention( scale: Optional[float] = None, max_compressed_seqlen: Optional[int] = None, ) -> torch.Tensor: - """Joint local + compressed sparse attention with autograd. - - Full packed-sequence prefill: query [T,H,D], local_kv [T,D], compressed_kv - [Tc,D], sink [H]. Q/KV have already had the model's norm/RoPE applied. - The high-level layer repacks dense SBHD/BSHD inputs into tokens packed in - batch-major order; THD inputs already use this token order. There is no compressed Q. - cu_seqlens describes both query and local KV; cu_seqlens_comp describes - valid compressed KV tokens, excluding output padding. Values are the first - 512 channels of each shared KV token representation (Q/K width may also be 576). - - Local positions satisfy max(0,q-window_size+1) <= k <= q per sequence. - Compressed block j is eligible when (j+1)*ratio <= q+1. With indices (CSA), - use its selected, valid, causal compressed KV tokens; without indices (HCA), - use every eligible compressed KV token. Input indices address packed compressed_kv - only, before adding the local KV offset. - - One cuDNN sparse-attention call normalizes the combined visible entries with - the sink and provides the KV-only LSE consumed by cuDNN backward. Output is - [T,H,512], before output unrotation/projection. Scale defaults to D**-0.5. - No dense attention mask is part of this interface. HCA must provide - max_compressed_seqlen to bound its dense compressed-history metadata without - reading a CUDA prefix on host. + """Run joint local and compressed KV attention with autograd. + + This full-sequence interface requires one local KV token for each query token. + The caller prepares projected, normalized and rotated Q/KV. DSv4Attention + converts batched inputs to packed token order before calling this function. + See the sparse_attention README for supported dtypes and devices. + + Parameters + ---------- + query : torch.Tensor + Contiguous packed ``thd`` queries ``[T,64,D]``, with D=512 or 576. + local_kv : torch.Tensor + Contiguous local KV ``[T,D]``, in the same sequence/token order as query. + compressed_kv : torch.Tensor + Contiguous packed compressed KV ``[Tc,D]``. Only indices below + ``cu_seqlens_comp[-1]`` are valid; any remaining output is padding. + sink : torch.Tensor + FP32 sink logits ``[64]``, one per attention head. + cu_seqlens, cu_seqlens_comp : torch.Tensor + Contiguous CUDA INT32 ``[B+1]`` cumulative query/local KV and compressed KV + token counts. Start at zero; adjacent differences give sequence lengths. + window_size : int + Positive local window size including the current query token: local + positions satisfy ``max(0,q-window_size+1) <= k <= q`` within each sequence. + ratio : int + Positive compression block length. Compressed token j in a sequence is + eligible when ``(j+1)*ratio <= q+1`` for query position q. + indices : torch.Tensor, optional + CSA: CUDA INT32 ``[T,K]`` global token indices into packed compressed_kv + only, with -1 for invalid entries. The helper checks sequence bounds and + causality, then adds T for concatenated local + compressed KV. None selects + HCA behavior: include all eligible compressed tokens. See the example in + ``_prepare_sparse_kv_indices`` for the index spaces. + scale : float, optional + Attention score multiplier. None uses ``query.shape[-1] ** -0.5``. + max_compressed_seqlen : int, optional + Required for HCA: upper bound on complete compressed tokens in any sequence, + used to build the index lists without reading CUDA sequence counts on host. + Unused when indices is supplied. + + Returns + ------- + torch.Tensor + Attention output ``[T,64,512]`` before inverse RoPE/output projection. + Values use the first 512 KV channels. One softmax includes the visible local + and compressed KV tokens plus the sink. Autograd returns gradients for + query, both KV tensors and sink; selection indices have no gradients. """ if query.ndim != 3 or query.shape[1] != 64: raise ValueError("cuDNN sparse attention requires query [T,64,D].") @@ -474,7 +587,9 @@ def sparse_attention( if local_kv.shape[-1] != query.shape[-1] or compressed_kv.shape[-1] != query.shape[-1]: raise ValueError("Q and both KV tensors must have the same QK dimension.") if local_kv.shape[0] != query.shape[0]: - raise ValueError("Full-sequence attention requires one local KV token per query token.") + raise ValueError( + "Full-sequence DSv4 attention requires equal query and local KV token counts." + ) if sink.shape != (query.shape[1],): raise ValueError("sink must have shape [64].") if window_size <= 0 or ratio <= 0: @@ -487,7 +602,7 @@ def sparse_attention( raise ValueError("CSA indices must be CUDA INT32 on the query device.") if scale is None: scale = query.shape[-1] ** -0.5 - selected = _prepare_sparse_indices( + selected = _prepare_sparse_kv_indices( query, cu_seqlens, cu_seqlens_comp, diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py index 800d6ff42e8..58be2880b4e 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py @@ -61,7 +61,7 @@ def _apply_rotary_eager(x, cos, sin, cu_seqlens=None): token_indices = torch.arange(x.shape[0], device=x.device) sequences = torch.bucketize(token_indices, cu_seqlens[1:], right=True) positions = token_indices - cu_seqlens[sequences] - # Compressed tensors may have unused capacity past the final prefix. + # Compressed tensors may have output padding after cu_seqlens[-1]. positions = torch.where(token_indices < cu_seqlens[-1], positions, 0) cos, sin = cos[0, positions], sin[0, positions] width = cos.shape[-1] diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index 24921dc0647..8b2b45c77ed 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -25,7 +25,14 @@ def _compiled_q_rms_norm(q: torch.Tensor, eps: float) -> torch.Tensor: def _packed_prefixes(hidden_states, cu_seqlens, max_seqlen, ratio): - """Build compressed sequence prefixes and output capacity.""" + """Build compressed sequence boundaries and an allocation length for packed thd input. + + Subtract adjacent cu_seqlens values to get sequence lengths, divide each by + ratio with rounding down, then prepend zero to their cumulative sum. TE allocates + floor(total_input_tokens / ratio) outputs without reading the device count on + the host; this can exceed the sum of per-sequence compressed counts. Return + (cu_comp, total_comp, max_seqlen). + """ if not isinstance(cu_seqlens, torch.Tensor): raise ValueError("THD requires contiguous INT32 cu_seqlens_q [B+1] on the input device.") if ( @@ -39,9 +46,8 @@ def _packed_prefixes(hidden_states, cu_seqlens, max_seqlen, ratio): if not isinstance(max_seqlen, int) or max_seqlen < ratio: raise ValueError("THD requires integer max_seqlen_q >= compression_ratio.") # The sum of per-sequence floors can be smaller than floor(T / ratio). - # Allocate that safe capacity without reading CUDA prefix contents on host; - # cuDNN uses the compressed prefix to exclude output padding beyond the valid - # compressed KV tokens. + # TE allocates that upper bound without reading the CUDA count on the host; + # cuDNN uses cu_comp to exclude output padding beyond the valid compressed tokens. lengths = cu_seqlens[1:] - cu_seqlens[:-1] cu_comp = torch.cat((cu_seqlens.new_zeros(1), (lengths // ratio).cumsum(0, dtype=torch.int32))) total_comp = hidden_states.shape[0] // ratio @@ -53,9 +59,55 @@ def _packed_prefixes(hidden_states, cu_seqlens, max_seqlen, ratio): class DSv4Attention(torch.nn.Module): """DSv4 CSA/HCA attention in ``sbhd``, ``bshd``, or packed ``thd`` format. - Composes TE projections/norms, Compressor, Indexer, and RoPE with cuDNN - sparse attention. See the sparse_attention README for support and training - requirements. Shared-input projections are fused. + Composes TE projections/norms, Compressor, Indexer and RoPE with cuDNN sparse + attention. Shared-input projections are fused. See the sparse_attention README + for support and training requirements. + + Parameters + ---------- + hidden_size : int + Input and output hidden-state width. + q_lora_rank : int + Width of the query-down projection and its residual. + layer_type : str + ``compressed_sparse_attention`` (CSA, with an indexer) or + ``heavily_compressed_attention`` (HCA, without top-k selection). + head_dim : int + Attention head width; currently 512, with 64 heads. + rope_head_dim : int + Even number of trailing channels rotated by RoPE; at least 2, no greater + than head_dim, and no greater than 128 for CSA's index queries/keys. + sliding_window : int + Local causal attention window size, including the current token. + compression_ratio : int + Number of input tokens per compressed block. + o_groups : int + Number of output-projection groups; must divide 64 * head_dim. + o_lora_rank : int + Intermediate output-projection width per group. + index_n_heads : int, default = 64 + CSA index heads, either 32 or 64; unused for HCA. + index_topk : int, default = 512 + Maximum compressed KV tokens selected per CSA query; unused for HCA. + rope_theta : float, default = 160000.0 + Frequency base for the default rotary embeddings; ignored with custom rope. + rope : torch.nn.Module, optional + Custom module called as ``rope(sequence_length, device)`` returning token + and compressed ``(cos, sin)`` pairs. None uses the default embeddings. + rms_norm_eps : float, default = 1e-6 + Epsilon for query and KV normalization. + max_seqlen : int, optional + Initial length of the default RoPE cache. The cache grows when needed; + this does not replace forward's max_seqlen_q for packed ``thd`` input. + device : torch.device or str, default = "cuda" + Device on which to allocate parameters and default RoPE buffers. + params_dtype : torch.dtype, default = torch.float32 + Dtype used to initialize parameters; activation support is described in + the README independently of parameter initialization. + qkv_format : str, default = "sbhd" + Hidden-state layout: ``[S,B,hidden_size]``, ``[B,S,hidden_size]``, or + ``[T,hidden_size]`` for sbhd, bshd or thd. Sequence-major input is transposed + before calling the batched/packed Compressor and Indexer components. """ def __init__( @@ -180,7 +232,7 @@ def forward( max_seqlen_q: Optional[int] = None, return_indexer_context: bool = False, ): - """Run full self-attention and optionally expose single-sequence CSA context. + """Run full DSv4 sparse-attention and optionally expose single-sequence CSA context. Parameters ---------- @@ -188,9 +240,10 @@ def forward( Input ``[S,B,hidden_size]`` for ``sbhd``, ``[B,S,hidden_size]`` for ``bshd``, or ``[T,hidden_size]`` for packed ``thd``. cu_seqlens_q : torch.Tensor, optional - Contiguous CUDA INT32 ``[B+1]`` prefixes, required only for ``thd``. - Prefixes start at zero, end at T, and contain no padding/global offsets; - their contents are caller-owned and are not checked on the host. + Contiguous CUDA INT32 ``[B+1]`` cumulative sequence token counts, required + only for ``thd``. Start at zero, end at T, and do not decrease; adjacent + differences are sequence lengths. Tokens are packed without gaps. + The caller supplies these values; they are not checked on the host. max_seqlen_q : int, optional Maximum sequence length, required only for ``thd``. return_indexer_context : bool, default = False diff --git a/transformer_engine/pytorch/attention/sparse_attention/indexer.py b/transformer_engine/pytorch/attention/sparse_attention/indexer.py index ba496ee11d6..fd4e77e543f 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/indexer.py +++ b/transformer_engine/pytorch/attention/sparse_attention/indexer.py @@ -14,12 +14,34 @@ class Indexer(torch.nn.Module): - """Select compressed KV entries using the indexer's own Q/K/weight projections. + """Select compressed KV tokens using the indexer's own Q/K/weight projections. - Accept batch-major or packed hidden states and query residuals, sequence - prefixes, and caller-provided RoPE tables. Return global compressed KV token - indices addressing compressed KV only; - ``return_context=True`` also returns live Q/K/head weights for a separate loss. + Accept batched [B,S,D] or packed thd [T,D] hidden states and matching query residuals. + Standalone calls do not interpret sequence-major [S,B,D] input; DSv4Attention + transposes that layout first. Return global compressed KV token indices; + return_context=True also returns live Q/K/head weights for a caller-owned loss. + See the sparse_attention README for support and training requirements. + + Parameters + ---------- + hidden_size : int + Input hidden-state width for compressed-key and head-weight projections. + q_lora_rank : int + Width of the query-down residual supplied to forward. + head_dim : int + Index query/key width; the current selection operation uses 128. + n_heads : int + Number of index heads, 32 or 64. Also sets the per-token scoring-weight width. + top_k : int + Maximum compressed KV tokens to select per query token. + ratio : int + Input tokens per compression block. Indexer uses overlapping compression. + eps : float + Epsilon for the compressed-key RMSNorm. + device : torch.device or str + Device on which to allocate parameters. + params_dtype : torch.dtype + Dtype used to initialize projection and compression parameters. """ def __init__( @@ -68,21 +90,24 @@ def forward( Parameters ---------- hidden_states : torch.Tensor - Batch-major ``[B,S,hidden_size]`` or packed ``[T,hidden_size]`` input. + Batched ``[B,S,hidden_size]`` with equal sequence length S, or packed ``thd`` + ``[T,hidden_size]`` with concatenated sequences. q_residual : torch.Tensor Query-down residual with matching leading axes and width ``q_lora_rank``. cu, cu_comp : torch.Tensor - CUDA INT32 ``[B+1]`` token/compressed prefixes. + CUDA INT32 ``[B+1]`` cumulative input/compressed token counts, starting + at zero. Each compressed increment is the input sequence length + divided by ratio, rounded down. token_rope, compressed_rope : tuple of torch.Tensor FP32 cosine/sine pairs ``[1,max_positions,1,rotary_width]`` for token and compressed-window positions, respectively. total_comp : int Allocated compressed output length: exactly ``B * (S // ratio)`` for - dense input; packed output may include padding after the valid + ``[B,S,hidden_size]`` input; packed ``thd`` output may include padding after the valid compressed KV tokens. max_seqlen : int - Maximum token sequence length used to bound selection. For dense - input this must equal S; for packed input use the maximum segment length. + Maximum sequence length used to bound selection. For ``[B,S,hidden_size]`` + input this must equal S; for packed ``thd`` input use the longest sequence length. return_context : bool, default = False Also return differentiable Q/K/head weights for a caller-owned loss. @@ -129,8 +154,8 @@ def forward( top_k=min(self.top_k, n_comp), ratio=self.ratio, max_seqlen=max_seqlen, - # cuDNN's compile bound covers the partial trailing ratio even - # though cu_comp contains only complete compressed blocks. + # Pass ceil(max_seqlen / ratio) as cuDNN max_seqlen_k; cu_comp still + # counts only complete blocks, using floor(length / ratio) per sequence. max_compressed_seqlen=(max_seqlen + self.ratio - 1) // self.ratio, scale=(self.head_dim * self.n_heads) ** -0.5, ) From 8f9b5982ce2c22c6bfff4913fea6ea099520708d Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Thu, 19 Nov 2026 02:53:01 -0800 Subject: [PATCH 32/37] Refine DSv4 tests and expose indexer loss helpers Keep representative RoPE and packed-attention cases to avoid repeated test work. Align RoPE entry point names with their callers and export the existing loss helpers so model code can train the indexer through the public API. Signed-off-by: Sudhakar Singh --- docs/api/pytorch.rst | 4 + tests/pytorch/attention/mla_rope_utils.py | 4 +- tests/pytorch/attention/test_fused_mla.py | 5 +- tests/pytorch/test_dsv4_attention.py | 164 ++++++------------ transformer_engine/common/triton/dsa_rope.py | 12 +- transformer_engine/pytorch/__init__.py | 2 + .../pytorch/attention/__init__.py | 10 +- .../pytorch/attention/fused_mla_q_uproj.py | 2 +- .../attention/sparse_attention/README.md | 11 +- .../attention/sparse_attention/__init__.py | 3 + .../attention/sparse_attention/dsa_rope.py | 12 +- .../sparse_attention/dsv4_attention.py | 8 +- .../attention/sparse_attention/indexer.py | 6 +- 13 files changed, 105 insertions(+), 138 deletions(-) diff --git a/docs/api/pytorch.rst b/docs/api/pytorch.rst index bdd14514df7..ea52de3e37e 100644 --- a/docs/api/pytorch.rst +++ b/docs/api/pytorch.rst @@ -34,6 +34,10 @@ PyTorch .. autoapiclass:: transformer_engine.pytorch.Indexer(hidden_size, q_lora_rank, head_dim, n_heads, top_k, ratio, eps, device, params_dtype, **kwargs) :members: forward +.. autoapifunction:: transformer_engine.pytorch.dense_indexer_loss_scores + +.. autoapifunction:: transformer_engine.pytorch.dense_indexer_loss_backward + .. autoapiclass:: transformer_engine.pytorch.GatedDeltaNetAttention(num_attention_heads, kv_channels, **kwargs) :members: forward diff --git a/tests/pytorch/attention/mla_rope_utils.py b/tests/pytorch/attention/mla_rope_utils.py index 0aedc863a05..daebee34939 100644 --- a/tests/pytorch/attention/mla_rope_utils.py +++ b/tests/pytorch/attention/mla_rope_utils.py @@ -18,8 +18,8 @@ try: import triton from transformer_engine.common.triton.dsa_rope import ( - _autotuned_mla_rope_fwd_inplace_kernel as rotary_fwd_q_kernel, - _autotuned_mla_rope_bwd_inplace_kernel as rotary_bwd_q_kernel, + rotary_fwd_q_kernel, + rotary_bwd_q_kernel, rotary_fwd_kv_kernel, rotary_bwd_kv_kernel, ) diff --git a/tests/pytorch/attention/test_fused_mla.py b/tests/pytorch/attention/test_fused_mla.py index 4a5e1fc6a3a..b0c66b0a44a 100644 --- a/tests/pytorch/attention/test_fused_mla.py +++ b/tests/pytorch/attention/test_fused_mla.py @@ -471,7 +471,10 @@ def test_inputs_on_non_current_device(): @pytest.mark.skipif(not HAVE_TRITON, reason="MLA RoPE requires Triton") def test_mla_rope_kv_equations_backward(): - """Separate MLA K/V assembly and positional-gradient reduction survive reuse.""" + """Compare MLA K/V assembly and input gradients with independent PyTorch equations. + + The positional key is shared across heads, so its gradient must sum their contributions. + """ if not torch.cuda.is_available(): pytest.skip("MLA RoPE requires CUDA") torch.manual_seed(59) diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index 2c08c47b1f9..bc8ebe7f9c7 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -7,6 +7,7 @@ import pytest import torch +from transformer_engine.pytorch import dense_indexer_loss_scores, dense_indexer_loss_backward from transformer_engine.pytorch.attention.sparse_attention import ( Compressor, Indexer, @@ -140,107 +141,64 @@ def test_dsa_rejects_unsupported_metadata(): # RoPE -@pytest.mark.parametrize("heads", [1, 64]) -def test_dsa_triton_rope_batched(monkeypatch, heads): - """The packed Triton kernels must reset positions at each BSHD sequence.""" +@pytest.mark.parametrize("layout", ["batched_q", "packed_kv"]) +def test_dsa_triton_rope_adapters(monkeypatch, layout): + """Compare production Q/KV adapter roles with eager outputs, gradients and aliasing.""" if not torch.cuda.is_available(): pytest.skip("Triton DSA RoPE requires CUDA") if dsa_rope._triton_rope_module() is None: pytest.skip("Triton DSL is unavailable") torch.manual_seed(23) - batch, seq, dim, width = 2, 17, 512, 64 - x = torch.randn(batch, seq, heads, dim, device="cuda", dtype=torch.bfloat16) - (cos, sin), _ = dsa_rope.rotary_embeddings(seq, 4, width, 160000.0, x.device) - cu = torch.arange(batch + 1, device=x.device, dtype=torch.int32) * seq + inplace = layout == "batched_q" + if inplace: + x = torch.randn(2, 17, 64, 512, device="cuda", dtype=torch.bfloat16) + boundaries, seq = [0, 17, 34], 17 + apply = dsa_rope._apply_rotary_query + else: + # Ragged lengths 3 and 5, followed by one padded compressed-token position. + x = torch.randn(9, 1, 512, device="cuda", dtype=torch.bfloat16) + boundaries, seq = [0, 3, 8], 5 + apply = dsa_rope.apply_dsa_rope + (cos, sin), _ = dsa_rope.rotary_embeddings(seq, 4, 64, 160000.0, x.device) + cu = torch.tensor(boundaries, device=x.device, dtype=torch.int32) gradient = torch.randn_like(x) eager = dsa_rope._apply_rotary_eager + reference_input = x.detach().clone().requires_grad_() + reference = eager(reference_input, cos, sin, cu) + if not inplace: + # Check eager packed indexing against separate sequences, including padding at position 0. + with torch.no_grad(): + expected = torch.cat( + [ + eager( + x[start:end].unsqueeze(0), cos[:, : end - start], sin[:, : end - start] + ).squeeze(0) + for start, end in ((0, 3), (3, 8), (8, 9)) + ] + ) + torch.testing.assert_close(reference, expected) + reference_grad = torch.autograd.grad(reference, reference_input, gradient)[0] + monkeypatch.setattr( dsa_rope, "_apply_rotary_eager", lambda *_: pytest.fail("Triton dispatch fell back to eager RoPE"), ) - candidate_input = x.detach().requires_grad_() - candidate = dsa_rope.apply_rotary(candidate_input, cos, sin, cu) - candidate_grad = torch.autograd.grad(candidate, candidate_input, gradient)[0] + candidate_source = x.detach().clone().requires_grad_() + candidate_input = candidate_source.clone() if inplace else candidate_source + candidate_ptr = candidate_input.data_ptr() + candidate = apply(candidate_input, cos, sin, cu) + candidate_grad = torch.autograd.grad(candidate, candidate_source, gradient)[0] - reference_input = x.detach().requires_grad_() - reference = eager(reference_input, cos, sin) - reference_grad = torch.autograd.grad(reference, reference_input, gradient)[0] - - assert candidate.data_ptr() != candidate_input.data_ptr() + assert (candidate.data_ptr() == candidate_ptr) == inplace torch.testing.assert_close(candidate, reference, atol=2e-2, rtol=2e-2) - grad_delta = candidate_grad.float() - reference_grad.float() - assert grad_delta.abs().max() <= 0.03125 - assert grad_delta.norm() / reference_grad.float().norm() < 0.002 - - query_source = x.detach().requires_grad_() - query_input = query_source.clone() - query_ptr = query_input.data_ptr() - query = dsa_rope._apply_rotary_query(query_input, cos, sin, cu) - query_grad = torch.autograd.grad(query, query_source, gradient)[0] - - assert query.data_ptr() == query_ptr - torch.testing.assert_close(query, reference, atol=2e-2, rtol=2e-2) - torch.testing.assert_close(query_grad, reference_grad, atol=0.03125, rtol=0.002) - - -def test_dsa_rope_non_power_of_two(monkeypatch): - """An accepted width of 96 must fall back for both rotary entry points.""" - if not torch.cuda.is_available(): - pytest.skip("DSA rotary dispatch requires CUDA") - if dsa_rope._triton_rope_module() is None: - pytest.skip("Triton DSL is unavailable") - torch.manual_seed(41) - x = torch.randn(17, 2, 128, device="cuda", dtype=torch.bfloat16) - (cos, sin), _ = dsa_rope.rotary_embeddings(17, 4, 96, 160000.0, x.device) - cu = torch.tensor([0, 17], device=x.device, dtype=torch.int32) - monkeypatch.setattr( - dsa_rope, "_launch_triton", lambda *_: pytest.fail("Width 96 reached Triton") - ) - assert not dsa_rope._can_use_triton(x, cos, sin, cu) - gradient = torch.randn_like(x) - reference_input = x.detach().requires_grad_() - reference = dsa_rope._apply_rotary_eager(reference_input, cos, sin, cu) - reference_grad = torch.autograd.grad(reference, reference_input, gradient)[0] - for apply in (dsa_rope.apply_rotary, dsa_rope._apply_rotary_query): - candidate_input = x.detach().requires_grad_() - candidate = apply(candidate_input, cos, sin, cu) - candidate_grad = torch.autograd.grad(candidate, candidate_input, gradient)[0] - torch.testing.assert_close(candidate, reference) - torch.testing.assert_close(candidate_grad, reference_grad) - - -def test_dsa_packed_rope_eager(): - """Reset per-sequence positions and safely rotate unused compressed capacity.""" - torch.manual_seed(31) - cu = torch.tensor([0, 3, 8], dtype=torch.int32) - (cos, sin), _ = dsa_rope.rotary_embeddings(5, 1, 4, 12345.0, "cpu") - x = torch.randn(9, 2, 8, requires_grad=True) - reference_input = x.detach().clone().requires_grad_() - actual = dsa_rope.apply_rotary(x, cos, sin, cu) - expected = torch.cat( - [ - dsa_rope._apply_rotary_eager( - reference_input[start:end].unsqueeze(0), - cos[:, : end - start], - sin[:, : end - start], - ).squeeze(0) - for start, end in ((0, 3), (3, 8), (8, 9)) - ] - ) - gradient = torch.randn_like(actual) - actual.backward(gradient) - expected.backward(gradient) - torch.testing.assert_close(actual, expected) - torch.testing.assert_close(x.grad, reference_input.grad) + torch.testing.assert_close(candidate_grad, reference_grad, atol=0.03125, rtol=0.002) @pytest.mark.parametrize("interleaved", [False, True]) -@pytest.mark.parametrize("inverse", [False, True]) -@pytest.mark.parametrize("cp_rank", [0, 1]) -def test_common_rope_layout_inverse_packed(interleaved, inverse, cp_rank): - """Independent pair equations cover shared lookup, both layouts and transpose.""" +def test_common_rope_layout_packed(interleaved): + """Check both Q output layouts and gradients for the current CP1/noninverse callers.""" if not torch.cuda.is_available(): pytest.skip("Triton RoPE requires CUDA") kernels = dsa_rope._triton_rope_module() @@ -249,21 +207,16 @@ def test_common_rope_layout_inverse_packed(interleaved, inverse, cp_rank): import triton torch.manual_seed(53) - # CP2 local lengths are 4 and 6, plus one padded token position. Each rank owns - # the first/last chunks prescribed by the zig-zag CP mapping. - positions = ( - [0, 1, 6, 7, 0, 1, 2, 9, 10, 11, 0] if cp_rank == 0 else [2, 3, 4, 5, 3, 4, 5, 6, 7, 8, 0] - ) - cu = torch.tensor([0, 8, 20], device="cuda", dtype=torch.int32) + # Two sequences of lengths 4 and 6, then one padded token using position zero. + positions = [0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 0] + cu = torch.tensor([0, 4, 10], device="cuda", dtype=torch.int32) x = torch.randn(11, 3, 80, device="cuda", dtype=torch.bfloat16) - # Noncontiguous rows ensure callers preserve the explicit table strides. - cos = torch.randn(12, 128, device="cuda")[:, ::2] - sin = torch.randn(12, 128, device="cuda")[:, ::2] + # Contiguous columns with a larger row stride exercise explicit table strides. + cos = torch.randn(6, 128, device="cuda")[:, :64] + sin = torch.randn(6, 128, device="cuda")[:, :64] reference_input = x.detach().requires_grad_() tail = reference_input[..., 16:].float() c, s = cos[positions, None], sin[positions, None] - if inverse: - s = -s left = tail[..., 0::2] * c[..., :32] - tail[..., 1::2] * s[..., :32] right = tail[..., 1::2] * c[..., 32:] + tail[..., 0::2] * s[..., 32:] rotated = ( @@ -274,13 +227,9 @@ def test_common_rope_layout_inverse_packed(interleaved, inverse, cp_rank): expected_grad = torch.autograd.grad(reference, reference_input, gradient)[0] candidate, actual_grad = x.clone(), gradient.clone() grid = lambda meta: (11, triton.cdiv(3, meta["BLOCK_H"])) - # The tables have strided columns; materialize those columns while retaining - # a larger row stride, which is the kernel's supported table contract. - cos = torch.stack((cos, cos), 1)[:, 0, :] - sin = torch.stack((sin, sin), 1)[:, 0, :] for kernel, tensor in ( - (kernels._autotuned_mla_rope_fwd_inplace_kernel, candidate), - (kernels._autotuned_mla_rope_bwd_inplace_kernel, actual_grad), + (kernels.rotary_fwd_q_kernel, candidate), + (kernels.rotary_bwd_q_kernel, actual_grad), ): kernel[grid]( tensor, @@ -297,9 +246,9 @@ def test_common_rope_layout_inverse_packed(interleaved, inverse, cp_rank): tensor.stride(1), cos.stride(0), sin.stride(0), - cp_rank, - 2, - INVERSE=inverse, + 0, + 1, + INVERSE=False, REMOVE_INTERLEAVING=interleaved, ) torch.testing.assert_close(candidate, reference, atol=0.03125, rtol=0.002) @@ -395,7 +344,7 @@ def test_dsa_indexer_backward(index_heads): ).contiguous() target = (comp_logits - full_lse[..., None]).exp().sum(2) target = target / target.sum(-1, keepdim=True).clamp_min(1e-30) - index_score, attn_score = dsa_cudnn.dense_indexer_loss_scores( + index_score, attn_score = dense_indexer_loss_scores( qi, ki, wi, @@ -423,7 +372,7 @@ def test_dsa_indexer_backward(index_heads): coefficient * upstream * (target * (target.clamp_min(1e-30).log() - log_prob)).sum() / seq ) reference_grads = torch.autograd.grad(loss, (qr, kr, wr)) - actual_grads = dsa_cudnn.dense_indexer_loss_backward( + actual_grads = dense_indexer_loss_backward( qi, ki, wi, @@ -491,7 +440,6 @@ def test_dsv4_thd_compressed_prefixes(): @pytest.mark.parametrize( "prefix,maximum", [ - (None, 8), ([0, 8], 8), (torch.tensor([0, 8], dtype=torch.int64), 8), (torch.tensor([[0, 8]], dtype=torch.int32), 8), @@ -526,8 +474,7 @@ def test_dsv4_thd_rejects_hidden_batch_axis_and_multi_sequence_context(): @pytest.mark.parametrize("variant,ratio", [("hca", 128), ("csa", 4)]) -@pytest.mark.parametrize("lengths", [[129, 257], [255, 383]]) -def test_dsv4_attention_thd(variant, ratio, lengths): +def test_dsv4_attention_thd(variant, ratio): """Packed full-layer output and gradients match independent dense sequences.""" if not torch.cuda.is_available() or torch.cuda.get_device_capability() not in ( (10, 0), @@ -538,6 +485,7 @@ def test_dsv4_attention_thd(variant, ratio, lengths): cudnn = pytest.importorskip("cudnn") if not hasattr(getattr(cudnn, "DSA", None), "sparse_attention_forward_wrapper"): pytest.skip("DSv4 requires cuDNN Frontend 1.29 sparse attention APIs") + lengths = [255, 383] torch.manual_seed(37) layer = dsv4_attention.DSv4Attention( hidden_size=32, diff --git a/transformer_engine/common/triton/dsa_rope.py b/transformer_engine/common/triton/dsa_rope.py index 39e9a67a4ea..a2fc012f8b4 100644 --- a/transformer_engine/common/triton/dsa_rope.py +++ b/transformer_engine/common/triton/dsa_rope.py @@ -48,7 +48,7 @@ def _get_thd_token_idx(cu_seqlens, pid_m, seq_num, cp_rank, cp_size): @triton.jit -def _mla_rope_fwd_inplace_kernel( +def _rotary_fwd_q_kernel( Q, COS, SIN, @@ -134,7 +134,7 @@ def _mla_rope_fwd_inplace_kernel( tl.store(Q + x_right_off, x_right, mask=mask) -_autotuned_mla_rope_fwd_inplace_kernel = triton.autotune( +rotary_fwd_q_kernel = triton.autotune( configs=[ triton.Config({"BLOCK_H": 1}), triton.Config({"BLOCK_H": 2}), @@ -147,11 +147,11 @@ def _mla_rope_fwd_inplace_kernel( ], key=["emb_dim", "head_num"], restore_value=["Q"], -)(_mla_rope_fwd_inplace_kernel) +)(_rotary_fwd_q_kernel) @triton.jit -def _mla_rope_bwd_inplace_kernel( +def _rotary_bwd_q_kernel( DO, COS, SIN, @@ -237,7 +237,7 @@ def _mla_rope_bwd_inplace_kernel( tl.store(DO + x_2_off, x_2, mask=mask) -_autotuned_mla_rope_bwd_inplace_kernel = triton.autotune( +rotary_bwd_q_kernel = triton.autotune( configs=[ triton.Config({"BLOCK_H": 1}), triton.Config({"BLOCK_H": 2}), @@ -250,7 +250,7 @@ def _mla_rope_bwd_inplace_kernel( ], key=["emb_dim", "head_num"], restore_value=["DO"], -)(_mla_rope_bwd_inplace_kernel) +)(_rotary_bwd_q_kernel) @triton.autotune( diff --git a/transformer_engine/pytorch/__init__.py b/transformer_engine/pytorch/__init__.py index 601dfe43e65..dfc91d923ac 100644 --- a/transformer_engine/pytorch/__init__.py +++ b/transformer_engine/pytorch/__init__.py @@ -33,6 +33,8 @@ DSv4Attention, Compressor, Indexer, + dense_indexer_loss_scores, + dense_indexer_loss_backward, ) from transformer_engine.pytorch.attention import GatedDeltaNetAttention from transformer_engine.pytorch.attention import GatedDeltaNet2Attention diff --git a/transformer_engine/pytorch/attention/__init__.py b/transformer_engine/pytorch/attention/__init__.py index aa8f21b2b36..f8b10ae300e 100644 --- a/transformer_engine/pytorch/attention/__init__.py +++ b/transformer_engine/pytorch/attention/__init__.py @@ -5,7 +5,13 @@ """Python interface for attention""" from .dot_product_attention import DotProductAttention -from .sparse_attention import DSv4Attention, Compressor, Indexer +from .sparse_attention import ( + DSv4Attention, + Compressor, + Indexer, + dense_indexer_loss_scores, + dense_indexer_loss_backward, +) from .linear_attention import ( GatedDeltaNetAttention, GatedDeltaNet2Attention, @@ -22,6 +28,8 @@ "DSv4Attention", "Compressor", "Indexer", + "dense_indexer_loss_scores", + "dense_indexer_loss_backward", "GatedDeltaNetAttention", "GatedDeltaNet2Attention", "GatedDeltaProductAttention", diff --git a/transformer_engine/pytorch/attention/fused_mla_q_uproj.py b/transformer_engine/pytorch/attention/fused_mla_q_uproj.py index 643d4287ba7..3cf46c3eb08 100644 --- a/transformer_engine/pytorch/attention/fused_mla_q_uproj.py +++ b/transformer_engine/pytorch/attention/fused_mla_q_uproj.py @@ -21,7 +21,7 @@ try: import triton from transformer_engine.common.triton.dsa_rope import ( - _autotuned_mla_rope_bwd_inplace_kernel as rotary_bwd_q_kernel, + rotary_bwd_q_kernel, ) except ImportError: triton = None diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index bb7cc586114..112b32f313d 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -21,7 +21,7 @@ DSv4Attention ├── Indexer (CSA only) │ ├── TE index-query/head-weight projections │ ├── own Compressor (index keys) -│ ├── dsa_rope.apply_rotary (query + index keys) +│ ├── dsa_rope.apply_dsa_rope (query + index keys) │ └── dsa_cudnn.select_indices -> cuDNN score + top-k ├── dsa_rope embeddings + rotary application ├── dsa_cudnn.sparse_attention -> cuDNN sparse attention @@ -154,9 +154,12 @@ single-sequence, caller-owned indexer loss. Without that objective, the indexer is not trained by the attention output alone. Pre-attention normalization, residuals, and loss calculation remain model-owned. -For B=1, the caller can train the indexer with the dense helper primitives: +For B=1, the caller can train the indexer with `dense_indexer_loss_scores` and +`dense_indexer_loss_backward`, exported alongside the three sparse-attention classes: ```python +from transformer_engine.pytorch import dense_indexer_loss_scores, dense_indexer_loss_backward + output, context = csa(hidden_states, return_indexer_context=True) qi, ki, wi = (context[name].unsqueeze(0) for name in ("index_q", "index_k", "index_w")) @@ -167,12 +170,12 @@ qa, kc = (context[name].unsqueeze(0) # The selected sparse-attention LSE excludes sink and is not this denominator. full_lse = ... index_scale = (qi.shape[2] * qi.shape[3]) ** -0.5 -index_scores, teacher_scores = dsa_cudnn.dense_indexer_loss_scores( +index_scores, teacher_scores = dense_indexer_loss_scores( qi, ki, wi, qa, kc, full_lse, ratio=csa.compression_ratio, index_scale=index_scale, attention_scale=csa.head_dim ** -0.5, ) -index_grads = dsa_cudnn.dense_indexer_loss_backward( +index_grads = dense_indexer_loss_backward( qi, ki, wi, index_scores, teacher_scores, torch.ones((), device=qi.device), ratio=csa.compression_ratio, index_scale=index_scale, diff --git a/transformer_engine/pytorch/attention/sparse_attention/__init__.py b/transformer_engine/pytorch/attention/sparse_attention/__init__.py index 93d93f14f7f..9ea6e5ba095 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/__init__.py +++ b/transformer_engine/pytorch/attention/sparse_attention/__init__.py @@ -7,9 +7,12 @@ from .dsv4_attention import DSv4Attention from .compressor import Compressor from .indexer import Indexer +from .dsa_cudnn import dense_indexer_loss_scores, dense_indexer_loss_backward __all__ = [ "DSv4Attention", "Compressor", "Indexer", + "dense_indexer_loss_scores", + "dense_indexer_loss_backward", ] diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py index 58be2880b4e..715423ecafa 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py @@ -113,11 +113,7 @@ def _apply_rotary_triton(x, cos, sin, cu_seqlens): def _launch_triton(packed, cos, sin, cu_seqlens, module, backward): - kernel = ( - module._autotuned_mla_rope_bwd_inplace_kernel - if backward - else module._autotuned_mla_rope_fwd_inplace_kernel - ) + kernel = module.rotary_bwd_q_kernel if backward else module.rotary_fwd_q_kernel heads, dim = packed.shape[-2:] def grid(meta): @@ -152,7 +148,7 @@ class _TritonRotary(torch.autograd.Function): def forward(ctx, x, cos, sin, cu_seqlens, module): """Rotate a copy and save the frequency tables for backward.""" # cuDNN attention saves its output for backward. Cloning also preserves - # apply_rotary's established out-of-place contract for every call site. + # apply_dsa_rope's established out-of-place contract for every call site. output = x.clone(memory_format=torch.contiguous_format) packed = output.reshape(-1, x.shape[-2], x.shape[-1]) _launch_triton(packed, cos, sin, cu_seqlens, module, False) @@ -198,7 +194,7 @@ def backward(ctx, grad_output): def _apply_rotary_query(query, cos, sin, cu_seqlens): """Rotate fresh normalized queries in place with Triton, avoiding a copy. - Unlike apply_rotary, this private path requires an input safe to mutate. + Unlike apply_dsa_rope, this private path requires an input safe to mutate. Its backward and eager fallback remain out of place. """ if not _can_use_triton(query, cos, sin, cu_seqlens): @@ -207,7 +203,7 @@ def _apply_rotary_query(query, cos, sin, cu_seqlens): return _TritonQueryRotary.apply(query, _rope_tables(cos), _rope_tables(sin), cu_seqlens, module) -def apply_rotary(x, cos, sin, cu_seqlens=None): +def apply_dsa_rope(x, cos, sin, cu_seqlens=None): """Rotate trailing interleaved pairs, using Triton when its contract is satisfied.""" if _can_use_triton(x, cos, sin, cu_seqlens): return _apply_rotary_triton(x, cos, sin, cu_seqlens) diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index 8b2b45c77ed..aaea7ae3a47 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -12,7 +12,7 @@ from .compressor import Compressor from .dsa_cudnn import sparse_attention as _sparse_attention -from .dsa_rope import _apply_rotary_query, _DSv4RotaryEmbedding, apply_rotary +from .dsa_rope import _apply_rotary_query, _DSv4RotaryEmbedding, apply_dsa_rope from .indexer import Indexer __all__ = ["DSv4Attention"] @@ -315,9 +315,9 @@ def forward( .reshape(-1, self.num_heads, self.head_dim) .contiguous() ) - local_kv = apply_rotary(self.kv_norm(local_kv_projected).unsqueeze(-2), *token_rope, cu) + local_kv = apply_dsa_rope(self.kv_norm(local_kv_projected).unsqueeze(-2), *token_rope, cu) local_kv = local_kv.reshape(-1, self.head_dim).contiguous() - compressed_kv = apply_rotary( + compressed_kv = apply_dsa_rope( self.compressor(hidden_states, cu, cu_comp, total_comp=total_comp).reshape( *comp_shape, 1, self.head_dim ), @@ -360,7 +360,7 @@ def forward( max_compressed_seqlen=None if self.is_csa else seq // self.compression_ratio, ).reshape(*token_shape, self.num_heads, self.head_dim) cos, sin = token_rope - output = apply_rotary(output, cos, -sin, cu).reshape(*token_shape, self.o_groups, -1) + output = apply_dsa_rope(output, cos, -sin, cu).reshape(*token_shape, self.o_groups, -1) weight = self.o_a_proj.weight.reshape(self.o_groups, self.o_lora_rank, -1) grouped = torch.einsum("...gd,grd->...gr", output, weight).flatten(-2) result = self.o_b_proj(grouped) diff --git a/transformer_engine/pytorch/attention/sparse_attention/indexer.py b/transformer_engine/pytorch/attention/sparse_attention/indexer.py index fd4e77e543f..bd1c36b7085 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/indexer.py +++ b/transformer_engine/pytorch/attention/sparse_attention/indexer.py @@ -10,7 +10,7 @@ from .compressor import Compressor from .dsa_cudnn import select_indices -from .dsa_rope import apply_rotary +from .dsa_rope import apply_dsa_rope class Indexer(torch.nn.Module): @@ -127,7 +127,7 @@ def forward( ) # The singleton axis is the shared index-key head, not a tunable head count. key = ( - apply_rotary( + apply_dsa_rope( index_compressed.reshape(*comp_shape, 1, self.head_dim), *compressed_rope, cu_comp, @@ -139,7 +139,7 @@ def forward( *hidden_states.shape[:-1], self.n_heads, self.head_dim ) query = ( - apply_rotary(query, *token_rope, cu) + apply_dsa_rope(query, *token_rope, cu) .reshape(-1, self.n_heads, self.head_dim) .contiguous() ) From 8aeb32d120116107442645409aec33c6b1674e0e Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Thu, 19 Nov 2026 15:13:29 -0800 Subject: [PATCH 33/37] Consolidate DSA RoPE application and expose its cache Use one RoPE adapter and autograd function so callers can opt into reusing a disposable query buffer while preserving inputs by default. Expose the table cache for standalone sparse attention components and honor the requested device on cache hits. Keep the existing adapter parity cases and exercise cache migration in the CSA integration test. Simplify the MLA KV test name and explain how it assembles keys and values. Signed-off-by: Sudhakar Singh --- tests/pytorch/attention/test_fused_mla.py | 9 +- tests/pytorch/test_dsv4_attention.py | 16 ++- .../attention/sparse_attention/README.md | 3 +- .../attention/sparse_attention/dsa_rope.py | 122 ++++++++++-------- .../sparse_attention/dsv4_attention.py | 8 +- 5 files changed, 87 insertions(+), 71 deletions(-) diff --git a/tests/pytorch/attention/test_fused_mla.py b/tests/pytorch/attention/test_fused_mla.py index b0c66b0a44a..e96488e8a42 100644 --- a/tests/pytorch/attention/test_fused_mla.py +++ b/tests/pytorch/attention/test_fused_mla.py @@ -470,10 +470,13 @@ def test_inputs_on_non_current_device(): @pytest.mark.skipif(not HAVE_TRITON, reason="MLA RoPE requires Triton") -def test_mla_rope_kv_equations_backward(): - """Compare MLA K/V assembly and input gradients with independent PyTorch equations. +def test_mla_rope_kv(): + """Compare MLA keys, values, and input gradients with independent PyTorch equations. - The positional key is shared across heads, so its gradient must sum their contributions. + Split the KV input into per-head key and value channels. Rotate the shared + positional key for each token and append it to every head's key. The value + channels pass through unchanged. Backward sums the positional-key gradients + across heads. """ if not torch.cuda.is_available(): pytest.skip("MLA RoPE requires CUDA") diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index bc8ebe7f9c7..145423d41ca 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -153,17 +153,15 @@ def test_dsa_triton_rope_adapters(monkeypatch, layout): if inplace: x = torch.randn(2, 17, 64, 512, device="cuda", dtype=torch.bfloat16) boundaries, seq = [0, 17, 34], 17 - apply = dsa_rope._apply_rotary_query else: # Ragged lengths 3 and 5, followed by one padded compressed-token position. x = torch.randn(9, 1, 512, device="cuda", dtype=torch.bfloat16) boundaries, seq = [0, 3, 8], 5 - apply = dsa_rope.apply_dsa_rope (cos, sin), _ = dsa_rope.rotary_embeddings(seq, 4, 64, 160000.0, x.device) cu = torch.tensor(boundaries, device=x.device, dtype=torch.int32) gradient = torch.randn_like(x) - eager = dsa_rope._apply_rotary_eager + eager = dsa_rope.apply_dsa_rope_eager reference_input = x.detach().clone().requires_grad_() reference = eager(reference_input, cos, sin, cu) if not inplace: @@ -182,13 +180,13 @@ def test_dsa_triton_rope_adapters(monkeypatch, layout): monkeypatch.setattr( dsa_rope, - "_apply_rotary_eager", + "apply_dsa_rope_eager", lambda *_: pytest.fail("Triton dispatch fell back to eager RoPE"), ) candidate_source = x.detach().clone().requires_grad_() candidate_input = candidate_source.clone() if inplace else candidate_source candidate_ptr = candidate_input.data_ptr() - candidate = apply(candidate_input, cos, sin, cu) + candidate = dsa_rope.apply_dsa_rope(candidate_input, cos, sin, cu, allow_inplace=inplace) candidate_grad = torch.autograd.grad(candidate, candidate_source, gradient)[0] assert (candidate.data_ptr() == candidate_ptr) == inplace @@ -501,7 +499,10 @@ def test_dsv4_attention_thd(variant, ratio): o_lora_rank=8, index_topk=8, rope=( - dsa_rope._DSv4RotaryEmbedding(ratio, 64, 12345.0, "cuda") if variant == "csa" else None + # A warm CPU cache must move on a same-length CUDA request. + dsa_rope.DSv4RotaryEmbedding(ratio, 64, 12345.0, "cpu", max_seqlen=max(lengths)) + if variant == "csa" + else None ), params_dtype=torch.bfloat16, qkv_format="thd", @@ -510,6 +511,9 @@ def test_dsv4_attention_thd(variant, ratio): cu = torch.tensor([0, lengths[0], sum(lengths)], device=x.device, dtype=torch.int32) actual = layer(x, cu_seqlens_q=cu, max_seqlen_q=max(lengths)) assert actual.shape == x.shape + if variant == "csa": + assert layer.rope.cos.device == x.device + assert layer.rope.cos.dtype == torch.float32 gradient = torch.randn_like(actual) actual.backward(gradient) packed_grads = { diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index 112b32f313d..2acd8b601be 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -82,7 +82,8 @@ compressed tokens. `_prepare_sparse_kv_indices` documents a small example of the index spaces. Selection indices address only packed compressed KV before the local KV offset is applied. -`dsa_rope` supplies token/compressed-position embeddings and rotary application. +`dsa_rope.DSv4RotaryEmbedding` caches token/compressed-position embeddings. +`dsa_rope.apply_dsa_rope` applies them. A caller may override the default embeddings with a `rope` module returning token and compressed `(cos, sin)` pairs. diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py index 715423ecafa..5355a08cedd 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py @@ -8,6 +8,8 @@ import torch +from transformer_engine.pytorch.utils import canonicalize_device + def rotary_embeddings(seq, ratio, width, theta, device): """Return token and compressed-window (cos, sin) pairs in FP32.""" @@ -19,8 +21,25 @@ def rotary_embeddings(seq, ratio, width, theta, device): return (cos, sin), (cos[:, window_positions], sin[:, window_positions]) -class _DSv4RotaryEmbedding(torch.nn.Module): - """Cache shared FP32 token frequencies; compressed positions are window starts.""" +class DSv4RotaryEmbedding(torch.nn.Module): + """Cache FP32 token frequencies and compressed-window start frequencies. + + Buffers are not saved in state_dict. Module.to() moves the cache while keeping + its frequencies in FP32, even when a different floating-point dtype is requested. + + Parameters + ---------- + ratio : int + Number of input tokens per compressed window. + width : int + Even rotary dimension. + theta : float + Rotary frequency base. + device : torch.device or str + Device for the optional initial cache. + max_seqlen : int, optional + Initial cache length. Otherwise allocate lazily on the first forward call. + """ def __init__(self, ratio, width, theta, device, max_seqlen=None): super().__init__() @@ -41,11 +60,30 @@ def _apply(self, fn): return self def forward(self, seq, device): - """Return cached token and compressed-window frequency pairs.""" + """Return FP32 token and compressed ``(cos, sin)`` pairs. + + Parameters + ---------- + seq : int + Token sequence length. Grow the cache if needed. + device : torch.device or str + Requested output device. Move an existing cache when it differs. + + Returns + ------- + tuple + Token pair with shape [1, seq, 1, width] and compressed pair with shape + [1, seq // ratio, 1, width]. Compressed positions are window starts + 0, ratio, 2 * ratio, ... . Returned tensors are views of the cache. + """ + device = canonicalize_device(device) if self.cos is None or self.cos.shape[1] < seq: (self.cos, self.sin), _ = rotary_embeddings( seq, self.ratio, self.width, self.theta, device ) + elif self.cos.device != device: + self.cos = self.cos.to(device=device) + self.sin = self.sin.to(device=device) n_comp = seq // self.ratio return (self.cos[:, :seq], self.sin[:, :seq]), ( self.cos[:, : n_comp * self.ratio : self.ratio], @@ -53,7 +91,7 @@ def forward(self, seq, device): ) -def _apply_rotary_eager(x, cos, sin, cu_seqlens=None): +def apply_dsa_rope_eager(x, cos, sin, cu_seqlens=None): """Rotate trailing interleaved pairs in FP32, then restore input dtype.""" if x.ndim == 3: if cu_seqlens is None: @@ -104,14 +142,6 @@ def _rope_tables(table): return torch.cat((pairs, pairs), dim=-1) -def _apply_rotary_triton(x, cos, sin, cu_seqlens): - """Apply Triton RoPE out of place so callers retain the eager aliasing contract.""" - module = _triton_rope_module() - if module is None: - raise RuntimeError("Triton RoPE is unavailable.") - return _TritonRotary.apply(x, _rope_tables(cos), _rope_tables(sin), cu_seqlens, module) - - def _launch_triton(packed, cos, sin, cu_seqlens, module, backward): kernel = module.rotary_bwd_q_kernel if backward else module.rotary_fwd_q_kernel heads, dim = packed.shape[-2:] @@ -142,19 +172,21 @@ def grid(meta): class _TritonRotary(torch.autograd.Function): - """Pair the retained in-place Triton kernels behind an out-of-place autograd API.""" + """Apply the shared Triton kernels with optional input mutation.""" @staticmethod - def forward(ctx, x, cos, sin, cu_seqlens, module): - """Rotate a copy and save the frequency tables for backward.""" - # cuDNN attention saves its output for backward. Cloning also preserves - # apply_dsa_rope's established out-of-place contract for every call site. - output = x.clone(memory_format=torch.contiguous_format) + def forward(ctx, x, cos, sin, cu_seqlens, module, allow_inplace): + """Rotate a contiguous buffer and save the frequency tables for backward.""" + # Contiguity ensures the packed reshape aliases the buffer we return. + inplace = allow_inplace and x.is_contiguous() + output = x if inplace else x.clone(memory_format=torch.contiguous_format) packed = output.reshape(-1, x.shape[-2], x.shape[-1]) _launch_triton(packed, cos, sin, cu_seqlens, module, False) ctx.save_for_backward(cos, sin, cu_seqlens) ctx.module = module ctx.shape = x.shape + if inplace: + ctx.mark_dirty(x) return output @staticmethod @@ -164,47 +196,23 @@ def backward(ctx, grad_output): grad_input = grad_output.clone(memory_format=torch.contiguous_format) packed = grad_input.reshape(-1, ctx.shape[-2], ctx.shape[-1]) _launch_triton(packed, cos, sin, cu_seqlens, ctx.module, True) - return grad_input, None, None, None, None + return grad_input, None, None, None, None, None -class _TritonQueryRotary(torch.autograd.Function): - """Rotate the fresh normalized query in place; keep backward out of place.""" +def apply_dsa_rope(x, cos, sin, cu_seqlens=None, *, allow_inplace=False): + """Rotate trailing interleaved pairs, using Triton when its contract is satisfied. - @staticmethod - def forward(ctx, query, cos, sin, cu_seqlens, module): - """Rotate the fresh query and record its in-place mutation.""" - packed = query.reshape(-1, query.shape[-2], query.shape[-1]) - _launch_triton(packed, cos, sin, cu_seqlens, module, False) - ctx.save_for_backward(cos, sin, cu_seqlens) - ctx.module = module - ctx.shape = query.shape - ctx.mark_dirty(query) - return query - - @staticmethod - def backward(ctx, grad_output): - """Apply the backward rotation without mutating the output gradient.""" - cos, sin, cu_seqlens = ctx.saved_tensors - grad_input = grad_output.clone(memory_format=torch.contiguous_format) - packed = grad_input.reshape(-1, ctx.shape[-2], ctx.shape[-1]) - _launch_triton(packed, cos, sin, cu_seqlens, ctx.module, True) - return grad_input, None, None, None, None - - -def _apply_rotary_query(query, cos, sin, cu_seqlens): - """Rotate fresh normalized queries in place with Triton, avoiding a copy. - - Unlike apply_dsa_rope, this private path requires an input safe to mutate. - Its backward and eager fallback remain out of place. + By default, preserve the input. With allow_inplace=True, the caller must supply + a fresh buffer safe for autograd mutation; only contiguous Triton inputs are + mutated. Noncontiguous inputs and the eager fallback remain out of place. """ - if not _can_use_triton(query, cos, sin, cu_seqlens): - return _apply_rotary_eager(query, cos, sin, cu_seqlens) - module = _triton_rope_module() - return _TritonQueryRotary.apply(query, _rope_tables(cos), _rope_tables(sin), cu_seqlens, module) - - -def apply_dsa_rope(x, cos, sin, cu_seqlens=None): - """Rotate trailing interleaved pairs, using Triton when its contract is satisfied.""" if _can_use_triton(x, cos, sin, cu_seqlens): - return _apply_rotary_triton(x, cos, sin, cu_seqlens) - return _apply_rotary_eager(x, cos, sin, cu_seqlens) + return _TritonRotary.apply( + x, + _rope_tables(cos), + _rope_tables(sin), + cu_seqlens, + _triton_rope_module(), + allow_inplace, + ) + return apply_dsa_rope_eager(x, cos, sin, cu_seqlens) diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index aaea7ae3a47..e27bc76a0e7 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -12,7 +12,7 @@ from .compressor import Compressor from .dsa_cudnn import sparse_attention as _sparse_attention -from .dsa_rope import _apply_rotary_query, _DSv4RotaryEmbedding, apply_dsa_rope +from .dsa_rope import DSv4RotaryEmbedding, apply_dsa_rope from .indexer import Indexer __all__ = ["DSv4Attention"] @@ -207,11 +207,11 @@ def __init__( self.sliding_window = sliding_window # A model can supply its RoPE frequencies (for example YaRN) without # duplicating that policy here. It must return the same token/compressed - # (cos, sin) pairs as _DSv4RotaryEmbedding.forward. + # (cos, sin) pairs as DSv4RotaryEmbedding.forward. self.rope = ( rope if rope is not None - else _DSv4RotaryEmbedding( + else DSv4RotaryEmbedding( compression_ratio, rope_head_dim, rope_theta, device, max_seqlen ) ) @@ -311,7 +311,7 @@ def forward( # Query-up norm is unweighted; TE RMSNorm would add a learned scale. q = _compiled_q_rms_norm(q, self.rms_norm_eps) q = ( - _apply_rotary_query(q, *token_rope, cu) + apply_dsa_rope(q, *token_rope, cu, allow_inplace=True) .reshape(-1, self.num_heads, self.head_dim) .contiguous() ) From f51dfb8282a5e1d641eb09cc621cc0d81601437b Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Fri, 20 Nov 2026 02:23:05 -0800 Subject: [PATCH 34/37] Consolidate DSA and MLA RoPE adapters Share table construction and the Triton Q launch across DSA and MLA while preserving each path's layout and mutation contract. Keep standalone RoPE coverage together and zero-fill masked MLA heads so partial blocks contribute correctly to the positional-key gradient. Signed-off-by: Sudhakar Singh --- tests/pytorch/attention/mla_rope_utils.py | 372 ----------- tests/pytorch/attention/test_fused_mla.py | 57 +- .../attention/test_linear_mxfp8_attention.py | 8 +- tests/pytorch/test_dsv4_attention.py | 121 +--- tests/pytorch/test_fused_rope.py | 173 ++++++ .../triton/{dsa_rope.py => dsa_mla_rope.py} | 5 +- .../pytorch/attention/fused_mla_q_uproj.py | 2 +- transformer_engine/pytorch/attention/rope.py | 581 +++++++++++++++++- .../attention/sparse_attention/README.md | 9 +- .../attention/sparse_attention/dsa_rope.py | 218 ------- .../sparse_attention/dsv4_attention.py | 9 +- .../attention/sparse_attention/indexer.py | 2 +- 12 files changed, 784 insertions(+), 773 deletions(-) delete mode 100644 tests/pytorch/attention/mla_rope_utils.py rename transformer_engine/common/triton/{dsa_rope.py => dsa_mla_rope.py} (99%) delete mode 100644 transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py diff --git a/tests/pytorch/attention/mla_rope_utils.py b/tests/pytorch/attention/mla_rope_utils.py deleted file mode 100644 index daebee34939..00000000000 --- a/tests/pytorch/attention/mla_rope_utils.py +++ /dev/null @@ -1,372 +0,0 @@ -# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# See LICENSE for license information. - -"""MLA RoPE for DSv3 671B - Triton forward and backward kernels. - -Source: Megatron-LM megatron/core/fusions/fused_mla_yarn_rope_apply.py -Falls back to pure PyTorch when Triton is unavailable. - -Note: DSv3 uses YaRN-scaled RoPE for long-context extrapolation. This test -intentionally uses plain RoPE (base=10000) because it only validates MXFP8 -attention path wiring, tensor shapes, forward/backward flow, and relative BF16 -vs MXFP8 behavior. Both reference and MXFP8 paths use the same RoPE tables. -""" - -import torch - -try: - import triton - from transformer_engine.common.triton.dsa_rope import ( - rotary_fwd_q_kernel, - rotary_bwd_q_kernel, - rotary_fwd_kv_kernel, - rotary_bwd_kv_kernel, - ) - - HAVE_TRITON = True -except ImportError: - HAVE_TRITON = False - -HEAD_DIM_ROPE = 64 -HEAD_DIM_NOPE = 128 -HEAD_DIM_V = 128 -ROTARY_BASE = 10000 - - -def build_rope_tables( - seq_len: int, - emb_dim: int = HEAD_DIM_ROPE, - base: int = ROTARY_BASE, - device: torch.device = None, -) -> tuple[torch.Tensor, torch.Tensor]: - inv_freq = 1.0 / ( - base ** (torch.arange(0, emb_dim, 2, dtype=torch.float32, device=device) / emb_dim) - ) - t = torch.arange(seq_len, device=device, dtype=torch.float32) - freqs = torch.outer(t, inv_freq) - freqs = torch.cat([freqs, freqs], dim=-1) - return torch.cos(freqs).contiguous(), torch.sin(freqs).contiguous() - - -if HAVE_TRITON: - - def _flattened_token_stride(tensor: torch.Tensor) -> int: - if tensor.dim() == 4: - return tensor.stride(1) - return tensor.stride(0) - - class _MLARoPEQTriton(torch.autograd.Function): - @staticmethod - def forward(ctx, q, cos, sin, head_dim_nope, head_dim_rope): - s, b, nheads, _ = q.shape - total = s * b - - grid_q = lambda META: (total, triton.cdiv(nheads, META["BLOCK_H"])) - rotary_fwd_q_kernel[grid_q]( - q, - cos, - sin, - head_dim_nope, - head_dim_rope, - nheads, - b, - None, - None, - None, - _flattened_token_stride(q), - q.stride(2), - cos.stride(0), - sin.stride(0), - 0, - 1, - INVERSE=False, - REMOVE_INTERLEAVING=False, - ) - - ctx.save_for_backward(cos, sin) - ctx.head_dim_nope = head_dim_nope - ctx.head_dim_rope = head_dim_rope - ctx.nheads = nheads - ctx.s = s - ctx.b = b - return q - - @staticmethod - def backward(ctx, dq): - cos, sin = ctx.saved_tensors - s, b, nheads = ctx.s, ctx.b, ctx.nheads - total = s * b - - grid_q = lambda META: (total, triton.cdiv(nheads, META["BLOCK_H"])) - rotary_bwd_q_kernel[grid_q]( - dq, - cos, - sin, - ctx.head_dim_nope, - ctx.head_dim_rope, - nheads, - b, - None, - None, - None, - _flattened_token_stride(dq), - dq.stride(2), - cos.stride(0), - sin.stride(0), - 0, - 1, - INVERSE=False, - REMOVE_INTERLEAVING=False, - ) - return dq, None, None, None, None - - class _MLARoPEKVTriton(torch.autograd.Function): - @staticmethod - def forward(ctx, kv, k_pos_emb, cos, sin, head_dim_nope, head_dim_rope, head_dim_v): - s, b, nheads, _ = kv.shape - total = s * b - - o_key = kv.new_empty(s, b, nheads, head_dim_nope + head_dim_rope) - o_value = kv.new_empty(s, b, nheads, head_dim_v) - grid_kv = lambda META: (total, triton.cdiv(nheads, META["BLOCK_H"])) - rotary_fwd_kv_kernel[grid_kv]( - kv, - k_pos_emb, - o_key, - o_value, - cos, - sin, - head_dim_rope, - head_dim_nope, - head_dim_v, - nheads, - b, - None, - None, - _flattened_token_stride(kv), - kv.stride(2), - _flattened_token_stride(k_pos_emb), - _flattened_token_stride(o_key), - o_key.stride(2), - _flattened_token_stride(o_value), - o_value.stride(2), - 0, - 1, - ) - - ctx.save_for_backward(cos, sin) - ctx.head_dim_nope = head_dim_nope - ctx.head_dim_rope = head_dim_rope - ctx.head_dim_v = head_dim_v - ctx.nheads = nheads - ctx.s = s - ctx.b = b - return o_key, o_value - - @staticmethod - def backward(ctx, dk_out, dv_out): - cos, sin = ctx.saved_tensors - s, b, nheads = ctx.s, ctx.b, ctx.nheads - ndp, ndr, ndv = ctx.head_dim_nope, ctx.head_dim_rope, ctx.head_dim_v - total = s * b - - d_kv = dk_out.new_empty(s, b, nheads, ndp + ndv) - d_emb = dk_out.new_empty(s, b, 1, ndr) - grid_kv = lambda META: (total, triton.cdiv(nheads, META["BLOCK_H"])) - rotary_bwd_kv_kernel[grid_kv]( - dk_out, - dv_out, - d_kv, - d_emb, - cos, - sin, - ndr, - ndp, - ndv, - nheads, - b, - None, - None, - _flattened_token_stride(dk_out), - dk_out.stride(2), - _flattened_token_stride(dv_out), - dv_out.stride(2), - _flattened_token_stride(d_kv), - d_kv.stride(2), - _flattened_token_stride(d_emb), - 0, - 1, - ) - return d_kv, d_emb, None, None, None, None, None - - -def _apply_mla_rope_q_with_tables( - q: torch.Tensor, - cos_table: torch.Tensor, - sin_table: torch.Tensor, - head_dim_nope: int = HEAD_DIM_NOPE, - head_dim_rope: int = HEAD_DIM_ROPE, -) -> torch.Tensor: - if HAVE_TRITON: - return _MLARoPEQTriton.apply( - q, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - ) - return _apply_pytorch_q(q, cos_table, sin_table, head_dim_nope, head_dim_rope) - - -def _apply_mla_rope_kv_with_tables( - kv: torch.Tensor, - k_pos_emb: torch.Tensor, - cos_table: torch.Tensor, - sin_table: torch.Tensor, - head_dim_nope: int = HEAD_DIM_NOPE, - head_dim_rope: int = HEAD_DIM_ROPE, - head_dim_v: int = HEAD_DIM_V, -) -> tuple[torch.Tensor, torch.Tensor]: - if HAVE_TRITON: - return _MLARoPEKVTriton.apply( - kv, - k_pos_emb, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - head_dim_v, - ) - return _apply_pytorch_kv( - kv, - k_pos_emb, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - head_dim_v, - ) - - -def apply_mla_rope_q( - q: torch.Tensor, - head_dim_nope: int = HEAD_DIM_NOPE, - head_dim_rope: int = HEAD_DIM_ROPE, - base: int = ROTARY_BASE, - cos_table: torch.Tensor | None = None, - sin_table: torch.Tensor | None = None, -) -> torch.Tensor: - if cos_table is None or sin_table is None: - s = q.shape[0] - cos_table, sin_table = build_rope_tables( - s, - emb_dim=head_dim_rope, - base=base, - device=q.device, - ) - return _apply_mla_rope_q_with_tables( - q, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - ) - - -def apply_mla_rope_kv( - kv: torch.Tensor, - k_pos_emb: torch.Tensor, - head_dim_nope: int = HEAD_DIM_NOPE, - head_dim_rope: int = HEAD_DIM_ROPE, - head_dim_v: int = HEAD_DIM_V, - base: int = ROTARY_BASE, - cos_table: torch.Tensor | None = None, - sin_table: torch.Tensor | None = None, -) -> tuple[torch.Tensor, torch.Tensor]: - if cos_table is None or sin_table is None: - s = kv.shape[0] - cos_table, sin_table = build_rope_tables( - s, - emb_dim=head_dim_rope, - base=base, - device=kv.device, - ) - return _apply_mla_rope_kv_with_tables( - kv, - k_pos_emb, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - head_dim_v, - ) - - -def apply_mla_rope( - q: torch.Tensor, - kv: torch.Tensor, - k_pos_emb: torch.Tensor, - head_dim_nope: int = HEAD_DIM_NOPE, - head_dim_rope: int = HEAD_DIM_ROPE, - head_dim_v: int = HEAD_DIM_V, - base: int = ROTARY_BASE, - cos_table: torch.Tensor | None = None, - sin_table: torch.Tensor | None = None, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - if cos_table is None or sin_table is None: - s = q.shape[0] - cos_table, sin_table = build_rope_tables( - s, - emb_dim=head_dim_rope, - base=base, - device=q.device, - ) - q = _apply_mla_rope_q_with_tables(q, cos_table, sin_table, head_dim_nope, head_dim_rope) - k, v = _apply_mla_rope_kv_with_tables( - kv, - k_pos_emb, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - head_dim_v, - ) - return q, k, v - - -def _rotate_interleaved_to_neox( - x: torch.Tensor, cos_table: torch.Tensor, sin_table: torch.Tensor -) -> torch.Tensor: - cos_ = cos_table[:, None, None, :].to(x.dtype) - sin_ = sin_table[:, None, None, :].to(x.dtype) - half_dim = x.shape[-1] // 2 - x_1 = x[..., 0::2] - x_2 = x[..., 1::2] - x_left = x_1 * cos_[..., :half_dim] - x_2 * sin_[..., :half_dim] - x_right = x_2 * cos_[..., half_dim:] + x_1 * sin_[..., half_dim:] - return torch.cat((x_left, x_right), dim=-1) - - -def _apply_pytorch_q(q, cos_table, sin_table, head_dim_nope, head_dim_rope): - q_nope = q[..., :head_dim_nope] - q_rope = q[..., head_dim_nope : head_dim_nope + head_dim_rope] - q_rope = _rotate_interleaved_to_neox(q_rope, cos_table, sin_table) - return torch.cat((q_nope, q_rope), dim=-1) - - -def _apply_pytorch_kv( - kv, - k_pos_emb, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - head_dim_v, -): - k_nope = kv[..., :head_dim_nope] - v = kv[..., head_dim_nope : head_dim_nope + head_dim_v] - k_rope = _rotate_interleaved_to_neox(k_pos_emb, cos_table, sin_table).expand( - -1, -1, kv.shape[2], -1 - ) - return torch.cat((k_nope, k_rope), dim=-1), v diff --git a/tests/pytorch/attention/test_fused_mla.py b/tests/pytorch/attention/test_fused_mla.py index e96488e8a42..52edf667586 100644 --- a/tests/pytorch/attention/test_fused_mla.py +++ b/tests/pytorch/attention/test_fused_mla.py @@ -6,7 +6,7 @@ - FusedMLAQUpProjRopeQuant: query up-projection, RoPE and MXFP8 quantization (cuDNN FE). - fused_mla_kv_rope_mxfp8: key/value split, key RoPE and MXFP8 quantization. The reference - applies Megatron-LM's MLA key/value RoPE (the Triton kernel in mla_rope_utils) and quantizes + applies Megatron-LM's MLA key/value RoPE (the shared Triton kernels) and quantizes key and value with mxfp8_quantize_only; the fused kernel must produce the same bytes. Run: @@ -14,8 +14,6 @@ """ import os -import pathlib -import sys import pytest import torch @@ -30,9 +28,11 @@ from transformer_engine.pytorch.cpp_extensions import general_gemm as _fused_general_gemm from transformer_engine.pytorch.tensor.mxfp8_tensor import MXFP8Quantizer, MXFP8Tensor -_current_file = pathlib.Path(__file__).resolve() -sys.path = [str(_current_file.parent)] + sys.path -from mla_rope_utils import HAVE_TRITON, apply_mla_rope_kv, build_rope_tables +from transformer_engine.pytorch.attention.rope import ( + _triton_rope_module, + apply_mla_rope_kv, + build_mla_rope_tables, +) # All tests in this file require GB200/SM100 hardware. pytestmark = pytest.mark.launch_on_gb200 @@ -337,7 +337,7 @@ def test_fused_mla_q_uproj_autograd() -> None: _kv_rope_skip_reason = reason_for_no_mxfp8 elif not sm100: _kv_rope_skip_reason = "The fused kernel requires SM 10.0+." -elif not HAVE_TRITON: +elif _triton_rope_module() is None: _kv_rope_skip_reason = "The reference MLA RoPE needs Triton." else: _kv_rope_skip_reason = "" @@ -366,7 +366,7 @@ def _kv_rope_inputs(s, b, h, cos_dtype, seed=1234): torch.bfloat16 ) k_rope = kv_down[..., KV_LORA_RANK:].unsqueeze(-2) - cos, sin = build_rope_tables(s, emb_dim=HEAD_DIM_ROPE, device="cuda") + cos, sin = build_mla_rope_tables(s, emb_dim=HEAD_DIM_ROPE, device="cuda") # A magnitude scale as in YaRN, so that the products are not exact. cos, sin = (cos * 1.2).to(cos_dtype), (sin * 1.2).to(cos_dtype) return kv, k_rope, cos, sin @@ -409,7 +409,7 @@ def test_contiguous_k_rope_and_longer_rope_tables(): """A contiguous [s, b, emb_dim] k_rope and cos / sin with more rows than tokens.""" s, b, h = 256, 2, 4 kv, k_rope, _, _ = _kv_rope_inputs(s, b, h, torch.bfloat16) - cos, sin = build_rope_tables(2 * s, emb_dim=HEAD_DIM_ROPE, device="cuda") + cos, sin = build_mla_rope_tables(2 * s, emb_dim=HEAD_DIM_ROPE, device="cuda") cos, sin = cos.to(torch.bfloat16), sin.to(torch.bfloat16) ref_key, ref_value = _kv_rope_reference(kv, k_rope, cos[:s], sin[:s]) key, value = fused_mla_kv_rope_mxfp8(kv, k_rope.squeeze(-2).contiguous(), cos, sin, 128) @@ -467,42 +467,3 @@ def test_inputs_on_non_current_device(): _assert_identical(value, ref_value, "value") with pytest.raises(RuntimeError, match="same device"): fused_mla_kv_rope_mxfp8(kv, k_rope.to("cuda:0"), cos, sin, HEAD_DIM_V) - - -@pytest.mark.skipif(not HAVE_TRITON, reason="MLA RoPE requires Triton") -def test_mla_rope_kv(): - """Compare MLA keys, values, and input gradients with independent PyTorch equations. - - Split the KV input into per-head key and value channels. Rotate the shared - positional key for each token and append it to every head's key. The value - channels pass through unchanged. Backward sums the positional-key gradients - across heads. - """ - if not torch.cuda.is_available(): - pytest.skip("MLA RoPE requires CUDA") - torch.manual_seed(59) - kv = torch.randn(7, 2, 3, 32, device="cuda", dtype=torch.bfloat16, requires_grad=True) - positional = torch.randn(7, 2, 1, 64, device="cuda", dtype=torch.bfloat16, requires_grad=True) - cos, sin = build_rope_tables(7, emb_dim=64, device=kv.device) - key, value = apply_mla_rope_kv( - kv, - positional, - cos_table=cos, - sin_table=sin, - head_dim_nope=16, - head_dim_v=16, - ) - ref_kv = kv.detach().requires_grad_() - ref_pos = positional.detach().requires_grad_() - c, s = cos[:, None, None], sin[:, None, None] - left = ref_pos[..., 0::2].float() * c[..., :32] - ref_pos[..., 1::2].float() * s[..., :32] - right = ref_pos[..., 1::2].float() * c[..., 32:] + ref_pos[..., 0::2].float() * s[..., 32:] - ref_key = torch.cat( - (ref_kv[..., :16], torch.cat((left, right), -1).to(kv.dtype).expand(-1, -1, 3, -1)), -1 - ) - ref_value = ref_kv[..., 16:] - gradients = (torch.randn_like(key), torch.randn_like(value)) - actual_grads = torch.autograd.grad((key, value), (kv, positional), gradients) - ref_grads = torch.autograd.grad((ref_key, ref_value), (ref_kv, ref_pos), gradients) - for actual, expected in zip((key, value, *actual_grads), (ref_key, ref_value, *ref_grads)): - torch.testing.assert_close(actual, expected, atol=0.03125, rtol=0.02) diff --git a/tests/pytorch/attention/test_linear_mxfp8_attention.py b/tests/pytorch/attention/test_linear_mxfp8_attention.py index e5cb1c8a655..f673b493f21 100644 --- a/tests/pytorch/attention/test_linear_mxfp8_attention.py +++ b/tests/pytorch/attention/test_linear_mxfp8_attention.py @@ -35,7 +35,7 @@ _current_file = pathlib.Path(__file__).resolve() sys.path = [str(_current_file.parent.parent)] + sys.path from utils import ModelConfig, compare_and_assert, get_available_attention_backends -from mla_rope_utils import apply_mla_rope, build_rope_tables +from transformer_engine.pytorch.attention.rope import apply_mla_rope, build_mla_rope_tables try: @@ -288,7 +288,7 @@ def test_accuracy(self, batch_size: int, seq_len: int) -> None: _set_seed() baseline_modules, mxfp8_modules = _build_modules() x = torch.randn(seq_len, batch_size, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda") - rope_tables = build_rope_tables(seq_len, device=x.device) + rope_tables = build_mla_rope_tables(seq_len, device=x.device) q_bf16, k_bf16, v_bf16, out_bf16 = _run_forward_bf16(baseline_modules, x, rope_tables) q_mxfp8, k_mxfp8, v_mxfp8, out_mxfp8 = _run_forward_mxfp8( @@ -374,7 +374,7 @@ def test_backward(self, batch_size: int, seq_len: int) -> None: device="cuda", requires_grad=True, ) - rope_tables = build_rope_tables(seq_len, device=x.device) + rope_tables = build_mla_rope_tables(seq_len, device=x.device) *_, out_mxfp8 = _run_forward_mxfp8(mxfp8_modules, x, fp8_recipe, rope_tables) out_mxfp8.sum().backward() @@ -408,7 +408,7 @@ def test_performance(self, batch_size: int, seq_len: int) -> None: device="cuda", requires_grad=True, ) - rope_tables = build_rope_tables(seq_len, device=x.device) + rope_tables = build_mla_rope_tables(seq_len, device=x.device) mxfp8_fprop_ms, mxfp8_bprop_ms = _benchmark_training_step( _run_forward_mxfp8, mxfp8_modules, x, fp8_recipe, rope_tables diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index 145423d41ca..9627d2450bb 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -8,12 +8,12 @@ import torch from transformer_engine.pytorch import dense_indexer_loss_scores, dense_indexer_loss_backward +from transformer_engine.pytorch.attention import rope from transformer_engine.pytorch.attention.sparse_attention import ( Compressor, Indexer, compressor, dsa_cudnn, - dsa_rope, dsv4_attention, ) @@ -138,121 +138,6 @@ def test_dsa_rejects_unsupported_metadata(): ) -# RoPE - - -@pytest.mark.parametrize("layout", ["batched_q", "packed_kv"]) -def test_dsa_triton_rope_adapters(monkeypatch, layout): - """Compare production Q/KV adapter roles with eager outputs, gradients and aliasing.""" - if not torch.cuda.is_available(): - pytest.skip("Triton DSA RoPE requires CUDA") - if dsa_rope._triton_rope_module() is None: - pytest.skip("Triton DSL is unavailable") - torch.manual_seed(23) - inplace = layout == "batched_q" - if inplace: - x = torch.randn(2, 17, 64, 512, device="cuda", dtype=torch.bfloat16) - boundaries, seq = [0, 17, 34], 17 - else: - # Ragged lengths 3 and 5, followed by one padded compressed-token position. - x = torch.randn(9, 1, 512, device="cuda", dtype=torch.bfloat16) - boundaries, seq = [0, 3, 8], 5 - (cos, sin), _ = dsa_rope.rotary_embeddings(seq, 4, 64, 160000.0, x.device) - cu = torch.tensor(boundaries, device=x.device, dtype=torch.int32) - gradient = torch.randn_like(x) - - eager = dsa_rope.apply_dsa_rope_eager - reference_input = x.detach().clone().requires_grad_() - reference = eager(reference_input, cos, sin, cu) - if not inplace: - # Check eager packed indexing against separate sequences, including padding at position 0. - with torch.no_grad(): - expected = torch.cat( - [ - eager( - x[start:end].unsqueeze(0), cos[:, : end - start], sin[:, : end - start] - ).squeeze(0) - for start, end in ((0, 3), (3, 8), (8, 9)) - ] - ) - torch.testing.assert_close(reference, expected) - reference_grad = torch.autograd.grad(reference, reference_input, gradient)[0] - - monkeypatch.setattr( - dsa_rope, - "apply_dsa_rope_eager", - lambda *_: pytest.fail("Triton dispatch fell back to eager RoPE"), - ) - candidate_source = x.detach().clone().requires_grad_() - candidate_input = candidate_source.clone() if inplace else candidate_source - candidate_ptr = candidate_input.data_ptr() - candidate = dsa_rope.apply_dsa_rope(candidate_input, cos, sin, cu, allow_inplace=inplace) - candidate_grad = torch.autograd.grad(candidate, candidate_source, gradient)[0] - - assert (candidate.data_ptr() == candidate_ptr) == inplace - torch.testing.assert_close(candidate, reference, atol=2e-2, rtol=2e-2) - torch.testing.assert_close(candidate_grad, reference_grad, atol=0.03125, rtol=0.002) - - -@pytest.mark.parametrize("interleaved", [False, True]) -def test_common_rope_layout_packed(interleaved): - """Check both Q output layouts and gradients for the current CP1/noninverse callers.""" - if not torch.cuda.is_available(): - pytest.skip("Triton RoPE requires CUDA") - kernels = dsa_rope._triton_rope_module() - if kernels is None: - pytest.skip("Triton DSL is unavailable") - import triton - - torch.manual_seed(53) - # Two sequences of lengths 4 and 6, then one padded token using position zero. - positions = [0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 0] - cu = torch.tensor([0, 4, 10], device="cuda", dtype=torch.int32) - x = torch.randn(11, 3, 80, device="cuda", dtype=torch.bfloat16) - # Contiguous columns with a larger row stride exercise explicit table strides. - cos = torch.randn(6, 128, device="cuda")[:, :64] - sin = torch.randn(6, 128, device="cuda")[:, :64] - reference_input = x.detach().requires_grad_() - tail = reference_input[..., 16:].float() - c, s = cos[positions, None], sin[positions, None] - left = tail[..., 0::2] * c[..., :32] - tail[..., 1::2] * s[..., :32] - right = tail[..., 1::2] * c[..., 32:] + tail[..., 0::2] * s[..., 32:] - rotated = ( - torch.stack((left, right), -1).flatten(-2) if interleaved else torch.cat((left, right), -1) - ) - reference = torch.cat((reference_input[..., :16], rotated.to(x.dtype)), -1) - gradient = torch.randn_like(x) - expected_grad = torch.autograd.grad(reference, reference_input, gradient)[0] - candidate, actual_grad = x.clone(), gradient.clone() - grid = lambda meta: (11, triton.cdiv(3, meta["BLOCK_H"])) - for kernel, tensor in ( - (kernels.rotary_fwd_q_kernel, candidate), - (kernels.rotary_bwd_q_kernel, actual_grad), - ): - kernel[grid]( - tensor, - cos, - sin, - 16, - 64, - 3, - 1, - 2, - cu, - None, - tensor.stride(0), - tensor.stride(1), - cos.stride(0), - sin.stride(0), - 0, - 1, - INVERSE=False, - REMOVE_INTERLEAVING=interleaved, - ) - torch.testing.assert_close(candidate, reference, atol=0.03125, rtol=0.002) - torch.testing.assert_close(actual_grad, expected_grad, atol=0.03125, rtol=0.002) - - # Compressor @@ -307,7 +192,7 @@ def test_dsa_indexer_backward(index_heads): q_residual = torch.randn(1, seq, 16, device="cuda", dtype=torch.bfloat16) cu = torch.tensor([0, seq], device=x.device, dtype=torch.int32) cu_comp = torch.tensor([0, seq // ratio], device=x.device, dtype=torch.int32) - token_rope, compressed_rope = dsa_rope.rotary_embeddings(seq, ratio, 64, 160000.0, x.device) + token_rope, compressed_rope = rope.build_dsv4_rope_tables(seq, ratio, 64, 160000.0, x.device) ids, query, key, weights = layer( x, q_residual, @@ -500,7 +385,7 @@ def test_dsv4_attention_thd(variant, ratio): index_topk=8, rope=( # A warm CPU cache must move on a same-length CUDA request. - dsa_rope.DSv4RotaryEmbedding(ratio, 64, 12345.0, "cpu", max_seqlen=max(lengths)) + rope.DSv4RotaryEmbedding(ratio, 64, 12345.0, "cpu", max_seqlen=max(lengths)) if variant == "csa" else None ), diff --git a/tests/pytorch/test_fused_rope.py b/tests/pytorch/test_fused_rope.py index 5cc4fc0f8a0..dbae0c8286e 100644 --- a/tests/pytorch/test_fused_rope.py +++ b/tests/pytorch/test_fused_rope.py @@ -5,8 +5,12 @@ import math import torch import pytest +from transformer_engine.pytorch.attention import rope from transformer_engine.pytorch.attention.rope import ( RotaryPositionEmbedding, + _triton_rope_module, + apply_mla_rope_kv, + build_mla_rope_tables, apply_rotary_pos_emb, apply_fused_qkv_rotary_pos_emb, ) @@ -497,3 +501,172 @@ def test_rotary_position_embedding_forward_with_autocast_gives_same_result_as_wi atol=1e-8, rtol=1e-8, ) + + +@pytest.mark.parametrize("layout", ["batched_q", "packed_kv"]) +def test_dsa_triton_rope_adapters(monkeypatch, layout): + """Compare production Q/KV adapter roles with eager outputs, gradients and aliasing.""" + if not torch.cuda.is_available(): + pytest.skip("Triton DSA RoPE requires CUDA") + if rope._triton_rope_module() is None: + pytest.skip("Triton DSL is unavailable") + torch.manual_seed(23) + inplace = layout == "batched_q" + if inplace: + x = torch.randn(2, 17, 64, 512, device="cuda", dtype=torch.bfloat16) + boundaries, seq = [0, 17, 34], 17 + else: + # Ragged lengths 3 and 5, followed by one padded compressed-token position. + x = torch.randn(9, 1, 512, device="cuda", dtype=torch.bfloat16) + boundaries, seq = [0, 3, 8], 5 + (cos, sin), _ = rope.build_dsv4_rope_tables(seq, 4, 64, 160000.0, x.device) + cu = torch.tensor(boundaries, device=x.device, dtype=torch.int32) + gradient = torch.randn_like(x) + + eager = rope.apply_dsa_rope_eager + reference_input = x.detach().clone().requires_grad_() + reference = eager(reference_input, cos, sin, cu) + if not inplace: + # Check eager packed indexing against separate sequences, including padding at position 0. + with torch.no_grad(): + expected = torch.cat( + [ + eager( + x[start:end].unsqueeze(0), cos[:, : end - start], sin[:, : end - start] + ).squeeze(0) + for start, end in ((0, 3), (3, 8), (8, 9)) + ] + ) + torch.testing.assert_close(reference, expected) + reference_grad = torch.autograd.grad(reference, reference_input, gradient)[0] + + if inplace: + with pytest.raises(ValueError, match="power of two"): + eager(x, cos[..., :48], sin[..., :48], cu) + with pytest.raises(ValueError, match="allow_inplace=True"): + rope.apply_dsa_rope(x, cos, sin, allow_inplace=True) + with pytest.raises(ValueError, match="allow_inplace=True"): + rope.apply_dsa_rope(x.transpose(0, 1), cos, sin, cu, allow_inplace=True) + with pytest.raises(ValueError, match="allow_inplace=True"): + rope.apply_dsa_rope(reference_input, cos, sin, cu, allow_inplace=True) + torch.testing.assert_close(reference_input, x, rtol=0, atol=0) + + monkeypatch.setattr( + rope, + "apply_dsa_rope_eager", + lambda *_: pytest.fail("Triton dispatch fell back to eager RoPE"), + ) + candidate_source = x.detach().clone().requires_grad_() + candidate_input = candidate_source.clone() if inplace else candidate_source + candidate_ptr = candidate_input.data_ptr() + candidate = rope.apply_dsa_rope(candidate_input, cos, sin, cu, allow_inplace=inplace) + candidate_grad = torch.autograd.grad(candidate, candidate_source, gradient)[0] + + assert (candidate.data_ptr() == candidate_ptr) == inplace + torch.testing.assert_close(candidate, reference, atol=2e-2, rtol=2e-2) + torch.testing.assert_close(candidate_grad, reference_grad, atol=0.03125, rtol=0.002) + + +@pytest.mark.parametrize("interleaved", [False, True]) +def test_dsa_mla_rope_q_kernel_packed(interleaved): + """Check both Q output layouts and gradients for the current CP1/noninverse callers.""" + if not torch.cuda.is_available(): + pytest.skip("Triton RoPE requires CUDA") + kernels = rope._triton_rope_module() + if kernels is None: + pytest.skip("Triton DSL is unavailable") + import triton + + torch.manual_seed(53) + # Two sequences of lengths 4 and 6, then one padded token using position zero. + positions = [0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 0] + cu = torch.tensor([0, 4, 10], device="cuda", dtype=torch.int32) + x = torch.randn(11, 3, 80, device="cuda", dtype=torch.bfloat16) + # Contiguous columns with a larger row stride exercise explicit table strides. + cos = torch.randn(6, 128, device="cuda")[:, :64] + sin = torch.randn(6, 128, device="cuda")[:, :64] + reference_input = x.detach().requires_grad_() + tail = reference_input[..., 16:].float() + c, s = cos[positions, None], sin[positions, None] + left = tail[..., 0::2] * c[..., :32] - tail[..., 1::2] * s[..., :32] + right = tail[..., 1::2] * c[..., 32:] + tail[..., 0::2] * s[..., 32:] + rotated = ( + torch.stack((left, right), -1).flatten(-2) if interleaved else torch.cat((left, right), -1) + ) + reference = torch.cat((reference_input[..., :16], rotated.to(x.dtype)), -1) + gradient = torch.randn_like(x) + expected_grad = torch.autograd.grad(reference, reference_input, gradient)[0] + candidate, actual_grad = x.clone(), gradient.clone() + grid = lambda meta: (11, triton.cdiv(3, meta["BLOCK_H"])) + for kernel, tensor in ( + (kernels.rotary_fwd_q_kernel, candidate), + (kernels.rotary_bwd_q_kernel, actual_grad), + ): + kernel[grid]( + tensor, + cos, + sin, + 16, + 64, + 3, + 1, + 2, + cu, + None, + tensor.stride(0), + tensor.stride(1), + cos.stride(0), + sin.stride(0), + 0, + 1, + INVERSE=False, + REMOVE_INTERLEAVING=interleaved, + ) + torch.testing.assert_close(candidate, reference, atol=0.03125, rtol=0.002) + torch.testing.assert_close(actual_grad, expected_grad, atol=0.03125, rtol=0.002) + + +@pytest.mark.launch_on_gb200 +@pytest.mark.skipif(_triton_rope_module() is None, reason="MLA RoPE requires Triton") +def test_mla_rope_kv(monkeypatch): + """Compare MLA keys, values, and input gradients with independent PyTorch equations. + + Split the KV input into per-head key and value channels. Rotate the shared + positional key for each token and append it to every head's key. The value + channels pass through unchanged. Backward sums the positional-key gradients + across heads. + """ + if not torch.cuda.is_available(): + pytest.skip("MLA RoPE requires CUDA") + kernels = _triton_rope_module() + # Force a partial block for the three heads without changing the shared autotune cache. + backward_kernel = kernels.triton.autotune( + configs=[kernels.triton.Config({"BLOCK_H": 4})], key=[] + )(kernels.rotary_bwd_kv_kernel.fn) + monkeypatch.setattr(kernels, "rotary_bwd_kv_kernel", backward_kernel) + torch.manual_seed(59) + kv = torch.randn(7, 2, 3, 32, device="cuda", dtype=torch.bfloat16, requires_grad=True) + positional = torch.randn(7, 2, 1, 64, device="cuda", dtype=torch.bfloat16, requires_grad=True) + cos, sin = build_mla_rope_tables(7, emb_dim=64, device=kv.device) + key, value = apply_mla_rope_kv( + kv, + positional, + cos_table=cos, + sin_table=sin, + head_dim_nope=16, + head_dim_v=16, + ) + ref_kv = kv.detach().requires_grad_() + ref_pos = positional.detach().requires_grad_() + c, s = cos[:, None, None], sin[:, None, None] + left = ref_pos[..., 0::2].float() * c[..., :32] - ref_pos[..., 1::2].float() * s[..., :32] + right = ref_pos[..., 1::2].float() * c[..., 32:] + ref_pos[..., 0::2].float() * s[..., 32:] + ref_key = torch.cat( + (ref_kv[..., :16], torch.cat((left, right), -1).to(kv.dtype).expand(-1, -1, 3, -1)), -1 + ) + ref_value = ref_kv[..., 16:] + gradients = (torch.randn_like(key), torch.randn_like(value)) + actual_grads = torch.autograd.grad((key, value), (kv, positional), gradients) + ref_grads = torch.autograd.grad((ref_key, ref_value), (ref_kv, ref_pos), gradients) + for actual, expected in zip((key, value, *actual_grads), (ref_key, ref_value, *ref_grads)): + torch.testing.assert_close(actual, expected, atol=0.03125, rtol=0.02) diff --git a/transformer_engine/common/triton/dsa_rope.py b/transformer_engine/common/triton/dsa_mla_rope.py similarity index 99% rename from transformer_engine/common/triton/dsa_rope.py rename to transformer_engine/common/triton/dsa_mla_rope.py index a2fc012f8b4..c53b7fa0102 100644 --- a/transformer_engine/common/triton/dsa_rope.py +++ b/transformer_engine/common/triton/dsa_mla_rope.py @@ -404,8 +404,9 @@ def rotary_bwd_kv_kernel( mask_i = head_offsets_i[:, None] < head_num x_left_off = x_off + tl.arange(0, emb_dim // 2)[None, :] x_right_off = x_left_off + emb_dim // 2 - x_left_accum += tl.load(dK_ptr_i + x_left_off, mask=mask_i) - x_right_accum += tl.load(dK_ptr_i + x_right_off, mask=mask_i) + # Masked heads must contribute zero to the positional-key gradient. + x_left_accum += tl.load(dK_ptr_i + x_left_off, mask=mask_i, other=0) + x_right_accum += tl.load(dK_ptr_i + x_right_off, mask=mask_i, other=0) x_left_accum = tl.sum(x_left_accum, axis=0) x_right_accum = tl.sum(x_right_accum, axis=0) x_left_accum = x_left_accum.to(dEMB.dtype.element_ty) diff --git a/transformer_engine/pytorch/attention/fused_mla_q_uproj.py b/transformer_engine/pytorch/attention/fused_mla_q_uproj.py index 427f52371f0..56d3e822e72 100644 --- a/transformer_engine/pytorch/attention/fused_mla_q_uproj.py +++ b/transformer_engine/pytorch/attention/fused_mla_q_uproj.py @@ -21,7 +21,7 @@ try: import triton - from transformer_engine.common.triton.dsa_rope import ( + from transformer_engine.common.triton.dsa_mla_rope import ( rotary_bwd_q_kernel, ) except ImportError: diff --git a/transformer_engine/pytorch/attention/rope.py b/transformer_engine/pytorch/attention/rope.py index 83369547f1f..1c79bf92ee6 100644 --- a/transformer_engine/pytorch/attention/rope.py +++ b/transformer_engine/pytorch/attention/rope.py @@ -5,15 +5,25 @@ """ Rotary Position Embedding implementation of different types along with helper functions """ +from functools import lru_cache from typing import Optional, Tuple, Union, List import torch import transformer_engine_torch as tex from transformer_engine.pytorch.cpp_extensions.fused_attn import QKVFormat from transformer_engine.pytorch.device import te_device_type +from transformer_engine.pytorch.utils import canonicalize_device -__all__ = ["RotaryPositionEmbedding", "apply_rotary_pos_emb", "apply_fused_qkv_rotary_pos_emb"] +__all__ = [ + "RotaryPositionEmbedding", + "apply_rotary_pos_emb", + "apply_fused_qkv_rotary_pos_emb", + "DSv4RotaryEmbedding", + "build_dsv4_rope_tables", + "apply_dsa_rope", + "apply_dsa_rope_eager", +] class RotaryPositionEmbedding(torch.nn.Module): @@ -540,3 +550,572 @@ def apply_fused_qkv_rotary_pos_emb( cp_size, cp_rank, ) + + +def _build_rope_tables(seq_len, inv_freq, *, interleaved): + """Build FP32 cosine/sine tables for the DSv4 and MLA wrappers. + + Callers supply FP32 inverse frequencies [rotary_width / 2]. Both outputs have + shape [seq_len, rotary_width]. Interleaved layout repeats compact values + [a, b] as [a, a, b, b] for DSv4; otherwise duplicate halves as [a, b, a, b] + for MLA. + """ + positions = torch.arange(seq_len, device=inv_freq.device, dtype=torch.float32) + angles = torch.outer(positions, inv_freq) + cos, sin = angles.cos(), angles.sin() + if interleaved: + return cos.repeat_interleave(2, dim=-1), sin.repeat_interleave(2, dim=-1) + return torch.cat((cos, cos), dim=-1), torch.cat((sin, sin), dim=-1) + + +# DSA: trailing interleaved pairs with packed sequence positions. + + +def _validate_dsa_rope_width(width, head_dim=None): + """Require the power-of-two geometry used by the shared Triton kernels.""" + if width < 4 or width & (width - 1): + raise ValueError("DSv4 RoPE width must be a power of two >= 4.") + if head_dim is not None and width > head_dim: + raise ValueError("DSv4 RoPE width must not exceed the input head dimension.") + + +def build_dsv4_rope_tables(seq, ratio, width, theta, device): + """Return FP32 token/window pairs for a power-of-two rotary width >= 4.""" + _validate_dsa_rope_width(width) + inv_freq = theta ** (-torch.arange(0, width, 2, device=device, dtype=torch.float32) / width) + cos, sin = _build_rope_tables(seq, inv_freq, interleaved=True) + cos, sin = cos[None, :, None, :], sin[None, :, None, :] + window_positions = torch.arange(seq // ratio, device=device) * ratio + return (cos, sin), (cos[:, window_positions], sin[:, window_positions]) + + +class DSv4RotaryEmbedding(torch.nn.Module): + """Cache FP32 token frequencies and compressed-window start frequencies. + + Buffers are not saved in state_dict. Module.to() moves the cache while keeping + its frequencies in FP32, even when a different floating-point dtype is requested. + + Parameters + ---------- + ratio : int + Number of input tokens per compressed window. + width : int + Rotary dimension, a power of two >= 4. + theta : float + Rotary frequency base. + device : torch.device or str + Device for the optional initial cache. + max_seqlen : int, optional + Initial cache length. Otherwise allocate lazily on the first forward call. + """ + + def __init__(self, ratio, width, theta, device, max_seqlen=None): + super().__init__() + _validate_dsa_rope_width(width) + self.ratio, self.width, self.theta = ratio, width, theta + cos, sin = (None, None) + if max_seqlen is not None: + (cos, sin), _ = build_dsv4_rope_tables(max_seqlen, ratio, width, theta, device) + self.register_buffer("cos", cos, persistent=False) + self.register_buffer("sin", sin, persistent=False) + + def _apply(self, fn): + cos, sin = self.cos, self.sin + super()._apply(fn) + if cos is not None: + # Module.to(bfloat16) must not round the reusable frequencies. + self.cos = cos.to(device=self.cos.device) + self.sin = sin.to(device=self.sin.device) + return self + + def forward(self, seq, device): + """Return FP32 token and compressed ``(cos, sin)`` pairs. + + Parameters + ---------- + seq : int + Token sequence length. Grow the cache if needed. + device : torch.device or str + Requested output device. Move an existing cache when it differs. + + Returns + ------- + tuple + Token pair with shape [1, seq, 1, width] and compressed pair with shape + [1, seq // ratio, 1, width]. Compressed positions are window starts + 0, ratio, 2 * ratio, ... . Returned tensors are views of the cache. + """ + device = canonicalize_device(device) + if self.cos is None or self.cos.shape[1] < seq: + (self.cos, self.sin), _ = build_dsv4_rope_tables( + seq, self.ratio, self.width, self.theta, device + ) + elif self.cos.device != device: + self.cos = self.cos.to(device=device) + self.sin = self.sin.to(device=device) + n_comp = seq // self.ratio + return (self.cos[:, :seq], self.sin[:, :seq]), ( + self.cos[:, : n_comp * self.ratio : self.ratio], + self.sin[:, : n_comp * self.ratio : self.ratio], + ) + + +def apply_dsa_rope_eager(x, cos, sin, cu_seqlens=None): + """Rotate trailing pairs in FP32 for a power-of-two width >= 4, then restore dtype.""" + width = cos.shape[-1] + _validate_dsa_rope_width(width, x.shape[-1]) + if x.ndim == 3: + if cu_seqlens is None: + raise ValueError("Packed RoPE requires cu_seqlens.") + token_indices = torch.arange(x.shape[0], device=x.device) + sequences = torch.bucketize(token_indices, cu_seqlens[1:], right=True) + positions = token_indices - cu_seqlens[sequences] + # Compressed tensors may have output padding after cu_seqlens[-1]. + positions = torch.where(token_indices < cu_seqlens[-1], positions, 0) + cos, sin = cos[0, positions], sin[0, positions] + tail = x[..., -width:] + pair = torch.stack((-tail[..., 1::2], tail[..., 0::2]), -1).flatten(-2) + rotated = (tail.float() * cos + pair.float() * sin).to(x.dtype) + return torch.cat((x[..., :-width], rotated), -1) + + +@lru_cache(maxsize=1) +def _triton_rope_module(): + """Load the optional Triton kernels shared by DSA and MLA RoPE.""" + try: + from transformer_engine.common.triton import dsa_mla_rope + except ImportError: + return None + return dsa_mla_rope + + +def _can_use_dsa_rope_triton(x, cos, sin, cu_seqlens): + """Keep unsupported inputs on the behaviorally identical eager path.""" + return ( + cu_seqlens is not None + and x.is_cuda + and x.ndim in (3, 4) + and x.stride(-1) == 1 + and cos.shape == sin.shape + and not cos.requires_grad + and not sin.requires_grad + and _triton_rope_module() is not None + ) + + +def _dsa_triton_rope_table(table): + """Convert TE's repeated pairs to the Triton kernel's duplicated-half tables.""" + pairs = table[0, :, 0, ::2] + return torch.cat((pairs, pairs), dim=-1) + + +def _launch_rope_q_triton( + x, cos, sin, head_dim_nope, head_dim_rope, cu_seqlens, interleaved, module, backward +): + kernel = module.rotary_bwd_q_kernel if backward else module.rotary_fwd_q_kernel + heads = x.shape[-2] + if cu_seqlens is not None: + x = x.reshape(-1, heads, x.shape[-1]) + total, batch, sequences = x.shape[0], 1, cu_seqlens.numel() - 1 + token_stride, head_stride = x.stride(0), x.stride(1) + else: + seq_len, batch, heads, _ = x.shape + total, sequences = seq_len * batch, None + token_stride, head_stride = x.stride(1), x.stride(2) + + def grid(meta): + return total, module.triton.cdiv(heads, meta["BLOCK_H"]) + + kernel[grid]( + x, + cos, + sin, + head_dim_nope, + head_dim_rope, + heads, + batch, + sequences, + cu_seqlens, + None, + token_stride, + head_stride, + cos.stride(0), + sin.stride(0), + 0, + 1, + INVERSE=False, + REMOVE_INTERLEAVING=interleaved, + ) + + +class _RoPEQTriton(torch.autograd.Function): + """Apply Q RoPE with independent positioning, layout and mutation policies.""" + + @staticmethod + def forward(ctx, x, cos, sin, head_dim_nope, head_dim_rope, cu_seqlens, interleaved, mutation): + """Rotate a copy or the input according to the wrapper's mutation policy.""" + output = x.clone(memory_format=torch.contiguous_format) if mutation == "copy" else x + module = _triton_rope_module() + _launch_rope_q_triton( + output, cos, sin, head_dim_nope, head_dim_rope, cu_seqlens, interleaved, module, False + ) + ctx.save_for_backward(cos, sin, cu_seqlens) + ctx.module = module + ctx.head_dim_nope, ctx.head_dim_rope = head_dim_nope, head_dim_rope + ctx.interleaved, ctx.mutation = interleaved, mutation + if mutation == "marked_inplace": + ctx.mark_dirty(x) + return output + + @staticmethod + def backward(ctx, grad_output): + """Preserve DSA's gradient copy and MLA's existing gradient mutation.""" + cos, sin, cu_seqlens = ctx.saved_tensors + # Legacy MLA mutates input without mark_dirty and also reuses incoming gradients. + # Keep that behavior separate from tensor layout and sequence positioning. + grad_input = ( + grad_output + if ctx.mutation == "legacy_inplace" + else grad_output.clone(memory_format=torch.contiguous_format) + ) + _launch_rope_q_triton( + grad_input, + cos, + sin, + ctx.head_dim_nope, + ctx.head_dim_rope, + cu_seqlens, + ctx.interleaved, + ctx.module, + True, + ) + return grad_input, None, None, None, None, None, None, None + + +def apply_dsa_rope(x, cos, sin, cu_seqlens=None, *, allow_inplace=False): + """Rotate trailing interleaved pairs, using Triton when its contract is satisfied. + + By default, preserve the input. With allow_inplace=True, the caller must supply + a fresh buffer safe for autograd mutation. This requires a contiguous Triton + input that is not a leaf requiring gradients, otherwise raise ValueError. + Both paths require a power-of-two rotary width >= 4 that fits the input head. + """ + use_triton = _can_use_dsa_rope_triton(x, cos, sin, cu_seqlens) + if allow_inplace and ( + not use_triton or not x.is_contiguous() or (x.is_leaf and x.requires_grad) + ): + raise ValueError( + "allow_inplace=True requires a contiguous Triton-compatible input " + "that is not a leaf requiring gradients." + ) + if use_triton: + _validate_dsa_rope_width(cos.shape[-1], x.shape[-1]) + return _RoPEQTriton.apply( + x, + _dsa_triton_rope_table(cos), + _dsa_triton_rope_table(sin), + x.shape[-1] - cos.shape[-1], + cos.shape[-1], + cu_seqlens, + True, + "marked_inplace" if allow_inplace else "copy", + ) + return apply_dsa_rope_eager(x, cos, sin, cu_seqlens) + + +# MLA adapters from Megatron-LM fused_mla_yarn_rope_apply.py. +# The default tables use plain RoPE (base=10000), not YaRN scaling. +# Unlike DSA, these SBHD adapters convert interleaved input to split-half output. + + +_MLA_HEAD_DIM_ROPE = 64 +_MLA_HEAD_DIM_NOPE = 128 +_MLA_HEAD_DIM_V = 128 +_MLA_ROTARY_BASE = 10000 + + +def build_mla_rope_tables( + seq_len: int, + emb_dim: int = _MLA_HEAD_DIM_ROPE, + base: int = _MLA_ROTARY_BASE, + device: torch.device = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Build FP32 duplicated-half MLA tables with plain RoPE frequencies.""" + inv_freq = 1.0 / ( + base ** (torch.arange(0, emb_dim, 2, dtype=torch.float32, device=device) / emb_dim) + ) + cos, sin = _build_rope_tables(seq_len, inv_freq, interleaved=False) + return cos.contiguous(), sin.contiguous() + + +def _mla_flattened_token_stride(tensor: torch.Tensor) -> int: + if tensor.dim() == 4: + return tensor.stride(1) + return tensor.stride(0) + + +class _MLARoPEKVTriton(torch.autograd.Function): + @staticmethod + def forward(ctx, kv, k_pos_emb, cos, sin, head_dim_nope, head_dim_rope, head_dim_v): + """Split SBHD KV and append shared rotated position channels to each key head.""" + s, b, nheads, _ = kv.shape + total = s * b + module = _triton_rope_module() + + o_key = kv.new_empty(s, b, nheads, head_dim_nope + head_dim_rope) + o_value = kv.new_empty(s, b, nheads, head_dim_v) + + def grid_kv(meta): + return total, module.triton.cdiv(nheads, meta["BLOCK_H"]) + + module.rotary_fwd_kv_kernel[grid_kv]( + kv, + k_pos_emb, + o_key, + o_value, + cos, + sin, + head_dim_rope, + head_dim_nope, + head_dim_v, + nheads, + b, + None, + None, + _mla_flattened_token_stride(kv), + kv.stride(2), + _mla_flattened_token_stride(k_pos_emb), + _mla_flattened_token_stride(o_key), + o_key.stride(2), + _mla_flattened_token_stride(o_value), + o_value.stride(2), + 0, + 1, + ) + + ctx.save_for_backward(cos, sin) + ctx.head_dim_nope = head_dim_nope + ctx.head_dim_rope = head_dim_rope + ctx.head_dim_v = head_dim_v + ctx.nheads = nheads + ctx.s = s + ctx.b = b + return o_key, o_value + + @staticmethod + def backward(ctx, dk_out, dv_out): + """Gather KV gradients and reduce shared positional gradients across heads.""" + cos, sin = ctx.saved_tensors + s, b, nheads = ctx.s, ctx.b, ctx.nheads + ndp, ndr, ndv = ctx.head_dim_nope, ctx.head_dim_rope, ctx.head_dim_v + total = s * b + module = _triton_rope_module() + + d_kv = dk_out.new_empty(s, b, nheads, ndp + ndv) + d_emb = dk_out.new_empty(s, b, 1, ndr) + + def grid_kv(meta): + return total, module.triton.cdiv(nheads, meta["BLOCK_H"]) + + module.rotary_bwd_kv_kernel[grid_kv]( + dk_out, + dv_out, + d_kv, + d_emb, + cos, + sin, + ndr, + ndp, + ndv, + nheads, + b, + None, + None, + _mla_flattened_token_stride(dk_out), + dk_out.stride(2), + _mla_flattened_token_stride(dv_out), + dv_out.stride(2), + _mla_flattened_token_stride(d_kv), + d_kv.stride(2), + _mla_flattened_token_stride(d_emb), + 0, + 1, + ) + return d_kv, d_emb, None, None, None, None, None + + +def _apply_mla_rope_q_with_tables( + q: torch.Tensor, + cos_table: torch.Tensor, + sin_table: torch.Tensor, + head_dim_nope: int = _MLA_HEAD_DIM_NOPE, + head_dim_rope: int = _MLA_HEAD_DIM_ROPE, +) -> torch.Tensor: + if _triton_rope_module() is not None: + return _RoPEQTriton.apply( + q, + cos_table, + sin_table, + head_dim_nope, + head_dim_rope, + None, + False, + "legacy_inplace", + ) + return _apply_mla_rope_q_eager(q, cos_table, sin_table, head_dim_nope, head_dim_rope) + + +def _apply_mla_rope_kv_with_tables( + kv: torch.Tensor, + k_pos_emb: torch.Tensor, + cos_table: torch.Tensor, + sin_table: torch.Tensor, + head_dim_nope: int = _MLA_HEAD_DIM_NOPE, + head_dim_rope: int = _MLA_HEAD_DIM_ROPE, + head_dim_v: int = _MLA_HEAD_DIM_V, +) -> tuple[torch.Tensor, torch.Tensor]: + if _triton_rope_module() is not None: + return _MLARoPEKVTriton.apply( + kv, + k_pos_emb, + cos_table, + sin_table, + head_dim_nope, + head_dim_rope, + head_dim_v, + ) + return _apply_mla_rope_kv_eager( + kv, + k_pos_emb, + cos_table, + sin_table, + head_dim_nope, + head_dim_rope, + head_dim_v, + ) + + +def apply_mla_rope_q( + q: torch.Tensor, + head_dim_nope: int = _MLA_HEAD_DIM_NOPE, + head_dim_rope: int = _MLA_HEAD_DIM_ROPE, + base: int = _MLA_ROTARY_BASE, + cos_table: torch.Tensor | None = None, + sin_table: torch.Tensor | None = None, +) -> torch.Tensor: + """Rotate trailing SBHD query channels from interleaved to split-half order.""" + if cos_table is None or sin_table is None: + s = q.shape[0] + cos_table, sin_table = build_mla_rope_tables( + s, + emb_dim=head_dim_rope, + base=base, + device=q.device, + ) + return _apply_mla_rope_q_with_tables( + q, + cos_table, + sin_table, + head_dim_nope, + head_dim_rope, + ) + + +def apply_mla_rope_kv( + kv: torch.Tensor, + k_pos_emb: torch.Tensor, + head_dim_nope: int = _MLA_HEAD_DIM_NOPE, + head_dim_rope: int = _MLA_HEAD_DIM_ROPE, + head_dim_v: int = _MLA_HEAD_DIM_V, + base: int = _MLA_ROTARY_BASE, + cos_table: torch.Tensor | None = None, + sin_table: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Split SBHD KV and rotate shared positional keys into split-half order.""" + if cos_table is None or sin_table is None: + s = kv.shape[0] + cos_table, sin_table = build_mla_rope_tables( + s, + emb_dim=head_dim_rope, + base=base, + device=kv.device, + ) + return _apply_mla_rope_kv_with_tables( + kv, + k_pos_emb, + cos_table, + sin_table, + head_dim_nope, + head_dim_rope, + head_dim_v, + ) + + +def apply_mla_rope( + q: torch.Tensor, + kv: torch.Tensor, + k_pos_emb: torch.Tensor, + head_dim_nope: int = _MLA_HEAD_DIM_NOPE, + head_dim_rope: int = _MLA_HEAD_DIM_ROPE, + head_dim_v: int = _MLA_HEAD_DIM_V, + base: int = _MLA_ROTARY_BASE, + cos_table: torch.Tensor | None = None, + sin_table: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Apply the MLA query and shared-key rotations using the same tables.""" + if cos_table is None or sin_table is None: + s = q.shape[0] + cos_table, sin_table = build_mla_rope_tables( + s, + emb_dim=head_dim_rope, + base=base, + device=q.device, + ) + q = _apply_mla_rope_q_with_tables(q, cos_table, sin_table, head_dim_nope, head_dim_rope) + k, v = _apply_mla_rope_kv_with_tables( + kv, + k_pos_emb, + cos_table, + sin_table, + head_dim_nope, + head_dim_rope, + head_dim_v, + ) + return q, k, v + + +def _rotate_mla_interleaved_to_neox( + x: torch.Tensor, cos_table: torch.Tensor, sin_table: torch.Tensor +) -> torch.Tensor: + cos_ = cos_table[:, None, None, :].to(x.dtype) + sin_ = sin_table[:, None, None, :].to(x.dtype) + half_dim = x.shape[-1] // 2 + x_1 = x[..., 0::2] + x_2 = x[..., 1::2] + x_left = x_1 * cos_[..., :half_dim] - x_2 * sin_[..., :half_dim] + x_right = x_2 * cos_[..., half_dim:] + x_1 * sin_[..., half_dim:] + return torch.cat((x_left, x_right), dim=-1) + + +def _apply_mla_rope_q_eager(q, cos_table, sin_table, head_dim_nope, head_dim_rope): + q_nope = q[..., :head_dim_nope] + q_rope = q[..., head_dim_nope : head_dim_nope + head_dim_rope] + q_rope = _rotate_mla_interleaved_to_neox(q_rope, cos_table, sin_table) + return torch.cat((q_nope, q_rope), dim=-1) + + +def _apply_mla_rope_kv_eager( + kv, + k_pos_emb, + cos_table, + sin_table, + head_dim_nope, + head_dim_rope, # pylint: disable=unused-argument + head_dim_v, +): + k_nope = kv[..., :head_dim_nope] + v = kv[..., head_dim_nope : head_dim_nope + head_dim_v] + k_rope = _rotate_mla_interleaved_to_neox(k_pos_emb, cos_table, sin_table).expand( + -1, -1, kv.shape[2], -1 + ) + return torch.cat((k_nope, k_rope), dim=-1), v diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index 2acd8b601be..4aba1334759 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -21,9 +21,9 @@ DSv4Attention ├── Indexer (CSA only) │ ├── TE index-query/head-weight projections │ ├── own Compressor (index keys) -│ ├── dsa_rope.apply_dsa_rope (query + index keys) +│ ├── rope.apply_dsa_rope (query + index keys) │ └── dsa_cudnn.select_indices -> cuDNN score + top-k -├── dsa_rope embeddings + rotary application +├── rope embeddings + rotary application ├── dsa_cudnn.sparse_attention -> cuDNN sparse attention └── TE grouped output projection ``` @@ -82,8 +82,9 @@ compressed tokens. `_prepare_sparse_kv_indices` documents a small example of the index spaces. Selection indices address only packed compressed KV before the local KV offset is applied. -`dsa_rope.DSv4RotaryEmbedding` caches token/compressed-position embeddings. -`dsa_rope.apply_dsa_rope` applies them. +`transformer_engine.pytorch.attention.rope` provides the rotary utilities. +`rope.DSv4RotaryEmbedding` caches token/compressed-position embeddings. +`rope.apply_dsa_rope` applies them. A caller may override the default embeddings with a `rope` module returning token and compressed `(cos, sin)` pairs. diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py deleted file mode 100644 index 5355a08cedd..00000000000 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_rope.py +++ /dev/null @@ -1,218 +0,0 @@ -# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# See LICENSE for license information. - -"""Trailing-channel, interleaved DSA RoPE for full-sequence attention.""" - -from functools import lru_cache - -import torch - -from transformer_engine.pytorch.utils import canonicalize_device - - -def rotary_embeddings(seq, ratio, width, theta, device): - """Return token and compressed-window (cos, sin) pairs in FP32.""" - inv_freq = theta ** (-torch.arange(0, width, 2, device=device, dtype=torch.float32) / width) - angles = torch.outer(torch.arange(seq, device=device, dtype=torch.float32), inv_freq) - cos = angles.cos().repeat_interleave(2, dim=-1)[None, :, None, :] - sin = angles.sin().repeat_interleave(2, dim=-1)[None, :, None, :] - window_positions = torch.arange(seq // ratio, device=device) * ratio - return (cos, sin), (cos[:, window_positions], sin[:, window_positions]) - - -class DSv4RotaryEmbedding(torch.nn.Module): - """Cache FP32 token frequencies and compressed-window start frequencies. - - Buffers are not saved in state_dict. Module.to() moves the cache while keeping - its frequencies in FP32, even when a different floating-point dtype is requested. - - Parameters - ---------- - ratio : int - Number of input tokens per compressed window. - width : int - Even rotary dimension. - theta : float - Rotary frequency base. - device : torch.device or str - Device for the optional initial cache. - max_seqlen : int, optional - Initial cache length. Otherwise allocate lazily on the first forward call. - """ - - def __init__(self, ratio, width, theta, device, max_seqlen=None): - super().__init__() - self.ratio, self.width, self.theta = ratio, width, theta - cos, sin = (None, None) - if max_seqlen is not None: - (cos, sin), _ = rotary_embeddings(max_seqlen, ratio, width, theta, device) - self.register_buffer("cos", cos, persistent=False) - self.register_buffer("sin", sin, persistent=False) - - def _apply(self, fn): - cos, sin = self.cos, self.sin - super()._apply(fn) - if cos is not None: - # Module.to(bfloat16) must not round the reusable frequencies. - self.cos = cos.to(device=self.cos.device) - self.sin = sin.to(device=self.sin.device) - return self - - def forward(self, seq, device): - """Return FP32 token and compressed ``(cos, sin)`` pairs. - - Parameters - ---------- - seq : int - Token sequence length. Grow the cache if needed. - device : torch.device or str - Requested output device. Move an existing cache when it differs. - - Returns - ------- - tuple - Token pair with shape [1, seq, 1, width] and compressed pair with shape - [1, seq // ratio, 1, width]. Compressed positions are window starts - 0, ratio, 2 * ratio, ... . Returned tensors are views of the cache. - """ - device = canonicalize_device(device) - if self.cos is None or self.cos.shape[1] < seq: - (self.cos, self.sin), _ = rotary_embeddings( - seq, self.ratio, self.width, self.theta, device - ) - elif self.cos.device != device: - self.cos = self.cos.to(device=device) - self.sin = self.sin.to(device=device) - n_comp = seq // self.ratio - return (self.cos[:, :seq], self.sin[:, :seq]), ( - self.cos[:, : n_comp * self.ratio : self.ratio], - self.sin[:, : n_comp * self.ratio : self.ratio], - ) - - -def apply_dsa_rope_eager(x, cos, sin, cu_seqlens=None): - """Rotate trailing interleaved pairs in FP32, then restore input dtype.""" - if x.ndim == 3: - if cu_seqlens is None: - raise ValueError("Packed RoPE requires cu_seqlens.") - token_indices = torch.arange(x.shape[0], device=x.device) - sequences = torch.bucketize(token_indices, cu_seqlens[1:], right=True) - positions = token_indices - cu_seqlens[sequences] - # Compressed tensors may have output padding after cu_seqlens[-1]. - positions = torch.where(token_indices < cu_seqlens[-1], positions, 0) - cos, sin = cos[0, positions], sin[0, positions] - width = cos.shape[-1] - tail = x[..., -width:] - pair = torch.stack((-tail[..., 1::2], tail[..., 0::2]), -1).flatten(-2) - rotated = (tail.float() * cos + pair.float() * sin).to(x.dtype) - return torch.cat((x[..., :-width], rotated), -1) - - -@lru_cache(maxsize=1) -def _triton_rope_module(): - """Load the optional Triton DSA RoPE implementation.""" - try: - from transformer_engine.common.triton import dsa_rope - except ImportError: - return None - return dsa_rope - - -def _can_use_triton(x, cos, sin, cu_seqlens): - """Keep unsupported inputs on the behaviorally identical eager path.""" - return ( - cu_seqlens is not None - and x.is_cuda - and x.ndim in (3, 4) - and x.stride(-1) == 1 - and cos.shape == sin.shape - and cos.shape[-1] >= 4 - # The Triton kernels use tl.arange over half the rotary width. - and cos.shape[-1] & (cos.shape[-1] - 1) == 0 - and not cos.requires_grad - and not sin.requires_grad - and _triton_rope_module() is not None - ) - - -def _rope_tables(table): - """Convert TE's repeated pairs to the Triton kernel's duplicated-half tables.""" - pairs = table[0, :, 0, ::2] - return torch.cat((pairs, pairs), dim=-1) - - -def _launch_triton(packed, cos, sin, cu_seqlens, module, backward): - kernel = module.rotary_bwd_q_kernel if backward else module.rotary_fwd_q_kernel - heads, dim = packed.shape[-2:] - - def grid(meta): - return packed.shape[0], module.triton.cdiv(heads, meta["BLOCK_H"]) - - kernel[grid]( - packed, - cos, - sin, - dim - cos.shape[-1], - cos.shape[-1], - heads, - 1, - cu_seqlens.numel() - 1, - cu_seqlens, - None, - packed.stride(0), - packed.stride(1), - cos.stride(0), - sin.stride(0), - 0, - 1, - INVERSE=False, - REMOVE_INTERLEAVING=True, - ) - - -class _TritonRotary(torch.autograd.Function): - """Apply the shared Triton kernels with optional input mutation.""" - - @staticmethod - def forward(ctx, x, cos, sin, cu_seqlens, module, allow_inplace): - """Rotate a contiguous buffer and save the frequency tables for backward.""" - # Contiguity ensures the packed reshape aliases the buffer we return. - inplace = allow_inplace and x.is_contiguous() - output = x if inplace else x.clone(memory_format=torch.contiguous_format) - packed = output.reshape(-1, x.shape[-2], x.shape[-1]) - _launch_triton(packed, cos, sin, cu_seqlens, module, False) - ctx.save_for_backward(cos, sin, cu_seqlens) - ctx.module = module - ctx.shape = x.shape - if inplace: - ctx.mark_dirty(x) - return output - - @staticmethod - def backward(ctx, grad_output): - """Apply the backward rotation to a copy of the output gradient.""" - cos, sin, cu_seqlens = ctx.saved_tensors - grad_input = grad_output.clone(memory_format=torch.contiguous_format) - packed = grad_input.reshape(-1, ctx.shape[-2], ctx.shape[-1]) - _launch_triton(packed, cos, sin, cu_seqlens, ctx.module, True) - return grad_input, None, None, None, None, None - - -def apply_dsa_rope(x, cos, sin, cu_seqlens=None, *, allow_inplace=False): - """Rotate trailing interleaved pairs, using Triton when its contract is satisfied. - - By default, preserve the input. With allow_inplace=True, the caller must supply - a fresh buffer safe for autograd mutation; only contiguous Triton inputs are - mutated. Noncontiguous inputs and the eager fallback remain out of place. - """ - if _can_use_triton(x, cos, sin, cu_seqlens): - return _TritonRotary.apply( - x, - _rope_tables(cos), - _rope_tables(sin), - cu_seqlens, - _triton_rope_module(), - allow_inplace, - ) - return apply_dsa_rope_eager(x, cos, sin, cu_seqlens) diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py index e27bc76a0e7..d653d520a45 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -12,7 +12,7 @@ from .compressor import Compressor from .dsa_cudnn import sparse_attention as _sparse_attention -from .dsa_rope import DSv4RotaryEmbedding, apply_dsa_rope +from ..rope import DSv4RotaryEmbedding, _validate_dsa_rope_width, apply_dsa_rope from .indexer import Indexer __all__ = ["DSv4Attention"] @@ -75,7 +75,7 @@ class DSv4Attention(torch.nn.Module): head_dim : int Attention head width; currently 512, with 64 heads. rope_head_dim : int - Even number of trailing channels rotated by RoPE; at least 2, no greater + Number of trailing channels rotated by RoPE, a power of two >= 4 no greater than head_dim, and no greater than 128 for CSA's index queries/keys. sliding_window : int Local causal attention window size, including the current token. @@ -140,8 +140,9 @@ def __init__( raise ValueError("Only DSv4 CSA and HCA are supported.") if qkv_format not in ("sbhd", "bshd", "thd"): raise ValueError("qkv_format must be 'sbhd', 'bshd', or 'thd'.") - if head_dim != 512 or rope_head_dim < 2 or rope_head_dim % 2 or rope_head_dim > head_dim: - raise ValueError("head_dim must be 512; rope_head_dim must be even and fit.") + if head_dim != 512: + raise ValueError("head_dim must be 512.") + _validate_dsa_rope_width(rope_head_dim, head_dim) if any( v <= 0 for v in ( diff --git a/transformer_engine/pytorch/attention/sparse_attention/indexer.py b/transformer_engine/pytorch/attention/sparse_attention/indexer.py index bd1c36b7085..21b2cea604b 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/indexer.py +++ b/transformer_engine/pytorch/attention/sparse_attention/indexer.py @@ -10,7 +10,7 @@ from .compressor import Compressor from .dsa_cudnn import select_indices -from .dsa_rope import apply_dsa_rope +from ..rope import apply_dsa_rope class Indexer(torch.nn.Module): From dfe286abf6a7c39d1e61f0f8938afff9fea567b4 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Fri, 20 Nov 2026 02:23:05 -0800 Subject: [PATCH 35/37] Use the DeepSeek APE name in Compressor Use ape for the learned additive position embedding in the module, cuDNN wrapper, and examples. The experimental compressor state-dict key now matches the reference implementation. Signed-off-by: Sudhakar Singh --- tests/pytorch/test_dsv4_attention.py | 2 +- .../pytorch/attention/sparse_attention/README.md | 8 ++++---- .../pytorch/attention/sparse_attention/compressor.py | 9 +++++---- .../pytorch/attention/sparse_attention/dsa_cudnn.py | 4 ++-- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index 9627d2450bb..4d27e3dd029 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -147,7 +147,7 @@ def test_dsa_compressor_clears_unused_capacity(monkeypatch): torch.nn.Module.__init__(layer) layer.fused_proj = torch.nn.Identity() layer.kv_norm = torch.nn.RMSNorm(4) - layer.position_bias = torch.nn.Parameter(torch.zeros(4, 4)) + layer.ape = torch.nn.Parameter(torch.zeros(4, 4)) layer.projected, layer.ratio, layer.overlap, layer.weight_width = 4, 4, False, 0 monkeypatch.setattr( compressor, diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index 4aba1334759..74957b684be 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -15,7 +15,7 @@ discrete. The layer maps hidden states to hidden states with the same shape. DSv4Attention ├── TE query/local-KV projections + norms ├── Compressor (HCA or CSA) -│ ├── TE KV/gate projections + learned position bias +│ ├── TE KV/gate projections + learned additive position embedding (APE) │ ├── dsa_cudnn.compress -> cuDNN compressor │ └── TE RMSNorm ├── Indexer (CSA only) @@ -31,13 +31,13 @@ DSv4Attention ## Inside DSv4Attention The following sketch expands the cuDNN calls underneath Compressor, Indexer, and attention. -Query and local KV are already projected, normalized, and rotated; tensors are contiguous CUDA, with FP32 position bias/sink and CUDA INT32 sequence prefixes. +Query and local KV are already projected, normalized, and rotated; tensors are contiguous CUDA, with FP32 APE/sink and CUDA INT32 sequence prefixes. ```python import transformer_engine.pytorch.attention.sparse_attention.dsa_cudnn as dsa_cudnn compressed_kv = dsa_cudnn.compress( - kv, gates, position_bias.float(), cu, cu_comp, + kv, gates, ape.float(), cu, cu_comp, ratio=ratio, overlap=is_csa, total_comp=total_comp, ) compressed_kv = ... # Apply RMSNorm and RoPE to compressed KV. @@ -45,7 +45,7 @@ compressed_kv = ... # Apply RMSNorm and RoPE to compressed KV. indices = None if is_csa: index_key = dsa_cudnn.compress( - index_kv, index_gates, index_position_bias.float(), cu, cu_comp, + index_kv, index_gates, index_ape.float(), cu, cu_comp, ratio=ratio, overlap=True, total_comp=total_comp, ) index_key = ... # Apply RMSNorm and RoPE to compressed index keys. diff --git a/transformer_engine/pytorch/attention/sparse_attention/compressor.py b/transformer_engine/pytorch/attention/sparse_attention/compressor.py index b5a3b770802..d23435ac178 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/compressor.py +++ b/transformer_engine/pytorch/attention/sparse_attention/compressor.py @@ -20,7 +20,8 @@ class Compressor(torch.nn.Module): separate learned KV/gate projections for the current and preceding blocks. Neighboring compressed KV entries therefore share an entire source block, but use different projections of its tokens. - Weights are computed per KV channel as ``softmax(gates + position_bias)``. + Weights are computed per KV channel as ``softmax(gates + ape)``, where + ``ape`` is the learned additive position embedding. For ``ratio=2`` and ``[a b c d e f g h i j k]``: @@ -81,7 +82,7 @@ def __init__( # Fuse the KV, gate, and optional index-weight projections of hidden states. # Keep one contiguous weight for the shared projection call. self.fused_proj = Linear(hidden_size, 2 * projected + weight_width, **kw) - self.position_bias = torch.nn.Parameter( + self.ape = torch.nn.Parameter( torch.zeros(ratio, projected, device=device, dtype=params_dtype) ) self.kv_norm = RMSNorm(width, eps=eps, device=device, dtype=params_dtype) @@ -126,8 +127,8 @@ def forward(self, hidden_states, cu_seqlens, cu_seqlens_comp, *, total_comp): compressed_kv = compress( kv.reshape(-1, self.projected).contiguous(), gate.reshape(-1, self.projected).contiguous(), - # cuDNN requires FP32 bias; the cast preserves gradients to the parameter. - self.position_bias.float(), + # cuDNN requires FP32 APE. The cast preserves gradients to the parameter. + self.ape.float(), cu_seqlens, cu_seqlens_comp, ratio=self.ratio, diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py index e4d1989eb01..d9ab8cc94fd 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py @@ -136,8 +136,8 @@ def compress( Contiguous projected KV and gate scores of shape ``[T,coff*D]`` on the same device. ``coff`` is 2 when ``overlap=True``, otherwise 1. ape : torch.Tensor - FP32 learned position bias ``[ratio,coff*D]``, added to gate scores - before softmax over each compression block. + FP32 learned additive position embedding ``[ratio,coff*D]``, + added to gate scores before softmax over each compression block. cu_seqlens, cu_seqlens_comp : torch.Tensor Contiguous CUDA INT32 ``[B+1]`` cumulative input/compressed token counts. Start at zero; each increment is the corresponding sequence's token From b6b5b71c9e4103979125b35b3fa787fa5c549d77 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Fri, 20 Nov 2026 02:23:05 -0800 Subject: [PATCH 36/37] Test DSv4 auxiliary Indexer gradient isolation The standalone Indexer tests check numerical backward. Exercise the single-sequence context returned by the full layer to verify that the dense loss helpers train the Indexer without sending gradients to the input or other layer parameters. Signed-off-by: Sudhakar Singh --- tests/pytorch/test_dsv4_attention.py | 103 ++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 16 deletions(-) diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py index 4d27e3dd029..cb826fef32d 100644 --- a/tests/pytorch/test_dsv4_attention.py +++ b/tests/pytorch/test_dsv4_attention.py @@ -166,9 +166,7 @@ def test_dsa_compressor_clears_unused_capacity(monkeypatch): # Indexer -@pytest.mark.parametrize("index_heads", [32, 64]) -def test_dsa_indexer_backward(index_heads): - """Train the standalone Indexer and its Compressor against a dense KL reference.""" +def _require_dense_indexer_training(): if not torch.cuda.is_available() or torch.cuda.get_device_capability() not in ( (10, 0), (10, 3), @@ -185,6 +183,34 @@ def test_dsa_indexer_backward(index_heads): ) ): pytest.skip("cuDNN dense indexer training APIs are unavailable") + return cudnn + + +def _dense_teacher_logits(query, local_kv, compressed_kv, sink, *, ratio, window, scale): + query = query.float() + pos = torch.arange(query.shape[1], device=query.device) + valid = (torch.arange(compressed_kv.shape[1], device=query.device) + 1) * ratio <= ( + pos[:, None] + 1 + ) + local_mask = (pos[None, :] <= pos[:, None]) & (pos[None, :] > pos[:, None] - window) + comp_logits = ( + torch.einsum("bshd,bkd->bshk", query, compressed_kv.float()) * scale + ).masked_fill(~valid[None, :, None, :], -torch.inf) + local_logits = (torch.einsum("bshd,kd->bshk", query, local_kv.float()) * scale).masked_fill( + ~local_mask[None, :, None, :], -torch.inf + ) + # The denominator covers the local window, all eligible compressed KV, and the sink. + full_lse = torch.logaddexp( + torch.logaddexp(local_logits.logsumexp(-1), comp_logits.logsumexp(-1)), + sink[None, None, :], + ).contiguous() + return comp_logits, full_lse, valid + + +@pytest.mark.parametrize("index_heads", [32, 64]) +def test_dsa_indexer_backward(index_heads): + """Train the standalone Indexer and its Compressor against a dense KL reference.""" + _require_dense_indexer_training() torch.manual_seed(43) seq, ratio, window = 256, 4, 32 layer = Indexer(32, 16, 128, index_heads, 8, ratio, 1e-6, "cuda", torch.bfloat16) @@ -210,21 +236,10 @@ def test_dsa_indexer_backward(index_heads): kc = (torch.randn(1, seq // ratio, 512, device=x.device, dtype=torch.bfloat16) * 0.1).float() local_kv = (torch.randn(seq, 512, device=x.device, dtype=torch.bfloat16) * 0.1).float() sink = torch.randn(64, device=x.device) - pos = torch.arange(seq, device=x.device) - valid = (torch.arange(seq // ratio, device=x.device) + 1) * ratio <= pos[:, None] + 1 - local_mask = (pos[None, :] <= pos[:, None]) & (pos[None, :] > pos[:, None] - window) scale_i, scale_a = (index_heads * 128) ** -0.5, 512**-0.5 - comp_logits = (torch.einsum("bshd,bkd->bshk", qa, kc) * scale_a).masked_fill( - ~valid[None, :, None, :], -torch.inf - ) - local_logits = (torch.einsum("bshd,kd->bshk", qa, local_kv) * scale_a).masked_fill( - ~local_mask[None, :, None, :], -torch.inf + comp_logits, full_lse, valid = _dense_teacher_logits( + qa, local_kv, kc, sink, ratio=ratio, window=window, scale=scale_a ) - # The dense teacher denominator includes local keys, ALL compressed keys, and sink. - full_lse = torch.logaddexp( - torch.logaddexp(local_logits.logsumexp(-1), comp_logits.logsumexp(-1)), - sink[None, None, :], - ).contiguous() target = (comp_logits - full_lse[..., None]).exp().sum(2) target = target / target.sum(-1, keepdim=True).clamp_min(1e-30) index_score, attn_score = dense_indexer_loss_scores( @@ -298,6 +313,62 @@ def test_dsa_indexer_backward(index_heads): # DSv4 layer wiring +def test_dsv4_indexer_auxiliary_backward(): + """Backpropagate through the returned CSA context and isolate Indexer gradients.""" + cudnn = _require_dense_indexer_training() + if not hasattr(cudnn.DSA, "sparse_attention_forward_wrapper"): + pytest.skip("cuDNN sparse attention APIs are unavailable") + + torch.manual_seed(43) + seq, ratio, window = 256, 4, 32 + layer = dsv4_attention.DSv4Attention( + hidden_size=32, + q_lora_rank=16, + layer_type="compressed_sparse_attention", + head_dim=512, + rope_head_dim=64, + sliding_window=window, + compression_ratio=ratio, + o_groups=8, + o_lora_rank=8, + index_n_heads=32, + index_topk=8, + params_dtype=torch.bfloat16, + qkv_format="bshd", + ) + x = torch.randn(1, seq, 32, device="cuda", dtype=torch.bfloat16, requires_grad=True) + _, context = layer(x, return_indexer_context=True) + qi, ki, wi = (context[name].unsqueeze(0) for name in ("index_q", "index_k", "index_w")) + qa, kc = (context[name].unsqueeze(0) for name in ("attn_q", "compressed_kv")) + scale_i, scale_a = (qi.shape[-2] * qi.shape[-1]) ** -0.5, qa.shape[-1] ** -0.5 + _, full_lse, _ = _dense_teacher_logits( + qa, context["local_kv"], kc, context["sink"], ratio=ratio, window=window, scale=scale_a + ) + index_score, attn_score = dense_indexer_loss_scores( + qi, ki, wi, qa, kc, full_lse, ratio=ratio, index_scale=scale_i, attention_scale=scale_a + ) + grads = dense_indexer_loss_backward( + qi, + ki, + wi, + index_score, + attn_score, + torch.ones((), device=x.device), + ratio=ratio, + index_scale=scale_i, + loss_coeff=0.01, + ) + torch.autograd.backward((qi, ki, wi), grads) + + assert x.grad is None + for name, param in layer.named_parameters(): + if name.startswith("indexer."): + assert param.grad is not None, name + assert torch.isfinite(param.grad).all() and torch.count_nonzero(param.grad) > 0, name + else: + assert param.grad is None, name + + def test_dsv4_attention_rejects_fp8(monkeypatch): from transformer_engine.pytorch.quantization import FP8GlobalStateManager From ce449d76ba86e2ca0a7dd041e6d21db0ac74e6e4 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Fri, 20 Nov 2026 03:40:18 -0800 Subject: [PATCH 37/37] Clarify shared DSA and MLA RoPE kernel usage Keep Q kernel attribution in the shared module. Name the forward and backward entry points in the DSv4 and MLA documentation and identify the separate KV kernel used by the MLA reference tests. Signed-off-by: Sudhakar Singh --- tests/pytorch/attention/test_fused_mla.py | 4 ++-- transformer_engine/common/triton/dsa_mla_rope.py | 9 +++++---- .../pytorch/attention/fused_mla_q_uproj.py | 6 +++--- transformer_engine/pytorch/attention/rope.py | 4 ++-- .../pytorch/attention/sparse_attention/README.md | 9 ++++++--- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/tests/pytorch/attention/test_fused_mla.py b/tests/pytorch/attention/test_fused_mla.py index 52edf667586..e4d6bd970d1 100644 --- a/tests/pytorch/attention/test_fused_mla.py +++ b/tests/pytorch/attention/test_fused_mla.py @@ -6,8 +6,8 @@ - FusedMLAQUpProjRopeQuant: query up-projection, RoPE and MXFP8 quantization (cuDNN FE). - fused_mla_kv_rope_mxfp8: key/value split, key RoPE and MXFP8 quantization. The reference - applies Megatron-LM's MLA key/value RoPE (the shared Triton kernels) and quantizes - key and value with mxfp8_quantize_only; the fused kernel must produce the same bytes. + uses ``rotary_fwd_kv_kernel`` to split K/V and rotate the positional key, then calls + ``mxfp8_quantize_only``. The fused kernel must produce the same bytes. Run: pytest tests/pytorch/attention/test_fused_mla.py -v diff --git a/transformer_engine/common/triton/dsa_mla_rope.py b/transformer_engine/common/triton/dsa_mla_rope.py index c53b7fa0102..1772b4c2f15 100644 --- a/transformer_engine/common/triton/dsa_mla_rope.py +++ b/transformer_engine/common/triton/dsa_mla_rope.py @@ -2,11 +2,12 @@ # # See LICENSE for license information. -"""Shared DSA and MLA RoPE kernels from NVIDIA/Megatron-LM. +"""Triton RoPE kernels for DSA and MLA. -Query kernels: megatron/core/fusions/fused_mla_yarn_rope_apply.py at -c4129afff4867ca89b1fb7b4b911128424cdfbe8. Separate MLA KV kernels were moved -from TE's mla_rope_utils.py; framework adapters own tensor preparation/autograd. +The shared Q RoPE forward and backward kernels come from Megatron-LM's +megatron/core/fusions/fused_mla_yarn_rope_apply.py at +c4129afff4867ca89b1fb7b4b911128424cdfbe8. The separate MLA KV kernels came +from TE's mla_rope_utils.py. Framework adapters prepare tensors and handle autograd. """ import triton diff --git a/transformer_engine/pytorch/attention/fused_mla_q_uproj.py b/transformer_engine/pytorch/attention/fused_mla_q_uproj.py index 56d3e822e72..cb999340fde 100644 --- a/transformer_engine/pytorch/attention/fused_mla_q_uproj.py +++ b/transformer_engine/pytorch/attention/fused_mla_q_uproj.py @@ -4,8 +4,7 @@ """Fused MLA Q up-projection + per-head RoPE + MXFP8 quantize. -The MLA RoPE Triton kernels are taken from: -Megatron-LM megatron/core/fusions/fused_mla_yarn_rope_apply.py +cuDNN applies RoPE during fused query forward. Backward uses ``rotary_bwd_q_kernel``. """ from __future__ import annotations @@ -356,7 +355,8 @@ def backward(ctx, dq): tokens = s * b act_dtype = ctx.act_dtype - # --- RoPE backward (unchanged: bf16, same rotary_bwd_q_kernel as the unfused path) --- + # Apply BF16 RoPE backward with rotary_bwd_q_kernel. + # The DSA and MLA adapters also use this kernel. dq3 = dq.reshape(tokens, nh, q_head_dim).contiguous() def grid(META): diff --git a/transformer_engine/pytorch/attention/rope.py b/transformer_engine/pytorch/attention/rope.py index 1c79bf92ee6..c7a7f2505d6 100644 --- a/transformer_engine/pytorch/attention/rope.py +++ b/transformer_engine/pytorch/attention/rope.py @@ -572,7 +572,7 @@ def _build_rope_tables(seq_len, inv_freq, *, interleaved): def _validate_dsa_rope_width(width, head_dim=None): - """Require the power-of-two geometry used by the shared Triton kernels.""" + """Require the power-of-two rotary width used by the Triton Q RoPE kernels.""" if width < 4 or width & (width - 1): raise ValueError("DSv4 RoPE width must be a power of two >= 4.") if head_dim is not None and width > head_dim: @@ -681,7 +681,7 @@ def apply_dsa_rope_eager(x, cos, sin, cu_seqlens=None): @lru_cache(maxsize=1) def _triton_rope_module(): - """Load the optional Triton kernels shared by DSA and MLA RoPE.""" + """Load optional Triton Q RoPE kernels for DSA and MLA and separate MLA KV kernels.""" try: from transformer_engine.common.triton import dsa_mla_rope except ImportError: diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md index 74957b684be..5e5b7b73060 100644 --- a/transformer_engine/pytorch/attention/sparse_attention/README.md +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -1,9 +1,12 @@ # DSv4 attention API `DSv4Attention` is an experimental API in TE that composes cuDNN compression, -top-k indexing, and sparse attention kernels with Megatron's DSA RoPE kernels -through reusable building blocks such as `Compressor` and `Indexer`. cuDNN -supplies compression and sparse attention forward and backward kernels +top-k indexing, and sparse attention kernels with Triton RoPE through reusable +building blocks such as `Compressor` and `Indexer`. DSv4 uses `rotary_fwd_q_kernel` +and `rotary_bwd_q_kernel` for both Q and KV rotation. MLA's `apply_mla_rope_q` +uses the same forward and backward kernels for Q. + +cuDNN supplies compression and sparse attention forward and backward kernels and separate indexer-loss score/backward primitives; top-k selection itself is discrete. The layer maps hidden states to hidden states with the same shape. `Compressor` and `Indexer` can also be composed into other attention layers;