diff --git a/docs/api/pytorch.rst b/docs/api/pytorch.rst index 2d8cc87827e..424b4ee46dc 100644 --- a/docs/api/pytorch.rst +++ b/docs/api/pytorch.rst @@ -25,6 +25,19 @@ PyTorch .. autoapiclass:: transformer_engine.pytorch.DotProductAttention(num_attention_heads, kv_channels, **kwargs) :members: forward, set_context_parallel_group +.. autoapiclass:: transformer_engine.pytorch.DSv4Attention(hidden_size, q_lora_rank, **kwargs) + :members: forward + +.. autoapiclass:: transformer_engine.pytorch.Compressor(hidden_size, width, ratio, overlap, eps, device, params_dtype, **kwargs) + :members: forward + +.. autoapiclass:: transformer_engine.pytorch.Indexer(hidden_size, q_lora_rank, head_dim, n_heads, top_k, ratio, eps, device, params_dtype, **kwargs) + :members: forward + +.. autoapifunction:: transformer_engine.pytorch.dense_indexer_loss_scores + +.. autoapifunction:: transformer_engine.pytorch.dense_indexer_loss_backward + .. autoapiclass:: transformer_engine.pytorch.GatedDeltaNetAttention(num_attention_heads, kv_channels, **kwargs) :members: forward diff --git a/qa/L0_pytorch_unittest/test.sh b/qa/L0_pytorch_unittest/test.sh index 4697880310f..d7f463a79c1 100644 --- a/qa/L0_pytorch_unittest/test.sh +++ b/qa/L0_pytorch_unittest/test.sh @@ -71,6 +71,7 @@ python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_identity_quantiz NVTE_ALLOW_UNSAFE_PICKLE_EXTRA_STATE=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_attention.xml $TE_PATH/tests/pytorch/attention/test_attention.py || test_fail "test_attention.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_attention_backend_selection.xml $TE_PATH/tests/pytorch/attention/test_attention_backend_selection.py || test_fail "test_attention_backend_selection.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_flex_attention.xml $TE_PATH/tests/pytorch/attention/test_flex_attention.py || test_fail "test_flex_attention.py" +python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_dsv4_attention.xml $TE_PATH/tests/pytorch/test_dsv4_attention.py || test_fail "test_dsv4_attention.py" NVTE_GDN_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_gdn_attention.xml $TE_PATH/tests/pytorch/attention/test_gdn_attention.py || test_fail "test_gdn_attention.py" NVTE_GDN2_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_gdn2_attention.xml $TE_PATH/tests/pytorch/attention/test_gdn2_attention.py || test_fail "test_gdn2_attention.py" NVTE_GDP_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_gdp_attention.xml $TE_PATH/tests/pytorch/attention/test_gdp_attention.py || test_fail "test_gdp_attention.py" diff --git a/tests/pytorch/attention/mla_rope_utils.py b/tests/pytorch/attention/mla_rope_utils.py deleted file mode 100644 index 90eebfc66ae..00000000000 --- a/tests/pytorch/attention/mla_rope_utils.py +++ /dev/null @@ -1,668 +0,0 @@ -# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# See LICENSE for license information. - -"""MLA RoPE for DSv3 671B - Triton forward and backward kernels. - -Source: Megatron-LM megatron/core/fusions/fused_mla_yarn_rope_apply.py -Falls back to pure PyTorch when Triton is unavailable. - -Note: DSv3 uses YaRN-scaled RoPE for long-context extrapolation. This test -intentionally uses plain RoPE (base=10000) because it only validates MXFP8 -attention path wiring, tensor shapes, forward/backward flow, and relative BF16 -vs MXFP8 behavior. Both reference and MXFP8 paths use the same RoPE tables. -""" - -import torch - -try: - import triton - import triton.language as tl - - HAVE_TRITON = True -except ImportError: - HAVE_TRITON = False - -HEAD_DIM_ROPE = 64 -HEAD_DIM_NOPE = 128 -HEAD_DIM_V = 128 -ROTARY_BASE = 10000 - - -def build_rope_tables( - seq_len: int, - emb_dim: int = HEAD_DIM_ROPE, - base: int = ROTARY_BASE, - device: torch.device = None, -) -> tuple[torch.Tensor, torch.Tensor]: - inv_freq = 1.0 / ( - base ** (torch.arange(0, emb_dim, 2, dtype=torch.float32, device=device) / emb_dim) - ) - t = torch.arange(seq_len, device=device, dtype=torch.float32) - freqs = torch.outer(t, inv_freq) - freqs = torch.cat([freqs, freqs], dim=-1) - return torch.cos(freqs).contiguous(), torch.sin(freqs).contiguous() - - -if HAVE_TRITON: - - # Not used for non-packed batches; kept for THD compatibility. - @triton.jit - def _get_thd_token_idx(cu_seqlens, pid_m, seq_num, cp_rank, cp_size): - token_idx = -1 - this_seq_len = 0 - seq_idx = 0 - last_cum_seqlen = tl.load(cu_seqlens) // cp_size - while seq_idx < seq_num: - cur_cum_seqlen = tl.load(cu_seqlens + seq_idx + 1) // cp_size - if token_idx == -1 and cur_cum_seqlen > pid_m: - token_idx = pid_m - last_cum_seqlen - this_seq_len = cur_cum_seqlen - last_cum_seqlen - last_cum_seqlen = cur_cum_seqlen - seq_idx += 1 - if cp_size > 1: - if token_idx < this_seq_len // 2: - token_idx = token_idx + cp_rank * this_seq_len // 2 - else: - token_idx = (token_idx - this_seq_len // 2) + ( - 2 * cp_size - cp_rank - 1 - ) * this_seq_len // 2 - return token_idx - - @triton.autotune( - configs=[ - triton.Config({"BLOCK_H": 1}), - triton.Config({"BLOCK_H": 2}), - triton.Config({"BLOCK_H": 4}), - triton.Config({"BLOCK_H": 8}), - triton.Config({"BLOCK_H": 16}), - triton.Config({"BLOCK_H": 32}), - triton.Config({"BLOCK_H": 64}), - triton.Config({"BLOCK_H": 128}), - ], - key=["emb_dim", "head_num"], - restore_value=["Q"], - ) - @triton.jit - def rotary_fwd_q_kernel( - Q, - COS, - SIN, - qk_head_dim, - emb_dim: tl.constexpr, - head_num: tl.constexpr, - batch_size, - seq_num, - cu_seqlens_q, - stride_x_seq, - stride_x_nheads, - cp_rank, - cp_size, - BLOCK_H: tl.constexpr, - ): - pid_m = tl.program_id(axis=0) - pid_head = tl.program_id(axis=1) - if cu_seqlens_q is None: - token_idx = pid_m // batch_size - else: - token_idx = _get_thd_token_idx(cu_seqlens_q, pid_m, seq_num, cp_rank, cp_size) - cos_left = tl.load(COS + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - sin_left = tl.load(SIN + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - cos_right = tl.load(COS + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) - sin_right = tl.load(SIN + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) - cos_left = cos_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - sin_left = sin_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - cos_right = cos_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - sin_right = sin_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - head_offsets = pid_head * BLOCK_H + tl.arange(0, BLOCK_H) - Q = Q + pid_m * stride_x_seq - x_off = head_offsets[:, None] * stride_x_nheads + qk_head_dim - mask = head_offsets[:, None] < head_num - x_1_off = x_off + tl.arange(0, emb_dim // 2)[None, :] * 2 - x_2_off = x_1_off + 1 - x_1 = tl.load(Q + x_1_off, mask=mask) - x_2 = tl.load(Q + x_2_off, mask=mask) - x_left = x_1 * cos_left - x_2 * sin_left - x_right = x_2 * cos_right + x_1 * sin_right - x_left_off = x_off + tl.arange(0, emb_dim // 2)[None, :] - x_right_off = x_left_off + emb_dim // 2 - tl.store(Q + x_left_off, x_left, mask=mask) - tl.store(Q + x_right_off, x_right, mask=mask) - - @triton.autotune( - configs=[ - triton.Config({"BLOCK_H": 1}), - triton.Config({"BLOCK_H": 2}), - triton.Config({"BLOCK_H": 4}), - triton.Config({"BLOCK_H": 8}), - triton.Config({"BLOCK_H": 16}), - triton.Config({"BLOCK_H": 32}), - triton.Config({"BLOCK_H": 64}), - triton.Config({"BLOCK_H": 128}), - ], - key=["emb_dim", "head_num"], - restore_value=["DO"], - ) - @triton.jit - def rotary_bwd_q_kernel( - DO, - COS, - SIN, - qk_head_dim, - emb_dim: tl.constexpr, - head_num: tl.constexpr, - batch_size, - seq_num, - cu_seqlens_q, - stride_x_seq, - stride_x_nheads, - cp_rank, - cp_size, - BLOCK_H: tl.constexpr, - ): - pid_m = tl.program_id(axis=0) - pid_head = tl.program_id(axis=1) - if cu_seqlens_q is None: - token_idx = pid_m // batch_size - else: - token_idx = _get_thd_token_idx(cu_seqlens_q, pid_m, seq_num, cp_rank, cp_size) - cos_left = tl.load(COS + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - sin_left = tl.load(SIN + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - cos_right = tl.load(COS + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) - sin_right = tl.load(SIN + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) - cos_left = cos_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - sin_left = sin_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - cos_right = cos_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - sin_right = sin_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - head_offsets = pid_head * BLOCK_H + tl.arange(0, BLOCK_H) - DO = DO + pid_m * stride_x_seq - x_off = head_offsets[:, None] * stride_x_nheads + qk_head_dim - mask = head_offsets[:, None] < head_num - x_left_off = x_off + tl.arange(0, emb_dim // 2)[None, :] - x_right_off = x_left_off + emb_dim // 2 - x_left = tl.load(DO + x_left_off, mask=mask) - x_right = tl.load(DO + x_right_off, mask=mask) - x_1 = x_left * cos_left + x_right * sin_right - x_2 = -x_left * sin_left + x_right * cos_right - x_1_off = x_off + tl.arange(0, emb_dim // 2)[None, :] * 2 - x_2_off = x_1_off + 1 - tl.store(DO + x_1_off, x_1, mask=mask) - tl.store(DO + x_2_off, x_2, mask=mask) - - @triton.autotune( - configs=[ - triton.Config({"BLOCK_H": 1}), - triton.Config({"BLOCK_H": 2}), - triton.Config({"BLOCK_H": 4}), - triton.Config({"BLOCK_H": 8}), - triton.Config({"BLOCK_H": 16}), - triton.Config({"BLOCK_H": 32}), - triton.Config({"BLOCK_H": 64}), - triton.Config({"BLOCK_H": 128}), - ], - key=["emb_dim", "k_dim", "v_dim", "head_num"], - ) - @triton.jit - def rotary_fwd_kv_kernel( - KV, - K_POS_EMB, - O_KEY, - O_VALUE, - COS, - SIN, - emb_dim: tl.constexpr, - k_dim: tl.constexpr, - v_dim: tl.constexpr, - head_num: tl.constexpr, - batch_size, - seq_num, - cu_seqlens_kv, - stride_kv_seq, - stride_kv_nheads, - stride_emb_seq, - stride_k_seq, - stride_k_nheads, - stride_v_seq, - stride_v_nheads, - cp_rank, - cp_size, - BLOCK_H: tl.constexpr, - ): - pid_m = tl.program_id(axis=0) - pid_head = tl.program_id(axis=1) - if cu_seqlens_kv is None: - token_idx = pid_m // batch_size - else: - token_idx = _get_thd_token_idx(cu_seqlens_kv, pid_m, seq_num, cp_rank, cp_size) - cos_left = tl.load(COS + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - sin_left = tl.load(SIN + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - cos_right = tl.load(COS + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) - sin_right = tl.load(SIN + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) - head_offsets = pid_head * BLOCK_H + tl.arange(0, BLOCK_H) - KV_ptr = KV + pid_m * stride_kv_seq - kv_off = head_offsets[:, None] * stride_kv_nheads - mask = head_offsets[:, None] < head_num - k_in_off = kv_off + tl.arange(0, k_dim)[None, :] - v_in_off = kv_off + k_dim + tl.arange(0, v_dim)[None, :] - k = tl.load(KV_ptr + k_in_off, mask=mask) - v = tl.load(KV_ptr + v_in_off, mask=mask) - K_ptr = O_KEY + pid_m * stride_k_seq + pid_head * BLOCK_H * stride_k_nheads - V_ptr = O_VALUE + pid_m * stride_v_seq + pid_head * BLOCK_H * stride_v_nheads - k_out_off = tl.arange(0, BLOCK_H)[:, None] * stride_k_nheads + tl.arange(0, k_dim)[None, :] - v_out_off = tl.arange(0, BLOCK_H)[:, None] * stride_v_nheads + tl.arange(0, v_dim)[None, :] - tl.store(K_ptr + k_out_off, k, mask=mask) - tl.store(V_ptr + v_out_off, v, mask=mask) - EMB = K_POS_EMB + pid_m * stride_emb_seq - x_1 = tl.load(EMB + tl.arange(0, emb_dim // 2) * 2) - x_2 = tl.load(EMB + tl.arange(0, emb_dim // 2) * 2 + 1) - x_left = x_1 * cos_left - x_2 * sin_left - x_right = x_2 * cos_right + x_1 * sin_right - x_left = x_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - x_right = x_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - x_left_off = ( - tl.arange(0, BLOCK_H)[:, None] * stride_k_nheads - + k_dim - + tl.arange(0, emb_dim // 2)[None, :] - ) - x_right_off = x_left_off + emb_dim // 2 - tl.store(K_ptr + x_left_off, x_left, mask=mask) - tl.store(K_ptr + x_right_off, x_right, mask=mask) - - @triton.autotune( - configs=[ - triton.Config({"BLOCK_H": 1}), - triton.Config({"BLOCK_H": 2}), - triton.Config({"BLOCK_H": 4}), - triton.Config({"BLOCK_H": 8}), - triton.Config({"BLOCK_H": 16}), - triton.Config({"BLOCK_H": 32}), - triton.Config({"BLOCK_H": 64}), - triton.Config({"BLOCK_H": 128}), - ], - key=["emb_dim", "k_dim", "v_dim", "head_num"], - ) - @triton.jit - def rotary_bwd_kv_kernel( - dK, - dV, - dKV, - dEMB, - COS, - SIN, - emb_dim: tl.constexpr, - k_dim: tl.constexpr, - v_dim: tl.constexpr, - head_num: tl.constexpr, - batch_size, - seq_num, - cu_seqlens_kv, - stride_dk_seq, - stride_dk_nheads, - stride_dv_seq, - stride_dv_nheads, - stride_dkv_seq, - stride_dkv_nheads, - stride_demb_seq, - cp_rank, - cp_size, - BLOCK_H: tl.constexpr, - ): - pid_m = tl.program_id(axis=0) - pid_head = tl.program_id(axis=1) - if cu_seqlens_kv is None: - token_idx = pid_m // batch_size - else: - token_idx = _get_thd_token_idx(cu_seqlens_kv, pid_m, seq_num, cp_rank, cp_size) - head_offsets = pid_head * BLOCK_H + tl.arange(0, BLOCK_H) - dKV_ptr = dKV + pid_m * stride_dkv_seq - dkv_off = head_offsets[:, None] * stride_dkv_nheads - mask = head_offsets[:, None] < head_num - dk_out_off = dkv_off + tl.arange(0, k_dim)[None, :] - dv_out_off = dkv_off + k_dim + tl.arange(0, v_dim)[None, :] - dK_ptr = dK + pid_m * stride_dk_seq + pid_head * BLOCK_H * stride_dk_nheads - dV_ptr = dV + pid_m * stride_dv_seq + pid_head * BLOCK_H * stride_dv_nheads - dk_in_off = tl.arange(0, BLOCK_H)[:, None] * stride_dk_nheads + tl.arange(0, k_dim)[None, :] - dv_in_off = tl.arange(0, BLOCK_H)[:, None] * stride_dv_nheads + tl.arange(0, v_dim)[None, :] - dk = tl.load(dK_ptr + dk_in_off, mask=mask) - dv = tl.load(dV_ptr + dv_in_off, mask=mask) - tl.store(dKV_ptr + dk_out_off, dk, mask=mask) - tl.store(dKV_ptr + dv_out_off, dv, mask=mask) - if pid_head == 0: - x_left_accum = tl.zeros((BLOCK_H, emb_dim // 2), dtype=tl.float32) - x_right_accum = tl.zeros((BLOCK_H, emb_dim // 2), dtype=tl.float32) - for i in tl.static_range(triton.cdiv(head_num, BLOCK_H)): - head_offsets_i = i * BLOCK_H + tl.arange(0, BLOCK_H) - dK_ptr_i = dK + pid_m * stride_dk_seq - x_off = head_offsets_i[:, None] * stride_dk_nheads + k_dim - mask_i = head_offsets_i[:, None] < head_num - x_left_off = x_off + tl.arange(0, emb_dim // 2)[None, :] - x_right_off = x_left_off + emb_dim // 2 - x_left_accum += tl.load(dK_ptr_i + x_left_off, mask=mask_i) - x_right_accum += tl.load(dK_ptr_i + x_right_off, mask=mask_i) - x_left_accum = tl.sum(x_left_accum, axis=0) - x_right_accum = tl.sum(x_right_accum, axis=0) - x_left_accum = x_left_accum.to(dEMB.dtype.element_ty) - x_right_accum = x_right_accum.to(dEMB.dtype.element_ty) - cos_left = tl.load(COS + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - sin_left = tl.load(SIN + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - cos_right = tl.load( - COS + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2) - ) - sin_right = tl.load( - SIN + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2) - ) - x_1 = x_left_accum * cos_left + x_right_accum * sin_right - x_2 = -x_left_accum * sin_left + x_right_accum * cos_right - dEMB_ptr = dEMB + pid_m * stride_demb_seq - tl.store(dEMB_ptr + tl.arange(0, emb_dim // 2) * 2, x_1) - tl.store(dEMB_ptr + tl.arange(0, emb_dim // 2) * 2 + 1, x_2) - - def _flattened_token_stride(tensor: torch.Tensor) -> int: - if tensor.dim() == 4: - return tensor.stride(1) - return tensor.stride(0) - - class _MLARoPEQTriton(torch.autograd.Function): - @staticmethod - def forward(ctx, q, cos, sin, head_dim_nope, head_dim_rope): - s, b, nheads, _ = q.shape - total = s * b - - grid_q = lambda META: (total, triton.cdiv(nheads, META["BLOCK_H"])) - rotary_fwd_q_kernel[grid_q]( - q, - cos, - sin, - head_dim_nope, - head_dim_rope, - nheads, - b, - None, - None, - _flattened_token_stride(q), - q.stride(2), - 0, - 1, - ) - - ctx.save_for_backward(cos, sin) - ctx.head_dim_nope = head_dim_nope - ctx.head_dim_rope = head_dim_rope - ctx.nheads = nheads - ctx.s = s - ctx.b = b - return q - - @staticmethod - def backward(ctx, dq): - cos, sin = ctx.saved_tensors - s, b, nheads = ctx.s, ctx.b, ctx.nheads - total = s * b - - grid_q = lambda META: (total, triton.cdiv(nheads, META["BLOCK_H"])) - rotary_bwd_q_kernel[grid_q]( - dq, - cos, - sin, - ctx.head_dim_nope, - ctx.head_dim_rope, - nheads, - b, - None, - None, - _flattened_token_stride(dq), - dq.stride(2), - 0, - 1, - ) - return dq, None, None, None, None - - class _MLARoPEKVTriton(torch.autograd.Function): - @staticmethod - def forward(ctx, kv, k_pos_emb, cos, sin, head_dim_nope, head_dim_rope, head_dim_v): - s, b, nheads, _ = kv.shape - total = s * b - - o_key = kv.new_empty(s, b, nheads, head_dim_nope + head_dim_rope) - o_value = kv.new_empty(s, b, nheads, head_dim_v) - grid_kv = lambda META: (total, triton.cdiv(nheads, META["BLOCK_H"])) - rotary_fwd_kv_kernel[grid_kv]( - kv, - k_pos_emb, - o_key, - o_value, - cos, - sin, - head_dim_rope, - head_dim_nope, - head_dim_v, - nheads, - b, - None, - None, - _flattened_token_stride(kv), - kv.stride(2), - _flattened_token_stride(k_pos_emb), - _flattened_token_stride(o_key), - o_key.stride(2), - _flattened_token_stride(o_value), - o_value.stride(2), - 0, - 1, - ) - - ctx.save_for_backward(cos, sin) - ctx.head_dim_nope = head_dim_nope - ctx.head_dim_rope = head_dim_rope - ctx.head_dim_v = head_dim_v - ctx.nheads = nheads - ctx.s = s - ctx.b = b - return o_key, o_value - - @staticmethod - def backward(ctx, dk_out, dv_out): - cos, sin = ctx.saved_tensors - s, b, nheads = ctx.s, ctx.b, ctx.nheads - ndp, ndr, ndv = ctx.head_dim_nope, ctx.head_dim_rope, ctx.head_dim_v - total = s * b - - d_kv = dk_out.new_empty(s, b, nheads, ndp + ndv) - d_emb = dk_out.new_empty(s, b, 1, ndr) - grid_kv = lambda META: (total, triton.cdiv(nheads, META["BLOCK_H"])) - rotary_bwd_kv_kernel[grid_kv]( - dk_out, - dv_out, - d_kv, - d_emb, - cos, - sin, - ndr, - ndp, - ndv, - nheads, - b, - None, - None, - _flattened_token_stride(dk_out), - dk_out.stride(2), - _flattened_token_stride(dv_out), - dv_out.stride(2), - _flattened_token_stride(d_kv), - d_kv.stride(2), - _flattened_token_stride(d_emb), - 0, - 1, - ) - return d_kv, d_emb, None, None, None, None, None - - -def _apply_mla_rope_q_with_tables( - q: torch.Tensor, - cos_table: torch.Tensor, - sin_table: torch.Tensor, - head_dim_nope: int = HEAD_DIM_NOPE, - head_dim_rope: int = HEAD_DIM_ROPE, -) -> torch.Tensor: - if HAVE_TRITON: - return _MLARoPEQTriton.apply( - q, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - ) - return _apply_pytorch_q(q, cos_table, sin_table, head_dim_nope, head_dim_rope) - - -def _apply_mla_rope_kv_with_tables( - kv: torch.Tensor, - k_pos_emb: torch.Tensor, - cos_table: torch.Tensor, - sin_table: torch.Tensor, - head_dim_nope: int = HEAD_DIM_NOPE, - head_dim_rope: int = HEAD_DIM_ROPE, - head_dim_v: int = HEAD_DIM_V, -) -> tuple[torch.Tensor, torch.Tensor]: - if HAVE_TRITON: - return _MLARoPEKVTriton.apply( - kv, - k_pos_emb, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - head_dim_v, - ) - return _apply_pytorch_kv( - kv, - k_pos_emb, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - head_dim_v, - ) - - -def apply_mla_rope_q( - q: torch.Tensor, - head_dim_nope: int = HEAD_DIM_NOPE, - head_dim_rope: int = HEAD_DIM_ROPE, - base: int = ROTARY_BASE, - cos_table: torch.Tensor | None = None, - sin_table: torch.Tensor | None = None, -) -> torch.Tensor: - if cos_table is None or sin_table is None: - s = q.shape[0] - cos_table, sin_table = build_rope_tables( - s, - emb_dim=head_dim_rope, - base=base, - device=q.device, - ) - return _apply_mla_rope_q_with_tables( - q, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - ) - - -def apply_mla_rope_kv( - kv: torch.Tensor, - k_pos_emb: torch.Tensor, - head_dim_nope: int = HEAD_DIM_NOPE, - head_dim_rope: int = HEAD_DIM_ROPE, - head_dim_v: int = HEAD_DIM_V, - base: int = ROTARY_BASE, - cos_table: torch.Tensor | None = None, - sin_table: torch.Tensor | None = None, -) -> tuple[torch.Tensor, torch.Tensor]: - if cos_table is None or sin_table is None: - s = kv.shape[0] - cos_table, sin_table = build_rope_tables( - s, - emb_dim=head_dim_rope, - base=base, - device=kv.device, - ) - return _apply_mla_rope_kv_with_tables( - kv, - k_pos_emb, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - head_dim_v, - ) - - -def apply_mla_rope( - q: torch.Tensor, - kv: torch.Tensor, - k_pos_emb: torch.Tensor, - head_dim_nope: int = HEAD_DIM_NOPE, - head_dim_rope: int = HEAD_DIM_ROPE, - head_dim_v: int = HEAD_DIM_V, - base: int = ROTARY_BASE, - cos_table: torch.Tensor | None = None, - sin_table: torch.Tensor | None = None, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - if cos_table is None or sin_table is None: - s = q.shape[0] - cos_table, sin_table = build_rope_tables( - s, - emb_dim=head_dim_rope, - base=base, - device=q.device, - ) - q = _apply_mla_rope_q_with_tables(q, cos_table, sin_table, head_dim_nope, head_dim_rope) - k, v = _apply_mla_rope_kv_with_tables( - kv, - k_pos_emb, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - head_dim_v, - ) - return q, k, v - - -def _rotate_interleaved_to_neox( - x: torch.Tensor, cos_table: torch.Tensor, sin_table: torch.Tensor -) -> torch.Tensor: - cos_ = cos_table[:, None, None, :].to(x.dtype) - sin_ = sin_table[:, None, None, :].to(x.dtype) - half_dim = x.shape[-1] // 2 - x_1 = x[..., 0::2] - x_2 = x[..., 1::2] - x_left = x_1 * cos_[..., :half_dim] - x_2 * sin_[..., :half_dim] - x_right = x_2 * cos_[..., half_dim:] + x_1 * sin_[..., half_dim:] - return torch.cat((x_left, x_right), dim=-1) - - -def _apply_pytorch_q(q, cos_table, sin_table, head_dim_nope, head_dim_rope): - q_nope = q[..., :head_dim_nope] - q_rope = q[..., head_dim_nope : head_dim_nope + head_dim_rope] - q_rope = _rotate_interleaved_to_neox(q_rope, cos_table, sin_table) - return torch.cat((q_nope, q_rope), dim=-1) - - -def _apply_pytorch_kv( - kv, - k_pos_emb, - cos_table, - sin_table, - head_dim_nope, - head_dim_rope, - head_dim_v, -): - k_nope = kv[..., :head_dim_nope] - v = kv[..., head_dim_nope : head_dim_nope + head_dim_v] - k_rope = _rotate_interleaved_to_neox(k_pos_emb, cos_table, sin_table).expand( - -1, -1, kv.shape[2], -1 - ) - return torch.cat((k_nope, k_rope), dim=-1), v diff --git a/tests/pytorch/attention/test_fused_mla.py b/tests/pytorch/attention/test_fused_mla.py index 1fcd9a8e218..e4d6bd970d1 100644 --- a/tests/pytorch/attention/test_fused_mla.py +++ b/tests/pytorch/attention/test_fused_mla.py @@ -6,16 +6,14 @@ - FusedMLAQUpProjRopeQuant: query up-projection, RoPE and MXFP8 quantization (cuDNN FE). - fused_mla_kv_rope_mxfp8: key/value split, key RoPE and MXFP8 quantization. The reference - applies Megatron-LM's MLA key/value RoPE (the Triton kernel in mla_rope_utils) and quantizes - key and value with mxfp8_quantize_only; the fused kernel must produce the same bytes. + uses ``rotary_fwd_kv_kernel`` to split K/V and rotate the positional key, then calls + ``mxfp8_quantize_only``. The fused kernel must produce the same bytes. Run: pytest tests/pytorch/attention/test_fused_mla.py -v """ import os -import pathlib -import sys import pytest import torch @@ -30,9 +28,11 @@ from transformer_engine.pytorch.cpp_extensions import general_gemm as _fused_general_gemm from transformer_engine.pytorch.tensor.mxfp8_tensor import MXFP8Quantizer, MXFP8Tensor -_current_file = pathlib.Path(__file__).resolve() -sys.path = [str(_current_file.parent)] + sys.path -from mla_rope_utils import HAVE_TRITON, apply_mla_rope_kv, build_rope_tables +from transformer_engine.pytorch.attention.rope import ( + _triton_rope_module, + apply_mla_rope_kv, + build_mla_rope_tables, +) # All tests in this file require GB200/SM100 hardware. pytestmark = pytest.mark.launch_on_gb200 @@ -289,10 +289,15 @@ def test_fused_mla_q_uproj_autograd() -> None: 1, None, None, + None, dq3.stride(0), dq3.stride(1), + cos_flat.stride(0), + sin_flat.stride(0), 0, 1, + INVERSE=False, + REMOVE_INTERLEAVING=False, ) dq2d = dq3.reshape(tokens, PROJ_DIM).contiguous() gy_quantizer = MXFP8Quantizer(fp8_dtype=tex.DType.kFloat8E4M3, rowwise=True, columnwise=True) @@ -332,7 +337,7 @@ def test_fused_mla_q_uproj_autograd() -> None: _kv_rope_skip_reason = reason_for_no_mxfp8 elif not sm100: _kv_rope_skip_reason = "The fused kernel requires SM 10.0+." -elif not HAVE_TRITON: +elif _triton_rope_module() is None: _kv_rope_skip_reason = "The reference MLA RoPE needs Triton." else: _kv_rope_skip_reason = "" @@ -361,7 +366,7 @@ def _kv_rope_inputs(s, b, h, cos_dtype, seed=1234): torch.bfloat16 ) k_rope = kv_down[..., KV_LORA_RANK:].unsqueeze(-2) - cos, sin = build_rope_tables(s, emb_dim=HEAD_DIM_ROPE, device="cuda") + cos, sin = build_mla_rope_tables(s, emb_dim=HEAD_DIM_ROPE, device="cuda") # A magnitude scale as in YaRN, so that the products are not exact. cos, sin = (cos * 1.2).to(cos_dtype), (sin * 1.2).to(cos_dtype) return kv, k_rope, cos, sin @@ -404,7 +409,7 @@ def test_contiguous_k_rope_and_longer_rope_tables(): """A contiguous [s, b, emb_dim] k_rope and cos / sin with more rows than tokens.""" s, b, h = 256, 2, 4 kv, k_rope, _, _ = _kv_rope_inputs(s, b, h, torch.bfloat16) - cos, sin = build_rope_tables(2 * s, emb_dim=HEAD_DIM_ROPE, device="cuda") + cos, sin = build_mla_rope_tables(2 * s, emb_dim=HEAD_DIM_ROPE, device="cuda") cos, sin = cos.to(torch.bfloat16), sin.to(torch.bfloat16) ref_key, ref_value = _kv_rope_reference(kv, k_rope, cos[:s], sin[:s]) key, value = fused_mla_kv_rope_mxfp8(kv, k_rope.squeeze(-2).contiguous(), cos, sin, 128) diff --git a/tests/pytorch/attention/test_linear_mxfp8_attention.py b/tests/pytorch/attention/test_linear_mxfp8_attention.py index e5cb1c8a655..f673b493f21 100644 --- a/tests/pytorch/attention/test_linear_mxfp8_attention.py +++ b/tests/pytorch/attention/test_linear_mxfp8_attention.py @@ -35,7 +35,7 @@ _current_file = pathlib.Path(__file__).resolve() sys.path = [str(_current_file.parent.parent)] + sys.path from utils import ModelConfig, compare_and_assert, get_available_attention_backends -from mla_rope_utils import apply_mla_rope, build_rope_tables +from transformer_engine.pytorch.attention.rope import apply_mla_rope, build_mla_rope_tables try: @@ -288,7 +288,7 @@ def test_accuracy(self, batch_size: int, seq_len: int) -> None: _set_seed() baseline_modules, mxfp8_modules = _build_modules() x = torch.randn(seq_len, batch_size, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda") - rope_tables = build_rope_tables(seq_len, device=x.device) + rope_tables = build_mla_rope_tables(seq_len, device=x.device) q_bf16, k_bf16, v_bf16, out_bf16 = _run_forward_bf16(baseline_modules, x, rope_tables) q_mxfp8, k_mxfp8, v_mxfp8, out_mxfp8 = _run_forward_mxfp8( @@ -374,7 +374,7 @@ def test_backward(self, batch_size: int, seq_len: int) -> None: device="cuda", requires_grad=True, ) - rope_tables = build_rope_tables(seq_len, device=x.device) + rope_tables = build_mla_rope_tables(seq_len, device=x.device) *_, out_mxfp8 = _run_forward_mxfp8(mxfp8_modules, x, fp8_recipe, rope_tables) out_mxfp8.sum().backward() @@ -408,7 +408,7 @@ def test_performance(self, batch_size: int, seq_len: int) -> None: device="cuda", requires_grad=True, ) - rope_tables = build_rope_tables(seq_len, device=x.device) + rope_tables = build_mla_rope_tables(seq_len, device=x.device) mxfp8_fprop_ms, mxfp8_bprop_ms = _benchmark_training_step( _run_forward_mxfp8, mxfp8_modules, x, fp8_recipe, rope_tables diff --git a/tests/pytorch/test_dsv4_attention.py b/tests/pytorch/test_dsv4_attention.py new file mode 100644 index 00000000000..cb826fef32d --- /dev/null +++ b/tests/pytorch/test_dsv4_attention.py @@ -0,0 +1,503 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""DSA primitives and DSv4 layer wiring, including packed sequence boundaries.""" + +import pytest +import torch + +from transformer_engine.pytorch import dense_indexer_loss_scores, dense_indexer_loss_backward +from transformer_engine.pytorch.attention import rope +from transformer_engine.pytorch.attention.sparse_attention import ( + Compressor, + Indexer, + compressor, + dsa_cudnn, + dsv4_attention, +) + +# Sparse attention + + +@pytest.mark.parametrize("variant", ["hca", "csa"]) +@pytest.mark.parametrize("head_dim", [512, 576]) +def test_dsa_sparse_attention(variant, head_dim): + if not torch.cuda.is_available() or torch.cuda.get_device_capability() not in ( + (10, 0), + (10, 3), + (10, 7), + ): + pytest.skip("cuDNN sparse attention requires SM100, SM103, or SM107") + cudnn = pytest.importorskip("cudnn") + dsa = getattr(cudnn, "DSA", None) + if not all( + hasattr(dsa, operation) + for operation in ( + "sparse_attention_forward_wrapper", + "sparse_attention_backward_wrapper", + ) + ): + pytest.skip("cuDNN sparse attention APIs are unavailable") + torch.manual_seed(17) + ratio = 128 if variant == "hca" else 4 + lengths = [256, 128] + comp_lengths = [n // ratio for n in lengths] + total, total_comp = sum(lengths), sum(comp_lengths) + cu = torch.tensor([0, 256, total], device="cuda", dtype=torch.int32) + cu_comp = torch.tensor([0, comp_lengths[0], total_comp], device="cuda", dtype=torch.int32) + query = ( + torch.randn(total, 64, head_dim, device="cuda", dtype=torch.bfloat16) * 0.1 + ).requires_grad_() + local = ( + torch.randn(total, head_dim, device="cuda", dtype=torch.bfloat16) * 0.1 + ).requires_grad_() + compressed = ( + torch.randn(total_comp, head_dim, device="cuda", dtype=torch.bfloat16) * 0.1 + ).requires_grad_() + sink = torch.randn(64, device="cuda", requires_grad=True) + leaves = (query, local, compressed, sink) + reference_leaves = tuple(x.detach().float().requires_grad_() for x in leaves) + indices = None + if variant == "csa": + # With ratio=4, sequence one has 64 compressed tokens. Sequence two + # therefore selects packed indices 64, 66, ... rather than 0, 2, ... . + # Comparing with its separate reference catches a missing sequence offset. + indices = torch.full((total, 32), -1, device="cuda", dtype=torch.int32) + start = comp_start = 0 + for length, count in zip(lengths, comp_lengths): + for position in range(length): + ids = torch.arange( + comp_start, comp_start + (position + 1) // ratio, 2, device="cuda" + ) + indices[start + position, : ids.numel()] = ids + start += length + comp_start += count + + output = dsa_cudnn.sparse_attention( + *leaves, + cu, + cu_comp, + window_size=32, + ratio=ratio, + indices=indices, + max_compressed_seqlen=max(comp_lengths) if indices is None else None, + ) + q, k, c, s = reference_leaves + reference = [] + start = comp_start = 0 + for length, count in zip(lengths, comp_lengths): + kv = torch.cat((k[start : start + length], c[comp_start : comp_start + count])) + pos = torch.arange(length, device="cuda") + local_mask = (pos[None, :] <= pos[:, None]) & (pos[None, :] > pos[:, None] - 32) + comp_ids = torch.arange(count, device="cuda") + comp_mask = (comp_ids[None, :] + 1) * ratio <= pos[:, None] + 1 + if indices is not None: + comp_mask &= (comp_ids % 2 == 0)[None, :] + mask = torch.cat((local_mask, comp_mask), dim=-1) + logits = torch.einsum("thd,kd->thk", q[start : start + length], kv) * (head_dim**-0.5) + logits = logits.masked_fill(~mask[:, None, :], float("-inf")) + probabilities = torch.cat((logits, s[None, :, None].expand(length, -1, -1)), -1).softmax(-1) + reference.append(torch.einsum("thk,kd->thd", probabilities[..., :-1], kv[..., :512])) + start += length + comp_start += count + expected = torch.cat(reference) + torch.testing.assert_close(output.float(), expected, atol=1e-3, rtol=1e-2) + gradient = torch.randn_like(output) + output.backward(gradient) + expected.backward(gradient.float()) + for actual, ref in zip(leaves, reference_leaves): + assert actual.grad is not None and torch.isfinite(actual.grad).all() + # Relative L2 error tolerates BF16 reduction rounding near zero entries. + relative_error = (actual.grad.float() - ref.grad).norm() / ref.grad.norm().clamp_min(1e-8) + assert relative_error < 0.02, relative_error.item() + + +def test_dsa_rejects_unsupported_metadata(): + with pytest.raises(ValueError, match="positive"): + dsa_cudnn.sparse_attention( + torch.empty(1, 64, 512), + torch.empty(1, 512), + torch.empty(1, 512), + torch.empty(64), + None, + None, + window_size=0, + ratio=4, + ) + with pytest.raises(ValueError, match="query"): + dsa_cudnn.sparse_attention( + torch.empty(1, 4, 512), + None, + None, + None, + None, + None, + window_size=32, + ratio=4, + ) + + +# Compressor + + +def test_dsa_compressor_clears_unused_capacity(monkeypatch): + """Unused cuDNN output must not contaminate normalization weight gradients.""" + layer = Compressor.__new__(Compressor) + torch.nn.Module.__init__(layer) + layer.fused_proj = torch.nn.Identity() + layer.kv_norm = torch.nn.RMSNorm(4) + layer.ape = torch.nn.Parameter(torch.zeros(4, 4)) + layer.projected, layer.ratio, layer.overlap, layer.weight_width = 4, 4, False, 0 + monkeypatch.setattr( + compressor, + "compress", + lambda *args, **kwargs: torch.cat((torch.ones(1, 4), torch.full((1, 4), float("nan")))), + ) + cu = torch.tensor([0, 3, 8], dtype=torch.int32) + cu_comp = torch.tensor([0, 0, 1], dtype=torch.int32) + output = layer(torch.empty(8, 8), cu, cu_comp, total_comp=2) + output.sum().backward() + assert torch.isfinite(output).all() + assert torch.isfinite(layer.kv_norm.weight.grad).all() + assert torch.count_nonzero(output[1]) == 0 + + +# Indexer + + +def _require_dense_indexer_training(): + if not torch.cuda.is_available() or torch.cuda.get_device_capability() not in ( + (10, 0), + (10, 3), + (10, 7), + ): + pytest.skip("cuDNN indexer training requires SM100, SM103, or SM107") + cudnn = pytest.importorskip("cudnn") + if not all( + hasattr(getattr(cudnn, "DSA", None), op) + for op in ( + "dense_indexer_score_recompute_wrapper", + "dense_attn_score_recompute_wrapper", + "dense_indexer_backward_wrapper", + ) + ): + pytest.skip("cuDNN dense indexer training APIs are unavailable") + return cudnn + + +def _dense_teacher_logits(query, local_kv, compressed_kv, sink, *, ratio, window, scale): + query = query.float() + pos = torch.arange(query.shape[1], device=query.device) + valid = (torch.arange(compressed_kv.shape[1], device=query.device) + 1) * ratio <= ( + pos[:, None] + 1 + ) + local_mask = (pos[None, :] <= pos[:, None]) & (pos[None, :] > pos[:, None] - window) + comp_logits = ( + torch.einsum("bshd,bkd->bshk", query, compressed_kv.float()) * scale + ).masked_fill(~valid[None, :, None, :], -torch.inf) + local_logits = (torch.einsum("bshd,kd->bshk", query, local_kv.float()) * scale).masked_fill( + ~local_mask[None, :, None, :], -torch.inf + ) + # The denominator covers the local window, all eligible compressed KV, and the sink. + full_lse = torch.logaddexp( + torch.logaddexp(local_logits.logsumexp(-1), comp_logits.logsumexp(-1)), + sink[None, None, :], + ).contiguous() + return comp_logits, full_lse, valid + + +@pytest.mark.parametrize("index_heads", [32, 64]) +def test_dsa_indexer_backward(index_heads): + """Train the standalone Indexer and its Compressor against a dense KL reference.""" + _require_dense_indexer_training() + torch.manual_seed(43) + seq, ratio, window = 256, 4, 32 + layer = Indexer(32, 16, 128, index_heads, 8, ratio, 1e-6, "cuda", torch.bfloat16) + x = torch.randn(1, seq, 32, device="cuda", dtype=torch.bfloat16) + q_residual = torch.randn(1, seq, 16, device="cuda", dtype=torch.bfloat16) + cu = torch.tensor([0, seq], device=x.device, dtype=torch.int32) + cu_comp = torch.tensor([0, seq // ratio], device=x.device, dtype=torch.int32) + token_rope, compressed_rope = rope.build_dsv4_rope_tables(seq, ratio, 64, 160000.0, x.device) + ids, query, key, weights = layer( + x, + q_residual, + cu, + cu_comp, + token_rope, + compressed_rope, + total_comp=seq // ratio, + max_seqlen=seq, + return_context=True, + ) + assert ids.shape == (seq, 8) + qi, ki, wi = (t.unsqueeze(0) for t in (query, key, weights)) + qa = (torch.randn(1, seq, 64, 512, device=x.device, dtype=torch.bfloat16) * 0.1).float() + kc = (torch.randn(1, seq // ratio, 512, device=x.device, dtype=torch.bfloat16) * 0.1).float() + local_kv = (torch.randn(seq, 512, device=x.device, dtype=torch.bfloat16) * 0.1).float() + sink = torch.randn(64, device=x.device) + scale_i, scale_a = (index_heads * 128) ** -0.5, 512**-0.5 + comp_logits, full_lse, valid = _dense_teacher_logits( + qa, local_kv, kc, sink, ratio=ratio, window=window, scale=scale_a + ) + target = (comp_logits - full_lse[..., None]).exp().sum(2) + target = target / target.sum(-1, keepdim=True).clamp_min(1e-30) + index_score, attn_score = dense_indexer_loss_scores( + qi, + ki, + wi, + qa.to(torch.bfloat16), + kc.to(torch.bfloat16), + full_lse, + ratio=ratio, + index_scale=scale_i, + attention_scale=scale_a, + ) + actual_target = attn_score["out"] / attn_score["denom"].unsqueeze(-1).clamp_min(1e-30) + # Masked score-buffer entries are not valid teacher probabilities. + actual_target = actual_target.masked_fill(~valid[None], 0) + torch.testing.assert_close(actual_target, target, atol=2e-4, rtol=2e-2) + + qr, kr, wr = (t.detach().float().requires_grad_() for t in (qi, ki, wi)) + logits = (torch.einsum("bshd,bkd->bshk", qr, kr).relu() * wr[..., None]).sum(2) * scale_i + logits = logits.masked_fill(~valid[None], -torch.inf) + logits = torch.where(valid.any(-1)[None, :, None], logits, 0) + log_prob = logits.log_softmax(-1) + actual_log_prob = index_score["out"] - index_score["denom"].unsqueeze(-1) + torch.testing.assert_close(actual_log_prob[0][valid], log_prob[0][valid], atol=2e-3, rtol=2e-2) + coefficient, upstream = 0.01, 0.7 + loss = ( + coefficient * upstream * (target * (target.clamp_min(1e-30).log() - log_prob)).sum() / seq + ) + reference_grads = torch.autograd.grad(loss, (qr, kr, wr)) + actual_grads = dense_indexer_loss_backward( + qi, + ki, + wi, + index_score, + attn_score, + torch.tensor(upstream, device=x.device), + ratio=ratio, + index_scale=scale_i, + loss_coeff=coefficient, + ) + for actual, expected in zip(actual_grads, reference_grads): + assert actual.shape == expected.shape and torch.isfinite(actual).all() + error = (actual.float() - expected).norm() / expected.norm().clamp_min(1e-8) + assert error < 0.03, error.item() + parameters = tuple(layer.parameters()) + candidate = torch.autograd.grad((qi, ki, wi), parameters, actual_grads) + # TE Linear releases its backward workspace; compare using a fresh forward. + _, ref_q, ref_k, ref_w = layer( + x, + q_residual, + cu, + cu_comp, + token_rope, + compressed_rope, + total_comp=seq // ratio, + max_seqlen=seq, + return_context=True, + ) + reference = torch.autograd.grad( + (ref_q, ref_k, ref_w), + parameters, + tuple(g[0].to(t.dtype) for g, t in zip(reference_grads, (ref_q, ref_k, ref_w))), + ) + for actual, expected in zip(candidate, reference): + assert torch.isfinite(actual).all() and actual.float().norm() > 0 + error = (actual.float() - expected.float()).norm() / expected.float().norm().clamp_min(1e-8) + assert error < 0.03, error.item() + + +# DSv4 layer wiring + + +def test_dsv4_indexer_auxiliary_backward(): + """Backpropagate through the returned CSA context and isolate Indexer gradients.""" + cudnn = _require_dense_indexer_training() + if not hasattr(cudnn.DSA, "sparse_attention_forward_wrapper"): + pytest.skip("cuDNN sparse attention APIs are unavailable") + + torch.manual_seed(43) + seq, ratio, window = 256, 4, 32 + layer = dsv4_attention.DSv4Attention( + hidden_size=32, + q_lora_rank=16, + layer_type="compressed_sparse_attention", + head_dim=512, + rope_head_dim=64, + sliding_window=window, + compression_ratio=ratio, + o_groups=8, + o_lora_rank=8, + index_n_heads=32, + index_topk=8, + params_dtype=torch.bfloat16, + qkv_format="bshd", + ) + x = torch.randn(1, seq, 32, device="cuda", dtype=torch.bfloat16, requires_grad=True) + _, context = layer(x, return_indexer_context=True) + qi, ki, wi = (context[name].unsqueeze(0) for name in ("index_q", "index_k", "index_w")) + qa, kc = (context[name].unsqueeze(0) for name in ("attn_q", "compressed_kv")) + scale_i, scale_a = (qi.shape[-2] * qi.shape[-1]) ** -0.5, qa.shape[-1] ** -0.5 + _, full_lse, _ = _dense_teacher_logits( + qa, context["local_kv"], kc, context["sink"], ratio=ratio, window=window, scale=scale_a + ) + index_score, attn_score = dense_indexer_loss_scores( + qi, ki, wi, qa, kc, full_lse, ratio=ratio, index_scale=scale_i, attention_scale=scale_a + ) + grads = dense_indexer_loss_backward( + qi, + ki, + wi, + index_score, + attn_score, + torch.ones((), device=x.device), + ratio=ratio, + index_scale=scale_i, + loss_coeff=0.01, + ) + torch.autograd.backward((qi, ki, wi), grads) + + assert x.grad is None + for name, param in layer.named_parameters(): + if name.startswith("indexer."): + assert param.grad is not None, name + assert torch.isfinite(param.grad).all() and torch.count_nonzero(param.grad) > 0, name + else: + assert param.grad is None, name + + +def test_dsv4_attention_rejects_fp8(monkeypatch): + from transformer_engine.pytorch.quantization import FP8GlobalStateManager + + monkeypatch.setattr(FP8GlobalStateManager, "is_fp8_enabled", lambda: True) + layer = dsv4_attention.DSv4Attention.__new__(dsv4_attention.DSv4Attention) + torch.nn.Module.__init__(layer) + layer.qkv_format, layer.hidden_size = "sbhd", 8 + with pytest.raises(NotImplementedError, match="BF16 only"): + layer(torch.empty(4, 1, 8)) + + +def test_dsv4_thd_compressed_prefixes(): + """Floor each sequence independently, retaining a safe allocation capacity.""" + for lengths, capacity in (([129, 257], 96), ([131, 257], 97)): + x = torch.empty(sum(lengths), 8) + cu = torch.tensor([0, lengths[0], sum(lengths)], dtype=torch.int32) + cu_comp, total_comp, maximum = dsv4_attention._packed_prefixes(x, cu, 257, 4) + assert cu_comp.tolist() == [0, 32, 96] + assert total_comp == capacity + assert maximum == 257 + + +@pytest.mark.parametrize( + "prefix,maximum", + [ + ([0, 8], 8), + (torch.tensor([0, 8], dtype=torch.int64), 8), + (torch.tensor([[0, 8]], dtype=torch.int32), 8), + (torch.tensor([0], dtype=torch.int32), 8), + (torch.tensor([0, 0, 8, 8], dtype=torch.int32)[::2], 8), + (torch.tensor([0, 8], dtype=torch.int32), None), + (torch.tensor([0, 8], dtype=torch.int32), 0), + ], +) +def test_dsv4_thd_rejects_malformed_metadata(prefix, maximum): + with pytest.raises(ValueError, match="THD requires"): + dsv4_attention._packed_prefixes(torch.empty(8, 4), prefix, maximum, 4) + + +def test_dsv4_thd_rejects_hidden_batch_axis_and_multi_sequence_context(): + # These checks precede projections and do not require GPU-backed TE layers. + layer = dsv4_attention.DSv4Attention.__new__(dsv4_attention.DSv4Attention) + torch.nn.Module.__init__(layer) + layer.qkv_format, layer.hidden_size = "thd", 8 + layer.is_csa, layer.compression_ratio = True, 4 + with pytest.raises(ValueError, match="selected format"): + layer(torch.empty(16, 1, 8)) + with pytest.raises(ValueError, match="THD requires"): + layer(torch.empty(16, 8)) + with pytest.raises(ValueError, match="batch=1"): + layer( + torch.empty(16, 8), + cu_seqlens_q=torch.tensor([0, 8, 16], dtype=torch.int32), + max_seqlen_q=8, + return_indexer_context=True, + ) + + +@pytest.mark.parametrize("variant,ratio", [("hca", 128), ("csa", 4)]) +def test_dsv4_attention_thd(variant, ratio): + """Packed full-layer output and gradients match independent dense sequences.""" + if not torch.cuda.is_available() or torch.cuda.get_device_capability() not in ( + (10, 0), + (10, 3), + (10, 7), + ): + pytest.skip("DSv4 requires SM100, SM103, or SM107") + cudnn = pytest.importorskip("cudnn") + if not hasattr(getattr(cudnn, "DSA", None), "sparse_attention_forward_wrapper"): + pytest.skip("DSv4 requires cuDNN Frontend 1.29 sparse attention APIs") + lengths = [255, 383] + torch.manual_seed(37) + layer = dsv4_attention.DSv4Attention( + hidden_size=32, + q_lora_rank=16, + layer_type=( + "compressed_sparse_attention" if variant == "csa" else "heavily_compressed_attention" + ), + head_dim=512, + rope_head_dim=64, + sliding_window=32, + compression_ratio=ratio, + o_groups=8, + o_lora_rank=8, + index_topk=8, + rope=( + # A warm CPU cache must move on a same-length CUDA request. + rope.DSv4RotaryEmbedding(ratio, 64, 12345.0, "cpu", max_seqlen=max(lengths)) + if variant == "csa" + else None + ), + params_dtype=torch.bfloat16, + qkv_format="thd", + ) + x = torch.randn(sum(lengths), 32, device="cuda", dtype=torch.bfloat16, requires_grad=True) + cu = torch.tensor([0, lengths[0], sum(lengths)], device=x.device, dtype=torch.int32) + actual = layer(x, cu_seqlens_q=cu, max_seqlen_q=max(lengths)) + assert actual.shape == x.shape + if variant == "csa": + assert layer.rope.cos.device == x.device + assert layer.rope.cos.dtype == torch.float32 + gradient = torch.randn_like(actual) + actual.backward(gradient) + packed_grads = { + name: p.grad.clone() for name, p in layer.named_parameters() if p.grad is not None + } + layer.zero_grad(set_to_none=True) + layer.qkv_format = "sbhd" if variant == "hca" else "bshd" + batch_axis = 1 if layer.qkv_format == "sbhd" else 0 + reference_input = x.detach().clone().requires_grad_() + parts = [] + for part in reference_input.split(lengths): + if variant == "csa": + output, context = layer(part.unsqueeze(batch_axis), return_indexer_context=True) + assert all(context[name].requires_grad for name in ("index_q", "index_k", "index_w")) + assert not any( + context[name].requires_grad + for name in ("attn_q", "local_kv", "compressed_kv", "sink") + ) + else: + output = layer(part.unsqueeze(batch_axis)) + parts.append(output.squeeze(batch_axis)) + expected = torch.cat(parts) + expected.backward(gradient) + torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2) + for packed_grad, reference_grad in [(x.grad, reference_input.grad)] + [ + (packed_grads[name], p.grad) for name, p in layer.named_parameters() if name in packed_grads + ]: + assert reference_grad is not None + assert torch.isfinite(packed_grad).all() and torch.isfinite(reference_grad).all() + error = (packed_grad.float() - reference_grad.float()).norm() + assert error / reference_grad.float().norm().clamp_min(1e-8) < 0.03 diff --git a/tests/pytorch/test_fused_rope.py b/tests/pytorch/test_fused_rope.py index 5cc4fc0f8a0..dbae0c8286e 100644 --- a/tests/pytorch/test_fused_rope.py +++ b/tests/pytorch/test_fused_rope.py @@ -5,8 +5,12 @@ import math import torch import pytest +from transformer_engine.pytorch.attention import rope from transformer_engine.pytorch.attention.rope import ( RotaryPositionEmbedding, + _triton_rope_module, + apply_mla_rope_kv, + build_mla_rope_tables, apply_rotary_pos_emb, apply_fused_qkv_rotary_pos_emb, ) @@ -497,3 +501,172 @@ def test_rotary_position_embedding_forward_with_autocast_gives_same_result_as_wi atol=1e-8, rtol=1e-8, ) + + +@pytest.mark.parametrize("layout", ["batched_q", "packed_kv"]) +def test_dsa_triton_rope_adapters(monkeypatch, layout): + """Compare production Q/KV adapter roles with eager outputs, gradients and aliasing.""" + if not torch.cuda.is_available(): + pytest.skip("Triton DSA RoPE requires CUDA") + if rope._triton_rope_module() is None: + pytest.skip("Triton DSL is unavailable") + torch.manual_seed(23) + inplace = layout == "batched_q" + if inplace: + x = torch.randn(2, 17, 64, 512, device="cuda", dtype=torch.bfloat16) + boundaries, seq = [0, 17, 34], 17 + else: + # Ragged lengths 3 and 5, followed by one padded compressed-token position. + x = torch.randn(9, 1, 512, device="cuda", dtype=torch.bfloat16) + boundaries, seq = [0, 3, 8], 5 + (cos, sin), _ = rope.build_dsv4_rope_tables(seq, 4, 64, 160000.0, x.device) + cu = torch.tensor(boundaries, device=x.device, dtype=torch.int32) + gradient = torch.randn_like(x) + + eager = rope.apply_dsa_rope_eager + reference_input = x.detach().clone().requires_grad_() + reference = eager(reference_input, cos, sin, cu) + if not inplace: + # Check eager packed indexing against separate sequences, including padding at position 0. + with torch.no_grad(): + expected = torch.cat( + [ + eager( + x[start:end].unsqueeze(0), cos[:, : end - start], sin[:, : end - start] + ).squeeze(0) + for start, end in ((0, 3), (3, 8), (8, 9)) + ] + ) + torch.testing.assert_close(reference, expected) + reference_grad = torch.autograd.grad(reference, reference_input, gradient)[0] + + if inplace: + with pytest.raises(ValueError, match="power of two"): + eager(x, cos[..., :48], sin[..., :48], cu) + with pytest.raises(ValueError, match="allow_inplace=True"): + rope.apply_dsa_rope(x, cos, sin, allow_inplace=True) + with pytest.raises(ValueError, match="allow_inplace=True"): + rope.apply_dsa_rope(x.transpose(0, 1), cos, sin, cu, allow_inplace=True) + with pytest.raises(ValueError, match="allow_inplace=True"): + rope.apply_dsa_rope(reference_input, cos, sin, cu, allow_inplace=True) + torch.testing.assert_close(reference_input, x, rtol=0, atol=0) + + monkeypatch.setattr( + rope, + "apply_dsa_rope_eager", + lambda *_: pytest.fail("Triton dispatch fell back to eager RoPE"), + ) + candidate_source = x.detach().clone().requires_grad_() + candidate_input = candidate_source.clone() if inplace else candidate_source + candidate_ptr = candidate_input.data_ptr() + candidate = rope.apply_dsa_rope(candidate_input, cos, sin, cu, allow_inplace=inplace) + candidate_grad = torch.autograd.grad(candidate, candidate_source, gradient)[0] + + assert (candidate.data_ptr() == candidate_ptr) == inplace + torch.testing.assert_close(candidate, reference, atol=2e-2, rtol=2e-2) + torch.testing.assert_close(candidate_grad, reference_grad, atol=0.03125, rtol=0.002) + + +@pytest.mark.parametrize("interleaved", [False, True]) +def test_dsa_mla_rope_q_kernel_packed(interleaved): + """Check both Q output layouts and gradients for the current CP1/noninverse callers.""" + if not torch.cuda.is_available(): + pytest.skip("Triton RoPE requires CUDA") + kernels = rope._triton_rope_module() + if kernels is None: + pytest.skip("Triton DSL is unavailable") + import triton + + torch.manual_seed(53) + # Two sequences of lengths 4 and 6, then one padded token using position zero. + positions = [0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 0] + cu = torch.tensor([0, 4, 10], device="cuda", dtype=torch.int32) + x = torch.randn(11, 3, 80, device="cuda", dtype=torch.bfloat16) + # Contiguous columns with a larger row stride exercise explicit table strides. + cos = torch.randn(6, 128, device="cuda")[:, :64] + sin = torch.randn(6, 128, device="cuda")[:, :64] + reference_input = x.detach().requires_grad_() + tail = reference_input[..., 16:].float() + c, s = cos[positions, None], sin[positions, None] + left = tail[..., 0::2] * c[..., :32] - tail[..., 1::2] * s[..., :32] + right = tail[..., 1::2] * c[..., 32:] + tail[..., 0::2] * s[..., 32:] + rotated = ( + torch.stack((left, right), -1).flatten(-2) if interleaved else torch.cat((left, right), -1) + ) + reference = torch.cat((reference_input[..., :16], rotated.to(x.dtype)), -1) + gradient = torch.randn_like(x) + expected_grad = torch.autograd.grad(reference, reference_input, gradient)[0] + candidate, actual_grad = x.clone(), gradient.clone() + grid = lambda meta: (11, triton.cdiv(3, meta["BLOCK_H"])) + for kernel, tensor in ( + (kernels.rotary_fwd_q_kernel, candidate), + (kernels.rotary_bwd_q_kernel, actual_grad), + ): + kernel[grid]( + tensor, + cos, + sin, + 16, + 64, + 3, + 1, + 2, + cu, + None, + tensor.stride(0), + tensor.stride(1), + cos.stride(0), + sin.stride(0), + 0, + 1, + INVERSE=False, + REMOVE_INTERLEAVING=interleaved, + ) + torch.testing.assert_close(candidate, reference, atol=0.03125, rtol=0.002) + torch.testing.assert_close(actual_grad, expected_grad, atol=0.03125, rtol=0.002) + + +@pytest.mark.launch_on_gb200 +@pytest.mark.skipif(_triton_rope_module() is None, reason="MLA RoPE requires Triton") +def test_mla_rope_kv(monkeypatch): + """Compare MLA keys, values, and input gradients with independent PyTorch equations. + + Split the KV input into per-head key and value channels. Rotate the shared + positional key for each token and append it to every head's key. The value + channels pass through unchanged. Backward sums the positional-key gradients + across heads. + """ + if not torch.cuda.is_available(): + pytest.skip("MLA RoPE requires CUDA") + kernels = _triton_rope_module() + # Force a partial block for the three heads without changing the shared autotune cache. + backward_kernel = kernels.triton.autotune( + configs=[kernels.triton.Config({"BLOCK_H": 4})], key=[] + )(kernels.rotary_bwd_kv_kernel.fn) + monkeypatch.setattr(kernels, "rotary_bwd_kv_kernel", backward_kernel) + torch.manual_seed(59) + kv = torch.randn(7, 2, 3, 32, device="cuda", dtype=torch.bfloat16, requires_grad=True) + positional = torch.randn(7, 2, 1, 64, device="cuda", dtype=torch.bfloat16, requires_grad=True) + cos, sin = build_mla_rope_tables(7, emb_dim=64, device=kv.device) + key, value = apply_mla_rope_kv( + kv, + positional, + cos_table=cos, + sin_table=sin, + head_dim_nope=16, + head_dim_v=16, + ) + ref_kv = kv.detach().requires_grad_() + ref_pos = positional.detach().requires_grad_() + c, s = cos[:, None, None], sin[:, None, None] + left = ref_pos[..., 0::2].float() * c[..., :32] - ref_pos[..., 1::2].float() * s[..., :32] + right = ref_pos[..., 1::2].float() * c[..., 32:] + ref_pos[..., 0::2].float() * s[..., 32:] + ref_key = torch.cat( + (ref_kv[..., :16], torch.cat((left, right), -1).to(kv.dtype).expand(-1, -1, 3, -1)), -1 + ) + ref_value = ref_kv[..., 16:] + gradients = (torch.randn_like(key), torch.randn_like(value)) + actual_grads = torch.autograd.grad((key, value), (kv, positional), gradients) + ref_grads = torch.autograd.grad((ref_key, ref_value), (ref_kv, ref_pos), gradients) + for actual, expected in zip((key, value, *actual_grads), (ref_key, ref_value, *ref_grads)): + torch.testing.assert_close(actual, expected, atol=0.03125, rtol=0.02) diff --git a/transformer_engine/common/triton/dsa_mla_rope.py b/transformer_engine/common/triton/dsa_mla_rope.py new file mode 100644 index 00000000000..1772b4c2f15 --- /dev/null +++ b/transformer_engine/common/triton/dsa_mla_rope.py @@ -0,0 +1,423 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Triton RoPE kernels for DSA and MLA. + +The shared Q RoPE forward and backward kernels come from Megatron-LM's +megatron/core/fusions/fused_mla_yarn_rope_apply.py at +c4129afff4867ca89b1fb7b4b911128424cdfbe8. The separate MLA KV kernels came +from TE's mla_rope_utils.py. Framework adapters prepare tensors and handle autograd. +""" + +import triton +import triton.language as tl + + +@triton.jit +def _get_thd_token_idx(cu_seqlens, pid_m, seq_num, cp_rank, cp_size): + # Cast ``pid_m`` and ``cu_seqlens`` loads to a single shared dtype so + # the loop-body reassignments don't surface as + # "initial value is int32 but redefined as int64" in newer Triton + # versions (which promote ``// Python_int`` to int64). + pid_m = pid_m.to(tl.int64) + token_idx = tl.full((), -1, dtype=tl.int64) + this_seq_len = tl.full((), 0, dtype=tl.int64) + seq_idx = 0 + last_cum_seqlen = tl.load(cu_seqlens).to(tl.int64) // cp_size + while seq_idx < seq_num: + cur_cum_seqlen = tl.load(cu_seqlens + seq_idx + 1).to(tl.int64) // cp_size + if token_idx == -1 and cur_cum_seqlen > pid_m: + token_idx = pid_m - last_cum_seqlen + this_seq_len = cur_cum_seqlen - last_cum_seqlen + last_cum_seqlen = cur_cum_seqlen + seq_idx += 1 + # Padding tokens beyond cu_seqlens[-1] (from THD CUDA-graph padding) + # never match any sequence, leaving token_idx == -1. Clamp to 0 so + # the cos/sin table loads stay in-bounds; the wrong RoPE result is + # harmless because padding positions are excluded by loss_mask. + if token_idx == -1: + token_idx = tl.full((), 0, dtype=tl.int64) + if cp_size > 1: + if token_idx < this_seq_len // 2: + token_idx = token_idx + cp_rank * this_seq_len // 2 + else: + token_idx = (token_idx - this_seq_len // 2) + ( + 2 * cp_size - cp_rank - 1 + ) * this_seq_len // 2 + return token_idx + + +@triton.jit +def _rotary_fwd_q_kernel( + Q, + COS, + SIN, + nope_dim, + emb_dim: tl.constexpr, + head_num: tl.constexpr, + batch_size, + seq_num, + cu_seqlens_q, + position_ids, + stride_x_seq, + stride_x_nheads, + stride_cos_seq, + stride_sin_seq, + cp_rank, + cp_size, + INVERSE: tl.constexpr, + REMOVE_INTERLEAVING: tl.constexpr, + BLOCK_H: tl.constexpr, +): + """ + Forward pass: apply RoPE inplace to the trailing emb_dim elements. + Reads interleaved input; writes interleaved or split halves per REMOVE_INTERLEAVING. + + Input: + Q: [seq_len, batch_size, head_num, nope_dim + emb_dim] + or [total_seq_len, head_num, nope_dim + emb_dim] + COS/SIN: [max_seq_len, emb_dim] + + batch_size: batch size for sbhd format, not used for thd format + seq_num: number of sequences for thd format, not used for sbhd format + cu_seqlens_q: [seq_num + 1] accumulated sequence lengths for thd format + """ + pid_m = tl.program_id(axis=0) + pid_head = tl.program_id(axis=1) + + if position_ids is not None: + token_idx = tl.load(position_ids + pid_m) + elif cu_seqlens_q is None: + token_idx = pid_m // batch_size + else: + token_idx = _get_thd_token_idx(cu_seqlens_q, pid_m, seq_num, cp_rank, cp_size) + + cos_left = tl.load(COS + token_idx * stride_cos_seq + tl.arange(0, emb_dim // 2)) + sin_left = tl.load(SIN + token_idx * stride_sin_seq + tl.arange(0, emb_dim // 2)) + cos_right = tl.load( + COS + token_idx * stride_cos_seq + emb_dim // 2 + tl.arange(0, emb_dim // 2) + ) + sin_right = tl.load( + SIN + token_idx * stride_sin_seq + emb_dim // 2 + tl.arange(0, emb_dim // 2) + ) + if INVERSE: + sin_left = -sin_left + sin_right = -sin_right + cos_left = cos_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + sin_left = sin_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + cos_right = cos_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + sin_right = sin_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + + Q = Q + pid_m * stride_x_seq + pid_head * BLOCK_H * stride_x_nheads + + x_off = tl.arange(0, BLOCK_H)[:, None] * stride_x_nheads + nope_dim + mask = (pid_head * BLOCK_H + tl.arange(0, BLOCK_H))[:, None] < head_num + # x1 = t[..., 0::2], x2 = t[..., 1::2] + x_1_off = x_off + tl.arange(0, emb_dim // 2)[None, :] * 2 + x_2_off = x_1_off + 1 + x_1 = tl.load(Q + x_1_off, mask=mask) + x_2 = tl.load(Q + x_2_off, mask=mask) + + x_left = x_1 * cos_left - x_2 * sin_left + x_right = x_2 * cos_right + x_1 * sin_right + + if REMOVE_INTERLEAVING: + tl.store(Q + x_1_off, x_left, mask=mask) + tl.store(Q + x_2_off, x_right, mask=mask) + else: + # The interleaved input and split output layouts alias. Finish all loads + # before any warp stores to the overlapping destination addresses. + tl.debug_barrier() + x_left_off = x_off + tl.arange(0, emb_dim // 2)[None, :] + x_right_off = x_left_off + emb_dim // 2 + tl.store(Q + x_left_off, x_left, mask=mask) + tl.store(Q + x_right_off, x_right, mask=mask) + + +rotary_fwd_q_kernel = triton.autotune( + configs=[ + triton.Config({"BLOCK_H": 1}), + triton.Config({"BLOCK_H": 2}), + triton.Config({"BLOCK_H": 4}), + triton.Config({"BLOCK_H": 8}), + triton.Config({"BLOCK_H": 16}), + triton.Config({"BLOCK_H": 32}), + triton.Config({"BLOCK_H": 64}), + triton.Config({"BLOCK_H": 128}), + ], + key=["emb_dim", "head_num"], + restore_value=["Q"], +)(_rotary_fwd_q_kernel) + + +@triton.jit +def _rotary_bwd_q_kernel( + DO, + COS, + SIN, + nope_dim, + emb_dim: tl.constexpr, + head_num: tl.constexpr, + batch_size, + seq_num, + cu_seqlens_q, + position_ids, + stride_x_seq, + stride_x_nheads, + stride_cos_seq, + stride_sin_seq, + cp_rank, + cp_size, + INVERSE: tl.constexpr, + REMOVE_INTERLEAVING: tl.constexpr, + BLOCK_H: tl.constexpr, +): + """ + Backward pass: inverse RoPE inplace on the trailing emb_dim elements. + Reads interleaved or split halves per REMOVE_INTERLEAVING; writes interleaved gradients. + + Input: + DO: [seq_len, batch_size, head_num, nope_dim + emb_dim] + or [total_seq_len, head_num, nope_dim + emb_dim] + COS/SIN: [max_seq_len, emb_dim] + + batch_size, seq_num, and cu_seqlens_q are the same as in the forward pass + """ + pid_m = tl.program_id(axis=0) + pid_head = tl.program_id(axis=1) + + if position_ids is not None: + token_idx = tl.load(position_ids + pid_m) + elif cu_seqlens_q is None: + token_idx = pid_m // batch_size + else: + token_idx = _get_thd_token_idx(cu_seqlens_q, pid_m, seq_num, cp_rank, cp_size) + + cos_left = tl.load(COS + token_idx * stride_cos_seq + tl.arange(0, emb_dim // 2)) + sin_left = tl.load(SIN + token_idx * stride_sin_seq + tl.arange(0, emb_dim // 2)) + cos_right = tl.load( + COS + token_idx * stride_cos_seq + emb_dim // 2 + tl.arange(0, emb_dim // 2) + ) + sin_right = tl.load( + SIN + token_idx * stride_sin_seq + emb_dim // 2 + tl.arange(0, emb_dim // 2) + ) + if INVERSE: + sin_left = -sin_left + sin_right = -sin_right + cos_left = cos_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + sin_left = sin_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + cos_right = cos_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + sin_right = sin_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + + DO = DO + pid_m * stride_x_seq + pid_head * BLOCK_H * stride_x_nheads + + x_off = tl.arange(0, BLOCK_H)[:, None] * stride_x_nheads + nope_dim + mask = (pid_head * BLOCK_H + tl.arange(0, BLOCK_H))[:, None] < head_num + if REMOVE_INTERLEAVING: + x_1_off = x_off + tl.arange(0, emb_dim // 2)[None, :] * 2 + x_2_off = x_1_off + 1 + x_left = tl.load(DO + x_1_off, mask=mask) + x_right = tl.load(DO + x_2_off, mask=mask) + else: + x_left_off = x_off + tl.arange(0, emb_dim // 2)[None, :] + x_right_off = x_left_off + emb_dim // 2 + x_left = tl.load(DO + x_left_off, mask=mask) + x_right = tl.load(DO + x_right_off, mask=mask) + x_1_off = x_off + tl.arange(0, emb_dim // 2)[None, :] * 2 + x_2_off = x_1_off + 1 + + x_1 = x_left * cos_left + x_right * sin_right + x_2 = -x_left * sin_left + x_right * cos_right + + if not REMOVE_INTERLEAVING: + # The split input and interleaved output layouts alias. Finish all loads + # before any warp stores to the overlapping destination addresses. + tl.debug_barrier() + tl.store(DO + x_1_off, x_1, mask=mask) + tl.store(DO + x_2_off, x_2, mask=mask) + + +rotary_bwd_q_kernel = triton.autotune( + configs=[ + triton.Config({"BLOCK_H": 1}), + triton.Config({"BLOCK_H": 2}), + triton.Config({"BLOCK_H": 4}), + triton.Config({"BLOCK_H": 8}), + triton.Config({"BLOCK_H": 16}), + triton.Config({"BLOCK_H": 32}), + triton.Config({"BLOCK_H": 64}), + triton.Config({"BLOCK_H": 128}), + ], + key=["emb_dim", "head_num"], + restore_value=["DO"], +)(_rotary_bwd_q_kernel) + + +@triton.autotune( + configs=[ + triton.Config({"BLOCK_H": 1}), + triton.Config({"BLOCK_H": 2}), + triton.Config({"BLOCK_H": 4}), + triton.Config({"BLOCK_H": 8}), + triton.Config({"BLOCK_H": 16}), + triton.Config({"BLOCK_H": 32}), + triton.Config({"BLOCK_H": 64}), + triton.Config({"BLOCK_H": 128}), + ], + key=["emb_dim", "k_dim", "v_dim", "head_num"], +) +@triton.jit +def rotary_fwd_kv_kernel( + KV, + K_POS_EMB, + O_KEY, + O_VALUE, + COS, + SIN, + emb_dim: tl.constexpr, + k_dim: tl.constexpr, + v_dim: tl.constexpr, + head_num: tl.constexpr, + batch_size, + seq_num, + cu_seqlens_kv, + stride_kv_seq, + stride_kv_nheads, + stride_emb_seq, + stride_k_seq, + stride_k_nheads, + stride_v_seq, + stride_v_nheads, + cp_rank, + cp_size, + BLOCK_H: tl.constexpr, +): + """Rotate positional keys and assemble separate MLA key/value tensors.""" + pid_m = tl.program_id(axis=0) + pid_head = tl.program_id(axis=1) + if cu_seqlens_kv is None: + token_idx = pid_m // batch_size + else: + token_idx = _get_thd_token_idx(cu_seqlens_kv, pid_m, seq_num, cp_rank, cp_size) + cos_left = tl.load(COS + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) + sin_left = tl.load(SIN + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) + cos_right = tl.load(COS + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) + sin_right = tl.load(SIN + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) + head_offsets = pid_head * BLOCK_H + tl.arange(0, BLOCK_H) + KV_ptr = KV + pid_m * stride_kv_seq + kv_off = head_offsets[:, None] * stride_kv_nheads + mask = head_offsets[:, None] < head_num + k_in_off = kv_off + tl.arange(0, k_dim)[None, :] + v_in_off = kv_off + k_dim + tl.arange(0, v_dim)[None, :] + k = tl.load(KV_ptr + k_in_off, mask=mask) + v = tl.load(KV_ptr + v_in_off, mask=mask) + K_ptr = O_KEY + pid_m * stride_k_seq + pid_head * BLOCK_H * stride_k_nheads + V_ptr = O_VALUE + pid_m * stride_v_seq + pid_head * BLOCK_H * stride_v_nheads + k_out_off = tl.arange(0, BLOCK_H)[:, None] * stride_k_nheads + tl.arange(0, k_dim)[None, :] + v_out_off = tl.arange(0, BLOCK_H)[:, None] * stride_v_nheads + tl.arange(0, v_dim)[None, :] + tl.store(K_ptr + k_out_off, k, mask=mask) + tl.store(V_ptr + v_out_off, v, mask=mask) + EMB = K_POS_EMB + pid_m * stride_emb_seq + x_1 = tl.load(EMB + tl.arange(0, emb_dim // 2) * 2) + x_2 = tl.load(EMB + tl.arange(0, emb_dim // 2) * 2 + 1) + x_left = x_1 * cos_left - x_2 * sin_left + x_right = x_2 * cos_right + x_1 * sin_right + x_left = x_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + x_right = x_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) + x_left_off = ( + tl.arange(0, BLOCK_H)[:, None] * stride_k_nheads + + k_dim + + tl.arange(0, emb_dim // 2)[None, :] + ) + x_right_off = x_left_off + emb_dim // 2 + tl.store(K_ptr + x_left_off, x_left, mask=mask) + tl.store(K_ptr + x_right_off, x_right, mask=mask) + + +@triton.autotune( + configs=[ + triton.Config({"BLOCK_H": 1}), + triton.Config({"BLOCK_H": 2}), + triton.Config({"BLOCK_H": 4}), + triton.Config({"BLOCK_H": 8}), + triton.Config({"BLOCK_H": 16}), + triton.Config({"BLOCK_H": 32}), + triton.Config({"BLOCK_H": 64}), + triton.Config({"BLOCK_H": 128}), + ], + key=["emb_dim", "k_dim", "v_dim", "head_num"], +) +@triton.jit +def rotary_bwd_kv_kernel( + dK, + dV, + dKV, + dEMB, + COS, + SIN, + emb_dim: tl.constexpr, + k_dim: tl.constexpr, + v_dim: tl.constexpr, + head_num: tl.constexpr, + batch_size, + seq_num, + cu_seqlens_kv, + stride_dk_seq, + stride_dk_nheads, + stride_dv_seq, + stride_dv_nheads, + stride_dkv_seq, + stride_dkv_nheads, + stride_demb_seq, + cp_rank, + cp_size, + BLOCK_H: tl.constexpr, +): + """Propagate MLA K/V gradients and reduce shared positional-key gradients.""" + pid_m = tl.program_id(axis=0) + pid_head = tl.program_id(axis=1) + if cu_seqlens_kv is None: + token_idx = pid_m // batch_size + else: + token_idx = _get_thd_token_idx(cu_seqlens_kv, pid_m, seq_num, cp_rank, cp_size) + head_offsets = pid_head * BLOCK_H + tl.arange(0, BLOCK_H) + dKV_ptr = dKV + pid_m * stride_dkv_seq + dkv_off = head_offsets[:, None] * stride_dkv_nheads + mask = head_offsets[:, None] < head_num + dk_out_off = dkv_off + tl.arange(0, k_dim)[None, :] + dv_out_off = dkv_off + k_dim + tl.arange(0, v_dim)[None, :] + dK_ptr = dK + pid_m * stride_dk_seq + pid_head * BLOCK_H * stride_dk_nheads + dV_ptr = dV + pid_m * stride_dv_seq + pid_head * BLOCK_H * stride_dv_nheads + dk_in_off = tl.arange(0, BLOCK_H)[:, None] * stride_dk_nheads + tl.arange(0, k_dim)[None, :] + dv_in_off = tl.arange(0, BLOCK_H)[:, None] * stride_dv_nheads + tl.arange(0, v_dim)[None, :] + dk = tl.load(dK_ptr + dk_in_off, mask=mask) + dv = tl.load(dV_ptr + dv_in_off, mask=mask) + tl.store(dKV_ptr + dk_out_off, dk, mask=mask) + tl.store(dKV_ptr + dv_out_off, dv, mask=mask) + if pid_head == 0: + x_left_accum = tl.zeros((BLOCK_H, emb_dim // 2), dtype=tl.float32) + x_right_accum = tl.zeros((BLOCK_H, emb_dim // 2), dtype=tl.float32) + for i in tl.static_range(triton.cdiv(head_num, BLOCK_H)): + head_offsets_i = i * BLOCK_H + tl.arange(0, BLOCK_H) + dK_ptr_i = dK + pid_m * stride_dk_seq + x_off = head_offsets_i[:, None] * stride_dk_nheads + k_dim + mask_i = head_offsets_i[:, None] < head_num + x_left_off = x_off + tl.arange(0, emb_dim // 2)[None, :] + x_right_off = x_left_off + emb_dim // 2 + # Masked heads must contribute zero to the positional-key gradient. + x_left_accum += tl.load(dK_ptr_i + x_left_off, mask=mask_i, other=0) + x_right_accum += tl.load(dK_ptr_i + x_right_off, mask=mask_i, other=0) + x_left_accum = tl.sum(x_left_accum, axis=0) + x_right_accum = tl.sum(x_right_accum, axis=0) + x_left_accum = x_left_accum.to(dEMB.dtype.element_ty) + x_right_accum = x_right_accum.to(dEMB.dtype.element_ty) + cos_left = tl.load(COS + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) + sin_left = tl.load(SIN + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) + cos_right = tl.load(COS + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) + sin_right = tl.load(SIN + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) + x_1 = x_left_accum * cos_left + x_right_accum * sin_right + x_2 = -x_left_accum * sin_left + x_right_accum * cos_right + dEMB_ptr = dEMB + pid_m * stride_demb_seq + tl.store(dEMB_ptr + tl.arange(0, emb_dim // 2) * 2, x_1) + tl.store(dEMB_ptr + tl.arange(0, emb_dim // 2) * 2 + 1, x_2) diff --git a/transformer_engine/pytorch/__init__.py b/transformer_engine/pytorch/__init__.py index c408de1e52c..171873222a7 100644 --- a/transformer_engine/pytorch/__init__.py +++ b/transformer_engine/pytorch/__init__.py @@ -37,6 +37,13 @@ from transformer_engine.pytorch.module import destroy_ub from transformer_engine.pytorch.module import UserBufferQuantizationMode from transformer_engine.pytorch.attention import DotProductAttention +from transformer_engine.pytorch.attention import ( + DSv4Attention, + Compressor, + Indexer, + dense_indexer_loss_scores, + dense_indexer_loss_backward, +) from transformer_engine.pytorch.attention import GatedDeltaNetAttention from transformer_engine.pytorch.attention import GatedDeltaNet2Attention from transformer_engine.pytorch.attention import GatedDeltaProductAttention diff --git a/transformer_engine/pytorch/attention/__init__.py b/transformer_engine/pytorch/attention/__init__.py index e0c56087702..f8b10ae300e 100644 --- a/transformer_engine/pytorch/attention/__init__.py +++ b/transformer_engine/pytorch/attention/__init__.py @@ -5,6 +5,13 @@ """Python interface for attention""" from .dot_product_attention import DotProductAttention +from .sparse_attention import ( + DSv4Attention, + Compressor, + Indexer, + dense_indexer_loss_scores, + dense_indexer_loss_backward, +) from .linear_attention import ( GatedDeltaNetAttention, GatedDeltaNet2Attention, @@ -18,6 +25,11 @@ __all__ = [ "DotProductAttention", + "DSv4Attention", + "Compressor", + "Indexer", + "dense_indexer_loss_scores", + "dense_indexer_loss_backward", "GatedDeltaNetAttention", "GatedDeltaNet2Attention", "GatedDeltaProductAttention", diff --git a/transformer_engine/pytorch/attention/fused_mla_q_uproj.py b/transformer_engine/pytorch/attention/fused_mla_q_uproj.py index 43685f12a03..cb999340fde 100644 --- a/transformer_engine/pytorch/attention/fused_mla_q_uproj.py +++ b/transformer_engine/pytorch/attention/fused_mla_q_uproj.py @@ -4,8 +4,7 @@ """Fused MLA Q up-projection + per-head RoPE + MXFP8 quantize. -The MLA RoPE Triton kernels are taken from: -Megatron-LM megatron/core/fusions/fused_mla_yarn_rope_apply.py +cuDNN applies RoPE during fused query forward. Backward uses ``rotary_bwd_q_kernel``. """ from __future__ import annotations @@ -21,10 +20,12 @@ try: import triton - import triton.language as tl + from transformer_engine.common.triton.dsa_mla_rope import ( + rotary_bwd_q_kernel, + ) except ImportError: triton = None - tl = None + rotary_bwd_q_kernel = None from ..constants import MXFP8_BLOCK_SCALING_SIZE from ..distributed import get_distributed_world_size @@ -45,110 +46,6 @@ def _cudnn_frontend_version_supported() -> bool: return False -if triton is not None: - - @triton.jit - def _get_thd_token_idx(cu_seqlens, pid_m, seq_num, cp_rank, cp_size): - token_idx = -1 - this_seq_len = 0 - seq_idx = 0 - last_cum_seqlen = tl.load(cu_seqlens) // cp_size - while seq_idx < seq_num: - cur_cum_seqlen = tl.load(cu_seqlens + seq_idx + 1) // cp_size - if token_idx == -1 and cur_cum_seqlen > pid_m: - token_idx = pid_m - last_cum_seqlen - this_seq_len = cur_cum_seqlen - last_cum_seqlen - last_cum_seqlen = cur_cum_seqlen - seq_idx += 1 - if cp_size > 1: - if token_idx < this_seq_len // 2: - token_idx = token_idx + cp_rank * this_seq_len // 2 - else: - token_idx = (token_idx - this_seq_len // 2) + ( - 2 * cp_size - cp_rank - 1 - ) * this_seq_len // 2 - return token_idx - - @triton.autotune( - configs=[ - triton.Config({"BLOCK_H": 1}), - triton.Config({"BLOCK_H": 2}), - triton.Config({"BLOCK_H": 4}), - triton.Config({"BLOCK_H": 8}), - triton.Config({"BLOCK_H": 16}), - triton.Config({"BLOCK_H": 32}), - triton.Config({"BLOCK_H": 64}), - triton.Config({"BLOCK_H": 128}), - ], - key=["emb_dim", "head_num"], - restore_value=["DO"], - ) - @triton.jit - def rotary_bwd_q_kernel( - DO, - COS, - SIN, - qk_head_dim, - emb_dim: tl.constexpr, - head_num: tl.constexpr, - batch_size, - seq_num, - cu_seqlens_q, - stride_x_seq, - stride_x_nheads, - cp_rank, - cp_size, - BLOCK_H: tl.constexpr, - ): - """ - Triton kernel of the backward pass for applying YARN RoPE to MLA's query. - This kernel inplace modifies the input tensor DO. - - Input: - DO: [seq_len, batch_size, head_num, qk_head_dim + emb_dim] - or [total_seq_len, head_num, qk_head_dim + emb_dim] - COS/SIN: [max_seq_len, emb_dim] - - batch_size, seq_num, and cu_seqlens_q are the same as in the forward pass - """ - pid_m = tl.program_id(axis=0) - pid_head = tl.program_id(axis=1) - - if cu_seqlens_q is None: - token_idx = pid_m // batch_size - else: - token_idx = _get_thd_token_idx(cu_seqlens_q, pid_m, seq_num, cp_rank, cp_size) - - cos_left = tl.load(COS + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - sin_left = tl.load(SIN + token_idx * emb_dim + tl.arange(0, emb_dim // 2)) - cos_right = tl.load(COS + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) - sin_right = tl.load(SIN + token_idx * emb_dim + emb_dim // 2 + tl.arange(0, emb_dim // 2)) - cos_left = cos_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - sin_left = sin_left.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - cos_right = cos_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - sin_right = sin_right.expand_dims(0).broadcast_to(BLOCK_H, emb_dim // 2) - - DO = DO + pid_m * stride_x_seq + pid_head * BLOCK_H * stride_x_nheads - - x_off = tl.arange(0, BLOCK_H)[:, None] * stride_x_nheads + qk_head_dim - mask = x_off < head_num * stride_x_nheads - x_left_off = x_off + tl.arange(0, emb_dim // 2)[None, :] - x_right_off = x_left_off + emb_dim // 2 - x_left = tl.load(DO + x_left_off, mask=mask) - x_right = tl.load(DO + x_right_off, mask=mask) - - x_1 = x_left * cos_left + x_right * sin_right - x_2 = -x_left * sin_left + x_right * cos_right - - x_1_off = x_off + tl.arange(0, emb_dim // 2)[None, :] * 2 - x_2_off = x_1_off + 1 - tl.store(DO + x_1_off, x_1, mask=mask) - tl.store(DO + x_2_off, x_2, mask=mask) - -else: - rotary_bwd_q_kernel = None - - class FusedMLAQUpProjRopeQuant: """Wrapper for the cuDNN fused MLA Q up-proj + per-head RoPE + MXFP8 quantize kernel. @@ -458,7 +355,8 @@ def backward(ctx, dq): tokens = s * b act_dtype = ctx.act_dtype - # --- RoPE backward (unchanged: bf16, same rotary_bwd_q_kernel as the unfused path) --- + # Apply BF16 RoPE backward with rotary_bwd_q_kernel. + # The DSA and MLA adapters also use this kernel. dq3 = dq.reshape(tokens, nh, q_head_dim).contiguous() def grid(META): @@ -474,10 +372,15 @@ def grid(META): 1, None, None, + None, dq3.stride(0), dq3.stride(1), + cos.shape[-1], + sin.shape[-1], 0, 1, + INVERSE=False, + REMOVE_INTERLEAVING=False, ) # grad w.r.t. the (pre-RoPE) up-proj GEMM output; bf16. dq2d = dq3.reshape(tokens, nh * q_head_dim).contiguous() diff --git a/transformer_engine/pytorch/attention/rope.py b/transformer_engine/pytorch/attention/rope.py index 83369547f1f..c7a7f2505d6 100644 --- a/transformer_engine/pytorch/attention/rope.py +++ b/transformer_engine/pytorch/attention/rope.py @@ -5,15 +5,25 @@ """ Rotary Position Embedding implementation of different types along with helper functions """ +from functools import lru_cache from typing import Optional, Tuple, Union, List import torch import transformer_engine_torch as tex from transformer_engine.pytorch.cpp_extensions.fused_attn import QKVFormat from transformer_engine.pytorch.device import te_device_type +from transformer_engine.pytorch.utils import canonicalize_device -__all__ = ["RotaryPositionEmbedding", "apply_rotary_pos_emb", "apply_fused_qkv_rotary_pos_emb"] +__all__ = [ + "RotaryPositionEmbedding", + "apply_rotary_pos_emb", + "apply_fused_qkv_rotary_pos_emb", + "DSv4RotaryEmbedding", + "build_dsv4_rope_tables", + "apply_dsa_rope", + "apply_dsa_rope_eager", +] class RotaryPositionEmbedding(torch.nn.Module): @@ -540,3 +550,572 @@ def apply_fused_qkv_rotary_pos_emb( cp_size, cp_rank, ) + + +def _build_rope_tables(seq_len, inv_freq, *, interleaved): + """Build FP32 cosine/sine tables for the DSv4 and MLA wrappers. + + Callers supply FP32 inverse frequencies [rotary_width / 2]. Both outputs have + shape [seq_len, rotary_width]. Interleaved layout repeats compact values + [a, b] as [a, a, b, b] for DSv4; otherwise duplicate halves as [a, b, a, b] + for MLA. + """ + positions = torch.arange(seq_len, device=inv_freq.device, dtype=torch.float32) + angles = torch.outer(positions, inv_freq) + cos, sin = angles.cos(), angles.sin() + if interleaved: + return cos.repeat_interleave(2, dim=-1), sin.repeat_interleave(2, dim=-1) + return torch.cat((cos, cos), dim=-1), torch.cat((sin, sin), dim=-1) + + +# DSA: trailing interleaved pairs with packed sequence positions. + + +def _validate_dsa_rope_width(width, head_dim=None): + """Require the power-of-two rotary width used by the Triton Q RoPE kernels.""" + if width < 4 or width & (width - 1): + raise ValueError("DSv4 RoPE width must be a power of two >= 4.") + if head_dim is not None and width > head_dim: + raise ValueError("DSv4 RoPE width must not exceed the input head dimension.") + + +def build_dsv4_rope_tables(seq, ratio, width, theta, device): + """Return FP32 token/window pairs for a power-of-two rotary width >= 4.""" + _validate_dsa_rope_width(width) + inv_freq = theta ** (-torch.arange(0, width, 2, device=device, dtype=torch.float32) / width) + cos, sin = _build_rope_tables(seq, inv_freq, interleaved=True) + cos, sin = cos[None, :, None, :], sin[None, :, None, :] + window_positions = torch.arange(seq // ratio, device=device) * ratio + return (cos, sin), (cos[:, window_positions], sin[:, window_positions]) + + +class DSv4RotaryEmbedding(torch.nn.Module): + """Cache FP32 token frequencies and compressed-window start frequencies. + + Buffers are not saved in state_dict. Module.to() moves the cache while keeping + its frequencies in FP32, even when a different floating-point dtype is requested. + + Parameters + ---------- + ratio : int + Number of input tokens per compressed window. + width : int + Rotary dimension, a power of two >= 4. + theta : float + Rotary frequency base. + device : torch.device or str + Device for the optional initial cache. + max_seqlen : int, optional + Initial cache length. Otherwise allocate lazily on the first forward call. + """ + + def __init__(self, ratio, width, theta, device, max_seqlen=None): + super().__init__() + _validate_dsa_rope_width(width) + self.ratio, self.width, self.theta = ratio, width, theta + cos, sin = (None, None) + if max_seqlen is not None: + (cos, sin), _ = build_dsv4_rope_tables(max_seqlen, ratio, width, theta, device) + self.register_buffer("cos", cos, persistent=False) + self.register_buffer("sin", sin, persistent=False) + + def _apply(self, fn): + cos, sin = self.cos, self.sin + super()._apply(fn) + if cos is not None: + # Module.to(bfloat16) must not round the reusable frequencies. + self.cos = cos.to(device=self.cos.device) + self.sin = sin.to(device=self.sin.device) + return self + + def forward(self, seq, device): + """Return FP32 token and compressed ``(cos, sin)`` pairs. + + Parameters + ---------- + seq : int + Token sequence length. Grow the cache if needed. + device : torch.device or str + Requested output device. Move an existing cache when it differs. + + Returns + ------- + tuple + Token pair with shape [1, seq, 1, width] and compressed pair with shape + [1, seq // ratio, 1, width]. Compressed positions are window starts + 0, ratio, 2 * ratio, ... . Returned tensors are views of the cache. + """ + device = canonicalize_device(device) + if self.cos is None or self.cos.shape[1] < seq: + (self.cos, self.sin), _ = build_dsv4_rope_tables( + seq, self.ratio, self.width, self.theta, device + ) + elif self.cos.device != device: + self.cos = self.cos.to(device=device) + self.sin = self.sin.to(device=device) + n_comp = seq // self.ratio + return (self.cos[:, :seq], self.sin[:, :seq]), ( + self.cos[:, : n_comp * self.ratio : self.ratio], + self.sin[:, : n_comp * self.ratio : self.ratio], + ) + + +def apply_dsa_rope_eager(x, cos, sin, cu_seqlens=None): + """Rotate trailing pairs in FP32 for a power-of-two width >= 4, then restore dtype.""" + width = cos.shape[-1] + _validate_dsa_rope_width(width, x.shape[-1]) + if x.ndim == 3: + if cu_seqlens is None: + raise ValueError("Packed RoPE requires cu_seqlens.") + token_indices = torch.arange(x.shape[0], device=x.device) + sequences = torch.bucketize(token_indices, cu_seqlens[1:], right=True) + positions = token_indices - cu_seqlens[sequences] + # Compressed tensors may have output padding after cu_seqlens[-1]. + positions = torch.where(token_indices < cu_seqlens[-1], positions, 0) + cos, sin = cos[0, positions], sin[0, positions] + tail = x[..., -width:] + pair = torch.stack((-tail[..., 1::2], tail[..., 0::2]), -1).flatten(-2) + rotated = (tail.float() * cos + pair.float() * sin).to(x.dtype) + return torch.cat((x[..., :-width], rotated), -1) + + +@lru_cache(maxsize=1) +def _triton_rope_module(): + """Load optional Triton Q RoPE kernels for DSA and MLA and separate MLA KV kernels.""" + try: + from transformer_engine.common.triton import dsa_mla_rope + except ImportError: + return None + return dsa_mla_rope + + +def _can_use_dsa_rope_triton(x, cos, sin, cu_seqlens): + """Keep unsupported inputs on the behaviorally identical eager path.""" + return ( + cu_seqlens is not None + and x.is_cuda + and x.ndim in (3, 4) + and x.stride(-1) == 1 + and cos.shape == sin.shape + and not cos.requires_grad + and not sin.requires_grad + and _triton_rope_module() is not None + ) + + +def _dsa_triton_rope_table(table): + """Convert TE's repeated pairs to the Triton kernel's duplicated-half tables.""" + pairs = table[0, :, 0, ::2] + return torch.cat((pairs, pairs), dim=-1) + + +def _launch_rope_q_triton( + x, cos, sin, head_dim_nope, head_dim_rope, cu_seqlens, interleaved, module, backward +): + kernel = module.rotary_bwd_q_kernel if backward else module.rotary_fwd_q_kernel + heads = x.shape[-2] + if cu_seqlens is not None: + x = x.reshape(-1, heads, x.shape[-1]) + total, batch, sequences = x.shape[0], 1, cu_seqlens.numel() - 1 + token_stride, head_stride = x.stride(0), x.stride(1) + else: + seq_len, batch, heads, _ = x.shape + total, sequences = seq_len * batch, None + token_stride, head_stride = x.stride(1), x.stride(2) + + def grid(meta): + return total, module.triton.cdiv(heads, meta["BLOCK_H"]) + + kernel[grid]( + x, + cos, + sin, + head_dim_nope, + head_dim_rope, + heads, + batch, + sequences, + cu_seqlens, + None, + token_stride, + head_stride, + cos.stride(0), + sin.stride(0), + 0, + 1, + INVERSE=False, + REMOVE_INTERLEAVING=interleaved, + ) + + +class _RoPEQTriton(torch.autograd.Function): + """Apply Q RoPE with independent positioning, layout and mutation policies.""" + + @staticmethod + def forward(ctx, x, cos, sin, head_dim_nope, head_dim_rope, cu_seqlens, interleaved, mutation): + """Rotate a copy or the input according to the wrapper's mutation policy.""" + output = x.clone(memory_format=torch.contiguous_format) if mutation == "copy" else x + module = _triton_rope_module() + _launch_rope_q_triton( + output, cos, sin, head_dim_nope, head_dim_rope, cu_seqlens, interleaved, module, False + ) + ctx.save_for_backward(cos, sin, cu_seqlens) + ctx.module = module + ctx.head_dim_nope, ctx.head_dim_rope = head_dim_nope, head_dim_rope + ctx.interleaved, ctx.mutation = interleaved, mutation + if mutation == "marked_inplace": + ctx.mark_dirty(x) + return output + + @staticmethod + def backward(ctx, grad_output): + """Preserve DSA's gradient copy and MLA's existing gradient mutation.""" + cos, sin, cu_seqlens = ctx.saved_tensors + # Legacy MLA mutates input without mark_dirty and also reuses incoming gradients. + # Keep that behavior separate from tensor layout and sequence positioning. + grad_input = ( + grad_output + if ctx.mutation == "legacy_inplace" + else grad_output.clone(memory_format=torch.contiguous_format) + ) + _launch_rope_q_triton( + grad_input, + cos, + sin, + ctx.head_dim_nope, + ctx.head_dim_rope, + cu_seqlens, + ctx.interleaved, + ctx.module, + True, + ) + return grad_input, None, None, None, None, None, None, None + + +def apply_dsa_rope(x, cos, sin, cu_seqlens=None, *, allow_inplace=False): + """Rotate trailing interleaved pairs, using Triton when its contract is satisfied. + + By default, preserve the input. With allow_inplace=True, the caller must supply + a fresh buffer safe for autograd mutation. This requires a contiguous Triton + input that is not a leaf requiring gradients, otherwise raise ValueError. + Both paths require a power-of-two rotary width >= 4 that fits the input head. + """ + use_triton = _can_use_dsa_rope_triton(x, cos, sin, cu_seqlens) + if allow_inplace and ( + not use_triton or not x.is_contiguous() or (x.is_leaf and x.requires_grad) + ): + raise ValueError( + "allow_inplace=True requires a contiguous Triton-compatible input " + "that is not a leaf requiring gradients." + ) + if use_triton: + _validate_dsa_rope_width(cos.shape[-1], x.shape[-1]) + return _RoPEQTriton.apply( + x, + _dsa_triton_rope_table(cos), + _dsa_triton_rope_table(sin), + x.shape[-1] - cos.shape[-1], + cos.shape[-1], + cu_seqlens, + True, + "marked_inplace" if allow_inplace else "copy", + ) + return apply_dsa_rope_eager(x, cos, sin, cu_seqlens) + + +# MLA adapters from Megatron-LM fused_mla_yarn_rope_apply.py. +# The default tables use plain RoPE (base=10000), not YaRN scaling. +# Unlike DSA, these SBHD adapters convert interleaved input to split-half output. + + +_MLA_HEAD_DIM_ROPE = 64 +_MLA_HEAD_DIM_NOPE = 128 +_MLA_HEAD_DIM_V = 128 +_MLA_ROTARY_BASE = 10000 + + +def build_mla_rope_tables( + seq_len: int, + emb_dim: int = _MLA_HEAD_DIM_ROPE, + base: int = _MLA_ROTARY_BASE, + device: torch.device = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Build FP32 duplicated-half MLA tables with plain RoPE frequencies.""" + inv_freq = 1.0 / ( + base ** (torch.arange(0, emb_dim, 2, dtype=torch.float32, device=device) / emb_dim) + ) + cos, sin = _build_rope_tables(seq_len, inv_freq, interleaved=False) + return cos.contiguous(), sin.contiguous() + + +def _mla_flattened_token_stride(tensor: torch.Tensor) -> int: + if tensor.dim() == 4: + return tensor.stride(1) + return tensor.stride(0) + + +class _MLARoPEKVTriton(torch.autograd.Function): + @staticmethod + def forward(ctx, kv, k_pos_emb, cos, sin, head_dim_nope, head_dim_rope, head_dim_v): + """Split SBHD KV and append shared rotated position channels to each key head.""" + s, b, nheads, _ = kv.shape + total = s * b + module = _triton_rope_module() + + o_key = kv.new_empty(s, b, nheads, head_dim_nope + head_dim_rope) + o_value = kv.new_empty(s, b, nheads, head_dim_v) + + def grid_kv(meta): + return total, module.triton.cdiv(nheads, meta["BLOCK_H"]) + + module.rotary_fwd_kv_kernel[grid_kv]( + kv, + k_pos_emb, + o_key, + o_value, + cos, + sin, + head_dim_rope, + head_dim_nope, + head_dim_v, + nheads, + b, + None, + None, + _mla_flattened_token_stride(kv), + kv.stride(2), + _mla_flattened_token_stride(k_pos_emb), + _mla_flattened_token_stride(o_key), + o_key.stride(2), + _mla_flattened_token_stride(o_value), + o_value.stride(2), + 0, + 1, + ) + + ctx.save_for_backward(cos, sin) + ctx.head_dim_nope = head_dim_nope + ctx.head_dim_rope = head_dim_rope + ctx.head_dim_v = head_dim_v + ctx.nheads = nheads + ctx.s = s + ctx.b = b + return o_key, o_value + + @staticmethod + def backward(ctx, dk_out, dv_out): + """Gather KV gradients and reduce shared positional gradients across heads.""" + cos, sin = ctx.saved_tensors + s, b, nheads = ctx.s, ctx.b, ctx.nheads + ndp, ndr, ndv = ctx.head_dim_nope, ctx.head_dim_rope, ctx.head_dim_v + total = s * b + module = _triton_rope_module() + + d_kv = dk_out.new_empty(s, b, nheads, ndp + ndv) + d_emb = dk_out.new_empty(s, b, 1, ndr) + + def grid_kv(meta): + return total, module.triton.cdiv(nheads, meta["BLOCK_H"]) + + module.rotary_bwd_kv_kernel[grid_kv]( + dk_out, + dv_out, + d_kv, + d_emb, + cos, + sin, + ndr, + ndp, + ndv, + nheads, + b, + None, + None, + _mla_flattened_token_stride(dk_out), + dk_out.stride(2), + _mla_flattened_token_stride(dv_out), + dv_out.stride(2), + _mla_flattened_token_stride(d_kv), + d_kv.stride(2), + _mla_flattened_token_stride(d_emb), + 0, + 1, + ) + return d_kv, d_emb, None, None, None, None, None + + +def _apply_mla_rope_q_with_tables( + q: torch.Tensor, + cos_table: torch.Tensor, + sin_table: torch.Tensor, + head_dim_nope: int = _MLA_HEAD_DIM_NOPE, + head_dim_rope: int = _MLA_HEAD_DIM_ROPE, +) -> torch.Tensor: + if _triton_rope_module() is not None: + return _RoPEQTriton.apply( + q, + cos_table, + sin_table, + head_dim_nope, + head_dim_rope, + None, + False, + "legacy_inplace", + ) + return _apply_mla_rope_q_eager(q, cos_table, sin_table, head_dim_nope, head_dim_rope) + + +def _apply_mla_rope_kv_with_tables( + kv: torch.Tensor, + k_pos_emb: torch.Tensor, + cos_table: torch.Tensor, + sin_table: torch.Tensor, + head_dim_nope: int = _MLA_HEAD_DIM_NOPE, + head_dim_rope: int = _MLA_HEAD_DIM_ROPE, + head_dim_v: int = _MLA_HEAD_DIM_V, +) -> tuple[torch.Tensor, torch.Tensor]: + if _triton_rope_module() is not None: + return _MLARoPEKVTriton.apply( + kv, + k_pos_emb, + cos_table, + sin_table, + head_dim_nope, + head_dim_rope, + head_dim_v, + ) + return _apply_mla_rope_kv_eager( + kv, + k_pos_emb, + cos_table, + sin_table, + head_dim_nope, + head_dim_rope, + head_dim_v, + ) + + +def apply_mla_rope_q( + q: torch.Tensor, + head_dim_nope: int = _MLA_HEAD_DIM_NOPE, + head_dim_rope: int = _MLA_HEAD_DIM_ROPE, + base: int = _MLA_ROTARY_BASE, + cos_table: torch.Tensor | None = None, + sin_table: torch.Tensor | None = None, +) -> torch.Tensor: + """Rotate trailing SBHD query channels from interleaved to split-half order.""" + if cos_table is None or sin_table is None: + s = q.shape[0] + cos_table, sin_table = build_mla_rope_tables( + s, + emb_dim=head_dim_rope, + base=base, + device=q.device, + ) + return _apply_mla_rope_q_with_tables( + q, + cos_table, + sin_table, + head_dim_nope, + head_dim_rope, + ) + + +def apply_mla_rope_kv( + kv: torch.Tensor, + k_pos_emb: torch.Tensor, + head_dim_nope: int = _MLA_HEAD_DIM_NOPE, + head_dim_rope: int = _MLA_HEAD_DIM_ROPE, + head_dim_v: int = _MLA_HEAD_DIM_V, + base: int = _MLA_ROTARY_BASE, + cos_table: torch.Tensor | None = None, + sin_table: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Split SBHD KV and rotate shared positional keys into split-half order.""" + if cos_table is None or sin_table is None: + s = kv.shape[0] + cos_table, sin_table = build_mla_rope_tables( + s, + emb_dim=head_dim_rope, + base=base, + device=kv.device, + ) + return _apply_mla_rope_kv_with_tables( + kv, + k_pos_emb, + cos_table, + sin_table, + head_dim_nope, + head_dim_rope, + head_dim_v, + ) + + +def apply_mla_rope( + q: torch.Tensor, + kv: torch.Tensor, + k_pos_emb: torch.Tensor, + head_dim_nope: int = _MLA_HEAD_DIM_NOPE, + head_dim_rope: int = _MLA_HEAD_DIM_ROPE, + head_dim_v: int = _MLA_HEAD_DIM_V, + base: int = _MLA_ROTARY_BASE, + cos_table: torch.Tensor | None = None, + sin_table: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Apply the MLA query and shared-key rotations using the same tables.""" + if cos_table is None or sin_table is None: + s = q.shape[0] + cos_table, sin_table = build_mla_rope_tables( + s, + emb_dim=head_dim_rope, + base=base, + device=q.device, + ) + q = _apply_mla_rope_q_with_tables(q, cos_table, sin_table, head_dim_nope, head_dim_rope) + k, v = _apply_mla_rope_kv_with_tables( + kv, + k_pos_emb, + cos_table, + sin_table, + head_dim_nope, + head_dim_rope, + head_dim_v, + ) + return q, k, v + + +def _rotate_mla_interleaved_to_neox( + x: torch.Tensor, cos_table: torch.Tensor, sin_table: torch.Tensor +) -> torch.Tensor: + cos_ = cos_table[:, None, None, :].to(x.dtype) + sin_ = sin_table[:, None, None, :].to(x.dtype) + half_dim = x.shape[-1] // 2 + x_1 = x[..., 0::2] + x_2 = x[..., 1::2] + x_left = x_1 * cos_[..., :half_dim] - x_2 * sin_[..., :half_dim] + x_right = x_2 * cos_[..., half_dim:] + x_1 * sin_[..., half_dim:] + return torch.cat((x_left, x_right), dim=-1) + + +def _apply_mla_rope_q_eager(q, cos_table, sin_table, head_dim_nope, head_dim_rope): + q_nope = q[..., :head_dim_nope] + q_rope = q[..., head_dim_nope : head_dim_nope + head_dim_rope] + q_rope = _rotate_mla_interleaved_to_neox(q_rope, cos_table, sin_table) + return torch.cat((q_nope, q_rope), dim=-1) + + +def _apply_mla_rope_kv_eager( + kv, + k_pos_emb, + cos_table, + sin_table, + head_dim_nope, + head_dim_rope, # pylint: disable=unused-argument + head_dim_v, +): + k_nope = kv[..., :head_dim_nope] + v = kv[..., head_dim_nope : head_dim_nope + head_dim_v] + k_rope = _rotate_mla_interleaved_to_neox(k_pos_emb, cos_table, sin_table).expand( + -1, -1, kv.shape[2], -1 + ) + return torch.cat((k_nope, k_rope), dim=-1), v diff --git a/transformer_engine/pytorch/attention/sparse_attention/README.md b/transformer_engine/pytorch/attention/sparse_attention/README.md new file mode 100644 index 00000000000..5e5b7b73060 --- /dev/null +++ b/transformer_engine/pytorch/attention/sparse_attention/README.md @@ -0,0 +1,203 @@ +# DSv4 attention API + +`DSv4Attention` is an experimental API in TE that composes cuDNN compression, +top-k indexing, and sparse attention kernels with Triton RoPE through reusable +building blocks such as `Compressor` and `Indexer`. DSv4 uses `rotary_fwd_q_kernel` +and `rotary_bwd_q_kernel` for both Q and KV rotation. MLA's `apply_mla_rope_q` +uses the same forward and backward kernels for Q. + +cuDNN supplies compression and sparse attention forward and backward kernels +and separate indexer-loss score/backward primitives; top-k selection itself is +discrete. The layer maps hidden states to hidden states with the same shape. +`Compressor` and `Indexer` can also be composed into other attention layers; +`dsa_cudnn` exposes the underlying operations for prepared tensors. + +## Module hierarchy + +```text +DSv4Attention +├── TE query/local-KV projections + norms +├── Compressor (HCA or CSA) +│ ├── TE KV/gate projections + learned additive position embedding (APE) +│ ├── dsa_cudnn.compress -> cuDNN compressor +│ └── TE RMSNorm +├── Indexer (CSA only) +│ ├── TE index-query/head-weight projections +│ ├── own Compressor (index keys) +│ ├── rope.apply_dsa_rope (query + index keys) +│ └── dsa_cudnn.select_indices -> cuDNN score + top-k +├── rope embeddings + rotary application +├── dsa_cudnn.sparse_attention -> cuDNN sparse attention +└── TE grouped output projection +``` + +## Inside DSv4Attention + +The following sketch expands the cuDNN calls underneath Compressor, Indexer, and attention. +Query and local KV are already projected, normalized, and rotated; tensors are contiguous CUDA, with FP32 APE/sink and CUDA INT32 sequence prefixes. + +```python +import transformer_engine.pytorch.attention.sparse_attention.dsa_cudnn as dsa_cudnn + +compressed_kv = dsa_cudnn.compress( + kv, gates, ape.float(), cu, cu_comp, + ratio=ratio, overlap=is_csa, total_comp=total_comp, +) +compressed_kv = ... # Apply RMSNorm and RoPE to compressed KV. + +indices = None +if is_csa: + index_key = dsa_cudnn.compress( + index_kv, index_gates, index_ape.float(), cu, cu_comp, + ratio=ratio, overlap=True, total_comp=total_comp, + ) + index_key = ... # Apply RMSNorm and RoPE to compressed index keys. + indices = dsa_cudnn.select_indices( + index_query, index_key, index_weights, cu, cu_comp, + top_k=top_k, ratio=ratio, max_seqlen=max_seqlen, + max_compressed_seqlen=max_compressed_seqlen, + scale=(index_head_dim * index_n_heads) ** -0.5, + ) + +output = dsa_cudnn.sparse_attention( + query, local_kv, compressed_kv, sink.float(), cu, cu_comp, + window_size=window_size, ratio=ratio, indices=indices, + max_compressed_seqlen=max_compressed_seqlen if indices is None else None, +) +output = ... # Apply inverse RoPE and grouped output projection. +``` + +Batched inputs contain B sequences of the same length S. `DSv4Attention` accepts +`[S,B,hidden_size]` (sbhd) or `[B,S,hidden_size]` (bshd), and packs tokens in +batch-major order before these calls. Packed `[T,hidden_size]` (thd) inputs +concatenate sequences, with cumulative token counts identifying their boundaries. +Standalone `Compressor` and `Indexer` accept only batched `[B,S,D]` or packed +`thd` `[T,D]` input; callers must transpose sequence-major inputs first. Hidden +states have no explicit head axis; prepared queries have shape `[T,H,D]`. +Attention uses the original query +`[T,64,D]` and concatenated local/compressed KV `[T+Tc,D]`; there is no +compressed query. `T` is the total query/local KV token count. +`_prepare_sparse_kv_indices` creates a list of KV token indices for each query token, +combining causal local KV token indices with compressed KV token indices offset +by `T`; invalid entries are `-1`. CSA supplies global compressed KV token indices +from `select_indices`; HCA passes `indices=None` and includes all eligible +compressed tokens. `_prepare_sparse_kv_indices` documents a small example of these +index spaces. Selection indices address only packed compressed KV before the +local KV offset is applied. + +`transformer_engine.pytorch.attention.rope` provides the rotary utilities. +`rope.DSv4RotaryEmbedding` caches token/compressed-position embeddings. +`rope.apply_dsa_rope` applies them. +A caller may override the default embeddings with a `rope` module returning +token and compressed `(cos, sin)` pairs. + +## Usage + +CSA uses overlapping compression (see `Compressor`) and an Indexer to select +compressed KV entries. + +```python +import torch +from transformer_engine.pytorch.attention.sparse_attention import DSv4Attention + +csa = DSv4Attention( + hidden_size=4096, q_lora_rank=1536, + layer_type="compressed_sparse_attention", head_dim=512, rope_head_dim=64, + sliding_window=128, compression_ratio=4, o_groups=8, o_lora_rank=1024, + index_n_heads=32, index_topk=32, max_seqlen=2048, + params_dtype=torch.bfloat16, +) +output = csa(hidden_states) # [S, B, 4096] -> [S, B, 4096] +``` + +HCA uses non-overlapping compression and omits the Indexer: + +```python +hca = DSv4Attention( + hidden_size=4096, q_lora_rank=1536, + layer_type="heavily_compressed_attention", head_dim=512, rope_head_dim=64, + sliding_window=128, compression_ratio=128, o_groups=8, o_lora_rank=1024, + max_seqlen=2048, params_dtype=torch.bfloat16, +) +output = hca(hidden_states) # [S, B, 4096] -> [S, B, 4096] +``` + +`hidden_states` has shape `[S,B,hidden_size]` for `qkv_format="sbhd"` (default), +`[B,S,hidden_size]` for `"bshd"`, or `[T,hidden_size]` for `"thd"`. +Packed calls require contiguous CUDA INT32 +`cu_seqlens_q [B+1]` (from zero to `T`) and `max_seqlen_q`: + +```python +packed_csa = DSv4Attention( + hidden_size=4096, q_lora_rank=1536, + layer_type="compressed_sparse_attention", head_dim=512, rope_head_dim=64, + sliding_window=128, compression_ratio=4, o_groups=8, o_lora_rank=1024, + index_n_heads=32, index_topk=32, max_seqlen=2048, + params_dtype=torch.bfloat16, qkv_format="thd", +) +output = packed_csa(hidden_states, cu_seqlens_q=cu, max_seqlen_q=max_length) +``` + +## Note on support + +The layer admits SM100, SM103, and SM107 GPUs for full-sequence +forward/backward and requires BF16 activations. FP16 is unsupported by the +compression and indexer training dependencies. The operations require cuDNN +Frontend 1.29 or newer with its CSA/DSA APIs and are delivered as CuTe kernels. +Hardware validation includes prior SM100 runs and SM103/SM107 runs with +cuDNN Frontend 1.30.0, Cutlass DSL 4.8.0, CUDA 13.4, and cuDNN 9.27. + +The layer uses 64 heads of width 512 and requires at least one complete +compression window per sequence. Sequences start at position zero; +cache/decode, physical inter-sequence padding, TP/SP/CP, and FP8 are unsupported. +CSA and HCA include sliding-window attention; standalone sliding-window-only +layers should use `DotProductAttention` with `window_size` instead. + +Compression and attention support autograd; top-k selection is discrete. +CSA's `return_indexer_context=True` returns `(output, context)` for a +single-sequence, caller-owned indexer loss. Without that objective, the +indexer is not trained by the attention output alone. Pre-attention +normalization, residuals, and loss calculation remain model-owned. + +For B=1, the caller can train the indexer with `dense_indexer_loss_scores` and +`dense_indexer_loss_backward`, exported alongside the three sparse-attention classes: + +```python +from transformer_engine.pytorch import dense_indexer_loss_scores, dense_indexer_loss_backward + +output, context = csa(hidden_states, return_indexer_context=True) +qi, ki, wi = (context[name].unsqueeze(0) + for name in ("index_q", "index_k", "index_w")) +qa, kc = (context[name].unsqueeze(0) + for name in ("attn_q", "compressed_kv")) +# Caller computes detached FP32 full_lse [1,T,64] over the local window, +# ALL eligible compressed keys, and sink, using context's teacher tensors. +# The selected sparse-attention LSE excludes sink and is not this denominator. +full_lse = ... +index_scale = (qi.shape[2] * qi.shape[3]) ** -0.5 +index_scores, teacher_scores = dense_indexer_loss_scores( + qi, ki, wi, qa, kc, full_lse, + ratio=csa.compression_ratio, index_scale=index_scale, + attention_scale=csa.head_dim ** -0.5, +) +index_grads = dense_indexer_loss_backward( + qi, ki, wi, index_scores, teacher_scores, + torch.ones((), device=qi.device), + ratio=csa.compression_ratio, index_scale=index_scale, + loss_coeff=loss_coefficient, +) +torch.autograd.backward((qi, ki, wi), index_grads) +``` + +These helpers return score buffers and explicit gradients, not an autograd +loss. The model owns its loss coefficient and propagates its main objective +through `output` separately. Auxiliary gradients reach the live indexer +projections and compressor; they do not reach the shared hidden/query +projections or detached attention teacher. The dense-reference example in +`tests/pytorch/test_dsv4_attention.py` shows full teacher-LSE construction. + +Shared-input projections in `DSv4Attention` are fused: `q_a_kv_proj.weight` +stores query-down then local-KV rows; `compressor.fused_proj.weight` stores +KV then gate rows; the CSA `indexer.compressor.fused_proj.weight` additionally +stores head-weight rows last. These default checkpoint keys and shapes are +preserved. Separate-weight model imports must concatenate those rows explicitly. diff --git a/transformer_engine/pytorch/attention/sparse_attention/__init__.py b/transformer_engine/pytorch/attention/sparse_attention/__init__.py new file mode 100644 index 00000000000..9ea6e5ba095 --- /dev/null +++ b/transformer_engine/pytorch/attention/sparse_attention/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Sparse attention. This feature is experimental and subject to change.""" + +from .dsv4_attention import DSv4Attention +from .compressor import Compressor +from .indexer import Indexer +from .dsa_cudnn import dense_indexer_loss_scores, dense_indexer_loss_backward + +__all__ = [ + "DSv4Attention", + "Compressor", + "Indexer", + "dense_indexer_loss_scores", + "dense_indexer_loss_backward", +] diff --git a/transformer_engine/pytorch/attention/sparse_attention/compressor.py b/transformer_engine/pytorch/attention/sparse_attention/compressor.py new file mode 100644 index 00000000000..d23435ac178 --- /dev/null +++ b/transformer_engine/pytorch/attention/sparse_attention/compressor.py @@ -0,0 +1,145 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Learned projections and normalization around cuDNN compression.""" + +import torch + +from transformer_engine.pytorch.module import Linear, RMSNorm + +from .dsa_cudnn import compress + + +class Compressor(torch.nn.Module): + """Project KV/gates for HCA or CSA, compress with cuDNN, and apply RMSNorm. + + Non-overlapping compression combines each block of ``ratio`` tokens into + one compressed KV entry. Overlapping compression mixes the current block + with the entire preceding block to create one compressed KV entry, using + separate learned KV/gate projections for the current and preceding blocks. + Neighboring compressed KV entries therefore share an entire source block, + but use different projections of its tokens. + Weights are computed per KV channel as ``softmax(gates + ape)``, where + ``ape`` is the learned additive position embedding. + + For ``ratio=2`` and ``[a b c d e f g h i j k]``: + + - Non-overlapping windows: ``[a b]``, ``[c d]``, ``[e f]``, ``[g h]``, ``[i j]``. + - Overlapping windows: ``[a b]``, ``[a b c d]``, ``[c d e f]``, + ``[e f g h]``, ``[g h i j]``. + + The first overlapping entry masks the missing previous block. Both modes emit + ``floor(length / ratio)`` compressed KV tokens per sequence. Trailing tokens + that do not fill a complete block (``k`` here) are excluded from compression. + In DSv4Attention, they remain available through local sliding-window attention. + + Accept batched ``[B,S,D]`` or packed ``thd`` hidden states ``[T,D]`` with + cumulative sequence token counts. Standalone calls do not interpret + sequence-major ``[S,B,D]`` input; DSv4Attention performs that conversion. + Return ``[total_comp,width]`` compressed KV representations + before RoPE, plus projected head weights when ``weight_width`` is nonzero. + + See the sparse_attention README for support requirements. + + Parameters + ---------- + hidden_size : int + Input hidden-state width. + width : int + Width of each compressed KV representation. + ratio : int + Number of input tokens per compression block. + overlap : bool + Whether each output also uses the preceding complete block. + eps : float + Epsilon for RMSNorm after compression. + device : torch.device or str + Device on which to allocate parameters. + params_dtype : torch.dtype + Dtype used to initialize projection, position-bias and normalization parameters. + weight_width : int, default = 0 + Additional per-input-token head-weight projection width. Indexer sets this + to n_heads for its scoring weights; DSv4Attention's main KV compressor + leaves it at zero. + """ + + def __init__( + self, + hidden_size, + width, + ratio, + overlap, + eps, + device, + params_dtype, + *, + weight_width=0, + ): + super().__init__() + projected = width * (2 if overlap else 1) + kw = {"bias": False, "device": device, "params_dtype": params_dtype} + # Fuse the KV, gate, and optional index-weight projections of hidden states. + # Keep one contiguous weight for the shared projection call. + self.fused_proj = Linear(hidden_size, 2 * projected + weight_width, **kw) + self.ape = torch.nn.Parameter( + torch.zeros(ratio, projected, device=device, dtype=params_dtype) + ) + self.kv_norm = RMSNorm(width, eps=eps, device=device, dtype=params_dtype) + self.ratio, self.overlap = ratio, overlap + self.projected, self.weight_width = projected, weight_width + + def forward(self, hidden_states, cu_seqlens, cu_seqlens_comp, *, total_comp): + """Project and compress KV blocks before RoPE. + + Parameters + ---------- + hidden_states : torch.Tensor + Batched ``[B,S,hidden_size]`` with equal sequence length S, or packed ``thd`` + ``[T,hidden_size]`` with sequences concatenated along the token dimension. + cu_seqlens, cu_seqlens_comp : torch.Tensor + Contiguous CUDA INT32 ``[B+1]`` cumulative input/compressed token counts, + starting at zero. Each compressed count increment is the corresponding + input sequence length divided by ratio, rounded down. + total_comp : int + Allocated compressed output length. For ``[B,S,hidden_size]`` input, use + exactly ``B * (S // ratio)``; DSv4Attention allocates ``T // ratio`` for ``thd`` + input, which may include output padding after the + valid compressed tokens counted by ``cu_seqlens_comp[-1]``. + + Returns + ------- + torch.Tensor or tuple of torch.Tensor + Normalized compressed KV ``[total_comp,width]``. If ``weight_width > 0``, + return ``(compressed_kv, weights)`` with per-input-token head weights of + shape ``hidden_states.shape[:-1] + (weight_width,)``. Indexer uses this + projection for scoring; the main KV compressor does not. Only compressed + token indices below ``cu_seqlens_comp[-1]`` are valid; the rest is padding. + """ + projected = self.fused_proj(hidden_states) + widths = ( + (self.projected, self.projected, self.weight_width) + if self.weight_width + else (self.projected, self.projected) + ) + parts = projected.split(widths, dim=-1) + kv, gate = parts[:2] + compressed_kv = compress( + kv.reshape(-1, self.projected).contiguous(), + gate.reshape(-1, self.projected).contiguous(), + # cuDNN requires FP32 APE. The cast preserves gradients to the parameter. + self.ape.float(), + cu_seqlens, + cu_seqlens_comp, + ratio=self.ratio, + overlap=self.overlap, + total_comp=total_comp, + ) + if hidden_states.ndim == 2: + # cuDNN guarantees valid output only for the cu_seqlens_comp[-1] compressed tokens. + # Zero output from cu_seqlens_comp[-1] onward before RMSNorm so + # uninitialized values cannot affect weight gradients. + valid = torch.arange(total_comp, device=hidden_states.device) < cu_seqlens_comp[-1] + compressed_kv = compressed_kv.masked_fill(~valid[:, None], 0) + compressed = self.kv_norm(compressed_kv) + return (compressed, parts[2]) if self.weight_width else compressed diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py new file mode 100644 index 00000000000..d9ab8cc94fd --- /dev/null +++ b/transformer_engine/pytorch/attention/sparse_attention/dsa_cudnn.py @@ -0,0 +1,620 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""PyTorch wrappers for cuDNN DSA/CSA primitives.""" + +from importlib import import_module +from typing import Optional + +import torch +from torch.autograd.function import once_differentiable + +__all__ = [ + "compress", + "select_indices", + "sparse_attention", + "dense_indexer_loss_scores", + "dense_indexer_loss_backward", +] + + +def _namespace(name): + try: + return getattr(import_module("cudnn"), name) + except (ImportError, AttributeError) as exc: + raise ImportError( + f"Sparse attention requires nvidia-cudnn-frontend>=1.29.0 with the cudnn.{name}" + " namespace." + ) from exc + + +def _dsa_op(name): + try: + return getattr(_namespace("DSA"), name) + except AttributeError as exc: + raise ImportError( + f"Sparse attention requires cudnn.DSA.{name} from nvidia-cudnn-frontend>=1.29.0." + ) from exc + + +def _validate_tensors(reference, bf16, fp32, cu_seqlens, cu_seqlens_comp): + """Check tensor metadata without copying sequence boundaries to the CPU.""" + if not reference.is_cuda or torch.cuda.get_device_capability(reference.device) not in ( + (10, 0), + (10, 3), + (10, 7), + ): + raise ValueError( + "These cuDNN sparse attention kernels require an SM100, SM103, or SM107 GPU." + ) + for tensor in (reference, *bf16, *fp32, cu_seqlens, cu_seqlens_comp): + if tensor.device != reference.device or not tensor.is_contiguous(): + raise ValueError("Inputs must be contiguous on the same CUDA device.") + if any(t.dtype != torch.bfloat16 for t in (reference, *bf16)): + raise TypeError("Activations must be BF16.") + if any(t.dtype != torch.float32 for t in fp32): + raise TypeError("Position biases and sinks must be FP32.") + for offsets in (cu_seqlens, cu_seqlens_comp): + if offsets.dtype != torch.int32 or offsets.ndim != 1 or offsets.numel() < 2: + raise ValueError("Sequence prefixes must be INT32 [batch+1].") + if cu_seqlens.shape != cu_seqlens_comp.shape: + raise ValueError("Local and compressed prefixes must describe the same batch.") + + +class _Compress(torch.autograd.Function): + @staticmethod + def forward(ctx, kv, score, ape, cu_seqlens, cu_seqlens_comp, ratio, overlap, total_comp): + """Run cuDNN compression and save its backward inputs.""" + coff = 2 if overlap else 1 + if kv.ndim != 2 or kv.shape[-1] % coff: + raise ValueError("kv must have shape [tokens, coff * head_dim].") + head_dim = kv.shape[-1] // coff + result = _namespace("CSA").csa_compressor_forward_wrapper( + kv, + score, + ape, + cu_seqlens, + cu_seqlens_comp, + ratio=ratio, + head_dim=head_dim, + coff=coff, + total_comp=total_comp, + ) + ctx.save_for_backward(kv, score, ape, cu_seqlens, cu_seqlens_comp) + ctx.config = ratio, head_dim, coff + return result["out"] + + @staticmethod + @once_differentiable + def backward(ctx, grad_out): + """Propagate gradients through cuDNN compression.""" + ratio, head_dim, coff = ctx.config + # cuDNN owns scratch allocation and dAPE zeroing. Autograd may supply + # a strided/expanded gradient, while this kernel requires contiguous data. + result = _namespace("CSA").csa_compressor_backward_wrapper( + *ctx.saved_tensors, + grad_out.contiguous(), + ratio=ratio, + head_dim=head_dim, + coff=coff, + ) + return ( + result["grad_kv"], + result["grad_score"], + result["grad_ape"], + None, + None, + None, + None, + None, + ) + + +def compress( + kv: torch.Tensor, + score: torch.Tensor, + ape: torch.Tensor, + cu_seqlens: torch.Tensor, + cu_seqlens_comp: torch.Tensor, + *, + ratio: int, + overlap: bool, + total_comp: int, +) -> torch.Tensor: + """Compress token blocks with cuDNN forward and backward. + + Overlapping compression combines the current complete block with the previous + block; otherwise each block is compressed independently. Trailing input tokens + that do not fill a complete block are excluded from compression. This operation + accepts projected tensors; it does not apply the module's projections, RMSNorm + or RoPE. See the sparse_attention README for supported dtypes and devices. + + Parameters + ---------- + kv, score : torch.Tensor + Contiguous projected KV and gate scores of shape ``[T,coff*D]`` on the + same device. ``coff`` is 2 when ``overlap=True``, otherwise 1. + ape : torch.Tensor + FP32 learned additive position embedding ``[ratio,coff*D]``, + added to gate scores before softmax over each compression block. + cu_seqlens, cu_seqlens_comp : torch.Tensor + Contiguous CUDA INT32 ``[B+1]`` cumulative input/compressed token counts. + Start at zero; each increment is the corresponding sequence's token + count. Compressed counts are ``floor(sequence_length / ratio)``. + ratio : int + Positive number of input tokens per block producing one compressed token. + overlap : bool + Whether to include the previous block using its separate projections. + total_comp : int + Allocated compressed output length, at least ``cu_seqlens_comp[-1]``. + + Returns + ------- + torch.Tensor + Compressed KV ``[total_comp,D]`` before RMSNorm and RoPE. Only indices + below ``cu_seqlens_comp[-1]`` contain valid compressed tokens; remaining + output is padding. Autograd propagates gradients to kv, score and ape. + """ + if ratio <= 0 or total_comp < 0: + raise ValueError("ratio must be positive and total_comp nonnegative.") + if kv.ndim != 2 or score.shape != kv.shape or ape.shape != (ratio, kv.shape[-1]): + raise ValueError("Expected kv/score [T,coff*D] and ape [ratio,coff*D].") + _validate_tensors(kv, (score,), (ape,), cu_seqlens, cu_seqlens_comp) + return _Compress.apply( + kv, + score, + ape, + cu_seqlens, + cu_seqlens_comp, + ratio, + overlap, + total_comp, + ) + + +@torch.no_grad() +def select_indices( + query: torch.Tensor, + compressed_key: torch.Tensor, + weights: torch.Tensor, + cu_seqlens: torch.Tensor, + cu_seqlens_comp: torch.Tensor, + *, + top_k: int, + ratio: int, + max_seqlen: int, + max_compressed_seqlen: int, + scale: Optional[float] = None, +) -> torch.Tensor: + """Select compressed KV tokens using cuDNN's index scores and top-k operation. + + Inputs are the indexer's projected Q/K/head weights after compression/RoPE; + this operation does not project or compress inputs. It supports complete + sequences starting at position zero. See the sparse_attention README for + supported dtypes and devices. + + Parameters + ---------- + query : torch.Tensor + Contiguous index queries ``[T,H,128]``, with H=32 or 64. + compressed_key : torch.Tensor + Contiguous packed compressed index keys ``[Tc,128]``. + weights : torch.Tensor + Contiguous projected per-token head weights ``[T,H]``. + cu_seqlens, cu_seqlens_comp : torch.Tensor + Contiguous CUDA INT32 ``[B+1]`` cumulative query/compressed token counts, + starting at zero. Adjacent differences give each sequence's token count. + top_k : int + Positive maximum number of compressed KV token indices per query token. + ratio : int + Positive number of input tokens per compressed block. + max_seqlen : int + Maximum input sequence length, passed to cuDNN as ``max_seqlen_q``. + max_compressed_seqlen : int + Size passed to cuDNN as ``max_seqlen_k``. Use + ``(max_seqlen + ratio - 1) // ratio``, as Indexer does, even though + ``cu_seqlens_comp`` counts only complete compressed blocks. + scale : float, optional + Score multiplier. None uses ``query.shape[-1] ** -0.5``. Indexer includes + the positive head-normalization factor using + ``(head_dim * n_heads) ** -0.5``. Pass raw projected weights; scaling then + rounding weights back to their activation dtype can change close top-k results. + + Returns + ------- + torch.Tensor + CUDA INT32 ``[T,top_k]`` global indices into packed compressed_key, one + list per query token; -1 marks invalid entries. Indices address compressed + KV only, before adding the local KV offset. Selection has no autograd; + train the indexer with a separate caller-owned auxiliary loss. + """ + if query.ndim != 3 or query.shape[1] not in (32, 64) or query.shape[2] != 128: + raise ValueError("CSA index query must have shape [T,32 or 64,128].") + if compressed_key.ndim != 2 or compressed_key.shape[1] != 128: + raise ValueError("CSA index key must have shape [Tc,128].") + if weights.shape != query.shape[:2]: + raise ValueError("CSA index weights must have shape [T,H].") + if ratio <= 0 or top_k <= 0 or max_seqlen <= 0 or max_compressed_seqlen <= 0: + raise ValueError("CSA ratio, top_k and maximum sequence lengths must be positive.") + _validate_tensors(query, (compressed_key, weights), (), cu_seqlens, cu_seqlens_comp) + if scale is None: + scale = query.shape[-1] ** -0.5 + result = _dsa_op("indexer_forward_top_k_wrapper")( + query, + compressed_key.unsqueeze(1), + weights, + top_k=top_k, + ratio=ratio, + cu_seqlens_q=cu_seqlens, + cu_seqlens_k=cu_seqlens_comp, + max_seqlen_q=max_seqlen, + max_seqlen_k=max_compressed_seqlen, + precision="bf16", + topk_indices_global=True, + return_softmax=False, + sm_scale=scale, + ) + return result["indices"] + + +def _pad_index_heads(query, weights): + if query.ndim != 4 or query.shape[0] != 1 or query.shape[2] not in (32, 64): + raise ValueError("Dense indexer loss requires B1 [B,S,32 or 64,128] queries.") + if query.shape[-1] != 128 or weights.shape != query.shape[:3]: + raise ValueError("Dense indexer weights must match B1 index queries.") + if query.dtype != torch.bfloat16 or weights.dtype != torch.bfloat16: + raise TypeError("Dense indexer Q and weights must be BF16.") + pad = 64 - query.shape[2] + if pad: + return torch.nn.functional.pad(query, (0, 0, 0, pad)), torch.nn.functional.pad( + weights, (0, pad) + ) + return query, weights + + +def dense_indexer_loss_scores( + index_q, + index_k, + index_w, + attn_q, + compressed_kv, + full_lse, + *, + ratio: int, + index_scale: float, + attention_scale: float, +): + """Recompute indexer and teacher scores for a single-sequence auxiliary loss. + + See the sparse_attention README for support requirements and the loss example. + These primitives do not attach an autograd loss to the indexer tensors. + + Parameters + ---------- + index_q, index_k, index_w : torch.Tensor + Indexer query/key/head weights of shapes ``[1,S,H,128]``, ``[1,Sc,128]`` + and ``[1,S,H]``, with H=32 or 64. + attn_q, compressed_kv : torch.Tensor + Teacher attention queries ``[1,S,64,D]`` and compressed KV ``[1,Sc,D]``. + Use detached teacher tensors so the auxiliary loss trains only the indexer. + full_lse : torch.Tensor + Detached FP32 ``[1,S,64]`` log-sum-exp of teacher logits over the local + window, every eligible compressed KV token, and the sink. The saved sparse + attention LSE excludes the sink and covers only selected compressed tokens, + so it cannot be used as this dense teacher denominator. + ratio : int + Number of input tokens per compressed block. + index_scale, attention_scale : float + Score multipliers for the indexer and teacher attention, respectively. + + Returns + ------- + tuple of dict + ``(index_score, attn_score)`` cuDNN dictionaries with ``out`` and ``denom`` + tensors, passed to ``dense_indexer_loss_backward``. The caller owns the + auxiliary objective and propagation through its live indexer tensors. + """ + if any(t.dtype != torch.bfloat16 for t in (index_k, attn_q, compressed_kv)): + raise TypeError("Dense indexer keys and attention Q/K must be BF16.") + if full_lse.dtype != torch.float32 or full_lse.shape != attn_q.shape[:3]: + raise ValueError("full_lse must be FP32 [B,S,attention_heads].") + padded_q, padded_w = _pad_index_heads(index_q, index_w) + index_score = _dsa_op("dense_indexer_score_recompute_wrapper")( + padded_q, + index_k.unsqueeze(2), + padded_w, + sm_scale=index_scale, + ratio=ratio, + ) + attn_score = _dsa_op("dense_attn_score_recompute_wrapper")( + attn_q, + compressed_kv.unsqueeze(2), + full_lse, + attention_scale, + ratio=ratio, + ) + return index_score, attn_score + + +def dense_indexer_loss_backward( + index_q, + index_k, + index_w, + index_score, + attn_score, + grad_loss, + *, + ratio: int, + index_scale: float, + loss_coeff: float, +): + """Compute indexer gradients for KL loss averaged over query tokens, with B=1. + + Use the same tensors, compression ratio and index scale as in + ``dense_indexer_loss_scores``. See the sparse_attention README loss example. + + Parameters + ---------- + index_q, index_k, index_w : torch.Tensor + Live indexer query/key/head weights ``[1,S,H,128]``, ``[1,Sc,128]`` and + ``[1,S,H]``, with H=32 or 64. H=32 is padded internally for cuDNN. + index_score, attn_score : dict + Indexer and teacher dictionaries returned by ``dense_indexer_loss_scores``. + grad_loss : torch.Tensor + Scalar upstream gradient for the auxiliary loss. + ratio : int + Number of input tokens per compressed block. + index_scale : float + Index score multiplier used to compute index_score. + loss_coeff : float + Multiplier for the auxiliary loss and its gradients. + + Returns + ------- + tuple of torch.Tensor + ``(d_query, d_key, d_weights)`` matching the input tensor shapes. The caller + must propagate these gradients through the live indexer tensors, for example + with ``torch.autograd.backward``; this helper does not do that propagation. + """ + padded_q, padded_w = _pad_index_heads(index_q, index_w) + result = _dsa_op("dense_indexer_backward_wrapper")( + padded_q, + padded_w, + index_k, + attn_score["out"].clone(), + attn_score["denom"], + index_score["out"].clone(), + index_score["denom"], + grad_loss=grad_loss.contiguous(), + sm_scale=index_scale, + loss_coeff=loss_coeff, + ratio=ratio, + ) + heads = index_q.shape[2] + return ( + result["d_index_q"][:, :, :heads].contiguous(), + result["d_index_k"], + result["d_weights"][:, :, :heads].contiguous(), + ) + + +def _prepare_sparse_kv_indices( + query: torch.Tensor, + cu_seqlens: torch.Tensor, + cu_seqlens_comp: torch.Tensor, + window_size: int, + ratio: int, + indices: Optional[torch.Tensor], + max_compressed_seqlen: Optional[int], +) -> torch.Tensor: + """Build a list of KV token indices for each query token. + + Query is already in packed thd format [T,H,D]; only its token count and device are used. + This helper does not flatten it. Return INT32 [T,window_size+compressed_slots] + indices into torch.cat((local_kv, compressed_kv), dim=0). + + Local KV has T tokens. Compressed indices count from the start of the entire + packed compressed tensor, including preceding sequences. Add T to address + compressed KV after local KV in the concatenated tensor. Invalid entries are -1. + CSA supplies top-k indices from select_indices; HCA supplies indices=None and + this helper includes every eligible compressed token. + """ + total_q = query.shape[0] + device = query.device + lengths = cu_seqlens[1:] - cu_seqlens[:-1] + seq_ids = torch.repeat_interleave( + torch.arange(lengths.numel(), device=device), + lengths, + output_size=total_q, + ) + token = torch.arange(total_q, device=device) + seq_start = cu_seqlens[seq_ids] + position = token - seq_start + + local_position = position[:, None] - torch.arange( + window_size - 1, + -1, + -1, + device=device, + ) + local = seq_start[:, None] + local_position + local = torch.where(local_position >= 0, local, -1) + + comp_start = cu_seqlens_comp[seq_ids] + comp_end = cu_seqlens_comp[seq_ids + 1] + if indices is None: + if max_compressed_seqlen is None: + raise ValueError("max_compressed_seqlen is required for HCA attention.") + comp_position = torch.arange(max_compressed_seqlen, device=device)[None, :] + compressed = comp_start[:, None] + comp_position + valid = (compressed < comp_end[:, None]) & ( + (comp_position + 1) * ratio <= position[:, None] + 1 + ) + else: + if indices.ndim != 2 or indices.shape[0] != total_q: + raise ValueError("CSA indices must have shape [total_tokens, top_k].") + compressed = indices + comp_position = compressed - comp_start[:, None] + valid = ( + (compressed >= comp_start[:, None]) + & (compressed < comp_end[:, None]) + & ((comp_position + 1) * ratio <= position[:, None] + 1) + ) + # Example: lengths=[4,11,17], T=32; query 7 is position 3 of sequence two. + # With indices=None and max_compressed_seqlen=8 and 5, respectively: + # window=ratio=2: cu_comp=[0,2,7,15]; local=[6,7]; compressed=[2,3]. + # Combined KV indices: [6,7,34,35,-1,-1,-1,-1,-1,-1]. + # window=ratio=3: cu_comp=[0,1,4,9]; local=[5,6,7]; compressed=[1]. + # Combined KV indices: [5,6,7,33,-1,-1,-1,-1]. + # Ratio 3 illustrates only this helper's index arithmetic, not cuDNN compression support. + # Only compressed indices receive offset T; -1 marks invalid entries. + compressed = torch.where(valid, compressed + query.shape[0], -1) + # cuDNN accepts an arbitrary logical width and pads its internal kernel + # layout, so preserve only the model-visible local + compressed slots. + return torch.cat((local, compressed), dim=1).to(torch.int32).contiguous() + + +class _SparseAttention(torch.autograd.Function): + @staticmethod + def forward(ctx, query, kv, indices, sink, scale): + """Attend query [T,64,D] to concatenated KV [T+Tc,D] using INT32 KV token indices. + + indices [T,window_size+compressed_slots] contains one list per query token + addressing concatenated local + compressed KV, with -1 for invalid entries. + No compressed query is constructed. + """ + result = _dsa_op("sparse_attention_forward_wrapper")( + query, + kv, + indices, + attn_sink=sink, + softmax_scale=scale, + # Prefix LSE is only needed by the optional indexer-loss path. + indexer_topk=0, + ) + out, lse = result["out"], result["lse"] + ctx.save_for_backward(query, kv, out, lse, sink, indices) + ctx.scale = scale + return out + + @staticmethod + @once_differentiable + def backward(ctx, grad_out): + """Propagate output gradients through cuDNN sparse attention.""" + query, kv, out, lse, sink, indices = ctx.saved_tensors + result = _dsa_op("sparse_attention_backward_wrapper")( + query, + kv, + out, + grad_out.contiguous(), + lse, + sink, + indices, + softmax_scale=ctx.scale, + ) + return result["dq"], result["dkv"], None, result["d_sink"], None + + +def sparse_attention( + query: torch.Tensor, + local_kv: torch.Tensor, + compressed_kv: torch.Tensor, + sink: torch.Tensor, + cu_seqlens: torch.Tensor, + cu_seqlens_comp: torch.Tensor, + *, + window_size: int, + ratio: int, + indices: Optional[torch.Tensor] = None, + scale: Optional[float] = None, + max_compressed_seqlen: Optional[int] = None, +) -> torch.Tensor: + """Run joint local and compressed KV attention with autograd. + + This full-sequence interface requires one local KV token for each query token. + The caller prepares projected, normalized and rotated Q/KV. DSv4Attention + converts batched inputs to packed token order before calling this function. + See the sparse_attention README for supported dtypes and devices. + + Parameters + ---------- + query : torch.Tensor + Contiguous packed ``thd`` queries ``[T,64,D]``, with D=512 or 576. + local_kv : torch.Tensor + Contiguous local KV ``[T,D]``, in the same sequence/token order as query. + compressed_kv : torch.Tensor + Contiguous packed compressed KV ``[Tc,D]``. Only indices below + ``cu_seqlens_comp[-1]`` are valid; any remaining output is padding. + sink : torch.Tensor + FP32 sink logits ``[64]``, one per attention head. + cu_seqlens, cu_seqlens_comp : torch.Tensor + Contiguous CUDA INT32 ``[B+1]`` cumulative query/local KV and compressed KV + token counts. Start at zero; adjacent differences give sequence lengths. + window_size : int + Positive local window size including the current query token: local + positions satisfy ``max(0,q-window_size+1) <= k <= q`` within each sequence. + ratio : int + Positive compression block length. Compressed token j in a sequence is + eligible when ``(j+1)*ratio <= q+1`` for query position q. + indices : torch.Tensor, optional + CSA: CUDA INT32 ``[T,K]`` global token indices into packed compressed_kv + only, with -1 for invalid entries. The helper checks sequence bounds and + causality, then adds T for concatenated local + compressed KV. None selects + HCA behavior: include all eligible compressed tokens. See the example in + ``_prepare_sparse_kv_indices`` for the index spaces. + scale : float, optional + Attention score multiplier. None uses ``query.shape[-1] ** -0.5``. + max_compressed_seqlen : int, optional + Required for HCA: upper bound on complete compressed tokens in any sequence, + used to build the index lists without reading CUDA sequence counts on host. + Unused when indices is supplied. + + Returns + ------- + torch.Tensor + Attention output ``[T,64,512]`` before inverse RoPE/output projection. + Values use the first 512 KV channels. One softmax includes the visible local + and compressed KV tokens plus the sink. Autograd returns gradients for + query, both KV tensors and sink; selection indices have no gradients. + """ + if query.ndim != 3 or query.shape[1] != 64: + raise ValueError("cuDNN sparse attention requires query [T,64,D].") + if query.shape[-1] not in (512, 576): + raise ValueError("attention head dimension must be 512 or 576.") + if local_kv.ndim != 2 or compressed_kv.ndim != 2: + raise ValueError("local_kv and compressed_kv must be 2D packed tensors.") + if local_kv.shape[-1] != query.shape[-1] or compressed_kv.shape[-1] != query.shape[-1]: + raise ValueError("Q and both KV tensors must have the same QK dimension.") + if local_kv.shape[0] != query.shape[0]: + raise ValueError( + "Full-sequence DSv4 attention requires equal query and local KV token counts." + ) + if sink.shape != (query.shape[1],): + raise ValueError("sink must have shape [64].") + if window_size <= 0 or ratio <= 0: + raise ValueError("window_size and ratio must be positive.") + if indices is None and (max_compressed_seqlen is None or max_compressed_seqlen < 0): + raise ValueError("HCA requires a nonnegative max_compressed_seqlen.") + _validate_tensors(query, (local_kv, compressed_kv), (sink,), cu_seqlens, cu_seqlens_comp) + if indices is not None: + if indices.dtype != torch.int32 or indices.device != query.device: + raise ValueError("CSA indices must be CUDA INT32 on the query device.") + if scale is None: + scale = query.shape[-1] ** -0.5 + selected = _prepare_sparse_kv_indices( + query, + cu_seqlens, + cu_seqlens_comp, + window_size, + ratio, + indices, + max_compressed_seqlen, + ) + return _SparseAttention.apply( + query.contiguous(), + torch.cat((local_kv, compressed_kv), dim=0), + selected, + sink.contiguous(), + scale, + ) diff --git a/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py new file mode 100644 index 00000000000..d653d520a45 --- /dev/null +++ b/transformer_engine/pytorch/attention/sparse_attention/dsv4_attention.py @@ -0,0 +1,381 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""DSv4 CSA/HCA hidden-state attention layer.""" + +from typing import Optional + +import torch + +from transformer_engine.pytorch.module import Linear, RMSNorm + +from .compressor import Compressor +from .dsa_cudnn import sparse_attention as _sparse_attention +from ..rope import DSv4RotaryEmbedding, _validate_dsa_rope_width, apply_dsa_rope +from .indexer import Indexer + +__all__ = ["DSv4Attention"] + + +@torch.compile +def _compiled_q_rms_norm(q: torch.Tensor, eps: float) -> torch.Tensor: + """Compile DSv4's weightless per-head query normalization.""" + return torch.nn.functional.rms_norm(q, (q.shape[-1],), eps=eps) + + +def _packed_prefixes(hidden_states, cu_seqlens, max_seqlen, ratio): + """Build compressed sequence boundaries and an allocation length for packed thd input. + + Subtract adjacent cu_seqlens values to get sequence lengths, divide each by + ratio with rounding down, then prepend zero to their cumulative sum. TE allocates + floor(total_input_tokens / ratio) outputs without reading the device count on + the host; this can exceed the sum of per-sequence compressed counts. Return + (cu_comp, total_comp, max_seqlen). + """ + if not isinstance(cu_seqlens, torch.Tensor): + raise ValueError("THD requires contiguous INT32 cu_seqlens_q [B+1] on the input device.") + if ( + cu_seqlens.dtype != torch.int32 + or cu_seqlens.ndim != 1 + or cu_seqlens.numel() < 2 + or not cu_seqlens.is_contiguous() + or cu_seqlens.device != hidden_states.device + ): + raise ValueError("THD requires contiguous INT32 cu_seqlens_q [B+1] on the input device.") + if not isinstance(max_seqlen, int) or max_seqlen < ratio: + raise ValueError("THD requires integer max_seqlen_q >= compression_ratio.") + # The sum of per-sequence floors can be smaller than floor(T / ratio). + # TE allocates that upper bound without reading the CUDA count on the host; + # cuDNN uses cu_comp to exclude output padding beyond the valid compressed tokens. + lengths = cu_seqlens[1:] - cu_seqlens[:-1] + cu_comp = torch.cat((cu_seqlens.new_zeros(1), (lengths // ratio).cumsum(0, dtype=torch.int32))) + total_comp = hidden_states.shape[0] // ratio + if not total_comp: + raise ValueError("Sequences must contain at least one complete compression window.") + return cu_comp, total_comp, max_seqlen + + +class DSv4Attention(torch.nn.Module): + """DSv4 CSA/HCA attention in ``sbhd``, ``bshd``, or packed ``thd`` format. + + Composes TE projections/norms, Compressor, Indexer and RoPE with cuDNN sparse + attention. Shared-input projections are fused. See the sparse_attention README + for support and training requirements. + + Parameters + ---------- + hidden_size : int + Input and output hidden-state width. + q_lora_rank : int + Width of the query-down projection and its residual. + layer_type : str + ``compressed_sparse_attention`` (CSA, with an indexer) or + ``heavily_compressed_attention`` (HCA, without top-k selection). + head_dim : int + Attention head width; currently 512, with 64 heads. + rope_head_dim : int + Number of trailing channels rotated by RoPE, a power of two >= 4 no greater + than head_dim, and no greater than 128 for CSA's index queries/keys. + sliding_window : int + Local causal attention window size, including the current token. + compression_ratio : int + Number of input tokens per compressed block. + o_groups : int + Number of output-projection groups; must divide 64 * head_dim. + o_lora_rank : int + Intermediate output-projection width per group. + index_n_heads : int, default = 64 + CSA index heads, either 32 or 64; unused for HCA. + index_topk : int, default = 512 + Maximum compressed KV tokens selected per CSA query; unused for HCA. + rope_theta : float, default = 160000.0 + Frequency base for the default rotary embeddings; ignored with custom rope. + rope : torch.nn.Module, optional + Custom module called as ``rope(sequence_length, device)`` returning token + and compressed ``(cos, sin)`` pairs. None uses the default embeddings. + rms_norm_eps : float, default = 1e-6 + Epsilon for query and KV normalization. + max_seqlen : int, optional + Initial length of the default RoPE cache. The cache grows when needed; + this does not replace forward's max_seqlen_q for packed ``thd`` input. + device : torch.device or str, default = "cuda" + Device on which to allocate parameters and default RoPE buffers. + params_dtype : torch.dtype, default = torch.float32 + Dtype used to initialize parameters; activation support is described in + the README independently of parameter initialization. + qkv_format : str, default = "sbhd" + Hidden-state layout: ``[S,B,hidden_size]``, ``[B,S,hidden_size]``, or + ``[T,hidden_size]`` for sbhd, bshd or thd. Sequence-major input is transposed + before calling the batched/packed Compressor and Indexer components. + """ + + def __init__( + self, + hidden_size: int, + q_lora_rank: int, + *, + layer_type: str, + head_dim: int, + rope_head_dim: int, + sliding_window: int, + compression_ratio: int, + o_groups: int, + o_lora_rank: int, + index_n_heads: int = 64, + index_topk: int = 512, + rope_theta: float = 160000.0, + rope: Optional[torch.nn.Module] = None, + rms_norm_eps: float = 1e-6, + max_seqlen: Optional[int] = None, + device: str = "cuda", + params_dtype: torch.dtype = torch.float32, + qkv_format: str = "sbhd", + ): + super().__init__() + # Keep the current kernel geometry explicit without exposing untested sizes. + self.num_heads = 64 + self.index_head_dim = 128 + if layer_type not in ("compressed_sparse_attention", "heavily_compressed_attention"): + raise ValueError("Only DSv4 CSA and HCA are supported.") + if qkv_format not in ("sbhd", "bshd", "thd"): + raise ValueError("qkv_format must be 'sbhd', 'bshd', or 'thd'.") + if head_dim != 512: + raise ValueError("head_dim must be 512.") + _validate_dsa_rope_width(rope_head_dim, head_dim) + if any( + v <= 0 + for v in ( + hidden_size, + q_lora_rank, + sliding_window, + compression_ratio, + o_groups, + o_lora_rank, + ) + ): + raise ValueError("DSv4 dimensions, window size and ratio must be positive.") + if self.num_heads * head_dim % o_groups: + raise ValueError("o_groups must divide the attention output channels.") + is_csa = layer_type == "compressed_sparse_attention" + if is_csa and ( + index_n_heads not in (32, 64) or index_topk <= 0 or rope_head_dim > self.index_head_dim + ): + raise ValueError( + "CSA requires 32 or 64 index heads, positive top-k, and RoPE width <= 128." + ) + + kw = {"bias": False, "device": device, "params_dtype": params_dtype} + # Query-down and local-KV projections read the same hidden states. + # Keep slices explicit below while one contiguous weight feeds the GEMM. + self.q_a_kv_proj = Linear( + hidden_size, + q_lora_rank + head_dim, + **kw, + ) + # LayerNormLinear would normalize before projection; DSv4 normalizes after it. + self.q_a_norm = RMSNorm(q_lora_rank, eps=rms_norm_eps, device=device, dtype=params_dtype) + self.q_b_proj = Linear(q_lora_rank, self.num_heads * head_dim, **kw) + self.kv_norm = RMSNorm(head_dim, eps=rms_norm_eps, device=device, dtype=params_dtype) + self.compressor = Compressor( + hidden_size, + head_dim, + compression_ratio, + is_csa, + rms_norm_eps, + device, + params_dtype, + ) + if is_csa: + self.indexer = Indexer( + hidden_size, + q_lora_rank, + self.index_head_dim, + index_n_heads, + index_topk, + compression_ratio, + rms_norm_eps, + device, + params_dtype, + ) + # HF/Megatron use one independent block per output group. A normal TE + # Linear forward mixes groups, so use its weight in the block multiply. + self.o_a_proj = Linear(self.num_heads * head_dim // o_groups, o_groups * o_lora_rank, **kw) + self.o_b_proj = Linear(o_groups * o_lora_rank, hidden_size, **kw) + self.sinks = torch.nn.Parameter( + torch.zeros(self.num_heads, device=device, dtype=params_dtype) + ) + self.sliding_window = sliding_window + # A model can supply its RoPE frequencies (for example YaRN) without + # duplicating that policy here. It must return the same token/compressed + # (cos, sin) pairs as DSv4RotaryEmbedding.forward. + self.rope = ( + rope + if rope is not None + else DSv4RotaryEmbedding( + compression_ratio, rope_head_dim, rope_theta, device, max_seqlen + ) + ) + self.hidden_size, self.q_lora_rank = hidden_size, q_lora_rank + self.head_dim, self.rope_head_dim = head_dim, rope_head_dim + self.rope_theta = rope_theta if rope is None else None + self.rms_norm_eps = rms_norm_eps + self.compression_ratio = compression_ratio + self.o_groups, self.o_lora_rank = o_groups, o_lora_rank + self.is_csa = is_csa + self.qkv_format = qkv_format + + def forward( + self, + hidden_states: torch.Tensor, + *, + cu_seqlens_q: Optional[torch.Tensor] = None, + max_seqlen_q: Optional[int] = None, + return_indexer_context: bool = False, + ): + """Run full DSv4 sparse-attention and optionally expose single-sequence CSA context. + + Parameters + ---------- + hidden_states : torch.Tensor + Input ``[S,B,hidden_size]`` for ``sbhd``, ``[B,S,hidden_size]`` for + ``bshd``, or ``[T,hidden_size]`` for packed ``thd``. + cu_seqlens_q : torch.Tensor, optional + Contiguous CUDA INT32 ``[B+1]`` cumulative sequence token counts, required + only for ``thd``. Start at zero, end at T, and do not decrease; adjacent + differences are sequence lengths. Tokens are packed without gaps. + The caller supplies these values; they are not checked on the host. + max_seqlen_q : int, optional + Maximum sequence length, required only for ``thd``. + return_indexer_context : bool, default = False + Return context for a caller-owned CSA auxiliary loss; requires B=1. + + Returns + ------- + torch.Tensor or tuple + Output with the input shape, or ``(output, context)``. Context contains + live ``index_q[T,H,128]``, ``index_k[Tc,128]``, ``index_w[T,H]`` and + discrete compressed KV token indices ``indices[T,K]``, before adding + the local KV offset; H is ``index_n_heads``. + Teacher ``attn_q[T,64,512]``, ``local_kv[T,512]``, + ``compressed_kv[Tc,512]`` and ``sink[64]`` are detached. Indexer inputs + are detached from the shared hidden/query projections, so auxiliary + gradients train only the indexer tower. See the README loss example. + """ + packed = self.qkv_format == "thd" + if ( + hidden_states.ndim != (2 if packed else 3) + or hidden_states.shape[-1] != self.hidden_size + ): + raise ValueError("hidden_states must have the selected format and hidden_size.") + from transformer_engine.pytorch.quantization import FP8GlobalStateManager + + if FP8GlobalStateManager.is_fp8_enabled(): + raise NotImplementedError("DSv4Attention supports BF16 only; disable TE FP8 autocast.") + if return_indexer_context and not self.is_csa: + raise ValueError("Only CSA has an indexer context.") + if self.qkv_format == "sbhd": + # cuDNN consumes tokens packed in batch-major order. B=1 needs only a view; + # larger batches require a physical repack with the current core. + hidden_states = hidden_states.transpose(0, 1).contiguous() + if packed: + cu = cu_seqlens_q + cu_comp, total_comp, seq = _packed_prefixes( + hidden_states, cu, max_seqlen_q, self.compression_ratio + ) + batch = cu.numel() - 1 + comp_shape = (total_comp,) + else: + if cu_seqlens_q is not None or max_seqlen_q is not None: + raise ValueError("cu_seqlens_q and max_seqlen_q are only supported with THD input.") + batch, seq, _ = hidden_states.shape + n_comp = seq // self.compression_ratio + if not n_comp: + raise ValueError("Sequence must contain at least one complete compression window.") + cu = torch.arange(batch + 1, device=hidden_states.device, dtype=torch.int32) * seq + cu_comp = ( + torch.arange(batch + 1, device=hidden_states.device, dtype=torch.int32) * n_comp + ) + total_comp = batch * n_comp + comp_shape = (batch, n_comp) + if return_indexer_context and batch != 1: + raise ValueError("The CSA indexer context is currently supported for batch=1.") + token_shape = hidden_states.shape[:-1] + + token_rope, compressed_rope = self.rope(seq, hidden_states.device) + + q_a, local_kv_projected = self.q_a_kv_proj(hidden_states).split( + (self.q_lora_rank, self.head_dim), dim=-1 + ) + q_residual = self.q_a_norm(q_a) + q = self.q_b_proj(q_residual).reshape(*token_shape, self.num_heads, self.head_dim) + # Query-up norm is unweighted; TE RMSNorm would add a learned scale. + q = _compiled_q_rms_norm(q, self.rms_norm_eps) + q = ( + apply_dsa_rope(q, *token_rope, cu, allow_inplace=True) + .reshape(-1, self.num_heads, self.head_dim) + .contiguous() + ) + local_kv = apply_dsa_rope(self.kv_norm(local_kv_projected).unsqueeze(-2), *token_rope, cu) + local_kv = local_kv.reshape(-1, self.head_dim).contiguous() + compressed_kv = apply_dsa_rope( + self.compressor(hidden_states, cu, cu_comp, total_comp=total_comp).reshape( + *comp_shape, 1, self.head_dim + ), + *compressed_rope, + cu_comp, + ) + compressed_kv = compressed_kv.reshape(total_comp, self.head_dim).contiguous() + + indices = None + index_q = index_k = index_w = None + if self.is_csa: + # Megatron trains this tower with a separate loss, without sending + # its gradients into the shared hidden/query projections. + index_result = self.indexer( + hidden_states.detach(), + q_residual.detach(), + cu, + cu_comp, + token_rope, + compressed_rope, + total_comp=total_comp, + max_seqlen=seq, + return_context=return_indexer_context, + ) + if return_indexer_context: + indices, index_q, index_k, index_w = index_result + else: + indices = index_result + + output = _sparse_attention( + q, + local_kv, + compressed_kv, + self.sinks.float(), + cu, + cu_comp, + window_size=self.sliding_window, + ratio=self.compression_ratio, + indices=indices, + max_compressed_seqlen=None if self.is_csa else seq // self.compression_ratio, + ).reshape(*token_shape, self.num_heads, self.head_dim) + cos, sin = token_rope + output = apply_dsa_rope(output, cos, -sin, cu).reshape(*token_shape, self.o_groups, -1) + weight = self.o_a_proj.weight.reshape(self.o_groups, self.o_lora_rank, -1) + grouped = torch.einsum("...gd,grd->...gr", output, weight).flatten(-2) + result = self.o_b_proj(grouped) + if self.qkv_format == "sbhd": + result = result.transpose(0, 1).contiguous() + if return_indexer_context: + return result, { + "index_q": index_q, + "index_k": index_k, + "index_w": index_w, + "indices": indices, + "attn_q": q.detach(), + "local_kv": local_kv.detach(), + "compressed_kv": compressed_kv.detach(), + "sink": self.sinks.float().detach(), + } + return result diff --git a/transformer_engine/pytorch/attention/sparse_attention/indexer.py b/transformer_engine/pytorch/attention/sparse_attention/indexer.py new file mode 100644 index 00000000000..21b2cea604b --- /dev/null +++ b/transformer_engine/pytorch/attention/sparse_attention/indexer.py @@ -0,0 +1,162 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Index projections, compression, and cuDNN compressed KV selection.""" + +import torch + +from transformer_engine.pytorch.module import Linear + +from .compressor import Compressor +from .dsa_cudnn import select_indices +from ..rope import apply_dsa_rope + + +class Indexer(torch.nn.Module): + """Select compressed KV tokens using the indexer's own Q/K/weight projections. + + Accept batched [B,S,D] or packed thd [T,D] hidden states and matching query residuals. + Standalone calls do not interpret sequence-major [S,B,D] input; DSv4Attention + transposes that layout first. Return global compressed KV token indices; + return_context=True also returns live Q/K/head weights for a caller-owned loss. + See the sparse_attention README for support and training requirements. + + Parameters + ---------- + hidden_size : int + Input hidden-state width for compressed-key and head-weight projections. + q_lora_rank : int + Width of the query-down residual supplied to forward. + head_dim : int + Index query/key width; the current selection operation uses 128. + n_heads : int + Number of index heads, 32 or 64. Also sets the per-token scoring-weight width. + top_k : int + Maximum compressed KV tokens to select per query token. + ratio : int + Input tokens per compression block. Indexer uses overlapping compression. + eps : float + Epsilon for the compressed-key RMSNorm. + device : torch.device or str + Device on which to allocate parameters. + params_dtype : torch.dtype + Dtype used to initialize projection and compression parameters. + """ + + def __init__( + self, + hidden_size, + q_lora_rank, + head_dim, + n_heads, + top_k, + ratio, + eps, + device, + params_dtype, + ): + super().__init__() + self.compressor = Compressor( + hidden_size, + head_dim, + ratio, + True, + eps, + device, + params_dtype, + weight_width=n_heads, + ) + kw = {"bias": False, "device": device, "params_dtype": params_dtype} + self.q_proj = Linear(q_lora_rank, n_heads * head_dim, **kw) + self.head_dim, self.n_heads = head_dim, n_heads + self.top_k, self.ratio = top_k, ratio + + def forward( + self, + hidden_states, + q_residual, + cu, + cu_comp, + token_rope, + compressed_rope, + *, + total_comp, + max_seqlen, + return_context=False, + ): + """Select compressed KV entries and optionally expose live indexer tensors. + + Parameters + ---------- + hidden_states : torch.Tensor + Batched ``[B,S,hidden_size]`` with equal sequence length S, or packed ``thd`` + ``[T,hidden_size]`` with concatenated sequences. + q_residual : torch.Tensor + Query-down residual with matching leading axes and width ``q_lora_rank``. + cu, cu_comp : torch.Tensor + CUDA INT32 ``[B+1]`` cumulative input/compressed token counts, starting + at zero. Each compressed increment is the input sequence length + divided by ratio, rounded down. + token_rope, compressed_rope : tuple of torch.Tensor + FP32 cosine/sine pairs ``[1,max_positions,1,rotary_width]`` for + token and compressed-window positions, respectively. + total_comp : int + Allocated compressed output length: exactly ``B * (S // ratio)`` for + ``[B,S,hidden_size]`` input; packed ``thd`` output may include padding after the valid + compressed KV tokens. + max_seqlen : int + Maximum sequence length used to bound selection. For ``[B,S,hidden_size]`` + input this must equal S; for packed ``thd`` input use the longest sequence length. + return_context : bool, default = False + Also return differentiable Q/K/head weights for a caller-owned loss. + + Returns + ------- + torch.Tensor or tuple of torch.Tensor + INT32 ``[T,min(top_k,max_seqlen // ratio)]`` global compressed KV token + indices addressing compressed KV only, with invalid entries set to -1. + With context, return + ``(indices, query[T,n_heads,head_dim], key[total_comp,head_dim], + weights[T,n_heads])``. Selected token indices do not carry gradients. + """ + n_comp = max_seqlen // self.ratio + comp_shape = (total_comp,) if hidden_states.ndim == 2 else (hidden_states.shape[0], n_comp) + index_compressed, weights = self.compressor( + hidden_states, cu, cu_comp, total_comp=total_comp + ) + # The singleton axis is the shared index-key head, not a tunable head count. + key = ( + apply_dsa_rope( + index_compressed.reshape(*comp_shape, 1, self.head_dim), + *compressed_rope, + cu_comp, + ) + .reshape(total_comp, self.head_dim) + .contiguous() + ) + query = self.q_proj(q_residual).reshape( + *hidden_states.shape[:-1], self.n_heads, self.head_dim + ) + query = ( + apply_dsa_rope(query, *token_rope, cu) + .reshape(-1, self.n_heads, self.head_dim) + .contiguous() + ) + # Top-k token indices are discrete; training index projections needs a separate loss. + weights = weights.reshape(-1, self.n_heads).contiguous() + indices = select_indices( + query, + key, + weights, + cu, + cu_comp, + top_k=min(self.top_k, n_comp), + ratio=self.ratio, + max_seqlen=max_seqlen, + # Pass ceil(max_seqlen / ratio) as cuDNN max_seqlen_k; cu_comp still + # counts only complete blocks, using floor(length / ratio) per sequence. + max_compressed_seqlen=(max_seqlen + self.ratio - 1) // self.ratio, + scale=(self.head_dim * self.n_heads) ** -0.5, + ) + return (indices, query, key, weights) if return_context else indices