From 6e8155c42e222932c13c962e38b2ae067dd9fd06 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Thu, 5 Nov 2026 18:40:49 -0800 Subject: [PATCH 01/34] Add shared linear attention CP utilities Signed-off-by: Kshitij Lakhani --- tests/cpp/operator/CMakeLists.txt | 1 + tests/cpp/operator/test_linear_attention.cu | 79 ++++++++++++ transformer_engine/common/CMakeLists.txt | 1 + .../transformer_engine/linear_attention.h | 43 +++++++ .../common/linear_attention/__init__.py | 119 +++++++++++++++++ .../common/linear_attention/affine_prefix.cu | 121 ++++++++++++++++++ transformer_engine/pytorch/csrc/extensions.h | 4 + .../pytorch/csrc/extensions/attention.cpp | 23 ++++ .../pytorch/csrc/extensions/pybind.cpp | 3 + 9 files changed, 394 insertions(+) create mode 100644 tests/cpp/operator/test_linear_attention.cu create mode 100644 transformer_engine/common/include/transformer_engine/linear_attention.h create mode 100644 transformer_engine/common/linear_attention/__init__.py create mode 100644 transformer_engine/common/linear_attention/affine_prefix.cu diff --git a/tests/cpp/operator/CMakeLists.txt b/tests/cpp/operator/CMakeLists.txt index 599b2f892c1..36c368ff298 100644 --- a/tests/cpp/operator/CMakeLists.txt +++ b/tests/cpp/operator/CMakeLists.txt @@ -47,6 +47,7 @@ add_executable(test_operator test_swizzle_block_scaling.cu test_swap_first_dims.cu test_grouped_gemm.cu + test_linear_attention.cu ../test_common.cu) # Find required packages diff --git a/tests/cpp/operator/test_linear_attention.cu b/tests/cpp/operator/test_linear_attention.cu new file mode 100644 index 00000000000..aa395ecc162 --- /dev/null +++ b/tests/cpp/operator/test_linear_attention.cu @@ -0,0 +1,79 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +#include +#include + +#include +#include +#include + +#include "../test_common.h" + +namespace { + +class LinearAttentionPrefixTest : public ::testing::TestWithParam {}; + +TEST_P(LinearAttentionPrefixTest, OrderedAffineRecipes) { + using test::Tensor; + using transformer_engine::DType; + const size_t k = GetParam(), heads = 2, values = 5, row_size = heads * values * k; + Tensor seeds("seeds", std::vector{3, heads, values, k}, DType::kFloat32); + Tensor offsets("offsets", std::vector{6, heads, values, k}, DType::kFloat32); + Tensor matrices("matrices", std::vector{6, heads, k, k}, DType::kFloat32); + Tensor descriptors("descriptors", std::vector{6, 4}, DType::kInt32); + Tensor output("output", std::vector{6, heads, values, k}, DType::kFloat32); + std::mt19937 generator(317); + std::uniform_real_distribution distribution(-1.f, 1.f); + auto *s = seeds.rowwise_cpu_dptr(); + auto *h = offsets.rowwise_cpu_dptr(); + auto *t = matrices.rowwise_cpu_dptr(); + auto *d = descriptors.rowwise_cpu_dptr(); + for (size_t i = 0; i < 3 * row_size; ++i) s[i] = distribution(generator); + for (size_t i = 0; i < 6 * row_size; ++i) h[i] = distribution(generator); + for (size_t i = 0; i < 6 * heads * k * k; ++i) t[i] = distribution(generator) / sqrt(k); + const int rows[] = {0, 0, 0, 1, 1, 0, 3, 1, 2, 5, 3, -2, + -1, 0, 0, 1, -1, 0, 3, 1, -1, 5, 3, -2}; + std::copy(rows, rows + 24, d); + seeds.from_cpu(); + offsets.from_cpu(); + matrices.from_cpu(); + descriptors.from_cpu(); + nvte_linear_attention_affine_prefix(seeds.data(), offsets.data(), matrices.data(), + descriptors.data(), output.data(), nullptr); + ASSERT_EQ(cudaDeviceSynchronize(), cudaSuccess); + output.to_cpu(); + const auto *actual = output.rowwise_cpu_dptr(); + for (size_t span = 0; span < 6; ++span) { + for (size_t head = 0; head < heads; ++head) { + for (size_t value = 0; value < values; ++value) { + const size_t row = head * values + value; + std::vector state(k, 0.), next(k); + if (rows[span * 4] >= 0) { + const float *start = s + rows[span * 4] * row_size + row * k; + std::copy(start, start + k, state.begin()); + } + for (int step = 0; step < rows[span * 4 + 2]; ++step) { + const int record = rows[span * 4 + 1] + step * rows[span * 4 + 3]; + for (size_t col = 0; col < k; ++col) { + next[col] = h[record * row_size + row * k + col]; + for (size_t inner = 0; inner < k; ++inner) { + next[col] += state[inner] * t[(record * heads + head) * k * k + inner * k + col]; + } + } + state.swap(next); + } + for (size_t col = 0; col < k; ++col) { + ASSERT_NEAR(actual[span * row_size + row * k + col], state[col], 2e-5); + } + } + } + } +} + +INSTANTIATE_TEST_SUITE_P(Common, LinearAttentionPrefixTest, ::testing::Values(3, 64, 128, 257)); + +} // namespace diff --git a/transformer_engine/common/CMakeLists.txt b/transformer_engine/common/CMakeLists.txt index 39bd2083d9a..884c209b1e4 100644 --- a/transformer_engine/common/CMakeLists.txt +++ b/transformer_engine/common/CMakeLists.txt @@ -300,6 +300,7 @@ list(APPEND transformer_engine_cuda_sources transpose/swap_first_dims.cu dropout/dropout.cu fused_attn/context_parallel.cu + linear_attention/affine_prefix.cu fused_attn/kv_cache.cu fused_attn/fused_attn_f16_arbitrary_seqlen.cu fused_attn/fused_attn_fp8.cu diff --git a/transformer_engine/common/include/transformer_engine/linear_attention.h b/transformer_engine/common/include/transformer_engine/linear_attention.h new file mode 100644 index 00000000000..8b2be9e76c7 --- /dev/null +++ b/transformer_engine/common/include/transformer_engine/linear_attention.h @@ -0,0 +1,43 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +/*! \file linear_attention.h + * \brief Framework-independent linear-attention state composition. + */ +#ifndef TRANSFORMER_ENGINE_LINEAR_ATTENTION_H_ +#define TRANSFORMER_ENGINE_LINEAR_ATTENTION_H_ + +#include "transformer_engine.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/*! \brief Apply ordered affine summaries to selected recurrent states. + * + * Each output starts from a selected seed and repeatedly evaluates + * ``state = state @ transition[record] + additive[record]``. This operation + * serves both forward prefixes and backward suffixes; backward callers pass + * the backward-oriented transitions and a negative record stride. + * + * All tensors are contiguous device tensors. Seeds and additives are FP32 + * [seeds, heads, value_dim, key_dim] and [records, heads, value_dim, key_dim]. + * Transitions are FP32 [records, heads, key_dim, key_dim]. Descriptors are + * INT32 [outputs, 4], with rows (seed_index, first_record, count, stride). + * Outputs are FP32 [outputs, heads, value_dim, key_dim]. Referenced seed and + * record indices must be valid; count must be nonnegative. + * A seed_index of -1 means an implicit zero state. Zero-count descriptors + * copy their seed (or write zeros). No memory is allocated. + */ +void nvte_linear_attention_affine_prefix(const NVTETensor seeds, const NVTETensor additive, + const NVTETensor transitions, const NVTETensor descriptors, + NVTETensor output, cudaStream_t stream); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TRANSFORMER_ENGINE_LINEAR_ATTENTION_H_ diff --git a/transformer_engine/common/linear_attention/__init__.py b/transformer_engine/common/linear_attention/__init__.py new file mode 100644 index 00000000000..27bd5185960 --- /dev/null +++ b/transformer_engine/common/linear_attention/__init__.py @@ -0,0 +1,119 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Framework-independent metadata for contiguous linear-attention partitions.""" + +from bisect import bisect_right +from dataclasses import dataclass +from typing import Tuple + + +@dataclass(frozen=True) +class LinearAttentionFragment: + """Intersection of one independent document with one rank's token interval.""" + + document: int + start: int + end: int + first_rank: int + last_rank: int + + +@dataclass(frozen=True) +class LinearAttentionCPPartition: + """Host metadata, independent of tensor frameworks and attention flavors. + + Packed inputs intersect document intervals with contiguous rank intervals. + Dense inputs give every rank a contiguous, nearly equal slice of each document. + Empty documents have no fragments; their state is unchanged. + """ + + documents: Tuple[int, ...] + rank: int + size: int + fragments: Tuple[LinearAttentionFragment, ...] + local_cu_seqlens: Tuple[int, ...] + forward_descriptors: Tuple[Tuple[int, int, int, int], ...] + backward_descriptors: Tuple[Tuple[int, int, int, int], ...] + outgoing: Tuple[int, ...] + incoming: Tuple[int, ...] + terminal: Tuple[int, ...] + seed_owners: Tuple[int, ...] + empty_owned: Tuple[int, ...] + slots: int + communicates: bool + + +def build_linear_attention_partition(documents, size, rank, *, dense=False): + """Build ordered fragment intersections and prefix/suffix record ranges. + + ``documents`` are global cumulative offsets. Packed rank r owns + [floor(r*T/P), floor((r+1)*T/P)). Dense documents must all have the same + length. Descriptor rows are (document, first, count, stride). + Their record ranges never cross an independent document's boundary. + """ + documents = tuple(documents) + if size < 1 or not 0 <= rank < size: + raise ValueError("CP size must be positive and rank must lie inside the group.") + if len(documents) < 2 or documents[0] != 0: + raise ValueError("Global cumulative sequence lengths must start at zero.") + if any(a > b for a, b in zip(documents, documents[1:])): + raise ValueError("Global cumulative sequence lengths must be nondecreasing.") + total = documents[-1] + boundaries = tuple(r * total // size for r in range(size + 1)) + slots = len(documents) - 1 if dense else 1 + fragments, seeds, empty = [], [], [] + communicates = False + for doc, (start, end) in enumerate(zip(documents, documents[1:])): + if dense: + length = end - start + if length != documents[1]: + raise ValueError("Dense CP needs equal global document lengths.") + cuts = tuple(r * length // size for r in range(size + 1)) + first = min(size - 1, bisect_right(cuts, 0) - 1) if length else 0 + last = min(size - 1, bisect_right(cuts, length - 1) - 1) if length else 0 + left, right = start + cuts[rank], start + cuts[rank + 1] + else: + first = min(size - 1, bisect_right(boundaries, start) - 1) + last = min(size - 1, bisect_right(boundaries, end - 1) - 1) if end > start else first + left, right = max(start, boundaries[rank]), min(end, boundaries[rank + 1]) + seeds.append(first) + communicates |= first != last + if start == end and first == rank: + empty.append(doc) + if left < right: + fragments.append(LinearAttentionFragment(doc, left, right, first, last)) + cu = [0] + forward, backward, outgoing, incoming, terminal = [], [], [], [], [] + for index, frag in enumerate(fragments): + cu.append(cu[-1] + frag.end - frag.start) + slot = frag.document if dense else 0 + forward.append( + (frag.document, frag.first_rank * slots + slot, rank - frag.first_rank, slots) + ) + backward.append( + (frag.document, frag.last_rank * slots + slot, frag.last_rank - rank, -slots) + ) + if rank < frag.last_rank: + outgoing.append(index) + else: + terminal.append(index) + if rank > frag.first_rank: + incoming.append(index) + return LinearAttentionCPPartition( + documents, + rank, + size, + tuple(fragments), + tuple(cu), + tuple(forward), + tuple(backward), + tuple(outgoing), + tuple(incoming), + tuple(terminal), + tuple(seeds), + tuple(empty), + slots, + communicates, + ) diff --git a/transformer_engine/common/linear_attention/affine_prefix.cu b/transformer_engine/common/linear_attention/affine_prefix.cu new file mode 100644 index 00000000000..0ae6f7cc4d0 --- /dev/null +++ b/transformer_engine/common/linear_attention/affine_prefix.cu @@ -0,0 +1,121 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +#include + +#include "../common.h" +#include "transformer_engine/linear_attention.h" + +namespace transformer_engine { +namespace { + +// Reuse transition loads across value rows, preserving FP32 accumulation. +constexpr int kRows = 4; + +__global__ void affine_prefix_kernel(const float *seeds, const float *additive, + const float *transitions, const int *descriptors, + float *output, size_t num_seeds, size_t num_records, + size_t heads, size_t value_dim, size_t key_dim) { + extern __shared__ float storage[]; + float *state = storage; + float *next = storage + kRows * key_dim; + const size_t tiles = (value_dim + kRows - 1) / kRows; + const size_t span = blockIdx.x / (heads * tiles); + const size_t head = (blockIdx.x / tiles) % heads; + const size_t first_value = (blockIdx.x % tiles) * kRows; + const int *desc = descriptors + span * 4; + const int seed = desc[0], first = desc[1], count = desc[2], stride = desc[3]; + assert((seed == -1 || (seed >= 0 && static_cast(seed) < num_seeds)) && count >= 0); + // An implicit zero seed makes the first matrix product unnecessary. + const int begin = seed == -1 && count > 0 ? 1 : 0; + for (size_t col = threadIdx.x; col < key_dim; col += blockDim.x) { + for (int row = 0; row < kRows; ++row) { + const size_t value = first_value + row; + float initial = 0.f; + if (value < value_dim) { + if (begin) { + assert(first >= 0 && static_cast(first) < num_records); + initial = additive[((first * heads + head) * value_dim + value) * key_dim + col]; + } else if (seed >= 0) { + initial = seeds[((seed * heads + head) * value_dim + value) * key_dim + col]; + } + } + state[row * key_dim + col] = initial; + } + } + __syncthreads(); + for (int step = begin; step < count; ++step) { + const int record = first + step * stride; + assert(record >= 0 && static_cast(record) < num_records); + const float *matrix = transitions + (record * heads + head) * key_dim * key_dim; + for (size_t col = threadIdx.x; col < key_dim; col += blockDim.x) { + float result[kRows]; + for (int row = 0; row < kRows; ++row) { + const size_t value = first_value + row; + result[row] = value < value_dim + ? additive[((record * heads + head) * value_dim + value) * key_dim + col] + : 0.f; + } + for (size_t inner = 0; inner < key_dim; ++inner) { + const float transition = matrix[inner * key_dim + col]; +#pragma unroll + for (int row = 0; row < kRows; ++row) { + result[row] = fmaf(state[row * key_dim + inner], transition, result[row]); + } + } + for (int row = 0; row < kRows; ++row) next[row * key_dim + col] = result[row]; + } + __syncthreads(); + float *swap = state; + state = next; + next = swap; + } + for (size_t col = threadIdx.x; col < key_dim; col += blockDim.x) { + for (int row = 0; row < kRows; ++row) { + const size_t value = first_value + row; + if (value < value_dim) { + output[((span * heads + head) * value_dim + value) * key_dim + col] = + state[row * key_dim + col]; + } + } + } +} + +} // namespace +} // namespace transformer_engine + +void nvte_linear_attention_affine_prefix(const NVTETensor seeds, const NVTETensor additive, + const NVTETensor transitions, const NVTETensor descriptors, + NVTETensor output, cudaStream_t stream) { + NVTE_API_CALL(nvte_linear_attention_affine_prefix); + using namespace transformer_engine; + const auto &s = convertNVTETensorCheck(seeds)->data; + const auto &a = convertNVTETensorCheck(additive)->data; + const auto &t = convertNVTETensorCheck(transitions)->data; + const auto &d = convertNVTETensorCheck(descriptors)->data; + auto &o = convertNVTETensorCheck(output)->data; + NVTE_CHECK(s.dtype == DType::kFloat32 && a.dtype == DType::kFloat32 && + t.dtype == DType::kFloat32 && o.dtype == DType::kFloat32, + "Affine states and summaries must be FP32."); + NVTE_CHECK(d.dtype == DType::kInt32, "Affine descriptors must be INT32."); + NVTE_CHECK(s.shape.size() == 4 && a.shape.size() == 4 && t.shape.size() == 4 && + o.shape.size() == 4 && d.shape.size() == 2, + "Invalid affine tensor ranks."); + const size_t heads = s.shape[1], value_dim = s.shape[2], key_dim = s.shape[3]; + NVTE_CHECK(heads > 0 && value_dim > 0 && key_dim > 0, "State dimensions must be positive."); + NVTE_CHECK((a.shape == std::vector{t.shape[0], heads, value_dim, key_dim}) && + (t.shape == std::vector{a.shape[0], heads, key_dim, key_dim}) && + (o.shape == std::vector{d.shape[0], heads, value_dim, key_dim}) && + d.shape[1] == 4, + "Inconsistent affine tensor shapes."); + if (o.shape[0] == 0) return; + const size_t tiles = o.shape[0] * heads * ((value_dim + kRows - 1) / kRows); + affine_prefix_kernel<<>>( + static_cast(s.dptr), static_cast(a.dptr), + static_cast(t.dptr), static_cast(d.dptr), + static_cast(o.dptr), s.shape[0], a.shape[0], heads, value_dim, key_dim); + NVTE_CHECK_CUDA(cudaGetLastError()); +} diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index 7bcd8430ce1..e9ef198dc6e 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -27,6 +27,10 @@ class CommOverlapP2P; namespace transformer_engine::pytorch { +at::Tensor linear_attention_affine_prefix(const at::Tensor &seeds, const at::Tensor &additive, + const at::Tensor &transitions, + const at::Tensor &descriptors); + /*************************************************************************************************** * Router fusion **************************************************************************************************/ diff --git a/transformer_engine/pytorch/csrc/extensions/attention.cpp b/transformer_engine/pytorch/csrc/extensions/attention.cpp index 98d55d2b8b5..d0f9f3d8fa6 100644 --- a/transformer_engine/pytorch/csrc/extensions/attention.cpp +++ b/transformer_engine/pytorch/csrc/extensions/attention.cpp @@ -7,9 +7,32 @@ #include "../extensions.h" #include "common.h" #include "pybind.h" +#include "transformer_engine/linear_attention.h" namespace transformer_engine::pytorch { +at::Tensor linear_attention_affine_prefix(const at::Tensor &seeds, const at::Tensor &additive, + const at::Tensor &transitions, + const at::Tensor &descriptors) { + for (const auto &tensor : {seeds, additive, transitions, descriptors}) { + NVTE_CHECK(tensor.is_cuda() && tensor.is_contiguous(), + "Affine prefix tensors must be contiguous CUDA tensors."); + NVTE_CHECK(tensor.device() == seeds.device(), "Affine prefix devices must agree."); + } + NVTE_CHECK(seeds.dim() == 4 && descriptors.dim() == 2, "Invalid affine prefix ranks."); + at::cuda::CUDAGuard device_guard(seeds.device()); + at::Tensor output = at::empty({descriptors.size(0), seeds.size(1), seeds.size(2), seeds.size(3)}, + seeds.options()); + auto s = makeTransformerEngineTensor(seeds); + auto a = makeTransformerEngineTensor(additive); + auto t = makeTransformerEngineTensor(transitions); + auto d = makeTransformerEngineTensor(descriptors); + auto o = makeTransformerEngineTensor(output); + nvte_linear_attention_affine_prefix(s.data(), a.data(), t.data(), d.data(), o.data(), + at::cuda::getCurrentCUDAStream()); + return output; +} + // get the fused attention backend std::tuple get_fused_attn_backend(const py::object &p) { FusedAttnConfigWrapper cfg; diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index 6fbed1436fb..2a354713102 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -646,6 +646,9 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { py::call_guard()); // Support THD format for Context Parallel + m.def("linear_attention_affine_prefix", + &transformer_engine::pytorch::linear_attention_affine_prefix, + "Apply ordered linear-attention state summaries", py::call_guard()); m.def("thd_read_half_tensor", &transformer_engine::pytorch::thd_read_half_tensor, "Read the first half(half_idx=0) or the second half(half_idx=1) of each sequence in a THD " "tensor", From bf36a95906a41cf444c0b030b5111dc6248ec245 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Thu, 5 Nov 2026 18:41:08 -0800 Subject: [PATCH 02/34] Add contiguous CP for PyTorch GDN Signed-off-by: Kshitij Lakhani --- .../attention/linear_attention/base.py | 45 +- .../linear_attention/context_parallel.py | 383 ++++++++++++++++++ .../pytorch/attention/linear_attention/gdn.py | 218 +++++++++- 3 files changed, 635 insertions(+), 11 deletions(-) create mode 100644 transformer_engine/pytorch/attention/linear_attention/context_parallel.py diff --git a/transformer_engine/pytorch/attention/linear_attention/base.py b/transformer_engine/pytorch/attention/linear_attention/base.py index f1dfe199eb4..3aafba5001d 100644 --- a/transformer_engine/pytorch/attention/linear_attention/base.py +++ b/transformer_engine/pytorch/attention/linear_attention/base.py @@ -65,6 +65,8 @@ def _needs_eager_linear_attention(call: Dict[str, Any]) -> Optional[str]: `call` maps the module's `forward` parameter names to the arguments this call passed, including `self`. """ + if getattr(call.get("self"), "cp_group", None) is not None: + return "linear-attention context parallelism" if call.get("checkpoint_core_attention", False): return "activation checkpointing of the attention" return None @@ -269,20 +271,16 @@ def _validate_initial_state( f"got {tuple(initial_state.shape)}." ) - def _run( + def _validate_inputs( self, query_layer: torch.Tensor, key_layer: torch.Tensor, value_layer: torch.Tensor, gates: Dict[str, torch.Tensor], - initial_state: Optional[torch.Tensor] = None, - *, + initial_state: Optional[torch.Tensor], qkv_format: str, - cu_seqlens: Optional[torch.Tensor] = None, - output_final_state: bool = False, - **kernel_kwargs: Any, - ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: - """Validate the inputs, run the variant's op, and restore the TE layout.""" + ) -> None: + """Validate tensor/layout contracts for local and context-parallel adapters.""" if qkv_format not in {"thd", "bshd", "sbhd"}: raise ValueError( f"{self.variant} attention only supports qkv_format={{'thd', 'bshd', 'sbhd'}}, " @@ -318,6 +316,37 @@ def _run( ) self._validate_gates(gates, query_layer) + def _prepare_cp_inputs(self, tensors, initial_state): + """Normalize a flavor's token-aligned CP inputs before state propagation.""" + return tensors, initial_state + + def _cp_forward_summary(self, query, key, value, gates, cu_seqlens, **kernel_kwargs): + """Return the fragment maps S_out = S_in @ T + H for this flavor.""" + raise NotImplementedError(f"{self.variant} does not implement CP forward summaries.") + + def _cp_backward_summary( + self, query, key, value, gates, grad_output, cu_seqlens, **kernel_kwargs + ): + """Return reverse state-gradient maps for this flavor.""" + raise NotImplementedError(f"{self.variant} does not implement CP backward summaries.") + + def _run( + self, + query_layer: torch.Tensor, + key_layer: torch.Tensor, + value_layer: torch.Tensor, + gates: Dict[str, torch.Tensor], + initial_state: Optional[torch.Tensor] = None, + *, + qkv_format: str, + cu_seqlens: Optional[torch.Tensor] = None, + output_final_state: bool = False, + **kernel_kwargs: Any, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + """Validate the inputs, run the variant's op, and restore the TE layout.""" + self._validate_inputs(query_layer, key_layer, value_layer, gates, initial_state, qkv_format) + device = query_layer.device + cu_seqlens, sequence_length = self._resolve_cu_seqlens(query_layer, qkv_format, cu_seqlens) batch_size = cu_seqlens.shape[0] - 1 self._validate_initial_state(initial_state, batch_size=batch_size, device=device) diff --git a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py new file mode 100644 index 00000000000..f12bba79a17 --- /dev/null +++ b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py @@ -0,0 +1,383 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Contiguous linear-attention CP: fragments, collectives and state gradients. + +Flavor adapters supply local attention and affine summary operations. The shared +CUDA operation composes states in document order. +""" + +from dataclasses import dataclass + +import torch +import torch.distributed as dist +from torch.autograd.function import once_differentiable + +import transformer_engine_torch as tex +from transformer_engine.common.linear_attention import ( + LinearAttentionCPPartition, + build_linear_attention_partition, +) +from .base import _to_thd, _validate_cu_seqlens + + +@dataclass +class _DevicePartition: + """Cached device metadata; the host partition has no framework dependency.""" + + host: LinearAttentionCPPartition + cu: torch.Tensor + forward: torch.Tensor + backward: torch.Tensor + forward_zero: torch.Tensor + backward_zero: torch.Tensor + docs: torch.Tensor + outgoing: torch.Tensor + incoming: torch.Tensor + terminal: torch.Tensor + seed_fragments: torch.Tensor + seed_docs: torch.Tensor + empty_docs: torch.Tensor + outgoing_cu: torch.Tensor + incoming_cu: torch.Tensor + + +def _device_partition(host, device): + """Materialize small metadata tensors once, outside the steady-state kernels.""" + + def indices(values): + return torch.tensor(values, device=device, dtype=torch.int64) + + def cumulative(selected): + offsets = [0] + for index in selected: + frag = host.fragments[index] + offsets.append(offsets[-1] + frag.end - frag.start) + return torch.tensor(offsets, device=device, dtype=torch.int32) + + first = [i for i, f in enumerate(host.fragments) if f.first_rank == host.rank] + forward = torch.tensor(host.forward_descriptors, device=device, dtype=torch.int32).reshape( + -1, 4 + ) + backward = torch.tensor(host.backward_descriptors, device=device, dtype=torch.int32).reshape( + -1, 4 + ) + forward_zero, backward_zero = forward.clone(), backward.clone() + forward_zero[:, 0] = -1 + backward_zero[:, 0] = -1 + return _DevicePartition( + host, + torch.tensor(host.local_cu_seqlens, device=device, dtype=torch.int32), + forward, + backward, + forward_zero, + backward_zero, + indices([f.document for f in host.fragments]), + indices(host.outgoing), + indices(host.incoming), + indices(host.terminal), + indices(first), + indices([host.fragments[i].document for i in first]), + indices(host.empty_owned), + cumulative(host.outgoing), + cumulative(host.incoming), + ) + + +def _gather_summaries(additive, transition, group): + """One fixed-size all-gather for both summary tensors, in CP group rank order.""" + slots, heads, value_dim, key_dim = additive.shape + size = dist.get_world_size(group) + packed = torch.cat((additive.flatten(), transition.flatten())) + gathered = torch.empty(size * packed.numel(), device=packed.device, dtype=packed.dtype) + dist.all_gather_into_tensor(gathered, packed, group=group) + gathered = gathered.view(size, -1) + split = additive.numel() + return ( + gathered[:, :split].contiguous().view(size * slots, heads, value_dim, key_dim), + gathered[:, split:].contiguous().view(size * slots, heads, key_dim, key_dim), + ) + + +def _summary_buffers(seed, slots): + """Unused slots are identity recipes, including ranks owning zero tokens.""" + _, heads, value_dim, key_dim = seed.shape + additive = seed.new_zeros((slots, heads, value_dim, key_dim)) + transition = torch.eye(key_dim, device=seed.device, dtype=seed.dtype) + return additive, transition.expand(slots, heads, key_dim, key_dim).contiguous() + + +def _selected_span(plan, selected): + """Selected spans are consecutive: all dense spans, or one packed edge span.""" + cu = plan.host.local_cu_seqlens + return slice(cu[selected[0]], cu[selected[-1] + 1]) + + +def _slot_indices(plan, indices): + """Dense slots are document IDs; packed partitions have only one edge slot.""" + if plan.host.slots == 1: + return torch.zeros_like(indices) + return plan.docs.index_select(0, indices) + + +def _local_attention(adapter, tensors, gate_names, cu, state, options, *, final=True): + """Call the flavor adapter with packed local inputs.""" + return adapter._call_op( + *tensors[:3], + dict(zip(gate_names, tensors[3:])), + cu, + initial_state=state, + output_final_state=final, + **options, + ) + + +class _LinearAttentionContextParallel(torch.autograd.Function): + """A distributed first-order VJP, with full local attention retained for FE backward. + + Summary APIs intentionally do not have autograd registrations. Differentiating + an all-gather of summaries therefore cannot recover cross-rank gradients. + The reverse summary exchange supplies the missing final-state adjoints to FE. + """ + + @staticmethod + def forward( + ctx, adapter, plan, group, options, final, gate_names, track_grad, initial_state, *tensors + ): + """Establish starting states, then retain the local attention graph.""" + ctx.set_materialize_grads(False) + q = tensors[0] + k = tensors[1] + v = tensors[2] + state_shape = (len(plan.host.documents) - 1, q.shape[1], v.shape[-1], k.shape[-1]) + seed = ( + q.new_zeros((1, *state_shape[1:]), dtype=torch.float32) + if initial_state is None + else initial_state + ) + additive, transition = _summary_buffers(seed, plan.host.slots) + if plan.host.outgoing: + span = _selected_span(plan, plan.host.outgoing) + selected = tuple(t[span] for t in tensors) + h, t = adapter._cp_forward_summary( + *selected[:3], dict(zip(gate_names, selected[3:])), plan.outgoing_cu, **options + ) + slots = _slot_indices(plan, plan.outgoing) + additive.index_copy_(0, slots, h) + transition.index_copy_(0, slots, t) + if plan.host.communicates: + additive, transition = _gather_summaries(additive, transition, group) + starting = tex.linear_attention_affine_prefix( + seed.contiguous(), + additive, + transition, + plan.forward if initial_state is not None else plan.forward_zero, + ) + # Retain FE's local graph only for training. This Function owns the + # distributed VJP and the outer state gradients. + with torch.set_grad_enabled(track_grad): + inputs = tuple(t.detach().requires_grad_(track_grad) for t in (*tensors, starting)) + if plan.host.fragments: + output, states = _local_attention( + adapter, inputs[:-1], gate_names, plan.cu, inputs[-1], options + ) + else: + output = v.new_empty((q.shape[0], q.shape[1], v.shape[-1])) + states = starting + terminal = seed.new_zeros(state_shape) if final else seed.new_empty((0,)) + if final: + terminal.index_copy_( + 0, + plan.docs.index_select(0, plan.terminal), + states.detach().index_select(0, plan.terminal), + ) + if initial_state is not None: + terminal.index_copy_(0, plan.empty_docs, seed.index_select(0, plan.empty_docs)) + dist.all_reduce(terminal, group=group) + if track_grad: + ctx.save_for_backward(*inputs, output, states) + ctx.plan, ctx.group, ctx.final = plan, group, final + ctx.adapter, ctx.options, ctx.gate_names = adapter, options, gate_names + ctx.has_initial_state = initial_state is not None + return output.detach(), terminal + + @staticmethod + @once_differentiable + def backward(ctx, grad_output, grad_final): + """Exchange reverse recipes and pass outgoing state adjoints to FE's VJP.""" + tensors = ctx.saved_tensors[:-3] + starting, output, states = ctx.saved_tensors[-3:] + q, k, v = tensors[:3] + plan, group = ctx.plan, ctx.group + if grad_output is None: + grad_output = torch.zeros_like(output) + grad_output = grad_output.contiguous() + shape = (len(plan.host.documents) - 1, q.shape[1], v.shape[-1], k.shape[-1]) + seed = starting.new_zeros(shape if ctx.final else (1, *shape[1:])) + if ctx.final: + if grad_final is not None: + seed.copy_(grad_final) + # Final states are replicated outputs: sum losses consuming any replica. + dist.all_reduce(seed, group=group) + additive, transition = _summary_buffers(seed, plan.host.slots) + if plan.host.incoming: + span = _selected_span(plan, plan.host.incoming) + selected = tuple(t[span] for t in tensors) + h, t = ctx.adapter._cp_backward_summary( + *selected[:3], + dict(zip(ctx.gate_names, selected[3:])), + grad_output[span], + plan.incoming_cu, + **ctx.options, + ) + slots = _slot_indices(plan, plan.incoming) + additive.index_copy_(0, slots, h) + transition.index_copy_(0, slots, t) + if plan.host.communicates: + additive, transition = _gather_summaries(additive, transition, group) + ending_grad = tex.linear_attention_affine_prefix( + seed, additive, transition, plan.backward if ctx.final else plan.backward_zero + ) + if plan.host.fragments: + with torch.enable_grad(): + grads = torch.autograd.grad( + (output, states), + (*tensors, starting), + (grad_output, ending_grad), + # The outer graph controls lifetime, including retain_graph=True. + retain_graph=True, + ) + else: + grads = tuple(torch.zeros_like(t) for t in (*tensors, starting)) + initial_grad = None + if ctx.has_initial_state: + initial_grad = seed.new_zeros(shape) + initial_grad.index_add_( + 0, plan.seed_docs, grads[-1].index_select(0, plan.seed_fragments) + ) + if ctx.final: + initial_grad.index_copy_(0, plan.empty_docs, seed.index_select(0, plan.empty_docs)) + # Each document's seed gradient belongs to its first rank. Do not + # replicate it here: if initial_state came from a previous CP call, + # that call's replicated-final-state VJP sums these owner gradients. + # Replicating now would count chained state gradients CP-size times. + return (None, None, None, None, None, None, None, initial_grad, *grads[:-1]) + + +def context_parallel_attention( + adapter, + q, + k, + v, + gates, + initial_state, + *, + group, + qkv_format, + cu_seqlens, + global_cu_seqlens, + output_final_state, + **kernel_kwargs, +): + """Validate local inputs, cache a partition, and restore the public output layout.""" + adapter._validate_inputs(q, k, v, gates, initial_state, qkv_format) + size, rank = dist.get_world_size(group), dist.get_rank(group) + tensors = tuple(_to_thd(t, qkv_format).contiguous() for t in (q, k, v, *gates.values())) + dense = qkv_format != "thd" + batch, length = None, None + if dense: + if cu_seqlens is not None: + raise ValueError("Dense linear-attention CP does not accept local cu_seqlens.") + batch, length = q.shape[:2] if qkv_format == "bshd" else (q.shape[1], q.shape[0]) + elif not isinstance(global_cu_seqlens, torch.Tensor): + raise ValueError( + "Linear-attention CP with THD requires global_cu_seqlens for all documents." + ) + if cu_seqlens is not None: + _validate_cu_seqlens(cu_seqlens, device=q.device, name="cu_seqlens") + if global_cu_seqlens is not None: + if not isinstance(global_cu_seqlens, torch.Tensor): + raise TypeError("global_cu_seqlens must be a torch.Tensor.") + if global_cu_seqlens.dtype != torch.int32 or global_cu_seqlens.dim() != 1: + raise ValueError("global_cu_seqlens must be a one-dimensional int32 tensor.") + metadata = (global_cu_seqlens, cu_seqlens) + cacheable = all(t is None or not torch.is_inference(t) for t in metadata) + key = ( + qkv_format, + tuple(q.shape[:2]), + id(global_cu_seqlens), + global_cu_seqlens._version if cacheable else None, + q.device, + size, + rank, + id(cu_seqlens), + None if cu_seqlens is None or not cacheable else cu_seqlens._version, + ) + else: + key, cacheable = (qkv_format, batch, length, q.device, size, rank), True + cache = getattr(adapter, "_cp_partition_cache", None) + if not cacheable: + # Inference tensors have no mutation counter, so revalidate their + # metadata instead of silently reusing a stale partition. + cache = None + if cache is None or cache[0] != key: + documents = ( + tuple(global_cu_seqlens.tolist()) + if global_cu_seqlens is not None + else tuple(i * length * size for i in range(batch + 1)) + ) + if dense and len(documents) != batch + 1: + raise ValueError("Dense global_cu_seqlens must describe the full batch.") + host = build_linear_attention_partition(documents, size, rank, dense=dense) + plan = _device_partition(host, q.device) + if cu_seqlens is not None: + if tuple(cu_seqlens.tolist()) != host.local_cu_seqlens: + raise ValueError("Local cu_seqlens does not match the contiguous CP fragments.") + # Hold tensor references to prevent object-ID reuse from invalidating the cache. + adapter._cp_partition_cache = (key, plan, global_cu_seqlens, cu_seqlens) + else: + plan = cache[1] + if tensors[0].shape[0] != plan.host.local_cu_seqlens[-1]: + raise ValueError("Local token count does not match this CP rank's token interval.") + adapter._validate_initial_state( + initial_state, batch_size=len(plan.host.documents) - 1, device=q.device + ) + tensors, initial_state = adapter._prepare_cp_inputs(tensors, initial_state) + if adapter._op is None: + adapter._op = adapter._resolve_op() + if ( + not plan.host.communicates + and initial_state is None + and not output_final_state + and plan.host.fragments + ): + output, states = _local_attention( + adapter, tensors, tuple(gates), plan.cu, None, kernel_kwargs, final=False + ) + else: + track_grad = torch.is_grad_enabled() and any( + t is not None and t.requires_grad for t in (*tensors, initial_state) + ) + output, states = _LinearAttentionContextParallel.apply( + adapter, + plan, + group, + kernel_kwargs, + output_final_state, + tuple(gates), + track_grad, + initial_state, + *tensors, + ) + output = output[..., : v.shape[-1]].contiguous() + if output_final_state: + states = states[..., : v.shape[-1], : k.shape[-1]].contiguous() + output = ( + output.reshape(*q.shape[:-2], q.shape[-2] * v.shape[-1]) + if qkv_format == "thd" + else output.reshape(batch, length, q.shape[-2] * v.shape[-1]) + ) + if qkv_format == "sbhd": + output = output.transpose(0, 1).contiguous() + return (output, states) if output_final_state else output diff --git a/transformer_engine/pytorch/attention/linear_attention/gdn.py b/transformer_engine/pytorch/attention/linear_attention/gdn.py index a9875f5e6b8..2b94c8400ad 100644 --- a/transformer_engine/pytorch/attention/linear_attention/gdn.py +++ b/transformer_engine/pytorch/attention/linear_attention/gdn.py @@ -10,6 +10,8 @@ from typing import Dict, Optional, Tuple, Union import torch +import torch.distributed as dist +import torch.nn.functional as F from transformer_engine.pytorch.constants import dist_group_type from transformer_engine.pytorch.jit import no_torch_dynamo @@ -21,6 +23,12 @@ ) +def _aligned(tensor): + """Clone offset views when FE needs a 16-byte-aligned address.""" + tensor = tensor.contiguous() + return tensor.clone() if tensor.data_ptr() % 16 else tensor + + class _GDNKernelAdapter(AlignedTimelineKernelAdapter): """Adapter from TransformerEngine attention layouts to cuDNN frontend GDN. @@ -74,6 +82,120 @@ def _call_op( use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, ) + def _prepare_cp_inputs(self, tensors, initial_state): + """Pad partial feature tiles to keep FE's state gradients correct.""" + # Some FE engines mishandle irregular tiles, such as K=32, V=96. + # Zero extension preserves the recurrence and the original scale. + q, k, v, g, beta = (_aligned(t) for t in tensors) + key_dim, value_dim = k.shape[-1], v.shape[-1] + key_pad = max(64, 1 << (key_dim - 1).bit_length()) - key_dim + value_pad = max(64, 1 << (value_dim - 1).bit_length()) - value_dim + if key_pad: + q, k = F.pad(q, (0, key_pad)), F.pad(k, (0, key_pad)) + if value_pad: + v = F.pad(v, (0, value_pad)) + if initial_state is not None and (key_pad or value_pad): + initial_state = F.pad(initial_state, (0, key_pad, 0, value_pad)) + return (q, k, v, g, beta), initial_state + + @staticmethod + def _summary_op(name): + """Resolve optional FE summary entry points without requiring them at import.""" + try: + from cudnn.linear_attention import ops + + return getattr(ops, name) + except (AttributeError, ImportError) as exc: + raise ImportError(f"cuDNN frontend does not provide {name}.") from exc + + def _cp_forward_summary(self, query, key, value, gates, cu_seqlens, **kernel_kwargs): + """Return H,T; use ordinary state passes if no summary engine supports this shape.""" + del query + k, v, g, beta = (_aligned(t) for t in (key, value, gates["g"], gates["beta"])) + try: + return self._summary_op("gated_delta_net_summary")( + k, v, g, beta, cu_seqlens, **kernel_kwargs + ) + except ImportError: + pass + except Exception as exc: # pylint: disable=broad-exception-caught + if type(exc).__name__ != "cudnnGraphNotSupportedError": + raise + # A zero seed gives H; an identity seed with zero values gives T. + additive = self._call_op( + k, + k, + v, + {"g": g, "beta": beta}, + cu_seqlens, + initial_state=None, + output_final_state=True, + **kernel_kwargs, + )[1] + count, heads, _, key_dim = additive.shape + identity = torch.eye(key_dim, device=k.device, dtype=torch.float32) + identity = identity.expand(count, heads, key_dim, key_dim).contiguous() + transition = self._call_op( + k, + k, + torch.zeros_like(k), + {"g": g, "beta": beta}, + cu_seqlens, + initial_state=identity, + output_final_state=True, + **kernel_kwargs, + )[1] + return additive, transition + + def _cp_backward_summary( + self, query, key, value, gates, grad_output, cu_seqlens, **kernel_kwargs + ): + """Return the reverse state-gradient recipe, including local output losses.""" + del value + q, k, g, beta, grad_output = ( + _aligned(t) for t in (query, key, gates["g"], gates["beta"], grad_output) + ) + try: + return self._summary_op("gated_delta_net_summary_bwd")( + q, + k, + g, + beta, + grad_output, + cu_seqlens, + scale=self.scale, + output_transition=True, + **kernel_kwargs, + ) + except ImportError: + pass + except Exception as exc: # pylint: disable=broad-exception-caught + if type(exc).__name__ != "cudnnGraphNotSupportedError": + raise + values = torch.zeros_like(grad_output) + with torch.enable_grad(): + seed = torch.zeros( + (cu_seqlens.numel() - 1, q.shape[1], grad_output.shape[-1], k.shape[-1]), + device=q.device, + dtype=torch.float32, + requires_grad=True, + ) + output = self._call_op( + q.detach(), + k.detach(), + values, + {"g": g.detach(), "beta": beta.detach()}, + cu_seqlens, + initial_state=seed, + output_final_state=False, + **kernel_kwargs, + )[0] + (additive,) = torch.autograd.grad(output, seed, grad_output) + _, transition = self._cp_forward_summary( + q, k, values, {"g": g, "beta": beta}, cu_seqlens, **kernel_kwargs + ) + return additive, transition.transpose(-1, -2).contiguous() + def forward( self, query_layer: torch.Tensor, @@ -87,8 +209,46 @@ def forward( cu_seqlens: Optional[torch.Tensor] = None, output_final_state: bool = False, use_qk_l2norm_in_kernel: bool = False, + cp_group: Optional[dist.ProcessGroup] = None, + global_cu_seqlens: Optional[torch.Tensor] = None, ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: """Run GDN and return a TE-layout output, optionally with the final state.""" + if cp_group is not None and dist.get_world_size(cp_group) > 1: + from .context_parallel import context_parallel_attention + + return context_parallel_attention( + self, + query_layer, + key_layer, + value_layer, + {"g": g, "beta": beta}, + initial_state, + group=cp_group, + qkv_format=qkv_format, + cu_seqlens=cu_seqlens, + global_cu_seqlens=global_cu_seqlens, + output_final_state=output_final_state, + use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, + ) + if global_cu_seqlens is not None: + if cp_group is None: + raise ValueError("global_cu_seqlens requires CP.") + if not isinstance(global_cu_seqlens, torch.Tensor): + raise TypeError("global_cu_seqlens must be a torch.Tensor.") + if global_cu_seqlens.dtype != torch.int32 or global_cu_seqlens.dim() != 1: + raise ValueError("global_cu_seqlens must be a one-dimensional int32 tensor.") + if qkv_format == "thd": + if cu_seqlens is not None and cu_seqlens.tolist() != global_cu_seqlens.tolist(): + raise ValueError("Local cu_seqlens must match the size-one global input.") + cu_seqlens = global_cu_seqlens.to(query_layer.device) + elif qkv_format in {"bshd", "sbhd"}: + batch, length = ( + query_layer.shape[:2] + if qkv_format == "bshd" + else (query_layer.shape[1], query_layer.shape[0]) + ) + if global_cu_seqlens.tolist() != [i * length for i in range(batch + 1)]: + raise ValueError("Dense global_cu_seqlens does not match the size-one input.") return self._run( query_layer, key_layer, @@ -121,9 +281,8 @@ class GatedDeltaNetAttention(LinearAttentionBase): `b` batch size, `h` the number of heads, `d` head size, and `t` the total number of tokens in a batch, with ``t = sum(s_i)`` for all sequences in the batch. Gated DeltaNet is - inherently causal; padded batches must use `qkv_format='thd'` with - `cu_seqlens` passed to `forward` to exclude padding tokens from the - recurrence. + inherently causal. Pack valid tokens from padded batches into + `qkv_format='thd'` and pass their `cu_seqlens` to `forward`. tp_size : int, default = 1 tensor parallel world size. tp_group : ProcessGroup, default = `None` @@ -135,6 +294,12 @@ class GatedDeltaNetAttention(LinearAttentionBase): scale for the Gated DeltaNet recurrence. Defaults to ``1.0 / sqrt(kv_channels)`` (or the query/key head size, if ``kv_channels`` is a tuple). + cp_group : ProcessGroup, default = `None` + Context parallel process group. ``None`` disables CP. Group rank + order must follow document token order. THD uses contiguous pieces + of the global packed buffer; BSHD/SBHD use contiguous pieces + of each sequence. Q/K/V and gates must already be local tensors. + Summary communication uses all-gather. No output gather is performed. """ def __init__( @@ -146,6 +311,8 @@ def __init__( tp_group: Optional[dist_group_type] = None, layer_number: Optional[int] = None, scale: Optional[float] = None, + *, + cp_group: Optional[dist.ProcessGroup] = None, ) -> None: super().__init__( num_attention_heads, @@ -163,6 +330,19 @@ def __init__( self.qk_head_dim, self.v_head_dim, ) + self.set_context_parallel_group(cp_group) + + def set_context_parallel_group(self, cp_group: Optional[dist.ProcessGroup]) -> None: + """Set or disable CP before forward, including after distributed initialization. + + A single process group is supported. This first implementation uses the + current CUDA stream and contiguous token ordering, without SDPA's dual + chunk layout or communication-stream configuration. + """ + if cp_group is not None and not isinstance(cp_group, dist.ProcessGroup): + raise TypeError("GDN cp_group must be one torch.distributed.ProcessGroup or None.") + self.cp_group = cp_group + self.gdn_attention._cp_partition_cache = None @no_torch_dynamo(when=_needs_eager_linear_attention) def forward( @@ -179,6 +359,7 @@ def forward( initial_state: Optional[torch.Tensor] = None, output_final_state: bool = False, use_qk_l2norm_in_kernel: bool = False, + global_cu_seqlens: Optional[torch.Tensor] = None, ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: """Apply Gated DeltaNet linear attention. @@ -209,6 +390,35 @@ def forward( If true, also return the final recurrent state. use_qk_l2norm_in_kernel : bool, default = `False` If true, L2-normalize Q and K inside the kernel before the recurrence. + global_cu_seqlens : Optional[torch.Tensor], default = `None` + Required for THD CP: global cumulative document lengths, int32 on + CPU or CUDA, identical on all ranks. Rank ``r`` supplies tokens + ``[r*T//P:(r+1)*T//P]`` from each packed input, where ``T`` is the + final global offset and ``P`` is CP size. No CP divisibility constraint + applies; pack tokens without gaps. Local ``cu_seqlens`` is derived; if + supplied, it must match the nonempty local document fragments. + Optional for dense CP: supply ``[0, S, 2*S, ..., B*S]`` for global + sequence length ``S`` when it is not divisible by CP size. Each + rank then supplies the sequence slice ``[r*S//P:(r+1)*S//P]`` from + every batch row. Without this metadata, dense CP assumes equal local + sequence lengths and infers ``S = local_sequence_length * P``. + + Notes + ----- + With CP, ``initial_state`` describes all global documents and must have + identical values on all ranks. Each document's initial-state gradient is + returned on its first owning rank, with zeros elsewhere; sum these + contributions over CP to inspect a replicated leaf state's full gradient. + This ownership also permits chaining final states into the next call. + Final states are replicated for all global documents. Their backward + sums gradients from every replica; divide a replicated final-state loss + by CP size to count it once. All ranks must call forward and backward with matching + options, even when their local loss or token count is zero. Only + first-order gradients are supported, matching the underlying FE path. + CP executes eagerly, including inside a ``torch.compile`` model. + Irregular feature sizes are internally extended with zeros and cropped + back, preserving the recurrence and original attention scale. No token + padding is introduced. """ if g is None or beta is None: raise ValueError( @@ -222,6 +432,8 @@ def forward( "cu_seqlens": cu_seqlens, "output_final_state": output_final_state, "use_qk_l2norm_in_kernel": use_qk_l2norm_in_kernel, + "cp_group": self.cp_group, + "global_cu_seqlens": global_cu_seqlens, } with self.prepare_forward_ctx(query_layer, allow_non_contiguous=True) as query_layer: return self._dispatch_attention( From bfd296a6fb7a58a261f6ec6fde4cac95563ab2b0 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Thu, 5 Nov 2026 18:42:42 -0800 Subject: [PATCH 03/34] Test GDN context parallelism Signed-off-by: Kshitij Lakhani --- qa/L1_pytorch_distributed_unittest/test.sh | 2 + tests/pytorch/attention/run_gdn_with_cp.py | 417 ++++++++++++++++++ .../attention/test_gdn_attention_with_cp.py | 165 +++++++ 3 files changed, 584 insertions(+) create mode 100644 tests/pytorch/attention/run_gdn_with_cp.py create mode 100644 tests/pytorch/attention/test_gdn_attention_with_cp.py diff --git a/qa/L1_pytorch_distributed_unittest/test.sh b/qa/L1_pytorch_distributed_unittest/test.sh index 41ab6aeb444..02a200d9602 100644 --- a/qa/L1_pytorch_distributed_unittest/test.sh +++ b/qa/L1_pytorch_distributed_unittest/test.sh @@ -43,6 +43,8 @@ else NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_attention_deterministic_with_cp.xml $TE_PATH/tests/pytorch/attention/test_attention_with_cp.py || test_fail "NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 test_attention_with_cp.py" fi +NVTE_GDN_TEST_REQUIRED=1 python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_gdn_attention_with_cp.xml $TE_PATH/tests/pytorch/attention/test_gdn_attention_with_cp.py || test_fail "test_gdn_attention_with_cp.py" + python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_sanity.xml $TE_PATH/tests/pytorch/distributed/test_sanity.py || test_fail "test_sanity.py" python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_parallel_cross_entropy.xml $TE_PATH/tests/pytorch/distributed/test_parallel_cross_entropy.py || test_fail "test_parallel_cross_entropy.py" python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_symmetric_memory.xml $TE_PATH/tests/pytorch/distributed/test_symmetric_memory.py || test_fail "test_symmetric_memory.py" diff --git a/tests/pytorch/attention/run_gdn_with_cp.py b/tests/pytorch/attention/run_gdn_with_cp.py new file mode 100644 index 00000000000..075622ceb92 --- /dev/null +++ b/tests/pytorch/attention/run_gdn_with_cp.py @@ -0,0 +1,417 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Distributed GDN CP numerical checks; launch with torch.distributed.run.""" + +import importlib.util +import os +import traceback +from contextlib import nullcontext +from unittest.mock import patch + +import pytest +import torch +import torch.distributed as dist +import torch.nn.functional as F + +from transformer_engine.pytorch import GatedDeltaNetAttention +from transformer_engine.pytorch.attention.linear_attention import context_parallel as cp +from test_gdn_attention import ( + _assert_rms_close, + _gdn_reference, + _FWD_TOL, + _STATE_TOL, + _BWD_TOL, +) + + +def _reference(inputs, cu, scale, norm): + """Independent FP64 recurrence, including unchanged states for empty documents.""" + q, k, v, g, beta, initial = inputs + outputs, states = [], [] + if norm: + q, k = F.normalize(q.double(), dim=-1), F.normalize(k.double(), dim=-1) + for doc, (start, end) in enumerate(zip(cu, cu[1:])): + if start == end: + states.append(initial[doc].double()) + continue + out, state = _gdn_reference( + q[start:end].unsqueeze(0), + k[start:end].unsqueeze(0), + v[start:end].unsqueeze(0), + g[start:end].unsqueeze(0), + beta[start:end].unsqueeze(0), + scale=scale, + initial_state=initial[doc : doc + 1], + cu_seqlens=torch.tensor([0, end - start], device=q.device, dtype=torch.int32), + ) + outputs.append(out[0]) + states.append(state[0]) + return (torch.cat(outputs) if outputs else v.double() * 0), torch.stack(states) + + +def _case( + group, + lengths, + *, + dtype=torch.bfloat16, + key_dim=64, + value_dim=64, + layout="thd", + norm=False, + initial=True, + final=True, + checkpoint=False, + scale=0.2, + last_loss=False, + fallback=False, + tp_group=None, + independent=False, + dense_global=False, + state_only=False, +): + """Compare each local output/gradient to the corresponding full-sequence result.""" + size, rank = dist.get_world_size(group), dist.get_rank(group) + if rank == 0: + print( + f"GDN CP case: lengths={lengths}, dtype={dtype}, K={key_dim}, V={value_dim}," + f" layout={layout}, norm={norm}, initial={initial}, final={final}," + f" checkpoint={checkpoint}, last_loss={last_loss}, fallback={fallback}," + f" independent={independent}", + flush=True, + ) + device = torch.device("cuda", int(os.environ["LOCAL_RANK"])) + torch.manual_seed(791 + (dist.get_rank(tp_group) if tp_group is not None else 0)) + heads = 2 + total = sum(lengths) + cu = [0] + for length in lengths: + cu.append(cu[-1] + length) + q = torch.randn(total, heads, key_dim, device=device, dtype=dtype) + k = torch.randn_like(q) + if not norm: + q, k = F.normalize(q, dim=-1), F.normalize(k, dim=-1) + v = torch.randn(total, heads, value_dim, device=device, dtype=dtype) + g = -torch.rand(total, heads, device=device) * 0.02 + beta = torch.rand(total, heads, device=device) * 0.7 + seed = torch.randn(len(lengths), heads, value_dim, key_dim, device=device) * 0.1 + if not initial: + seed.zero_() + full = tuple(t.detach().requires_grad_(True) for t in (q, k, v, g, beta, seed)) + global_cu = torch.tensor(cu, device=device, dtype=torch.int32) + if independent or total == 0: + ref_output, ref_states = _reference(full, cu, scale, norm) + else: + reference = GatedDeltaNetAttention( + heads, (key_dim, value_dim), qkv_format="thd", scale=scale + ) + ref_output, ref_states = reference( + *full[:5], + cu_seqlens=global_cu, + initial_state=full[5] if initial else None, + output_final_state=True, + use_qk_l2norm_in_kernel=norm, + ) + ref_output = ref_output.reshape(total, heads, value_dim) + grad_output = torch.randn_like(v) + if last_loss: + grad_output[: -max(1, total // size)] = 0 + grad_states = torch.randn_like(seed) * 0.1 if final else torch.zeros_like(seed) + loss = (ref_states * grad_states).sum() + if not state_only: + loss = loss + (ref_output * grad_output).sum() + ref_grads = torch.autograd.grad(loss, full, allow_unused=True) + ref_grads = tuple(torch.zeros_like(t) if d is None else d for t, d in zip(full, ref_grads)) + + def local(tensor): + if layout == "thd": + return tensor[rank * total // size : (rank + 1) * total // size] + batch, length = len(lengths), lengths[0] + tensor = tensor.reshape(batch, length, *tensor.shape[1:]) + tensor = tensor[:, rank * length // size : (rank + 1) * length // size] + return tensor.transpose(0, 1) if layout == "sbhd" else tensor + + inputs = tuple(local(t).detach().requires_grad_(True) for t in full[:5]) + state = full[5].detach().requires_grad_(True) if initial else None + tp_size = 1 if tp_group is None else dist.get_world_size(tp_group) + module = GatedDeltaNetAttention( + heads * tp_size, + (key_dim, value_dim), + qkv_format=layout, + scale=scale, + cp_group=group, + tp_group=tp_group, + ) + kwargs = dict( + global_cu_seqlens=global_cu if layout == "thd" or dense_global else None, + initial_state=state, + output_final_state=final, + checkpoint_core_attention=checkpoint, + use_qk_l2norm_in_kernel=norm, + ) + summary_patch = ( + patch.object( + module.gdn_attention, "_summary_op", side_effect=ImportError("Missing summary") + ) + if fallback + else nullcontext() + ) + with summary_patch: + result = module(*inputs, **kwargs) + output, states = result if final else (result, None) + expected = local(ref_output).reshape_as(output) + if output.numel(): + _assert_rms_close(output, expected, _FWD_TOL[dtype], "CP output") + if final: + _assert_rms_close(states, ref_states, _STATE_TOL[dtype], "CP final states") + local_loss = 0 if state_only else (output * local(grad_output).reshape_as(output)).sum() + if final: + local_loss = local_loss + (states * grad_states).sum() / size + local_loss.backward() + for name, tensor, grad in zip(("q", "k", "v", "g", "beta"), inputs, ref_grads[:5]): + if tensor.numel(): + expected = local(grad) + if expected.count_nonzero() == 0: + torch.testing.assert_close(tensor.grad, expected, atol=1e-6, rtol=0) + else: + _assert_rms_close(tensor.grad, expected, _BWD_TOL[dtype], f"CP d{name}") + if initial: + initial_grad = state.grad.detach().clone() + dist.all_reduce(initial_grad, group=group) + _assert_rms_close(initial_grad, ref_grads[5], _BWD_TOL[dtype], "CP initial-state gradient") + dist.barrier(group=group) + + +def _invalid_metadata(group): + """Reject incompatible packed metadata before entering a collective.""" + device = torch.device("cuda", int(os.environ["LOCAL_RANK"])) + size = dist.get_world_size(group) + q = torch.zeros(4, 2, 64, device=device, dtype=torch.bfloat16) + gate = torch.zeros(4, 2, device=device) + module = GatedDeltaNetAttention(2, 64, qkv_format="thd", cp_group=group) + invalid = ( + {}, + {"global_cu_seqlens": torch.tensor([0, 4 * size], dtype=torch.int64)}, + {"global_cu_seqlens": torch.tensor([0, 4 * size, 2 * size], dtype=torch.int32)}, + {"global_cu_seqlens": torch.tensor([0, 5 * size], dtype=torch.int32)}, + { + "global_cu_seqlens": torch.tensor([0, 4 * size], dtype=torch.int32), + "cu_seqlens": torch.tensor([0, 3], dtype=torch.int32, device=device), + }, + ) + for kwargs in invalid: + with pytest.raises(ValueError): + module(q, q, q, g=gate, beta=gate, **kwargs) + dist.barrier(group=group) + + +def _inference_and_deferred_group(group): + """Default scale, deferred configuration, and metadata created in inference mode.""" + size, rank = dist.get_world_size(group), dist.get_rank(group) + device = torch.device("cuda", int(os.environ["LOCAL_RANK"])) + torch.manual_seed(998) + q, k = ( + F.normalize(torch.randn(35, 2, 64, device=device, dtype=torch.bfloat16), dim=-1) + for _ in range(2) + ) + v = torch.randn_like(q) + g, beta = -torch.rand(35, 2, device=device) * 0.02, torch.rand(35, 2, device=device) * 0.7 + reference = GatedDeltaNetAttention(2, 64, qkv_format="thd") + module = GatedDeltaNetAttention(2, 64, qkv_format="thd") + module.set_context_parallel_group(group) + q, k, v, g, beta = (t.requires_grad_() for t in (q, k, v, g, beta)) + for context in (torch.no_grad, torch.inference_mode): + with context(): + global_cu = torch.tensor([0, 17, 35], device=device, dtype=torch.int32) + expected, expected_states = reference( + q, k, v, g, beta, cu_seqlens=global_cu, output_final_state=True + ) + lo, hi = rank * 35 // size, (rank + 1) * 35 // size + output, states = module( + q[lo:hi], + k[lo:hi], + v[lo:hi], + g[lo:hi], + beta[lo:hi], + global_cu_seqlens=global_cu, + output_final_state=True, + ) + assert not output.requires_grad and not states.requires_grad + _assert_rms_close(output, expected[lo:hi], _FWD_TOL[q.dtype], "inference output") + _assert_rms_close(states, expected_states, _STATE_TOL[q.dtype], "inference states") + dist.barrier(group=group) + + +def _state_chaining(group): + """Backpropagate through two CP calls joined by their replicated final states.""" + size, rank = dist.get_world_size(group), dist.get_rank(group) + device = torch.device("cuda", int(os.environ["LOCAL_RANK"])) + torch.manual_seed(932) + tokens, heads, dim = 35, 2, 64 + full = [] + for _ in range(2): + q, k = ( + F.normalize( + torch.randn(tokens, heads, dim, device=device, dtype=torch.bfloat16), dim=-1 + ) + for _ in range(2) + ) + v = torch.randn_like(q) + g = -torch.rand(tokens, heads, device=device) * 0.02 + beta = torch.rand(tokens, heads, device=device) * 0.7 + full.extend(t.detach().requires_grad_(True) for t in (q, k, v, g, beta)) + seed = (torch.randn(2, heads, dim, dim, device=device) * 0.1).requires_grad_(True) + cu = torch.tensor([0, 17, 35], device=device, dtype=torch.int32) + reference = GatedDeltaNetAttention(heads, dim, qkv_format="thd") + first, state = reference(*full[:5], cu_seqlens=cu, initial_state=seed, output_final_state=True) + second = reference(*full[5:], cu_seqlens=cu, initial_state=state) + grad_first, grad_second = torch.randn_like(first), torch.randn_like(second) + ((first * grad_first).sum() * 0.3 + (second * grad_second).sum()).backward() + lo, hi = rank * tokens // size, (rank + 1) * tokens // size + inputs = tuple(t[lo:hi].detach().requires_grad_(True) for t in full) + local_seed = seed.detach().requires_grad_(True) + module = GatedDeltaNetAttention(heads, dim, qkv_format="thd", cp_group=group) + local_first, state = module( + *inputs[:5], global_cu_seqlens=cu, initial_state=local_seed, output_final_state=True + ) + local_second = module(*inputs[5:], global_cu_seqlens=cu, initial_state=state) + loss = (local_first * grad_first[lo:hi]).sum() * 0.3 + loss = loss + (local_second * grad_second[lo:hi]).sum() + for retain in (True, False): + loss.backward(retain_graph=retain) + for actual, expected in zip(inputs, full): + _assert_rms_close( + actual.grad, + expected.grad[lo:hi], + _BWD_TOL[torch.bfloat16], + "chained token gradient", + ) + initial_grad = local_seed.grad.clone() + dist.all_reduce(initial_grad, group=group) + _assert_rms_close( + initial_grad, seed.grad, _BWD_TOL[torch.bfloat16], "chained initial-state gradient" + ) + for tensor in (*inputs, local_seed): + tensor.grad = None + dist.barrier(group=group) + + +def main(): + """Run one persistent process group to amortize FE compilation across cases.""" + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + dist.init_process_group("nccl") + group = dist.group.WORLD + size = dist.get_world_size(group) + cases = 0 + try: + for lengths in ( + [8, 4, 12], + [16, 4, 4], + [7, 2, 4, 5, 14], + [6, 6, 6, 6], + [0, 17, 0, 18, 0], + [1, 0, 1], + [0, 0], + ): + _case(group, lengths) + cases += 1 + for dtype in (torch.float16, torch.bfloat16): + for norm in (False, True): + _case( + group, + [size * 65 + 3, 7], + dtype=dtype, + value_dim=128, + norm=norm, + independent=True, + ) + cases += 1 + _case( + group, + [size * 65 + 3, 7], + dtype=torch.float16, + key_dim=128, + value_dim=64, + independent=True, + ) + cases += 1 + for layout, dtype in (("bshd", torch.bfloat16), ("sbhd", torch.float16)): + _case(group, [size * 65] * 2, dtype=dtype, layout=layout, norm=True) + cases += 1 + for layout in ("bshd", "sbhd"): + _case(group, [size * 17 + 3] * 2, layout=layout, dense_global=True) + cases += 1 + _case(group, [1, 1], layout="bshd", dense_global=True, checkpoint=True) + cases += 1 + for initial, final, checkpoint, last_loss in ( + (False, False, False, True), + (True, False, False, True), + (False, True, True, True), + (True, True, True, False), + ): + _case( + group, + [size * 67], + initial=initial, + final=final, + checkpoint=checkpoint, + last_loss=last_loss, + ) + cases += 1 + _case(group, [size * 65 + 1, 7], fallback=True, norm=True) + cases += 1 + _case(group, [size * 65 + 1, 7], state_only=True) + cases += 1 + # Irregular features are zero-extended internally because ordinary FE + # state gradients are incorrect for some unpadded cuTile geometries. + _case(group, [size * 65 + 1, 7], key_dim=32, value_dim=96, norm=True, independent=True) + cases += 1 + # Larger cuTile geometries have no dedicated FROST summary engine; + # exercise the real capability fallback, rather than a mocked error. + if importlib.util.find_spec("cuda.tile") is not None: + _case(group, [131, 7], key_dim=192, value_dim=160, independent=True) + cases += 1 + _inference_and_deferred_group(group) + cases += 1 + _state_chaining(group) + cases += 1 + _invalid_metadata(group) + cases += 1 + # Independent complete documents need no state exchange or prefix work. + with patch.object( + cp, "_gather_summaries", side_effect=AssertionError("Unexpected exchange") + ), patch.object( + cp.tex, + "linear_attention_affine_prefix", + side_effect=AssertionError("Unexpected prefix"), + ): + _case(group, [8] * size, initial=False, final=False) + cases += 1 + # A size-one group follows the ordinary local path, even while WORLD + # contains several ranks. Every process creates groups in the same order. + singletons = [dist.new_group([r]) for r in range(size)] + _case(singletons[dist.get_rank(group)], [17, 13]) + cases += 1 + if size == 4: + cp_groups = [dist.new_group(ranks) for ranks in ([0, 1], [2, 3])] + tp_groups = [dist.new_group(ranks) for ranks in ([0, 2], [1, 3])] + rank = dist.get_rank() + _case(cp_groups[rank // 2], [139, 13], tp_group=tp_groups[rank % 2]) + cases += 1 + if dist.get_rank() == 0: + print( + f"GDN CP: {cases} correctness cases passed with CP={size}", + flush=True, + ) + except BaseException: # pylint: disable=broad-exception-caught + # A failed rank must exit before a peer waits forever for its next + # collective. NCCL's orderly shutdown can wait on that missing peer. + traceback.print_exc() + os._exit(1) + else: + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/tests/pytorch/attention/test_gdn_attention_with_cp.py b/tests/pytorch/attention/test_gdn_attention_with_cp.py new file mode 100644 index 00000000000..f1c7bd4ea8b --- /dev/null +++ b/tests/pytorch/attention/test_gdn_attention_with_cp.py @@ -0,0 +1,165 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Contiguous GDN CP: host boundaries, common arithmetic, and multi-GPU numerics.""" + +import os +from pathlib import Path +import random +import subprocess +import sys +import tempfile + +import pytest +import torch + +from test_gdn_attention import _GDN_AVAILABLE +import transformer_engine +from transformer_engine.common.linear_attention import build_linear_attention_partition +import transformer_engine_torch as tex + + +@pytest.mark.parametrize("size", [1, 2, 4, 8]) +def test_partition_intersections(size): + """Recover all document intervals without dropping tokens or crossing documents.""" + rng = random.Random(301) + for _ in range(100): + lengths = [rng.randrange(0, 20) for _ in range(rng.randrange(1, 10))] + cu = [0] + for length in lengths: + cu.append(cu[-1] + length) + partitions = [build_linear_attention_partition(cu, size, rank) for rank in range(size)] + for doc, (start, end) in enumerate(zip(cu, cu[1:])): + fragments = [f for p in partitions for f in p.fragments if f.document == doc] + assert sum(f.end - f.start for f in fragments) == end - start + assert all(start <= f.start < f.end <= end for f in fragments) + assert all(a.end == b.start for a, b in zip(fragments, fragments[1:])) + for rank, partition in enumerate(partitions): + assert ( + partition.local_cu_seqlens[-1] + == (rank + 1) * cu[-1] // size - rank * cu[-1] // size + ) + assert len(partition.incoming) <= 1 + assert len(partition.outgoing) <= 1 + for f, forward, backward in zip( + partition.fragments, partition.forward_descriptors, partition.backward_descriptors + ): + assert forward[2] == rank - f.first_rank + assert backward[2] == f.last_rank - rank + # Every referenced recipe belongs to this same document. + for predecessor in range(f.first_rank, rank): + p = partitions[predecessor] + if p.fragments: + assert p.fragments[p.outgoing[0]].document == f.document + for successor in range(rank + 1, f.last_rank + 1): + p = partitions[successor] + # An empty rank contributes an identity recipe. + if p.fragments: + assert p.fragments[p.incoming[0]].document == f.document + + +@pytest.mark.parametrize("documents", [[1, 2], [0], [0, 2, 1]]) +def test_invalid_documents(documents): + """Reject malformed global offsets before computing communication descriptors.""" + with pytest.raises(ValueError): + build_linear_attention_partition(documents, 4, 0) + + +@pytest.mark.parametrize("size", [1, 2, 4, 8]) +@pytest.mark.parametrize("length", [0, 1, 5, 9]) +def test_dense_partition_intersections(size, length): + """Dense rows retain their document identity, including uneven and empty ranks.""" + partitions = [ + build_linear_attention_partition([0, length, 2 * length], size, rank, dense=True) + for rank in range(size) + ] + for doc in (0, 1): + spans = [f for p in partitions for f in p.fragments if f.document == doc] + assert sum(f.end - f.start for f in spans) == length + assert all(a.end == b.start for a, b in zip(spans, spans[1:])) + if length: + assert spans[0].start == doc * length + assert spans[-1].end == (doc + 1) * length + for rank, partition in enumerate(partitions): + local_length = (rank + 1) * length // size - rank * length // size + assert partition.local_cu_seqlens[-1] == 2 * local_length + assert partition.slots == 2 + assert len(partition.fragments) == (2 if local_length else 0) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") +@pytest.mark.parametrize("key_dim", [3, 64, 128, 257]) +def test_common_affine_prefix(key_dim): + """Nonsymmetric transitions, reversed order, zero prefixes, and independent seeds.""" + torch.manual_seed(22) + current = torch.cuda.current_device() + # Also exercise the binding's device guard: allocating on another GPU must + # neither launch on the current GPU nor change the caller's current device. + device = torch.device( + "cuda", (current + 1) % torch.cuda.device_count() if key_dim == 3 else current + ) + seed = torch.randn(3, 2, 5, key_dim, device=device) + additive = torch.randn(6, 2, 5, key_dim, device=device) + transitions = torch.randn(6, 2, key_dim, key_dim, device=device) / key_dim**0.5 + rows = [ + [0, 0, 0, 1], + [1, 0, 3, 1], + [2, 5, 3, -2], + [-1, 0, 0, 1], + [-1, 0, 3, 1], + [-1, 5, 3, -2], + ] + descriptors = torch.tensor(rows, device=device, dtype=torch.int32) + actual = tex.linear_attention_affine_prefix(seed, additive, transitions, descriptors) + assert torch.cuda.current_device() == current + expected = [] + for source, first, count, stride in rows: + x = ( + torch.zeros_like(seed[0], dtype=torch.float64) + if source == -1 + else seed[source].double() + ) + for step in range(count): + index = first + step * stride + x = x @ transitions[index].double() + additive[index].double() + expected.append(x) + torch.testing.assert_close(actual.double(), torch.stack(expected), atol=2e-5, rtol=2e-5) + + +@pytest.mark.parametrize("size", [2, 4]) +def test_gdn_with_cp(size): + """Run genuine NCCL CP with all token and state gradients checked numerically.""" + if not _GDN_AVAILABLE or torch.cuda.device_count() < size: + if os.getenv("NVTE_GDN_CP_TEST_REQUIRED", "0") == "1": + pytest.fail(f"Required GDN CP test needs FE GDN and {size} CUDA GPUs") + pytest.skip(f"GDN CP test requires FE GDN and {size} CUDA GPUs") + runner = Path(__file__).with_name("run_gdn_with_cp.py") + env = os.environ.copy() + # torchrun starts a script, so its sys.path[0] is the tests directory. + # Preserve the TE package that pytest imported, including source worktrees. + package_root = str(Path(transformer_engine.__file__).resolve().parents[1]) + env["PYTHONPATH"] = os.pathsep.join((package_root, env.get("PYTHONPATH", ""))) + with tempfile.TemporaryDirectory(prefix="te-gdn-cp-") as log_dir: + log_path = Path(log_dir) / "worker.log" + print(f"CP={size} progress log: {log_path}", flush=True) + with log_path.open("w", encoding="utf-8") as log: + result = subprocess.run( + [ + sys.executable, + "-m", + "torch.distributed.run", + "--standalone", + f"--nproc_per_node={size}", + str(runner), + ], + check=False, + text=True, + stdout=log, + stderr=subprocess.STDOUT, + timeout=1200, + env=env, + ) + output = log_path.read_text() + assert result.returncode == 0, output + print(output) From 61edd9b69f7aab73668aecdc0aca94bce2194870 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Thu, 5 Nov 2026 18:42:48 -0800 Subject: [PATCH 04/34] Document GDN context parallelism Signed-off-by: Kshitij Lakhani --- docs/api/pytorch.rst | 2 +- docs/features/other_optimizations/index.rst | 2 +- .../linear_attention_context_parallel.rst | 108 ++++++++++++++++++ 3 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 docs/features/other_optimizations/linear_attention_context_parallel.rst diff --git a/docs/api/pytorch.rst b/docs/api/pytorch.rst index 2d8cc87827e..561d3ea3ddd 100644 --- a/docs/api/pytorch.rst +++ b/docs/api/pytorch.rst @@ -26,7 +26,7 @@ PyTorch :members: forward, set_context_parallel_group .. autoapiclass:: transformer_engine.pytorch.GatedDeltaNetAttention(num_attention_heads, kv_channels, **kwargs) - :members: forward + :members: forward, set_context_parallel_group .. autoapiclass:: transformer_engine.pytorch.GatedDeltaNet2Attention(num_attention_heads, kv_channels, **kwargs) :members: forward diff --git a/docs/features/other_optimizations/index.rst b/docs/features/other_optimizations/index.rst index 05e89c4b05d..3946e111c8a 100644 --- a/docs/features/other_optimizations/index.rst +++ b/docs/features/other_optimizations/index.rst @@ -9,4 +9,4 @@ Other optimizations .. toctree:: cpu_offloading/cpu_offloading.rst - + linear_attention_context_parallel.rst diff --git a/docs/features/other_optimizations/linear_attention_context_parallel.rst b/docs/features/other_optimizations/linear_attention_context_parallel.rst new file mode 100644 index 00000000000..b7a464eed65 --- /dev/null +++ b/docs/features/other_optimizations/linear_attention_context_parallel.rst @@ -0,0 +1,108 @@ +.. + Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + + See LICENSE for license information. + +GDN context parallelism +======================= + +PyTorch :class:`~transformer_engine.pytorch.GatedDeltaNetAttention` supports +contiguous context parallelism (CP). Each rank receives local Q, K, V and gate +tensors. TE exchanges recurrent-state summaries so local attention produces +the same result as processing each complete document. Outputs stay on the +input token owners; there is no output all-gather. + +Set ``cp_group`` at construction, or call ``set_context_parallel_group`` before +forward. ``None`` disables CP. A group of size one follows the local path. +Group rank order defines token order within a document. All ranks must use +matching options and participate in forward and backward, including ranks +with no tokens or no local loss. + +Input ownership +--------------- + +For CP size ``P`` and group rank ``r``: + +* **THD:** rank ``r`` owns ``[r*T//P:(r+1)*T//P]`` of the global packed token + buffer. Pass the same global ``int32`` cumulative document lengths as + ``global_cu_seqlens`` on every rank. This metadata can be on CPU or CUDA; + local fragment boundaries are derived automatically. +* **BSHD/SBHD:** rank ``r`` owns ``[r*S//P:(r+1)*S//P]`` of every batch row. + With equal local sequence lengths, TE infers ``S``. For uneven slices, + also pass global offsets ``[0,S,2*S,...,B*S]``. + +Q, K, V, ``g`` and ``beta`` must follow the same ownership. TE does not +redistribute inputs or interpret SDPA's dual-chunk layout. A local conversion +between BSHD/SBHD and THD does not change which tokens a rank owns. + +Document lengths need not be divisible by CP size. THD sequences are packed +without gaps; the caller removes physical padding before calling LA. Empty +documents are allowed and retain their initial state. + +For example, lengths ``[8,4,12]`` with CP size four give: + +.. code-block:: text + + rank 0: A[0:6] + rank 1: A[6:8], B[0:4] + rank 2: C[0:6] + rank 3: C[6:12] + +Only A and C need cross-rank state propagation. Independent documents never +share memory. The initial collective can still broadcast unused summary slots. + +.. code-block:: python + + import torch + import transformer_engine.pytorch as te + + # cp is an initialized torch.distributed process group. The input pipeline + # supplies this rank's token interval for Q/K/V and both gates. + attn = te.GatedDeltaNetAttention( + num_attention_heads=32, + kv_channels=128, + qkv_format="thd", + cp_group=cp, + ) + global_cu = torch.tensor([0, 8, 12, 24], dtype=torch.int32) + output = attn( + q_local, k_local, v_local, g=g_local, beta=beta_local, + global_cu_seqlens=global_cu, + ) + +Recurrent states and gradients +------------------------------ + +The optional ``initial_state`` is FP32 with shape +``[global_documents, local_heads, value_dim, key_dim]`` and identical values +on every CP rank. Requested final states have the same global shape and are +replicated using a separate collective. Empty documents return their seed. + +Each initial-state gradient is returned on its document's first owning rank, +with zeros elsewhere. Sum these contributions over CP when inspecting a +replicated leaf state's full gradient. This ownership also permits state +chaining between CP calls. + +Final-state backward sums gradients from all replicas. Divide a loss applied +identically to replicated final states by CP size to count it once. A loss +over each rank's local attention outputs needs no such adjustment. + +Execution +--------- + +For a fragment, cuDNN produces the affine map ``S_out = S_in @ T + H``. +TE all-gathers the required summaries and composes them in document order to +obtain local starting states. Local cuDNN attention then runs on every rank. +Backward uses a reverse summary exchange to propagate state gradients. +Composition uses FP32 arithmetic in the common library. + +GDN CP supports FP16/BF16 inputs, FP32 gates and states, Q/K normalization, +activation checkpointing and tensor parallelism. Head dimensions and device +support follow the available cuDNN frontend engines. Where a dedicated summary +engine is unavailable, ordinary GDN state passes provide a slower fallback. + +CP runs eagerly, including inside ``torch.compile`` models, and supports +first-order gradients. Full-graph compilation and CUDA graph capture are +outside this implementation's supported contract. Other LA flavors, direct +dual-chunk ownership, headwise CP and whole-document redistribution are +separate features. From 392ba757306a8db8206b5f8e544c91d9cbf78894 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Tue, 10 Nov 2026 17:03:32 -0800 Subject: [PATCH 05/34] Document GDN context parallel metadata local and global cu seqlens Signed-off-by: Kshitij Lakhani --- .../linear_attention_context_parallel.rst | 6 ++-- .../pytorch/attention/linear_attention/gdn.py | 31 +++++++++++-------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/docs/features/other_optimizations/linear_attention_context_parallel.rst b/docs/features/other_optimizations/linear_attention_context_parallel.rst index b7a464eed65..c0ef68f0912 100644 --- a/docs/features/other_optimizations/linear_attention_context_parallel.rst +++ b/docs/features/other_optimizations/linear_attention_context_parallel.rst @@ -28,8 +28,10 @@ For CP size ``P`` and group rank ``r``: ``global_cu_seqlens`` on every rank. This metadata can be on CPU or CUDA; local fragment boundaries are derived automatically. * **BSHD/SBHD:** rank ``r`` owns ``[r*S//P:(r+1)*S//P]`` of every batch row. - With equal local sequence lengths, TE infers ``S``. For uneven slices, - also pass global offsets ``[0,S,2*S,...,B*S]``. + Here ``B`` is batch size and ``S`` is each row's full sequence length. + When ``S`` is divisible by ``P``, TE can infer ``S`` from the local length. + Otherwise, pass global document offsets ``[0,S,2*S,...,B*S]``. Rank slices + still follow the formula above; arbitrary rank splits are not supported. Q, K, V, ``g`` and ``beta`` must follow the same ownership. TE does not redistribute inputs or interpret SDPA's dual-chunk layout. A local conversion diff --git a/transformer_engine/pytorch/attention/linear_attention/gdn.py b/transformer_engine/pytorch/attention/linear_attention/gdn.py index 2b94c8400ad..97d7f2c0af5 100644 --- a/transformer_engine/pytorch/attention/linear_attention/gdn.py +++ b/transformer_engine/pytorch/attention/linear_attention/gdn.py @@ -335,9 +335,8 @@ def __init__( def set_context_parallel_group(self, cp_group: Optional[dist.ProcessGroup]) -> None: """Set or disable CP before forward, including after distributed initialization. - A single process group is supported. This first implementation uses the - current CUDA stream and contiguous token ordering, without SDPA's dual - chunk layout or communication-stream configuration. + A single process group is supported. CP uses the current CUDA stream + and contiguous token ordering. """ if cp_group is not None and not isinstance(cp_group, dist.ProcessGroup): raise TypeError("GDN cp_group must be one torch.distributed.ProcessGroup or None.") @@ -378,7 +377,12 @@ def forward( Overrides the module's configured `qkv_format` for this call. cu_seqlens : Optional[torch.Tensor], default = `None` Cumulative sequence lengths for packed (`thd`) inputs, of shape - `[batch_size + 1]` and dtype `torch.int32`. + `[batch_size + 1]` and dtype `torch.int32` without CP. With THD CP, + TE intersects the global documents with this rank's token interval + and derives cumulative lengths for the resulting local fragments. + Documents with no tokens on this rank are omitted. With THD CP, + this argument is optional; if supplied, its values must equal + those derived offsets. checkpoint_core_attention : bool, default = `False` If true, forward activations for this module are recomputed during the backward pass instead of saved. @@ -391,17 +395,18 @@ def forward( use_qk_l2norm_in_kernel : bool, default = `False` If true, L2-normalize Q and K inside the kernel before the recurrence. global_cu_seqlens : Optional[torch.Tensor], default = `None` - Required for THD CP: global cumulative document lengths, int32 on - CPU or CUDA, identical on all ranks. Rank ``r`` supplies tokens + Global cumulative document lengths, dtype `torch.int32`, identical + on all ranks. Required for THD CP: rank ``r`` supplies tokens ``[r*T//P:(r+1)*T//P]`` from each packed input, where ``T`` is the final global offset and ``P`` is CP size. No CP divisibility constraint - applies; pack tokens without gaps. Local ``cu_seqlens`` is derived; if - supplied, it must match the nonempty local document fragments. - Optional for dense CP: supply ``[0, S, 2*S, ..., B*S]`` for global - sequence length ``S`` when it is not divisible by CP size. Each - rank then supplies the sequence slice ``[r*S//P:(r+1)*S//P]`` from - every batch row. Without this metadata, dense CP assumes equal local - sequence lengths and infers ``S = local_sequence_length * P``. + applies; pack tokens without gaps. + For BSHD/SBHD CP, use ``[0, S, 2*S, ..., B*S]``, where ``B`` is + batch size and ``S`` is the full sequence length of each batch row. + These are document boundaries, not rank boundaries. Each rank must + supply ``[r*S//P:(r+1)*S//P]`` of every row. This metadata is required + when ``S`` is not divisible by ``P``; otherwise TE can infer + ``S = local_sequence_length * P``. Arbitrary rank splits are not + supported. Notes ----- From 72ce1a6c33afe9ebf9abda954e8ae9b66c791d33 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Tue, 10 Nov 2026 17:26:46 -0800 Subject: [PATCH 06/34] Unify GDN CP sequence metadata Signed-off-by: Kshitij Lakhani --- .../linear_attention_context_parallel.rst | 10 ++- tests/pytorch/attention/run_gdn_with_cp.py | 75 +++++++++++------- tests/pytorch/attention/test_gdn_attention.py | 9 ++- .../linear_attention/context_parallel.py | 45 ++++------- .../pytorch/attention/linear_attention/gdn.py | 78 ++++++++++--------- 5 files changed, 119 insertions(+), 98 deletions(-) diff --git a/docs/features/other_optimizations/linear_attention_context_parallel.rst b/docs/features/other_optimizations/linear_attention_context_parallel.rst index c0ef68f0912..a82d7a4dd79 100644 --- a/docs/features/other_optimizations/linear_attention_context_parallel.rst +++ b/docs/features/other_optimizations/linear_attention_context_parallel.rst @@ -25,7 +25,7 @@ For CP size ``P`` and group rank ``r``: * **THD:** rank ``r`` owns ``[r*T//P:(r+1)*T//P]`` of the global packed token buffer. Pass the same global ``int32`` cumulative document lengths as - ``global_cu_seqlens`` on every rank. This metadata can be on CPU or CUDA; + ``cu_seqlens`` on every rank. This metadata can be on CPU or CUDA; local fragment boundaries are derived automatically. * **BSHD/SBHD:** rank ``r`` owns ``[r*S//P:(r+1)*S//P]`` of every batch row. Here ``B`` is batch size and ``S`` is each row's full sequence length. @@ -33,6 +33,12 @@ For CP size ``P`` and group rank ``r``: Otherwise, pass global document offsets ``[0,S,2*S,...,B*S]``. Rank slices still follow the formula above; arbitrary rank splits are not supported. +``cu_seqlens`` always describes the complete logical documents, before CP +partitioning. Without CP, those boundaries describe the supplied input directly. +Dense inputs can omit the metadata; if supplied without CP, it must describe +the full rows as ``[0,S,2*S,...,B*S]``. TE keeps the derived local cumulative +lengths private. + Q, K, V, ``g`` and ``beta`` must follow the same ownership. TE does not redistribute inputs or interpret SDPA's dual-chunk layout. A local conversion between BSHD/SBHD and THD does not change which tokens a rank owns. @@ -69,7 +75,7 @@ share memory. The initial collective can still broadcast unused summary slots. global_cu = torch.tensor([0, 8, 12, 24], dtype=torch.int32) output = attn( q_local, k_local, v_local, g=g_local, beta=beta_local, - global_cu_seqlens=global_cu, + cu_seqlens=global_cu, ) Recurrent states and gradients diff --git a/tests/pytorch/attention/run_gdn_with_cp.py b/tests/pytorch/attention/run_gdn_with_cp.py index 075622ceb92..8d47ebd7e39 100644 --- a/tests/pytorch/attention/run_gdn_with_cp.py +++ b/tests/pytorch/attention/run_gdn_with_cp.py @@ -144,7 +144,11 @@ def local(tensor): tp_group=tp_group, ) kwargs = dict( - global_cu_seqlens=global_cu if layout == "thd" or dense_global else None, + cu_seqlens=( + (global_cu.cpu() if size == 1 else global_cu) + if layout == "thd" or dense_global + else None + ), initial_state=state, output_final_state=final, checkpoint_core_attention=checkpoint, @@ -192,13 +196,10 @@ def _invalid_metadata(group): module = GatedDeltaNetAttention(2, 64, qkv_format="thd", cp_group=group) invalid = ( {}, - {"global_cu_seqlens": torch.tensor([0, 4 * size], dtype=torch.int64)}, - {"global_cu_seqlens": torch.tensor([0, 4 * size, 2 * size], dtype=torch.int32)}, - {"global_cu_seqlens": torch.tensor([0, 5 * size], dtype=torch.int32)}, - { - "global_cu_seqlens": torch.tensor([0, 4 * size], dtype=torch.int32), - "cu_seqlens": torch.tensor([0, 3], dtype=torch.int32, device=device), - }, + {"cu_seqlens": torch.tensor([0, 4 * size], dtype=torch.int64)}, + {"cu_seqlens": torch.tensor([0, 4 * size, 2 * size], dtype=torch.int32)}, + {"cu_seqlens": torch.tensor([0, 5 * size], dtype=torch.int32)}, + {"cu_seqlens": torch.tensor([0, 4], dtype=torch.int32, device=device)}, ) for kwargs in invalid: with pytest.raises(ValueError): @@ -223,23 +224,28 @@ def _inference_and_deferred_group(group): q, k, v, g, beta = (t.requires_grad_() for t in (q, k, v, g, beta)) for context in (torch.no_grad, torch.inference_mode): with context(): - global_cu = torch.tensor([0, 17, 35], device=device, dtype=torch.int32) - expected, expected_states = reference( - q, k, v, g, beta, cu_seqlens=global_cu, output_final_state=True - ) - lo, hi = rank * 35 // size, (rank + 1) * 35 // size - output, states = module( - q[lo:hi], - k[lo:hi], - v[lo:hi], - g[lo:hi], - beta[lo:hi], - global_cu_seqlens=global_cu, - output_final_state=True, - ) - assert not output.requires_grad and not states.requires_grad - _assert_rms_close(output, expected[lo:hi], _FWD_TOL[q.dtype], "inference output") - _assert_rms_close(states, expected_states, _STATE_TOL[q.dtype], "inference states") + # CP accepts host metadata; inference tensors intentionally bypass + # the partition cache because they have no mutation counter. + metadata_device = "cpu" if context is torch.no_grad else device + global_cu = torch.tensor([0, 17, 35], device=metadata_device, dtype=torch.int32) + for boundary in (17, 19): + global_cu[1] = boundary + expected, expected_states = reference( + q, k, v, g, beta, cu_seqlens=global_cu.to(device), output_final_state=True + ) + lo, hi = rank * 35 // size, (rank + 1) * 35 // size + output, states = module( + q[lo:hi], + k[lo:hi], + v[lo:hi], + g[lo:hi], + beta[lo:hi], + cu_seqlens=global_cu, + output_final_state=True, + ) + assert not output.requires_grad and not states.requires_grad + _assert_rms_close(output, expected[lo:hi], _FWD_TOL[q.dtype], "inference output") + _assert_rms_close(states, expected_states, _STATE_TOL[q.dtype], "inference states") dist.barrier(group=group) @@ -273,9 +279,9 @@ def _state_chaining(group): local_seed = seed.detach().requires_grad_(True) module = GatedDeltaNetAttention(heads, dim, qkv_format="thd", cp_group=group) local_first, state = module( - *inputs[:5], global_cu_seqlens=cu, initial_state=local_seed, output_final_state=True + *inputs[:5], cu_seqlens=cu, initial_state=local_seed, output_final_state=True ) - local_second = module(*inputs[5:], global_cu_seqlens=cu, initial_state=state) + local_second = module(*inputs[5:], cu_seqlens=cu, initial_state=state) loss = (local_first * grad_first[lo:hi]).sum() * 0.3 loss = loss + (local_second * grad_second[lo:hi]).sum() for retain in (True, False): @@ -337,7 +343,14 @@ def main(): ) cases += 1 for layout, dtype in (("bshd", torch.bfloat16), ("sbhd", torch.float16)): - _case(group, [size * 65] * 2, dtype=dtype, layout=layout, norm=True) + _case( + group, + [size * 65] * 2, + dtype=dtype, + layout=layout, + norm=True, + dense_global=layout == "bshd", + ) cases += 1 for layout in ("bshd", "sbhd"): _case(group, [size * 17 + 3] * 2, layout=layout, dense_global=True) @@ -391,8 +404,10 @@ def main(): # A size-one group follows the ordinary local path, even while WORLD # contains several ranks. Every process creates groups in the same order. singletons = [dist.new_group([r]) for r in range(size)] - _case(singletons[dist.get_rank(group)], [17, 13]) - cases += 1 + singleton = singletons[dist.get_rank(group)] + _case(singleton, [17, 13]) + _case(singleton, [17, 17], layout="bshd", dense_global=True) + cases += 2 if size == 4: cp_groups = [dist.new_group(ranks) for ranks in ([0, 1], [2, 3])] tp_groups = [dist.new_group(ranks) for ranks in ([0, 2], [1, 3])] diff --git a/tests/pytorch/attention/test_gdn_attention.py b/tests/pytorch/attention/test_gdn_attention.py index 8b0b3898956..e99a012a275 100644 --- a/tests/pytorch/attention/test_gdn_attention.py +++ b/tests/pytorch/attention/test_gdn_attention.py @@ -325,7 +325,14 @@ def test_gdn_dense_layout(qkv_format, dtype, compile_model): if compile_model: torch._dynamo.reset() attention = torch.compile(attention, fullgraph=True) - output = attention(q, k, v, g=g, beta=beta, use_qk_l2norm_in_kernel=True) + # Exercise explicit full-document metadata in eager and compiled dense calls; + # the other dtype retains the shape-inferred path. + cu_seqlens = ( + torch.arange(batch + 1, device=q.device, dtype=torch.int32) * sequence + if dtype == torch.bfloat16 + else None + ) + output = attention(q, k, v, g=g, beta=beta, cu_seqlens=cu_seqlens, use_qk_l2norm_in_kernel=True) assert output.shape == expected.shape _assert_rms_close(output, expected, _FWD_TOL[dtype], "output") diff --git a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py index f12bba79a17..f2b2f47bea1 100644 --- a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py +++ b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py @@ -19,7 +19,7 @@ LinearAttentionCPPartition, build_linear_attention_partition, ) -from .base import _to_thd, _validate_cu_seqlens +from .base import _to_thd @dataclass @@ -276,43 +276,33 @@ def context_parallel_attention( group, qkv_format, cu_seqlens, - global_cu_seqlens, output_final_state, **kernel_kwargs, ): - """Validate local inputs, cache a partition, and restore the public output layout.""" + """Derive local fragment boundaries from full-document metadata and run CP.""" adapter._validate_inputs(q, k, v, gates, initial_state, qkv_format) size, rank = dist.get_world_size(group), dist.get_rank(group) tensors = tuple(_to_thd(t, qkv_format).contiguous() for t in (q, k, v, *gates.values())) dense = qkv_format != "thd" batch, length = None, None if dense: - if cu_seqlens is not None: - raise ValueError("Dense linear-attention CP does not accept local cu_seqlens.") batch, length = q.shape[:2] if qkv_format == "bshd" else (q.shape[1], q.shape[0]) - elif not isinstance(global_cu_seqlens, torch.Tensor): - raise ValueError( - "Linear-attention CP with THD requires global_cu_seqlens for all documents." - ) + elif cu_seqlens is None: + raise ValueError("Linear-attention CP with THD requires cu_seqlens for all documents.") if cu_seqlens is not None: - _validate_cu_seqlens(cu_seqlens, device=q.device, name="cu_seqlens") - if global_cu_seqlens is not None: - if not isinstance(global_cu_seqlens, torch.Tensor): - raise TypeError("global_cu_seqlens must be a torch.Tensor.") - if global_cu_seqlens.dtype != torch.int32 or global_cu_seqlens.dim() != 1: - raise ValueError("global_cu_seqlens must be a one-dimensional int32 tensor.") - metadata = (global_cu_seqlens, cu_seqlens) - cacheable = all(t is None or not torch.is_inference(t) for t in metadata) + if not isinstance(cu_seqlens, torch.Tensor): + raise TypeError("cu_seqlens must be a torch.Tensor.") + if cu_seqlens.dtype != torch.int32 or cu_seqlens.dim() != 1: + raise ValueError("cu_seqlens must be a one-dimensional int32 tensor.") + cacheable = not torch.is_inference(cu_seqlens) key = ( qkv_format, tuple(q.shape[:2]), - id(global_cu_seqlens), - global_cu_seqlens._version if cacheable else None, + id(cu_seqlens), + cu_seqlens._version if cacheable else None, q.device, size, rank, - id(cu_seqlens), - None if cu_seqlens is None or not cacheable else cu_seqlens._version, ) else: key, cacheable = (qkv_format, batch, length, q.device, size, rank), True @@ -323,19 +313,16 @@ def context_parallel_attention( cache = None if cache is None or cache[0] != key: documents = ( - tuple(global_cu_seqlens.tolist()) - if global_cu_seqlens is not None + tuple(cu_seqlens.tolist()) + if cu_seqlens is not None else tuple(i * length * size for i in range(batch + 1)) ) if dense and len(documents) != batch + 1: - raise ValueError("Dense global_cu_seqlens must describe the full batch.") + raise ValueError("Dense cu_seqlens must describe the full batch.") host = build_linear_attention_partition(documents, size, rank, dense=dense) plan = _device_partition(host, q.device) - if cu_seqlens is not None: - if tuple(cu_seqlens.tolist()) != host.local_cu_seqlens: - raise ValueError("Local cu_seqlens does not match the contiguous CP fragments.") - # Hold tensor references to prevent object-ID reuse from invalidating the cache. - adapter._cp_partition_cache = (key, plan, global_cu_seqlens, cu_seqlens) + # Hold the metadata tensor to prevent object-ID reuse in the cache key. + adapter._cp_partition_cache = (key, plan, cu_seqlens) else: plan = cache[1] if tensors[0].shape[0] != plan.host.local_cu_seqlens[-1]: diff --git a/transformer_engine/pytorch/attention/linear_attention/gdn.py b/transformer_engine/pytorch/attention/linear_attention/gdn.py index 97d7f2c0af5..88f5f620925 100644 --- a/transformer_engine/pytorch/attention/linear_attention/gdn.py +++ b/transformer_engine/pytorch/attention/linear_attention/gdn.py @@ -20,6 +20,7 @@ AlignedTimelineKernelAdapter, LinearAttentionBase, _needs_eager_linear_attention, + _validate_cu_seqlens, ) @@ -57,6 +58,20 @@ def _validate_gates( f"{expected_gate_shape}; got {tuple(g.shape)} and {tuple(beta.shape)}." ) + def _resolve_cu_seqlens(self, query_layer, qkv_format, cu_seqlens): + """Accept full-document metadata for dense inputs as well as packed inputs.""" + if qkv_format == "thd" or cu_seqlens is None: + return super()._resolve_cu_seqlens(query_layer, qkv_format, cu_seqlens) + _validate_cu_seqlens(cu_seqlens, device=query_layer.device, name="cu_seqlens") + batch, length = ( + query_layer.shape[:2] + if qkv_format == "bshd" + else (query_layer.shape[1], query_layer.shape[0]) + ) + if cu_seqlens.numel() != batch + 1: + raise ValueError("Dense cu_seqlens must have shape [batch_size + 1].") + return cu_seqlens, length + def _call_op( self, query_layer: torch.Tensor, @@ -210,7 +225,6 @@ def forward( output_final_state: bool = False, use_qk_l2norm_in_kernel: bool = False, cp_group: Optional[dist.ProcessGroup] = None, - global_cu_seqlens: Optional[torch.Tensor] = None, ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: """Run GDN and return a TE-layout output, optionally with the final state.""" if cp_group is not None and dist.get_world_size(cp_group) > 1: @@ -226,29 +240,25 @@ def forward( group=cp_group, qkv_format=qkv_format, cu_seqlens=cu_seqlens, - global_cu_seqlens=global_cu_seqlens, output_final_state=output_final_state, use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, ) - if global_cu_seqlens is not None: - if cp_group is None: - raise ValueError("global_cu_seqlens requires CP.") - if not isinstance(global_cu_seqlens, torch.Tensor): - raise TypeError("global_cu_seqlens must be a torch.Tensor.") - if global_cu_seqlens.dtype != torch.int32 or global_cu_seqlens.dim() != 1: - raise ValueError("global_cu_seqlens must be a one-dimensional int32 tensor.") + if cp_group is not None and cu_seqlens is not None: + if not isinstance(cu_seqlens, torch.Tensor): + raise TypeError("cu_seqlens must be a torch.Tensor.") + if cu_seqlens.dtype != torch.int32 or cu_seqlens.dim() != 1: + raise ValueError("cu_seqlens must be a one-dimensional int32 tensor.") if qkv_format == "thd": - if cu_seqlens is not None and cu_seqlens.tolist() != global_cu_seqlens.tolist(): - raise ValueError("Local cu_seqlens must match the size-one global input.") - cu_seqlens = global_cu_seqlens.to(query_layer.device) + cu_seqlens = cu_seqlens.to(query_layer.device) elif qkv_format in {"bshd", "sbhd"}: batch, length = ( query_layer.shape[:2] if qkv_format == "bshd" else (query_layer.shape[1], query_layer.shape[0]) ) - if global_cu_seqlens.tolist() != [i * length for i in range(batch + 1)]: - raise ValueError("Dense global_cu_seqlens does not match the size-one input.") + if cu_seqlens.tolist() != [i * length for i in range(batch + 1)]: + raise ValueError("Dense cu_seqlens does not match the size-one CP input.") + cu_seqlens = None return self._run( query_layer, key_layer, @@ -358,7 +368,6 @@ def forward( initial_state: Optional[torch.Tensor] = None, output_final_state: bool = False, use_qk_l2norm_in_kernel: bool = False, - global_cu_seqlens: Optional[torch.Tensor] = None, ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: """Apply Gated DeltaNet linear attention. @@ -376,13 +385,13 @@ def forward( qkv_format : Optional[str], default = `None` Overrides the module's configured `qkv_format` for this call. cu_seqlens : Optional[torch.Tensor], default = `None` - Cumulative sequence lengths for packed (`thd`) inputs, of shape - `[batch_size + 1]` and dtype `torch.int32` without CP. With THD CP, - TE intersects the global documents with this rank's token interval - and derives cumulative lengths for the resulting local fragments. - Documents with no tokens on this rank are omitted. With THD CP, - this argument is optional; if supplied, its values must equal - those derived offsets. + Cumulative document lengths, of shape `[batch_size + 1]` and dtype + `torch.int32`. Required for THD. With CP, describe the complete + documents across the CP group, identically on all ranks; TE derives + the local fragment boundaries. For BSHD/SBHD CP, this metadata is + optional for equal slices and required for uneven slices; see Notes. + Without CP, dense metadata is optional and must describe the + full rows as `[0, S, 2*S, ..., B*S]`. checkpoint_core_attention : bool, default = `False` If true, forward activations for this module are recomputed during the backward pass instead of saved. @@ -394,22 +403,20 @@ def forward( If true, also return the final recurrent state. use_qk_l2norm_in_kernel : bool, default = `False` If true, L2-normalize Q and K inside the kernel before the recurrence. - global_cu_seqlens : Optional[torch.Tensor], default = `None` - Global cumulative document lengths, dtype `torch.int32`, identical - on all ranks. Required for THD CP: rank ``r`` supplies tokens - ``[r*T//P:(r+1)*T//P]`` from each packed input, where ``T`` is the - final global offset and ``P`` is CP size. No CP divisibility constraint - applies; pack tokens without gaps. - For BSHD/SBHD CP, use ``[0, S, 2*S, ..., B*S]``, where ``B`` is - batch size and ``S`` is the full sequence length of each batch row. - These are document boundaries, not rank boundaries. Each rank must - supply ``[r*S//P:(r+1)*S//P]`` of every row. This metadata is required - when ``S`` is not divisible by ``P``; otherwise TE can infer - ``S = local_sequence_length * P``. Arbitrary rank splits are not - supported. Notes ----- + With THD CP, rank ``r`` supplies tokens ``[r*T//P:(r+1)*T//P]`` from + the global packed buffer, where ``T = cu_seqlens[-1]`` and ``P`` is CP + size. Document lengths need not be divisible by CP size; pack tokens + without gaps. + + For BSHD/SBHD CP, rank ``r`` supplies ``[r*S//P:(r+1)*S//P]`` of every + batch row. Here ``S`` is the full sequence length and ``B`` is batch size. + Supply ``cu_seqlens=[0, S, 2*S, ..., B*S]`` when ``S`` is not divisible + by ``P``; otherwise TE can infer ``S = local_sequence_length * P``. + Arbitrary rank splits are not supported. + With CP, ``initial_state`` describes all global documents and must have identical values on all ranks. Each document's initial-state gradient is returned on its first owning rank, with zeros elsewhere; sum these @@ -438,7 +445,6 @@ def forward( "output_final_state": output_final_state, "use_qk_l2norm_in_kernel": use_qk_l2norm_in_kernel, "cp_group": self.cp_group, - "global_cu_seqlens": global_cu_seqlens, } with self.prepare_forward_ctx(query_layer, allow_non_contiguous=True) as query_layer: return self._dispatch_attention( From 19b4ffee23032df8296cd573de21777031f8a96b Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Tue, 10 Nov 2026 17:54:26 -0800 Subject: [PATCH 07/34] Use descriptive linear attention CP metadata names Signed-off-by: Kshitij Lakhani --- .../attention/test_gdn_attention_with_cp.py | 41 +++++---- .../common/linear_attention/__init__.py | 88 ++++++++++--------- .../linear_attention/context_parallel.py | 36 ++++---- 3 files changed, 90 insertions(+), 75 deletions(-) diff --git a/tests/pytorch/attention/test_gdn_attention_with_cp.py b/tests/pytorch/attention/test_gdn_attention_with_cp.py index f1c7bd4ea8b..1e4a0eca30d 100644 --- a/tests/pytorch/attention/test_gdn_attention_with_cp.py +++ b/tests/pytorch/attention/test_gdn_attention_with_cp.py @@ -20,8 +20,8 @@ import transformer_engine_torch as tex -@pytest.mark.parametrize("size", [1, 2, 4, 8]) -def test_partition_intersections(size): +@pytest.mark.parametrize("cp_size", [1, 2, 4, 8]) +def test_partition_intersections(cp_size): """Recover all document intervals without dropping tokens or crossing documents.""" rng = random.Random(301) for _ in range(100): @@ -29,50 +29,55 @@ def test_partition_intersections(size): cu = [0] for length in lengths: cu.append(cu[-1] + length) - partitions = [build_linear_attention_partition(cu, size, rank) for rank in range(size)] + partitions = [ + build_linear_attention_partition(cu, cp_size=cp_size, cp_rank=cp_rank) + for cp_rank in range(cp_size) + ] for doc, (start, end) in enumerate(zip(cu, cu[1:])): fragments = [f for p in partitions for f in p.fragments if f.document == doc] assert sum(f.end - f.start for f in fragments) == end - start assert all(start <= f.start < f.end <= end for f in fragments) assert all(a.end == b.start for a, b in zip(fragments, fragments[1:])) - for rank, partition in enumerate(partitions): + for cp_rank, partition in enumerate(partitions): assert ( partition.local_cu_seqlens[-1] - == (rank + 1) * cu[-1] // size - rank * cu[-1] // size + == (cp_rank + 1) * cu[-1] // cp_size - cp_rank * cu[-1] // cp_size ) assert len(partition.incoming) <= 1 assert len(partition.outgoing) <= 1 for f, forward, backward in zip( partition.fragments, partition.forward_descriptors, partition.backward_descriptors ): - assert forward[2] == rank - f.first_rank - assert backward[2] == f.last_rank - rank + assert forward[2] == cp_rank - f.first_rank + assert backward[2] == f.last_rank - cp_rank # Every referenced recipe belongs to this same document. - for predecessor in range(f.first_rank, rank): + for predecessor in range(f.first_rank, cp_rank): p = partitions[predecessor] if p.fragments: assert p.fragments[p.outgoing[0]].document == f.document - for successor in range(rank + 1, f.last_rank + 1): + for successor in range(cp_rank + 1, f.last_rank + 1): p = partitions[successor] # An empty rank contributes an identity recipe. if p.fragments: assert p.fragments[p.incoming[0]].document == f.document -@pytest.mark.parametrize("documents", [[1, 2], [0], [0, 2, 1]]) -def test_invalid_documents(documents): +@pytest.mark.parametrize("cu_seqlens", [[1, 2], [0], [0, 2, 1]]) +def test_invalid_documents(cu_seqlens): """Reject malformed global offsets before computing communication descriptors.""" with pytest.raises(ValueError): - build_linear_attention_partition(documents, 4, 0) + build_linear_attention_partition(cu_seqlens, cp_size=4, cp_rank=0) -@pytest.mark.parametrize("size", [1, 2, 4, 8]) +@pytest.mark.parametrize("cp_size", [1, 2, 4, 8]) @pytest.mark.parametrize("length", [0, 1, 5, 9]) -def test_dense_partition_intersections(size, length): +def test_dense_partition_intersections(cp_size, length): """Dense rows retain their document identity, including uneven and empty ranks.""" partitions = [ - build_linear_attention_partition([0, length, 2 * length], size, rank, dense=True) - for rank in range(size) + build_linear_attention_partition( + [0, length, 2 * length], cp_size=cp_size, cp_rank=cp_rank, dense=True + ) + for cp_rank in range(cp_size) ] for doc in (0, 1): spans = [f for p in partitions for f in p.fragments if f.document == doc] @@ -81,8 +86,8 @@ def test_dense_partition_intersections(size, length): if length: assert spans[0].start == doc * length assert spans[-1].end == (doc + 1) * length - for rank, partition in enumerate(partitions): - local_length = (rank + 1) * length // size - rank * length // size + for cp_rank, partition in enumerate(partitions): + local_length = (cp_rank + 1) * length // cp_size - cp_rank * length // cp_size assert partition.local_cu_seqlens[-1] == 2 * local_length assert partition.slots == 2 assert len(partition.fragments) == (2 if local_length else 0) diff --git a/transformer_engine/common/linear_attention/__init__.py b/transformer_engine/common/linear_attention/__init__.py index 27bd5185960..df8911b426b 100644 --- a/transformer_engine/common/linear_attention/__init__.py +++ b/transformer_engine/common/linear_attention/__init__.py @@ -24,14 +24,14 @@ class LinearAttentionFragment: class LinearAttentionCPPartition: """Host metadata, independent of tensor frameworks and attention flavors. - Packed inputs intersect document intervals with contiguous rank intervals. - Dense inputs give every rank a contiguous, nearly equal slice of each document. + Packed (THD) inputs intersect documents with contiguous rank intervals. + Dense (BSHD/SBHD) inputs give each rank a nearly equal slice of every document. Empty documents have no fragments; their state is unchanged. """ - documents: Tuple[int, ...] - rank: int - size: int + cu_seqlens: Tuple[int, ...] + cp_rank: int + cp_size: int fragments: Tuple[LinearAttentionFragment, ...] local_cu_seqlens: Tuple[int, ...] forward_descriptors: Tuple[Tuple[int, int, int, int], ...] @@ -41,70 +41,78 @@ class LinearAttentionCPPartition: terminal: Tuple[int, ...] seed_owners: Tuple[int, ...] empty_owned: Tuple[int, ...] - slots: int + slots: int # One per rank for packed inputs; one per document for dense inputs. communicates: bool -def build_linear_attention_partition(documents, size, rank, *, dense=False): - """Build ordered fragment intersections and prefix/suffix record ranges. +def build_linear_attention_partition(cu_seqlens, cp_size, cp_rank, *, dense=False): + """Build one rank's fragment and summary metadata on the host. - ``documents`` are global cumulative offsets. Packed rank r owns - [floor(r*T/P), floor((r+1)*T/P)). Dense documents must all have the same - length. Descriptor rows are (document, first, count, stride). - Their record ranges never cross an independent document's boundary. + ``cu_seqlens`` contains full-document boundaries as host integers. For + P = ``cp_size``, packed rank r owns [r*T//P:(r+1)*T//P] of T total tokens. + Dense rank r owns [r*S//P:(r+1)*S//P] of each length-S document. + Descriptor rows are (document, first_record, count, stride). Each selects + summaries of one document, in forward or reverse order. """ - documents = tuple(documents) - if size < 1 or not 0 <= rank < size: + cu_seqlens = tuple(cu_seqlens) + if cp_size < 1 or not 0 <= cp_rank < cp_size: raise ValueError("CP size must be positive and rank must lie inside the group.") - if len(documents) < 2 or documents[0] != 0: + if len(cu_seqlens) < 2 or cu_seqlens[0] != 0: raise ValueError("Global cumulative sequence lengths must start at zero.") - if any(a > b for a, b in zip(documents, documents[1:])): + if any(a > b for a, b in zip(cu_seqlens, cu_seqlens[1:])): raise ValueError("Global cumulative sequence lengths must be nondecreasing.") - total = documents[-1] - boundaries = tuple(r * total // size for r in range(size + 1)) - slots = len(documents) - 1 if dense else 1 + total = cu_seqlens[-1] + boundaries = tuple(r * total // cp_size for r in range(cp_size + 1)) + slots = len(cu_seqlens) - 1 if dense else 1 fragments, seeds, empty = [], [], [] communicates = False - for doc, (start, end) in enumerate(zip(documents, documents[1:])): + for doc, (start, end) in enumerate(zip(cu_seqlens, cu_seqlens[1:])): if dense: length = end - start - if length != documents[1]: + if length != cu_seqlens[1]: raise ValueError("Dense CP needs equal global document lengths.") - cuts = tuple(r * length // size for r in range(size + 1)) - first = min(size - 1, bisect_right(cuts, 0) - 1) if length else 0 - last = min(size - 1, bisect_right(cuts, length - 1) - 1) if length else 0 - left, right = start + cuts[rank], start + cuts[rank + 1] + cuts = tuple(r * length // cp_size for r in range(cp_size + 1)) + first_rank = min(cp_size - 1, bisect_right(cuts, 0) - 1) if length else 0 + last_rank = min(cp_size - 1, bisect_right(cuts, length - 1) - 1) if length else 0 + fragment_start, fragment_end = start + cuts[cp_rank], start + cuts[cp_rank + 1] else: - first = min(size - 1, bisect_right(boundaries, start) - 1) - last = min(size - 1, bisect_right(boundaries, end - 1) - 1) if end > start else first - left, right = max(start, boundaries[rank]), min(end, boundaries[rank + 1]) - seeds.append(first) - communicates |= first != last - if start == end and first == rank: + first_rank = min(cp_size - 1, bisect_right(boundaries, start) - 1) + last_rank = ( + min(cp_size - 1, bisect_right(boundaries, end - 1) - 1) + if end > start + else first_rank + ) + fragment_start = max(start, boundaries[cp_rank]) + fragment_end = min(end, boundaries[cp_rank + 1]) + seeds.append(first_rank) + communicates |= first_rank != last_rank + if start == end and first_rank == cp_rank: empty.append(doc) - if left < right: - fragments.append(LinearAttentionFragment(doc, left, right, first, last)) + if fragment_start < fragment_end: + fragments.append( + LinearAttentionFragment(doc, fragment_start, fragment_end, first_rank, last_rank) + ) cu = [0] forward, backward, outgoing, incoming, terminal = [], [], [], [], [] for index, frag in enumerate(fragments): cu.append(cu[-1] + frag.end - frag.start) slot = frag.document if dense else 0 forward.append( - (frag.document, frag.first_rank * slots + slot, rank - frag.first_rank, slots) + (frag.document, frag.first_rank * slots + slot, cp_rank - frag.first_rank, slots) ) backward.append( - (frag.document, frag.last_rank * slots + slot, frag.last_rank - rank, -slots) + (frag.document, frag.last_rank * slots + slot, frag.last_rank - cp_rank, -slots) ) - if rank < frag.last_rank: + if cp_rank < frag.last_rank: outgoing.append(index) else: terminal.append(index) - if rank > frag.first_rank: + if cp_rank > frag.first_rank: incoming.append(index) return LinearAttentionCPPartition( - documents, - rank, - size, + cu_seqlens, + cp_rank, + cp_size, tuple(fragments), tuple(cu), tuple(forward), diff --git a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py index f2b2f47bea1..05aac6d2a6d 100644 --- a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py +++ b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py @@ -56,7 +56,7 @@ def cumulative(selected): offsets.append(offsets[-1] + frag.end - frag.start) return torch.tensor(offsets, device=device, dtype=torch.int32) - first = [i for i, f in enumerate(host.fragments) if f.first_rank == host.rank] + first = [i for i, f in enumerate(host.fragments) if f.first_rank == host.cp_rank] forward = torch.tensor(host.forward_descriptors, device=device, dtype=torch.int32).reshape( -1, 4 ) @@ -88,15 +88,15 @@ def cumulative(selected): def _gather_summaries(additive, transition, group): """One fixed-size all-gather for both summary tensors, in CP group rank order.""" slots, heads, value_dim, key_dim = additive.shape - size = dist.get_world_size(group) + cp_size = dist.get_world_size(group) packed = torch.cat((additive.flatten(), transition.flatten())) - gathered = torch.empty(size * packed.numel(), device=packed.device, dtype=packed.dtype) + gathered = torch.empty(cp_size * packed.numel(), device=packed.device, dtype=packed.dtype) dist.all_gather_into_tensor(gathered, packed, group=group) - gathered = gathered.view(size, -1) + gathered = gathered.view(cp_size, -1) split = additive.numel() return ( - gathered[:, :split].contiguous().view(size * slots, heads, value_dim, key_dim), - gathered[:, split:].contiguous().view(size * slots, heads, key_dim, key_dim), + gathered[:, :split].contiguous().view(cp_size * slots, heads, value_dim, key_dim), + gathered[:, split:].contiguous().view(cp_size * slots, heads, key_dim, key_dim), ) @@ -150,7 +150,7 @@ def forward( q = tensors[0] k = tensors[1] v = tensors[2] - state_shape = (len(plan.host.documents) - 1, q.shape[1], v.shape[-1], k.shape[-1]) + state_shape = (len(plan.host.cu_seqlens) - 1, q.shape[1], v.shape[-1], k.shape[-1]) seed = ( q.new_zeros((1, *state_shape[1:]), dtype=torch.float32) if initial_state is None @@ -213,7 +213,7 @@ def backward(ctx, grad_output, grad_final): if grad_output is None: grad_output = torch.zeros_like(output) grad_output = grad_output.contiguous() - shape = (len(plan.host.documents) - 1, q.shape[1], v.shape[-1], k.shape[-1]) + shape = (len(plan.host.cu_seqlens) - 1, q.shape[1], v.shape[-1], k.shape[-1]) seed = starting.new_zeros(shape if ctx.final else (1, *shape[1:])) if ctx.final: if grad_final is not None: @@ -281,7 +281,7 @@ def context_parallel_attention( ): """Derive local fragment boundaries from full-document metadata and run CP.""" adapter._validate_inputs(q, k, v, gates, initial_state, qkv_format) - size, rank = dist.get_world_size(group), dist.get_rank(group) + cp_size, cp_rank = dist.get_world_size(group), dist.get_rank(group) tensors = tuple(_to_thd(t, qkv_format).contiguous() for t in (q, k, v, *gates.values())) dense = qkv_format != "thd" batch, length = None, None @@ -301,25 +301,27 @@ def context_parallel_attention( id(cu_seqlens), cu_seqlens._version if cacheable else None, q.device, - size, - rank, + cp_size, + cp_rank, ) else: - key, cacheable = (qkv_format, batch, length, q.device, size, rank), True + key, cacheable = (qkv_format, batch, length, q.device, cp_size, cp_rank), True cache = getattr(adapter, "_cp_partition_cache", None) if not cacheable: # Inference tensors have no mutation counter, so revalidate their # metadata instead of silently reusing a stale partition. cache = None if cache is None or cache[0] != key: - documents = ( + document_cu_seqlens = ( tuple(cu_seqlens.tolist()) if cu_seqlens is not None - else tuple(i * length * size for i in range(batch + 1)) + else tuple(i * length * cp_size for i in range(batch + 1)) ) - if dense and len(documents) != batch + 1: + if dense and len(document_cu_seqlens) != batch + 1: raise ValueError("Dense cu_seqlens must describe the full batch.") - host = build_linear_attention_partition(documents, size, rank, dense=dense) + host = build_linear_attention_partition( + document_cu_seqlens, cp_size=cp_size, cp_rank=cp_rank, dense=dense + ) plan = _device_partition(host, q.device) # Hold the metadata tensor to prevent object-ID reuse in the cache key. adapter._cp_partition_cache = (key, plan, cu_seqlens) @@ -328,7 +330,7 @@ def context_parallel_attention( if tensors[0].shape[0] != plan.host.local_cu_seqlens[-1]: raise ValueError("Local token count does not match this CP rank's token interval.") adapter._validate_initial_state( - initial_state, batch_size=len(plan.host.documents) - 1, device=q.device + initial_state, batch_size=len(plan.host.cu_seqlens) - 1, device=q.device ) tensors, initial_state = adapter._prepare_cp_inputs(tensors, initial_state) if adapter._op is None: From a3e16ed1b08be90dec56f3904fb8184fe401f362 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Thu, 12 Nov 2026 14:32:08 -0800 Subject: [PATCH 08/34] Rename affine prefix seed terminology to initial memory Signed-off-by: Kshitij Lakhani --- .../transformer_engine/linear_attention.h | 17 +++++++++-------- .../common/linear_attention/affine_prefix.cu | 4 ++++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/transformer_engine/common/include/transformer_engine/linear_attention.h b/transformer_engine/common/include/transformer_engine/linear_attention.h index 8b2be9e76c7..dc00e036d17 100644 --- a/transformer_engine/common/include/transformer_engine/linear_attention.h +++ b/transformer_engine/common/include/transformer_engine/linear_attention.h @@ -18,19 +18,20 @@ extern "C" { /*! \brief Apply ordered affine summaries to selected recurrent states. * - * Each output starts from a selected seed and repeatedly evaluates + * Each output starts from a selected initial memory and repeatedly evaluates * ``state = state @ transition[record] + additive[record]``. This operation * serves both forward prefixes and backward suffixes; backward callers pass * the backward-oriented transitions and a negative record stride. * - * All tensors are contiguous device tensors. Seeds and additives are FP32 - * [seeds, heads, value_dim, key_dim] and [records, heads, value_dim, key_dim]. + * All tensors are contiguous device tensors. Initial memories (``seeds``) and + * additives are FP32 [initial_memories, heads, value_dim, key_dim] and + * [records, heads, value_dim, key_dim], respectively. * Transitions are FP32 [records, heads, key_dim, key_dim]. Descriptors are - * INT32 [outputs, 4], with rows (seed_index, first_record, count, stride). - * Outputs are FP32 [outputs, heads, value_dim, key_dim]. Referenced seed and - * record indices must be valid; count must be nonnegative. - * A seed_index of -1 means an implicit zero state. Zero-count descriptors - * copy their seed (or write zeros). No memory is allocated. + * INT32 [outputs, 4], with rows (initial_memory_index, first_record, count, stride). + * Outputs are FP32 [outputs, heads, value_dim, key_dim]. Referenced initial-memory + * and record indices must be valid; count must be nonnegative. + * An initial_memory_index of -1 means an implicit zero state. Zero-count + * descriptors copy their initial memory (or write zeros). No memory is allocated. */ void nvte_linear_attention_affine_prefix(const NVTETensor seeds, const NVTETensor additive, const NVTETensor transitions, const NVTETensor descriptors, diff --git a/transformer_engine/common/linear_attention/affine_prefix.cu b/transformer_engine/common/linear_attention/affine_prefix.cu index 0ae6f7cc4d0..07fffe1ce20 100644 --- a/transformer_engine/common/linear_attention/affine_prefix.cu +++ b/transformer_engine/common/linear_attention/affine_prefix.cu @@ -23,6 +23,7 @@ __global__ void affine_prefix_kernel(const float *seeds, const float *additive, float *state = storage; float *next = storage + kRows * key_dim; const size_t tiles = (value_dim + kRows - 1) / kRows; + // span indexes a local fragment; each fragment uses heads * tiles blocks. const size_t span = blockIdx.x / (heads * tiles); const size_t head = (blockIdx.x / tiles) % heads; const size_t first_value = (blockIdx.x % tiles) * kRows; @@ -38,6 +39,9 @@ __global__ void affine_prefix_kernel(const float *seeds, const float *additive, if (value < value_dim) { if (begin) { assert(first >= 0 && static_cast(first) < num_records); + // H[first, head, value, col]: summary record, head, value row, key column. + // Flat offset = first*(heads*value_dim*key_dim) + head*(value_dim*key_dim) + // + value*key_dim + col. initial = additive[((first * heads + head) * value_dim + value) * key_dim + col]; } else if (seed >= 0) { initial = seeds[((seed * heads + head) * value_dim + value) * key_dim + col]; From be431732889d3985929169a510926fd10cd48273 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Fri, 13 Nov 2026 18:36:50 -0800 Subject: [PATCH 09/34] Rename CP metadata and state names Signed-off-by: Kshitij Lakhani --- .../linear_attention/context_parallel.py | 210 +++++++++++------- 1 file changed, 129 insertions(+), 81 deletions(-) diff --git a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py index 05aac6d2a6d..7c4fade1f1f 100644 --- a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py +++ b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py @@ -23,10 +23,10 @@ @dataclass -class _DevicePartition: - """Cached device metadata; the host partition has no framework dependency.""" +class _CPMetadata: + """Logical partition and device tensors used to execute linear-attention CP.""" - host: LinearAttentionCPPartition + partition: LinearAttentionCPPartition cu: torch.Tensor forward: torch.Tensor backward: torch.Tensor @@ -36,15 +36,15 @@ class _DevicePartition: outgoing: torch.Tensor incoming: torch.Tensor terminal: torch.Tensor - seed_fragments: torch.Tensor - seed_docs: torch.Tensor + initial_state_fragments: torch.Tensor + initial_state_docs: torch.Tensor empty_docs: torch.Tensor outgoing_cu: torch.Tensor incoming_cu: torch.Tensor -def _device_partition(host, device): - """Materialize small metadata tensors once, outside the steady-state kernels.""" +def _build_cp_metadata(partition, device): + """Build device metadata for an existing logical partition.""" def indices(values): return torch.tensor(values, device=device, dtype=torch.int64) @@ -52,36 +52,36 @@ def indices(values): def cumulative(selected): offsets = [0] for index in selected: - frag = host.fragments[index] + frag = partition.fragments[index] offsets.append(offsets[-1] + frag.end - frag.start) return torch.tensor(offsets, device=device, dtype=torch.int32) - first = [i for i, f in enumerate(host.fragments) if f.first_rank == host.cp_rank] - forward = torch.tensor(host.forward_descriptors, device=device, dtype=torch.int32).reshape( - -1, 4 - ) - backward = torch.tensor(host.backward_descriptors, device=device, dtype=torch.int32).reshape( + first = [i for i, f in enumerate(partition.fragments) if f.first_rank == partition.cp_rank] + forward = torch.tensor(partition.forward_descriptors, device=device, dtype=torch.int32).reshape( -1, 4 ) + backward = torch.tensor( + partition.backward_descriptors, device=device, dtype=torch.int32 + ).reshape(-1, 4) forward_zero, backward_zero = forward.clone(), backward.clone() forward_zero[:, 0] = -1 backward_zero[:, 0] = -1 - return _DevicePartition( - host, - torch.tensor(host.local_cu_seqlens, device=device, dtype=torch.int32), + return _CPMetadata( + partition, + torch.tensor(partition.local_cu_seqlens, device=device, dtype=torch.int32), forward, backward, forward_zero, backward_zero, - indices([f.document for f in host.fragments]), - indices(host.outgoing), - indices(host.incoming), - indices(host.terminal), + indices([f.document for f in partition.fragments]), + indices(partition.outgoing), + indices(partition.incoming), + indices(partition.terminal), indices(first), - indices([host.fragments[i].document for i in first]), - indices(host.empty_owned), - cumulative(host.outgoing), - cumulative(host.incoming), + indices([partition.fragments[i].document for i in first]), + indices(partition.empty_owned), + cumulative(partition.outgoing), + cumulative(partition.incoming), ) @@ -100,25 +100,25 @@ def _gather_summaries(additive, transition, group): ) -def _summary_buffers(seed, slots): +def _summary_buffers(state_template, slots): """Unused slots are identity recipes, including ranks owning zero tokens.""" - _, heads, value_dim, key_dim = seed.shape - additive = seed.new_zeros((slots, heads, value_dim, key_dim)) - transition = torch.eye(key_dim, device=seed.device, dtype=seed.dtype) + _, heads, value_dim, key_dim = state_template.shape + additive = state_template.new_zeros((slots, heads, value_dim, key_dim)) + transition = torch.eye(key_dim, device=state_template.device, dtype=state_template.dtype) return additive, transition.expand(slots, heads, key_dim, key_dim).contiguous() -def _selected_span(plan, selected): +def _selected_span(metadata, selected): """Selected spans are consecutive: all dense spans, or one packed edge span.""" - cu = plan.host.local_cu_seqlens + cu = metadata.partition.local_cu_seqlens return slice(cu[selected[0]], cu[selected[-1] + 1]) -def _slot_indices(plan, indices): +def _slot_indices(metadata, indices): """Dense slots are document IDs; packed partitions have only one edge slot.""" - if plan.host.slots == 1: + if metadata.partition.slots == 1: return torch.zeros_like(indices) - return plan.docs.index_select(0, indices) + return metadata.docs.index_select(0, indices) def _local_attention(adapter, tensors, gate_names, cu, state, options, *, final=True): @@ -143,61 +143,88 @@ class _LinearAttentionContextParallel(torch.autograd.Function): @staticmethod def forward( - ctx, adapter, plan, group, options, final, gate_names, track_grad, initial_state, *tensors + ctx, + adapter, + metadata, + group, + options, + final, + gate_names, + track_grad, + initial_state, + *tensors, ): """Establish starting states, then retain the local attention graph.""" ctx.set_materialize_grads(False) + partition = metadata.partition q = tensors[0] k = tensors[1] v = tensors[2] - state_shape = (len(plan.host.cu_seqlens) - 1, q.shape[1], v.shape[-1], k.shape[-1]) - seed = ( + state_shape = ( + len(partition.cu_seqlens) - 1, + q.shape[1], + v.shape[-1], + k.shape[-1], + ) + # Zero-prefix descriptors need only one placeholder when no initial state is supplied. + initial_state_buffer = ( q.new_zeros((1, *state_shape[1:]), dtype=torch.float32) if initial_state is None else initial_state ) - additive, transition = _summary_buffers(seed, plan.host.slots) - if plan.host.outgoing: - span = _selected_span(plan, plan.host.outgoing) + additive, transition = _summary_buffers(initial_state_buffer, partition.slots) + if partition.outgoing: + span = _selected_span(metadata, partition.outgoing) selected = tuple(t[span] for t in tensors) h, t = adapter._cp_forward_summary( - *selected[:3], dict(zip(gate_names, selected[3:])), plan.outgoing_cu, **options + *selected[:3], + dict(zip(gate_names, selected[3:])), + metadata.outgoing_cu, + **options, ) - slots = _slot_indices(plan, plan.outgoing) + slots = _slot_indices(metadata, metadata.outgoing) additive.index_copy_(0, slots, h) transition.index_copy_(0, slots, t) - if plan.host.communicates: + if partition.communicates: additive, transition = _gather_summaries(additive, transition, group) starting = tex.linear_attention_affine_prefix( - seed.contiguous(), + initial_state_buffer.contiguous(), additive, transition, - plan.forward if initial_state is not None else plan.forward_zero, + (metadata.forward if initial_state is not None else metadata.forward_zero), ) # Retain FE's local graph only for training. This Function owns the # distributed VJP and the outer state gradients. with torch.set_grad_enabled(track_grad): inputs = tuple(t.detach().requires_grad_(track_grad) for t in (*tensors, starting)) - if plan.host.fragments: + if partition.fragments: output, states = _local_attention( - adapter, inputs[:-1], gate_names, plan.cu, inputs[-1], options + adapter, inputs[:-1], gate_names, metadata.cu, inputs[-1], options ) else: output = v.new_empty((q.shape[0], q.shape[1], v.shape[-1])) states = starting - terminal = seed.new_zeros(state_shape) if final else seed.new_empty((0,)) + terminal = ( + initial_state_buffer.new_zeros(state_shape) + if final + else initial_state_buffer.new_empty((0,)) + ) if final: terminal.index_copy_( 0, - plan.docs.index_select(0, plan.terminal), - states.detach().index_select(0, plan.terminal), + metadata.docs.index_select(0, metadata.terminal), + states.detach().index_select(0, metadata.terminal), ) if initial_state is not None: - terminal.index_copy_(0, plan.empty_docs, seed.index_select(0, plan.empty_docs)) + terminal.index_copy_( + 0, + metadata.empty_docs, + initial_state_buffer.index_select(0, metadata.empty_docs), + ) dist.all_reduce(terminal, group=group) if track_grad: ctx.save_for_backward(*inputs, output, states) - ctx.plan, ctx.group, ctx.final = plan, group, final + ctx.cp_metadata, ctx.group, ctx.final = metadata, group, final ctx.adapter, ctx.options, ctx.gate_names = adapter, options, gate_names ctx.has_initial_state = initial_state is not None return output.detach(), terminal @@ -209,37 +236,46 @@ def backward(ctx, grad_output, grad_final): tensors = ctx.saved_tensors[:-3] starting, output, states = ctx.saved_tensors[-3:] q, k, v = tensors[:3] - plan, group = ctx.plan, ctx.group + metadata, group = ctx.cp_metadata, ctx.group + partition = metadata.partition if grad_output is None: grad_output = torch.zeros_like(output) grad_output = grad_output.contiguous() - shape = (len(plan.host.cu_seqlens) - 1, q.shape[1], v.shape[-1], k.shape[-1]) - seed = starting.new_zeros(shape if ctx.final else (1, *shape[1:])) + shape = ( + len(partition.cu_seqlens) - 1, + q.shape[1], + v.shape[-1], + k.shape[-1], + ) + final_state_grad = starting.new_zeros(shape if ctx.final else (1, *shape[1:])) if ctx.final: if grad_final is not None: - seed.copy_(grad_final) + final_state_grad.copy_(grad_final) # Final states are replicated outputs: sum losses consuming any replica. - dist.all_reduce(seed, group=group) - additive, transition = _summary_buffers(seed, plan.host.slots) - if plan.host.incoming: - span = _selected_span(plan, plan.host.incoming) + dist.all_reduce(final_state_grad, group=group) + additive, transition = _summary_buffers(final_state_grad, partition.slots) + if partition.incoming: + span = _selected_span(metadata, partition.incoming) selected = tuple(t[span] for t in tensors) h, t = ctx.adapter._cp_backward_summary( *selected[:3], dict(zip(ctx.gate_names, selected[3:])), grad_output[span], - plan.incoming_cu, + metadata.incoming_cu, **ctx.options, ) - slots = _slot_indices(plan, plan.incoming) + slots = _slot_indices(metadata, metadata.incoming) additive.index_copy_(0, slots, h) transition.index_copy_(0, slots, t) - if plan.host.communicates: + if partition.communicates: additive, transition = _gather_summaries(additive, transition, group) ending_grad = tex.linear_attention_affine_prefix( - seed, additive, transition, plan.backward if ctx.final else plan.backward_zero + final_state_grad, + additive, + transition, + metadata.backward if ctx.final else metadata.backward_zero, ) - if plan.host.fragments: + if partition.fragments: with torch.enable_grad(): grads = torch.autograd.grad( (output, states), @@ -250,19 +286,25 @@ def backward(ctx, grad_output, grad_final): ) else: grads = tuple(torch.zeros_like(t) for t in (*tensors, starting)) - initial_grad = None + initial_state_grad = None if ctx.has_initial_state: - initial_grad = seed.new_zeros(shape) - initial_grad.index_add_( - 0, plan.seed_docs, grads[-1].index_select(0, plan.seed_fragments) + initial_state_grad = final_state_grad.new_zeros(shape) + initial_state_grad.index_add_( + 0, + metadata.initial_state_docs, + grads[-1].index_select(0, metadata.initial_state_fragments), ) if ctx.final: - initial_grad.index_copy_(0, plan.empty_docs, seed.index_select(0, plan.empty_docs)) - # Each document's seed gradient belongs to its first rank. Do not + initial_state_grad.index_copy_( + 0, + metadata.empty_docs, + final_state_grad.index_select(0, metadata.empty_docs), + ) + # Each document's initial-state gradient belongs to its first rank. Do not # replicate it here: if initial_state came from a previous CP call, # that call's replicated-final-state VJP sums these owner gradients. # Replicating now would count chained state gradients CP-size times. - return (None, None, None, None, None, None, None, initial_grad, *grads[:-1]) + return (None, None, None, None, None, None, None, initial_state_grad, *grads[:-1]) def context_parallel_attention( @@ -311,6 +353,7 @@ def context_parallel_attention( # Inference tensors have no mutation counter, so revalidate their # metadata instead of silently reusing a stale partition. cache = None + # Build and upload metadata only when the cached partition no longer matches. if cache is None or cache[0] != key: document_cu_seqlens = ( tuple(cu_seqlens.tolist()) @@ -319,30 +362,35 @@ def context_parallel_attention( ) if dense and len(document_cu_seqlens) != batch + 1: raise ValueError("Dense cu_seqlens must describe the full batch.") - host = build_linear_attention_partition( + partition = build_linear_attention_partition( document_cu_seqlens, cp_size=cp_size, cp_rank=cp_rank, dense=dense ) - plan = _device_partition(host, q.device) + metadata = _build_cp_metadata(partition, q.device) # Hold the metadata tensor to prevent object-ID reuse in the cache key. - adapter._cp_partition_cache = (key, plan, cu_seqlens) + adapter._cp_partition_cache = (key, metadata, cu_seqlens) else: - plan = cache[1] - if tensors[0].shape[0] != plan.host.local_cu_seqlens[-1]: + metadata = cache[1] + partition = metadata.partition + if tensors[0].shape[0] != partition.local_cu_seqlens[-1]: raise ValueError("Local token count does not match this CP rank's token interval.") adapter._validate_initial_state( - initial_state, batch_size=len(plan.host.cu_seqlens) - 1, device=q.device + initial_state, + batch_size=len(partition.cu_seqlens) - 1, + device=q.device, ) tensors, initial_state = adapter._prepare_cp_inputs(tensors, initial_state) + # Resolve the cuDNN operation once and reuse it on later calls. if adapter._op is None: adapter._op = adapter._resolve_op() + # Without shared documents or explicit states, use ordinary local autograd. if ( - not plan.host.communicates + not partition.communicates and initial_state is None and not output_final_state - and plan.host.fragments + and partition.fragments ): output, states = _local_attention( - adapter, tensors, tuple(gates), plan.cu, None, kernel_kwargs, final=False + adapter, tensors, tuple(gates), metadata.cu, None, kernel_kwargs, final=False ) else: track_grad = torch.is_grad_enabled() and any( @@ -350,7 +398,7 @@ def context_parallel_attention( ) output, states = _LinearAttentionContextParallel.apply( adapter, - plan, + metadata, group, kernel_kwargs, output_final_state, From 6c4d7cff0a96a758f8ef62d884dc8afc9f86ca6c Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Fri, 13 Nov 2026 20:09:13 -0800 Subject: [PATCH 10/34] Clarify linear attention CP state names Signed-off-by: Kshitij Lakhani --- .../linear_attention_context_parallel.rst | 2 +- tests/cpp/operator/test_linear_attention.cu | 32 ++--- tests/pytorch/attention/run_gdn_with_cp.py | 29 ++-- .../attention/test_gdn_attention_with_cp.py | 10 +- .../transformer_engine/linear_attention.h | 8 +- .../common/linear_attention/__init__.py | 8 +- .../common/linear_attention/affine_prefix.cu | 118 +++++++++------- .../linear_attention/context_parallel.py | 131 ++++++++++-------- .../pytorch/attention/linear_attention/gdn.py | 10 +- transformer_engine/pytorch/csrc/extensions.h | 4 +- .../pytorch/csrc/extensions/attention.cpp | 32 +++-- 11 files changed, 214 insertions(+), 170 deletions(-) diff --git a/docs/features/other_optimizations/linear_attention_context_parallel.rst b/docs/features/other_optimizations/linear_attention_context_parallel.rst index a82d7a4dd79..dc451ee771e 100644 --- a/docs/features/other_optimizations/linear_attention_context_parallel.rst +++ b/docs/features/other_optimizations/linear_attention_context_parallel.rst @@ -84,7 +84,7 @@ Recurrent states and gradients The optional ``initial_state`` is FP32 with shape ``[global_documents, local_heads, value_dim, key_dim]`` and identical values on every CP rank. Requested final states have the same global shape and are -replicated using a separate collective. Empty documents return their seed. +replicated using a separate collective. Empty documents return their initial state. Each initial-state gradient is returned on its document's first owning rank, with zeros elsewhere. Sum these contributions over CP when inspecting a diff --git a/tests/cpp/operator/test_linear_attention.cu b/tests/cpp/operator/test_linear_attention.cu index aa395ecc162..5b80c0dd4a5 100644 --- a/tests/cpp/operator/test_linear_attention.cu +++ b/tests/cpp/operator/test_linear_attention.cu @@ -21,16 +21,16 @@ TEST_P(LinearAttentionPrefixTest, OrderedAffineRecipes) { using test::Tensor; using transformer_engine::DType; const size_t k = GetParam(), heads = 2, values = 5, row_size = heads * values * k; - Tensor seeds("seeds", std::vector{3, heads, values, k}, DType::kFloat32); - Tensor offsets("offsets", std::vector{6, heads, values, k}, DType::kFloat32); - Tensor matrices("matrices", std::vector{6, heads, k, k}, DType::kFloat32); + Tensor initial_states("initial_states", std::vector{3, heads, values, k}, DType::kFloat32); + Tensor additive("additive", std::vector{6, heads, values, k}, DType::kFloat32); + Tensor transitions("transitions", std::vector{6, heads, k, k}, DType::kFloat32); Tensor descriptors("descriptors", std::vector{6, 4}, DType::kInt32); Tensor output("output", std::vector{6, heads, values, k}, DType::kFloat32); std::mt19937 generator(317); std::uniform_real_distribution distribution(-1.f, 1.f); - auto *s = seeds.rowwise_cpu_dptr(); - auto *h = offsets.rowwise_cpu_dptr(); - auto *t = matrices.rowwise_cpu_dptr(); + auto *s = initial_states.rowwise_cpu_dptr(); + auto *h = additive.rowwise_cpu_dptr(); + auto *t = transitions.rowwise_cpu_dptr(); auto *d = descriptors.rowwise_cpu_dptr(); for (size_t i = 0; i < 3 * row_size; ++i) s[i] = distribution(generator); for (size_t i = 0; i < 6 * row_size; ++i) h[i] = distribution(generator); @@ -38,26 +38,26 @@ TEST_P(LinearAttentionPrefixTest, OrderedAffineRecipes) { const int rows[] = {0, 0, 0, 1, 1, 0, 3, 1, 2, 5, 3, -2, -1, 0, 0, 1, -1, 0, 3, 1, -1, 5, 3, -2}; std::copy(rows, rows + 24, d); - seeds.from_cpu(); - offsets.from_cpu(); - matrices.from_cpu(); + initial_states.from_cpu(); + additive.from_cpu(); + transitions.from_cpu(); descriptors.from_cpu(); - nvte_linear_attention_affine_prefix(seeds.data(), offsets.data(), matrices.data(), + nvte_linear_attention_affine_prefix(initial_states.data(), additive.data(), transitions.data(), descriptors.data(), output.data(), nullptr); ASSERT_EQ(cudaDeviceSynchronize(), cudaSuccess); output.to_cpu(); const auto *actual = output.rowwise_cpu_dptr(); - for (size_t span = 0; span < 6; ++span) { + for (size_t fragment = 0; fragment < 6; ++fragment) { for (size_t head = 0; head < heads; ++head) { for (size_t value = 0; value < values; ++value) { const size_t row = head * values + value; std::vector state(k, 0.), next(k); - if (rows[span * 4] >= 0) { - const float *start = s + rows[span * 4] * row_size + row * k; + if (rows[fragment * 4] >= 0) { + const float *start = s + rows[fragment * 4] * row_size + row * k; std::copy(start, start + k, state.begin()); } - for (int step = 0; step < rows[span * 4 + 2]; ++step) { - const int record = rows[span * 4 + 1] + step * rows[span * 4 + 3]; + for (int step = 0; step < rows[fragment * 4 + 2]; ++step) { + const int record = rows[fragment * 4 + 1] + step * rows[fragment * 4 + 3]; for (size_t col = 0; col < k; ++col) { next[col] = h[record * row_size + row * k + col]; for (size_t inner = 0; inner < k; ++inner) { @@ -67,7 +67,7 @@ TEST_P(LinearAttentionPrefixTest, OrderedAffineRecipes) { state.swap(next); } for (size_t col = 0; col < k; ++col) { - ASSERT_NEAR(actual[span * row_size + row * k + col], state[col], 2e-5); + ASSERT_NEAR(actual[fragment * row_size + row * k + col], state[col], 2e-5); } } } diff --git a/tests/pytorch/attention/run_gdn_with_cp.py b/tests/pytorch/attention/run_gdn_with_cp.py index 8d47ebd7e39..658351fa02c 100644 --- a/tests/pytorch/attention/run_gdn_with_cp.py +++ b/tests/pytorch/attention/run_gdn_with_cp.py @@ -95,10 +95,10 @@ def _case( v = torch.randn(total, heads, value_dim, device=device, dtype=dtype) g = -torch.rand(total, heads, device=device) * 0.02 beta = torch.rand(total, heads, device=device) * 0.7 - seed = torch.randn(len(lengths), heads, value_dim, key_dim, device=device) * 0.1 + initial_state = torch.randn(len(lengths), heads, value_dim, key_dim, device=device) * 0.1 if not initial: - seed.zero_() - full = tuple(t.detach().requires_grad_(True) for t in (q, k, v, g, beta, seed)) + initial_state.zero_() + full = tuple(t.detach().requires_grad_(True) for t in (q, k, v, g, beta, initial_state)) global_cu = torch.tensor(cu, device=device, dtype=torch.int32) if independent or total == 0: ref_output, ref_states = _reference(full, cu, scale, norm) @@ -117,7 +117,9 @@ def _case( grad_output = torch.randn_like(v) if last_loss: grad_output[: -max(1, total // size)] = 0 - grad_states = torch.randn_like(seed) * 0.1 if final else torch.zeros_like(seed) + grad_states = ( + torch.randn_like(initial_state) * 0.1 if final else torch.zeros_like(initial_state) + ) loss = (ref_states * grad_states).sum() if not state_only: loss = loss + (ref_output * grad_output).sum() @@ -267,19 +269,21 @@ def _state_chaining(group): g = -torch.rand(tokens, heads, device=device) * 0.02 beta = torch.rand(tokens, heads, device=device) * 0.7 full.extend(t.detach().requires_grad_(True) for t in (q, k, v, g, beta)) - seed = (torch.randn(2, heads, dim, dim, device=device) * 0.1).requires_grad_(True) + initial_state = (torch.randn(2, heads, dim, dim, device=device) * 0.1).requires_grad_(True) cu = torch.tensor([0, 17, 35], device=device, dtype=torch.int32) reference = GatedDeltaNetAttention(heads, dim, qkv_format="thd") - first, state = reference(*full[:5], cu_seqlens=cu, initial_state=seed, output_final_state=True) + first, state = reference( + *full[:5], cu_seqlens=cu, initial_state=initial_state, output_final_state=True + ) second = reference(*full[5:], cu_seqlens=cu, initial_state=state) grad_first, grad_second = torch.randn_like(first), torch.randn_like(second) ((first * grad_first).sum() * 0.3 + (second * grad_second).sum()).backward() lo, hi = rank * tokens // size, (rank + 1) * tokens // size inputs = tuple(t[lo:hi].detach().requires_grad_(True) for t in full) - local_seed = seed.detach().requires_grad_(True) + local_initial_state = initial_state.detach().requires_grad_(True) module = GatedDeltaNetAttention(heads, dim, qkv_format="thd", cp_group=group) local_first, state = module( - *inputs[:5], cu_seqlens=cu, initial_state=local_seed, output_final_state=True + *inputs[:5], cu_seqlens=cu, initial_state=local_initial_state, output_final_state=True ) local_second = module(*inputs[5:], cu_seqlens=cu, initial_state=state) loss = (local_first * grad_first[lo:hi]).sum() * 0.3 @@ -293,12 +297,15 @@ def _state_chaining(group): _BWD_TOL[torch.bfloat16], "chained token gradient", ) - initial_grad = local_seed.grad.clone() + initial_grad = local_initial_state.grad.clone() dist.all_reduce(initial_grad, group=group) _assert_rms_close( - initial_grad, seed.grad, _BWD_TOL[torch.bfloat16], "chained initial-state gradient" + initial_grad, + initial_state.grad, + _BWD_TOL[torch.bfloat16], + "chained initial-state gradient", ) - for tensor in (*inputs, local_seed): + for tensor in (*inputs, local_initial_state): tensor.grad = None dist.barrier(group=group) diff --git a/tests/pytorch/attention/test_gdn_attention_with_cp.py b/tests/pytorch/attention/test_gdn_attention_with_cp.py index 1e4a0eca30d..82ea79530c8 100644 --- a/tests/pytorch/attention/test_gdn_attention_with_cp.py +++ b/tests/pytorch/attention/test_gdn_attention_with_cp.py @@ -96,7 +96,7 @@ def test_dense_partition_intersections(cp_size, length): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") @pytest.mark.parametrize("key_dim", [3, 64, 128, 257]) def test_common_affine_prefix(key_dim): - """Nonsymmetric transitions, reversed order, zero prefixes, and independent seeds.""" + """Nonsymmetric transitions, reversed order, zero prefixes, and independent initial states.""" torch.manual_seed(22) current = torch.cuda.current_device() # Also exercise the binding's device guard: allocating on another GPU must @@ -104,7 +104,7 @@ def test_common_affine_prefix(key_dim): device = torch.device( "cuda", (current + 1) % torch.cuda.device_count() if key_dim == 3 else current ) - seed = torch.randn(3, 2, 5, key_dim, device=device) + initial_states = torch.randn(3, 2, 5, key_dim, device=device) additive = torch.randn(6, 2, 5, key_dim, device=device) transitions = torch.randn(6, 2, key_dim, key_dim, device=device) / key_dim**0.5 rows = [ @@ -116,14 +116,14 @@ def test_common_affine_prefix(key_dim): [-1, 5, 3, -2], ] descriptors = torch.tensor(rows, device=device, dtype=torch.int32) - actual = tex.linear_attention_affine_prefix(seed, additive, transitions, descriptors) + actual = tex.linear_attention_affine_prefix(initial_states, additive, transitions, descriptors) assert torch.cuda.current_device() == current expected = [] for source, first, count, stride in rows: x = ( - torch.zeros_like(seed[0], dtype=torch.float64) + torch.zeros_like(initial_states[0], dtype=torch.float64) if source == -1 - else seed[source].double() + else initial_states[source].double() ) for step in range(count): index = first + step * stride diff --git a/transformer_engine/common/include/transformer_engine/linear_attention.h b/transformer_engine/common/include/transformer_engine/linear_attention.h index dc00e036d17..93e31f55e33 100644 --- a/transformer_engine/common/include/transformer_engine/linear_attention.h +++ b/transformer_engine/common/include/transformer_engine/linear_attention.h @@ -23,9 +23,9 @@ extern "C" { * serves both forward prefixes and backward suffixes; backward callers pass * the backward-oriented transitions and a negative record stride. * - * All tensors are contiguous device tensors. Initial memories (``seeds``) and - * additives are FP32 [initial_memories, heads, value_dim, key_dim] and - * [records, heads, value_dim, key_dim], respectively. + * All tensors are contiguous device tensors. Initial states are FP32 + * [initial_memories, heads, value_dim, key_dim]; additives are FP32 + * [records, heads, value_dim, key_dim]. * Transitions are FP32 [records, heads, key_dim, key_dim]. Descriptors are * INT32 [outputs, 4], with rows (initial_memory_index, first_record, count, stride). * Outputs are FP32 [outputs, heads, value_dim, key_dim]. Referenced initial-memory @@ -33,7 +33,7 @@ extern "C" { * An initial_memory_index of -1 means an implicit zero state. Zero-count * descriptors copy their initial memory (or write zeros). No memory is allocated. */ -void nvte_linear_attention_affine_prefix(const NVTETensor seeds, const NVTETensor additive, +void nvte_linear_attention_affine_prefix(const NVTETensor initial_states, const NVTETensor additive, const NVTETensor transitions, const NVTETensor descriptors, NVTETensor output, cudaStream_t stream); diff --git a/transformer_engine/common/linear_attention/__init__.py b/transformer_engine/common/linear_attention/__init__.py index df8911b426b..b9811d1e930 100644 --- a/transformer_engine/common/linear_attention/__init__.py +++ b/transformer_engine/common/linear_attention/__init__.py @@ -39,7 +39,7 @@ class LinearAttentionCPPartition: outgoing: Tuple[int, ...] incoming: Tuple[int, ...] terminal: Tuple[int, ...] - seed_owners: Tuple[int, ...] + initial_state_owners: Tuple[int, ...] empty_owned: Tuple[int, ...] slots: int # One per rank for packed inputs; one per document for dense inputs. communicates: bool @@ -64,7 +64,7 @@ def build_linear_attention_partition(cu_seqlens, cp_size, cp_rank, *, dense=Fals total = cu_seqlens[-1] boundaries = tuple(r * total // cp_size for r in range(cp_size + 1)) slots = len(cu_seqlens) - 1 if dense else 1 - fragments, seeds, empty = [], [], [] + fragments, initial_state_owners, empty = [], [], [] communicates = False for doc, (start, end) in enumerate(zip(cu_seqlens, cu_seqlens[1:])): if dense: @@ -84,7 +84,7 @@ def build_linear_attention_partition(cu_seqlens, cp_size, cp_rank, *, dense=Fals ) fragment_start = max(start, boundaries[cp_rank]) fragment_end = min(end, boundaries[cp_rank + 1]) - seeds.append(first_rank) + initial_state_owners.append(first_rank) communicates |= first_rank != last_rank if start == end and first_rank == cp_rank: empty.append(doc) @@ -120,7 +120,7 @@ def build_linear_attention_partition(cu_seqlens, cp_size, cp_rank, *, dense=Fals tuple(outgoing), tuple(incoming), tuple(terminal), - tuple(seeds), + tuple(initial_state_owners), tuple(empty), slots, communicates, diff --git a/transformer_engine/common/linear_attention/affine_prefix.cu b/transformer_engine/common/linear_attention/affine_prefix.cu index 07fffe1ce20..ae5a6e78163 100644 --- a/transformer_engine/common/linear_attention/affine_prefix.cu +++ b/transformer_engine/common/linear_attention/affine_prefix.cu @@ -15,73 +15,79 @@ namespace { // Reuse transition loads across value rows, preserving FP32 accumulation. constexpr int kRows = 4; -__global__ void affine_prefix_kernel(const float *seeds, const float *additive, +__global__ void affine_prefix_kernel(const float *initial_states, const float *additive, const float *transitions, const int *descriptors, - float *output, size_t num_seeds, size_t num_records, + float *output, size_t num_initial_states, size_t num_records, size_t heads, size_t value_dim, size_t key_dim) { extern __shared__ float storage[]; float *state = storage; - float *next = storage + kRows * key_dim; - const size_t tiles = (value_dim + kRows - 1) / kRows; - // span indexes a local fragment; each fragment uses heads * tiles blocks. - const size_t span = blockIdx.x / (heads * tiles); - const size_t head = (blockIdx.x / tiles) % heads; - const size_t first_value = (blockIdx.x % tiles) * kRows; - const int *desc = descriptors + span * 4; - const int seed = desc[0], first = desc[1], count = desc[2], stride = desc[3]; - assert((seed == -1 || (seed >= 0 && static_cast(seed) < num_seeds)) && count >= 0); - // An implicit zero seed makes the first matrix product unnecessary. - const int begin = seed == -1 && count > 0 ? 1 : 0; + float *next_state = storage + kRows * key_dim; + const size_t value_tiles = (value_dim + kRows - 1) / kRows; + // Each fragment/head pair owns one block per tile of value rows. + const size_t fragment = blockIdx.x / (heads * value_tiles); + const size_t head = (blockIdx.x / value_tiles) % heads; + const size_t first_value_row = (blockIdx.x % value_tiles) * kRows; + const int *descriptor = descriptors + fragment * 4; + const int initial_state_index = descriptor[0], first_record = descriptor[1], + record_count = descriptor[2], stride = descriptor[3]; + assert((initial_state_index == -1 || + (initial_state_index >= 0 && + static_cast(initial_state_index) < num_initial_states)) && + record_count >= 0); + // An implicit zero initial state makes the first matrix product unnecessary. + const int first_step = initial_state_index == -1 && record_count > 0 ? 1 : 0; for (size_t col = threadIdx.x; col < key_dim; col += blockDim.x) { for (int row = 0; row < kRows; ++row) { - const size_t value = first_value + row; + const size_t value = first_value_row + row; float initial = 0.f; if (value < value_dim) { - if (begin) { - assert(first >= 0 && static_cast(first) < num_records); - // H[first, head, value, col]: summary record, head, value row, key column. - // Flat offset = first*(heads*value_dim*key_dim) + head*(value_dim*key_dim) + if (first_step) { + assert(first_record >= 0 && static_cast(first_record) < num_records); + // H[first_record, head, value, col]: summary record, head, value row, key column. + // Flat offset = first_record*(heads*value_dim*key_dim) + head*(value_dim*key_dim) // + value*key_dim + col. - initial = additive[((first * heads + head) * value_dim + value) * key_dim + col]; - } else if (seed >= 0) { - initial = seeds[((seed * heads + head) * value_dim + value) * key_dim + col]; + initial = additive[((first_record * heads + head) * value_dim + value) * key_dim + col]; + } else if (initial_state_index >= 0) { + initial = + initial_states[((initial_state_index * heads + head) * value_dim + value) * key_dim + + col]; } } state[row * key_dim + col] = initial; } } __syncthreads(); - for (int step = begin; step < count; ++step) { - const int record = first + step * stride; + for (int step = first_step; step < record_count; ++step) { + const int record = first_record + step * stride; assert(record >= 0 && static_cast(record) < num_records); - const float *matrix = transitions + (record * heads + head) * key_dim * key_dim; + const float *transition_matrix = transitions + (record * heads + head) * key_dim * key_dim; for (size_t col = threadIdx.x; col < key_dim; col += blockDim.x) { float result[kRows]; for (int row = 0; row < kRows; ++row) { - const size_t value = first_value + row; + const size_t value = first_value_row + row; result[row] = value < value_dim ? additive[((record * heads + head) * value_dim + value) * key_dim + col] : 0.f; } for (size_t inner = 0; inner < key_dim; ++inner) { - const float transition = matrix[inner * key_dim + col]; + const float transition = transition_matrix[inner * key_dim + col]; #pragma unroll for (int row = 0; row < kRows; ++row) { result[row] = fmaf(state[row * key_dim + inner], transition, result[row]); } } - for (int row = 0; row < kRows; ++row) next[row * key_dim + col] = result[row]; + for (int row = 0; row < kRows; ++row) next_state[row * key_dim + col] = result[row]; } __syncthreads(); float *swap = state; - state = next; - next = swap; + state = next_state; + next_state = swap; } for (size_t col = threadIdx.x; col < key_dim; col += blockDim.x) { for (int row = 0; row < kRows; ++row) { - const size_t value = first_value + row; + const size_t value = first_value_row + row; if (value < value_dim) { - output[((span * heads + head) * value_dim + value) * key_dim + col] = + output[((fragment * heads + head) * value_dim + value) * key_dim + col] = state[row * key_dim + col]; } } @@ -91,35 +97,41 @@ __global__ void affine_prefix_kernel(const float *seeds, const float *additive, } // namespace } // namespace transformer_engine -void nvte_linear_attention_affine_prefix(const NVTETensor seeds, const NVTETensor additive, +void nvte_linear_attention_affine_prefix(const NVTETensor initial_states, const NVTETensor additive, const NVTETensor transitions, const NVTETensor descriptors, NVTETensor output, cudaStream_t stream) { NVTE_API_CALL(nvte_linear_attention_affine_prefix); using namespace transformer_engine; - const auto &s = convertNVTETensorCheck(seeds)->data; - const auto &a = convertNVTETensorCheck(additive)->data; - const auto &t = convertNVTETensorCheck(transitions)->data; - const auto &d = convertNVTETensorCheck(descriptors)->data; - auto &o = convertNVTETensorCheck(output)->data; - NVTE_CHECK(s.dtype == DType::kFloat32 && a.dtype == DType::kFloat32 && - t.dtype == DType::kFloat32 && o.dtype == DType::kFloat32, + const auto &initial_data = convertNVTETensorCheck(initial_states)->data; + const auto &additive_data = convertNVTETensorCheck(additive)->data; + const auto &transition_data = convertNVTETensorCheck(transitions)->data; + const auto &descriptor_data = convertNVTETensorCheck(descriptors)->data; + auto &output_data = convertNVTETensorCheck(output)->data; + NVTE_CHECK(initial_data.dtype == DType::kFloat32 && additive_data.dtype == DType::kFloat32 && + transition_data.dtype == DType::kFloat32 && output_data.dtype == DType::kFloat32, "Affine states and summaries must be FP32."); - NVTE_CHECK(d.dtype == DType::kInt32, "Affine descriptors must be INT32."); - NVTE_CHECK(s.shape.size() == 4 && a.shape.size() == 4 && t.shape.size() == 4 && - o.shape.size() == 4 && d.shape.size() == 2, + NVTE_CHECK(descriptor_data.dtype == DType::kInt32, "Affine descriptors must be INT32."); + NVTE_CHECK(initial_data.shape.size() == 4 && additive_data.shape.size() == 4 && + transition_data.shape.size() == 4 && output_data.shape.size() == 4 && + descriptor_data.shape.size() == 2, "Invalid affine tensor ranks."); - const size_t heads = s.shape[1], value_dim = s.shape[2], key_dim = s.shape[3]; + const size_t heads = initial_data.shape[1], value_dim = initial_data.shape[2], + key_dim = initial_data.shape[3]; NVTE_CHECK(heads > 0 && value_dim > 0 && key_dim > 0, "State dimensions must be positive."); - NVTE_CHECK((a.shape == std::vector{t.shape[0], heads, value_dim, key_dim}) && - (t.shape == std::vector{a.shape[0], heads, key_dim, key_dim}) && - (o.shape == std::vector{d.shape[0], heads, value_dim, key_dim}) && - d.shape[1] == 4, + NVTE_CHECK((additive_data.shape == + std::vector{transition_data.shape[0], heads, value_dim, key_dim}) && + (transition_data.shape == + std::vector{additive_data.shape[0], heads, key_dim, key_dim}) && + (output_data.shape == + std::vector{descriptor_data.shape[0], heads, value_dim, key_dim}) && + descriptor_data.shape[1] == 4, "Inconsistent affine tensor shapes."); - if (o.shape[0] == 0) return; - const size_t tiles = o.shape[0] * heads * ((value_dim + kRows - 1) / kRows); - affine_prefix_kernel<<>>( - static_cast(s.dptr), static_cast(a.dptr), - static_cast(t.dptr), static_cast(d.dptr), - static_cast(o.dptr), s.shape[0], a.shape[0], heads, value_dim, key_dim); + if (output_data.shape[0] == 0) return; + const size_t value_tiles = output_data.shape[0] * heads * ((value_dim + kRows - 1) / kRows); + affine_prefix_kernel<<>>( + static_cast(initial_data.dptr), static_cast(additive_data.dptr), + static_cast(transition_data.dptr), + static_cast(descriptor_data.dptr), static_cast(output_data.dptr), + initial_data.shape[0], additive_data.shape[0], heads, value_dim, key_dim); NVTE_CHECK_CUDA(cudaGetLastError()); } diff --git a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py index 7c4fade1f1f..55d9bd37c5f 100644 --- a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py +++ b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py @@ -27,20 +27,20 @@ class _CPMetadata: """Logical partition and device tensors used to execute linear-attention CP.""" partition: LinearAttentionCPPartition - cu: torch.Tensor - forward: torch.Tensor - backward: torch.Tensor - forward_zero: torch.Tensor - backward_zero: torch.Tensor - docs: torch.Tensor + cu_seqlens: torch.Tensor + forward_descriptors: torch.Tensor + backward_descriptors: torch.Tensor + forward_zero_descriptors: torch.Tensor + backward_zero_descriptors: torch.Tensor + document_indices: torch.Tensor outgoing: torch.Tensor incoming: torch.Tensor terminal: torch.Tensor initial_state_fragments: torch.Tensor - initial_state_docs: torch.Tensor - empty_docs: torch.Tensor - outgoing_cu: torch.Tensor - incoming_cu: torch.Tensor + initial_state_documents: torch.Tensor + empty_documents: torch.Tensor + outgoing_cu_seqlens: torch.Tensor + incoming_cu_seqlens: torch.Tensor def _build_cp_metadata(partition, device): @@ -57,22 +57,25 @@ def cumulative(selected): return torch.tensor(offsets, device=device, dtype=torch.int32) first = [i for i, f in enumerate(partition.fragments) if f.first_rank == partition.cp_rank] - forward = torch.tensor(partition.forward_descriptors, device=device, dtype=torch.int32).reshape( - -1, 4 - ) - backward = torch.tensor( + forward_descriptors = torch.tensor( + partition.forward_descriptors, device=device, dtype=torch.int32 + ).reshape(-1, 4) + backward_descriptors = torch.tensor( partition.backward_descriptors, device=device, dtype=torch.int32 ).reshape(-1, 4) - forward_zero, backward_zero = forward.clone(), backward.clone() - forward_zero[:, 0] = -1 - backward_zero[:, 0] = -1 + forward_zero_descriptors, backward_zero_descriptors = ( + forward_descriptors.clone(), + backward_descriptors.clone(), + ) + forward_zero_descriptors[:, 0] = -1 + backward_zero_descriptors[:, 0] = -1 return _CPMetadata( partition, torch.tensor(partition.local_cu_seqlens, device=device, dtype=torch.int32), - forward, - backward, - forward_zero, - backward_zero, + forward_descriptors, + backward_descriptors, + forward_zero_descriptors, + backward_zero_descriptors, indices([f.document for f in partition.fragments]), indices(partition.outgoing), indices(partition.incoming), @@ -110,25 +113,27 @@ def _summary_buffers(state_template, slots): def _selected_span(metadata, selected): """Selected spans are consecutive: all dense spans, or one packed edge span.""" - cu = metadata.partition.local_cu_seqlens - return slice(cu[selected[0]], cu[selected[-1] + 1]) + cu_seqlens = metadata.partition.local_cu_seqlens + return slice(cu_seqlens[selected[0]], cu_seqlens[selected[-1] + 1]) def _slot_indices(metadata, indices): """Dense slots are document IDs; packed partitions have only one edge slot.""" if metadata.partition.slots == 1: return torch.zeros_like(indices) - return metadata.docs.index_select(0, indices) + return metadata.document_indices.index_select(0, indices) -def _local_attention(adapter, tensors, gate_names, cu, state, options, *, final=True): +def _local_attention( + adapter, tensors, gate_names, cu_seqlens, state, options, *, output_final_state=True +): """Call the flavor adapter with packed local inputs.""" return adapter._call_op( *tensors[:3], dict(zip(gate_names, tensors[3:])), - cu, + cu_seqlens, initial_state=state, - output_final_state=final, + output_final_state=output_final_state, **options, ) @@ -148,7 +153,7 @@ def forward( metadata, group, options, - final, + output_final_state, gate_names, track_grad, initial_state, @@ -179,7 +184,7 @@ def forward( h, t = adapter._cp_forward_summary( *selected[:3], dict(zip(gate_names, selected[3:])), - metadata.outgoing_cu, + metadata.outgoing_cu_seqlens, **options, ) slots = _slot_indices(metadata, metadata.outgoing) @@ -187,44 +192,50 @@ def forward( transition.index_copy_(0, slots, t) if partition.communicates: additive, transition = _gather_summaries(additive, transition, group) - starting = tex.linear_attention_affine_prefix( + starting_states = tex.linear_attention_affine_prefix( initial_state_buffer.contiguous(), additive, transition, - (metadata.forward if initial_state is not None else metadata.forward_zero), + ( + metadata.forward_descriptors + if initial_state is not None + else metadata.forward_zero_descriptors + ), ) # Retain FE's local graph only for training. This Function owns the # distributed VJP and the outer state gradients. with torch.set_grad_enabled(track_grad): - inputs = tuple(t.detach().requires_grad_(track_grad) for t in (*tensors, starting)) + inputs = tuple( + t.detach().requires_grad_(track_grad) for t in (*tensors, starting_states) + ) if partition.fragments: output, states = _local_attention( - adapter, inputs[:-1], gate_names, metadata.cu, inputs[-1], options + adapter, inputs[:-1], gate_names, metadata.cu_seqlens, inputs[-1], options ) else: output = v.new_empty((q.shape[0], q.shape[1], v.shape[-1])) - states = starting + states = starting_states terminal = ( initial_state_buffer.new_zeros(state_shape) - if final + if output_final_state else initial_state_buffer.new_empty((0,)) ) - if final: + if output_final_state: terminal.index_copy_( 0, - metadata.docs.index_select(0, metadata.terminal), + metadata.document_indices.index_select(0, metadata.terminal), states.detach().index_select(0, metadata.terminal), ) if initial_state is not None: terminal.index_copy_( 0, - metadata.empty_docs, - initial_state_buffer.index_select(0, metadata.empty_docs), + metadata.empty_documents, + initial_state_buffer.index_select(0, metadata.empty_documents), ) dist.all_reduce(terminal, group=group) if track_grad: ctx.save_for_backward(*inputs, output, states) - ctx.cp_metadata, ctx.group, ctx.final = metadata, group, final + ctx.cp_metadata, ctx.group, ctx.output_final_state = metadata, group, output_final_state ctx.adapter, ctx.options, ctx.gate_names = adapter, options, gate_names ctx.has_initial_state = initial_state is not None return output.detach(), terminal @@ -234,7 +245,7 @@ def forward( def backward(ctx, grad_output, grad_final): """Exchange reverse recipes and pass outgoing state adjoints to FE's VJP.""" tensors = ctx.saved_tensors[:-3] - starting, output, states = ctx.saved_tensors[-3:] + starting_states, output, states = ctx.saved_tensors[-3:] q, k, v = tensors[:3] metadata, group = ctx.cp_metadata, ctx.group partition = metadata.partition @@ -247,8 +258,10 @@ def backward(ctx, grad_output, grad_final): v.shape[-1], k.shape[-1], ) - final_state_grad = starting.new_zeros(shape if ctx.final else (1, *shape[1:])) - if ctx.final: + final_state_grad = starting_states.new_zeros( + shape if ctx.output_final_state else (1, *shape[1:]) + ) + if ctx.output_final_state: if grad_final is not None: final_state_grad.copy_(grad_final) # Final states are replicated outputs: sum losses consuming any replica. @@ -261,7 +274,7 @@ def backward(ctx, grad_output, grad_final): *selected[:3], dict(zip(ctx.gate_names, selected[3:])), grad_output[span], - metadata.incoming_cu, + metadata.incoming_cu_seqlens, **ctx.options, ) slots = _slot_indices(metadata, metadata.incoming) @@ -269,36 +282,40 @@ def backward(ctx, grad_output, grad_final): transition.index_copy_(0, slots, t) if partition.communicates: additive, transition = _gather_summaries(additive, transition, group) - ending_grad = tex.linear_attention_affine_prefix( + ending_state_grad = tex.linear_attention_affine_prefix( final_state_grad, additive, transition, - metadata.backward if ctx.final else metadata.backward_zero, + ( + metadata.backward_descriptors + if ctx.output_final_state + else metadata.backward_zero_descriptors + ), ) if partition.fragments: with torch.enable_grad(): grads = torch.autograd.grad( (output, states), - (*tensors, starting), - (grad_output, ending_grad), + (*tensors, starting_states), + (grad_output, ending_state_grad), # The outer graph controls lifetime, including retain_graph=True. retain_graph=True, ) else: - grads = tuple(torch.zeros_like(t) for t in (*tensors, starting)) + grads = tuple(torch.zeros_like(t) for t in (*tensors, starting_states)) initial_state_grad = None if ctx.has_initial_state: initial_state_grad = final_state_grad.new_zeros(shape) initial_state_grad.index_add_( 0, - metadata.initial_state_docs, + metadata.initial_state_documents, grads[-1].index_select(0, metadata.initial_state_fragments), ) - if ctx.final: + if ctx.output_final_state: initial_state_grad.index_copy_( 0, - metadata.empty_docs, - final_state_grad.index_select(0, metadata.empty_docs), + metadata.empty_documents, + final_state_grad.index_select(0, metadata.empty_documents), ) # Each document's initial-state gradient belongs to its first rank. Do not # replicate it here: if initial_state came from a previous CP call, @@ -390,7 +407,13 @@ def context_parallel_attention( and partition.fragments ): output, states = _local_attention( - adapter, tensors, tuple(gates), metadata.cu, None, kernel_kwargs, final=False + adapter, + tensors, + tuple(gates), + metadata.cu_seqlens, + None, + kernel_kwargs, + output_final_state=False, ) else: track_grad = torch.is_grad_enabled() and any( diff --git a/transformer_engine/pytorch/attention/linear_attention/gdn.py b/transformer_engine/pytorch/attention/linear_attention/gdn.py index 88f5f620925..c47e9fc0996 100644 --- a/transformer_engine/pytorch/attention/linear_attention/gdn.py +++ b/transformer_engine/pytorch/attention/linear_attention/gdn.py @@ -136,7 +136,7 @@ def _cp_forward_summary(self, query, key, value, gates, cu_seqlens, **kernel_kwa except Exception as exc: # pylint: disable=broad-exception-caught if type(exc).__name__ != "cudnnGraphNotSupportedError": raise - # A zero seed gives H; an identity seed with zero values gives T. + # A zero initial state gives H; an identity initial state with zero values gives T. additive = self._call_op( k, k, @@ -189,7 +189,7 @@ def _cp_backward_summary( raise values = torch.zeros_like(grad_output) with torch.enable_grad(): - seed = torch.zeros( + initial_state = torch.zeros( (cu_seqlens.numel() - 1, q.shape[1], grad_output.shape[-1], k.shape[-1]), device=q.device, dtype=torch.float32, @@ -201,11 +201,11 @@ def _cp_backward_summary( values, {"g": g.detach(), "beta": beta.detach()}, cu_seqlens, - initial_state=seed, + initial_state=initial_state, output_final_state=False, **kernel_kwargs, )[0] - (additive,) = torch.autograd.grad(output, seed, grad_output) + (additive,) = torch.autograd.grad(output, initial_state, grad_output) _, transition = self._cp_forward_summary( q, k, values, {"g": g, "beta": beta}, cu_seqlens, **kernel_kwargs ) @@ -396,7 +396,7 @@ def forward( If true, forward activations for this module are recomputed during the backward pass instead of saved. initial_state : Optional[torch.Tensor], default = `None` - Recurrent state to seed the Gated DeltaNet recurrence with, of shape + Initial recurrent state for Gated DeltaNet, of shape `[batch_size, num_attention_heads // tp_size, v_head_dim, qk_head_dim]` and dtype `torch.float32`. output_final_state : bool, default = `False` diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index e9ef198dc6e..fd557620dff 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -27,8 +27,8 @@ class CommOverlapP2P; namespace transformer_engine::pytorch { -at::Tensor linear_attention_affine_prefix(const at::Tensor &seeds, const at::Tensor &additive, - const at::Tensor &transitions, +at::Tensor linear_attention_affine_prefix(const at::Tensor &initial_states, + const at::Tensor &additive, const at::Tensor &transitions, const at::Tensor &descriptors); /*************************************************************************************************** diff --git a/transformer_engine/pytorch/csrc/extensions/attention.cpp b/transformer_engine/pytorch/csrc/extensions/attention.cpp index d0f9f3d8fa6..9c22ef3c4ea 100644 --- a/transformer_engine/pytorch/csrc/extensions/attention.cpp +++ b/transformer_engine/pytorch/csrc/extensions/attention.cpp @@ -11,25 +11,27 @@ namespace transformer_engine::pytorch { -at::Tensor linear_attention_affine_prefix(const at::Tensor &seeds, const at::Tensor &additive, - const at::Tensor &transitions, +at::Tensor linear_attention_affine_prefix(const at::Tensor &initial_states, + const at::Tensor &additive, const at::Tensor &transitions, const at::Tensor &descriptors) { - for (const auto &tensor : {seeds, additive, transitions, descriptors}) { + for (const auto &tensor : {initial_states, additive, transitions, descriptors}) { NVTE_CHECK(tensor.is_cuda() && tensor.is_contiguous(), "Affine prefix tensors must be contiguous CUDA tensors."); - NVTE_CHECK(tensor.device() == seeds.device(), "Affine prefix devices must agree."); + NVTE_CHECK(tensor.device() == initial_states.device(), "Affine prefix devices must agree."); } - NVTE_CHECK(seeds.dim() == 4 && descriptors.dim() == 2, "Invalid affine prefix ranks."); - at::cuda::CUDAGuard device_guard(seeds.device()); - at::Tensor output = at::empty({descriptors.size(0), seeds.size(1), seeds.size(2), seeds.size(3)}, - seeds.options()); - auto s = makeTransformerEngineTensor(seeds); - auto a = makeTransformerEngineTensor(additive); - auto t = makeTransformerEngineTensor(transitions); - auto d = makeTransformerEngineTensor(descriptors); - auto o = makeTransformerEngineTensor(output); - nvte_linear_attention_affine_prefix(s.data(), a.data(), t.data(), d.data(), o.data(), - at::cuda::getCurrentCUDAStream()); + NVTE_CHECK(initial_states.dim() == 4 && descriptors.dim() == 2, "Invalid affine prefix ranks."); + at::cuda::CUDAGuard device_guard(initial_states.device()); + at::Tensor output = at::empty( + {descriptors.size(0), initial_states.size(1), initial_states.size(2), initial_states.size(3)}, + initial_states.options()); + auto initial_tensor = makeTransformerEngineTensor(initial_states); + auto additive_tensor = makeTransformerEngineTensor(additive); + auto transition_tensor = makeTransformerEngineTensor(transitions); + auto descriptor_tensor = makeTransformerEngineTensor(descriptors); + auto output_tensor = makeTransformerEngineTensor(output); + nvte_linear_attention_affine_prefix(initial_tensor.data(), additive_tensor.data(), + transition_tensor.data(), descriptor_tensor.data(), + output_tensor.data(), at::cuda::getCurrentCUDAStream()); return output; } From 0b299b6be10ab0090d5337cfbc513d456fedd38d Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Fri, 13 Nov 2026 20:09:27 -0800 Subject: [PATCH 11/34] Share CP metadata across attention modules Signed-off-by: Kshitij Lakhani --- .../linear_attention_context_parallel.rst | 14 +++++ tests/pytorch/attention/run_gdn_with_cp.py | 63 ++++++++++++++----- .../linear_attention/context_parallel.py | 36 ++++++++++- 3 files changed, 96 insertions(+), 17 deletions(-) diff --git a/docs/features/other_optimizations/linear_attention_context_parallel.rst b/docs/features/other_optimizations/linear_attention_context_parallel.rst index dc451ee771e..a18f2fb5cc0 100644 --- a/docs/features/other_optimizations/linear_attention_context_parallel.rst +++ b/docs/features/other_optimizations/linear_attention_context_parallel.rst @@ -114,3 +114,17 @@ first-order gradients. Full-graph compilation and CUDA graph capture are outside this implementation's supported contract. Other LA flavors, direct dual-chunk ownership, headwise CP and whole-document redistribution are separate features. + +Metadata preparation +-------------------- + +TE prepares partitions on the host and uploads the derived indices to the GPU. +CUDA ``cu_seqlens`` requires a synchronized host read on the first use. Pass CPU +metadata when it is already available from the input pipeline to avoid that read. + +Each attention module caches its most recent partition. A bounded shared cache +retains up to eight partitions per process so layers using the same metadata +on the same CUDA stream can reuse that preparation. Cache keys include tensor +identity and its mutation counter; changing the tensor in place invalidates +the entry. Newly allocated metadata requires preparation again. Tensors created +in inference mode bypass caching because they have no mutation counter. diff --git a/tests/pytorch/attention/run_gdn_with_cp.py b/tests/pytorch/attention/run_gdn_with_cp.py index 658351fa02c..8ca227bfba8 100644 --- a/tests/pytorch/attention/run_gdn_with_cp.py +++ b/tests/pytorch/attention/run_gdn_with_cp.py @@ -221,14 +221,18 @@ def _inference_and_deferred_group(group): v = torch.randn_like(q) g, beta = -torch.rand(35, 2, device=device) * 0.02, torch.rand(35, 2, device=device) * 0.7 reference = GatedDeltaNetAttention(2, 64, qkv_format="thd") - module = GatedDeltaNetAttention(2, 64, qkv_format="thd") - module.set_context_parallel_group(group) + modules = [GatedDeltaNetAttention(2, 64, qkv_format="thd") for _ in range(2)] + for module in modules: + module.set_context_parallel_group(group) q, k, v, g, beta = (t.requires_grad_() for t in (q, k, v, g, beta)) - for context in (torch.no_grad, torch.inference_mode): + for context, metadata_device in ( + (torch.no_grad, "cpu"), + (torch.no_grad, device), + (torch.inference_mode, device), + ): with context(): # CP accepts host metadata; inference tensors intentionally bypass # the partition cache because they have no mutation counter. - metadata_device = "cpu" if context is torch.no_grad else device global_cu = torch.tensor([0, 17, 35], device=metadata_device, dtype=torch.int32) for boundary in (17, 19): global_cu[1] = boundary @@ -236,18 +240,45 @@ def _inference_and_deferred_group(group): q, k, v, g, beta, cu_seqlens=global_cu.to(device), output_final_state=True ) lo, hi = rank * 35 // size, (rank + 1) * 35 // size - output, states = module( - q[lo:hi], - k[lo:hi], - v[lo:hi], - g[lo:hi], - beta[lo:hi], - cu_seqlens=global_cu, - output_final_state=True, - ) - assert not output.requires_grad and not states.requires_grad - _assert_rms_close(output, expected[lo:hi], _FWD_TOL[q.dtype], "inference output") - _assert_rms_close(states, expected_states, _STATE_TOL[q.dtype], "inference states") + # Reuse one batch across layers, then invalidate it in place. + for module in modules: + output, states = module( + q[lo:hi], + k[lo:hi], + v[lo:hi], + g[lo:hi], + beta[lo:hi], + cu_seqlens=global_cu, + output_final_state=True, + ) + assert not output.requires_grad and not states.requires_grad + _assert_rms_close( + output, expected[lo:hi], _FWD_TOL[q.dtype], "inference output" + ) + _assert_rms_close( + states, expected_states, _STATE_TOL[q.dtype], "inference states" + ) + # Prepare a separate entry when the same modules run on another stream. + stream = torch.cuda.Stream(device=device) + stream.wait_stream(torch.cuda.current_stream(device)) + with torch.no_grad(), torch.cuda.stream(stream): + global_cu = torch.tensor([0, 19, 35], dtype=torch.int32) + results = [ + module( + q[lo:hi], + k[lo:hi], + v[lo:hi], + g[lo:hi], + beta[lo:hi], + cu_seqlens=global_cu, + output_final_state=True, + ) + for module in modules + ] + stream.synchronize() + for output, states in results: + _assert_rms_close(output, expected[lo:hi], _FWD_TOL[q.dtype], "stream output") + _assert_rms_close(states, expected_states, _STATE_TOL[q.dtype], "stream states") dist.barrier(group=group) diff --git a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py index 55d9bd37c5f..32361c8f7b6 100644 --- a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py +++ b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py @@ -8,7 +8,9 @@ CUDA operation composes states in document order. """ +from collections import OrderedDict from dataclasses import dataclass +from threading import Lock import torch import torch.distributed as dist @@ -22,6 +24,30 @@ from .base import _to_thd +_CP_METADATA_CACHE_SIZE = 8 +_cp_metadata_cache = OrderedDict() +_cp_metadata_cache_lock = Lock() + + +def _get_shared_cp_metadata(key): + """Look up a prepared partition shared by attention modules on one stream.""" + with _cp_metadata_cache_lock: + entry = _cp_metadata_cache.get(key) + if entry is not None: + _cp_metadata_cache.move_to_end(key) + return entry + + +def _cache_cp_metadata(entry): + """Retain at most eight shared partitions, including their input metadata.""" + with _cp_metadata_cache_lock: + key = entry[0] + _cp_metadata_cache[key] = entry + _cp_metadata_cache.move_to_end(key) + if len(_cp_metadata_cache) > _CP_METADATA_CACHE_SIZE: + _cp_metadata_cache.popitem(last=False) + + @dataclass class _CPMetadata: """Logical partition and device tensors used to execute linear-attention CP.""" @@ -341,6 +367,7 @@ def context_parallel_attention( """Derive local fragment boundaries from full-document metadata and run CP.""" adapter._validate_inputs(q, k, v, gates, initial_state, qkv_format) cp_size, cp_rank = dist.get_world_size(group), dist.get_rank(group) + stream = torch.cuda.current_stream(q.device).cuda_stream tensors = tuple(_to_thd(t, qkv_format).contiguous() for t in (q, k, v, *gates.values())) dense = qkv_format != "thd" batch, length = None, None @@ -362,14 +389,18 @@ def context_parallel_attention( q.device, cp_size, cp_rank, + stream, ) else: - key, cacheable = (qkv_format, batch, length, q.device, cp_size, cp_rank), True + key, cacheable = (qkv_format, batch, length, q.device, cp_size, cp_rank, stream), True cache = getattr(adapter, "_cp_partition_cache", None) if not cacheable: # Inference tensors have no mutation counter, so revalidate their # metadata instead of silently reusing a stale partition. cache = None + elif cache is None or cache[0] != key: + # Layers using the same batch can share metadata without another host round trip. + cache = _get_shared_cp_metadata(key) # Build and upload metadata only when the cached partition no longer matches. if cache is None or cache[0] != key: document_cu_seqlens = ( @@ -385,9 +416,12 @@ def context_parallel_attention( metadata = _build_cp_metadata(partition, q.device) # Hold the metadata tensor to prevent object-ID reuse in the cache key. adapter._cp_partition_cache = (key, metadata, cu_seqlens) + if cacheable: + _cache_cp_metadata(adapter._cp_partition_cache) else: metadata = cache[1] partition = metadata.partition + adapter._cp_partition_cache = cache if tensors[0].shape[0] != partition.local_cu_seqlens[-1]: raise ValueError("Local token count does not match this CP rank's token interval.") adapter._validate_initial_state( From 87f84225c9f4fdf31c1baf154742654829a6a98c Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Fri, 13 Nov 2026 20:09:27 -0800 Subject: [PATCH 12/34] Document the linear attention CP adapter contract Signed-off-by: Kshitij Lakhani --- .../attention/linear_attention/context_parallel.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py index 32361c8f7b6..25a86ad756f 100644 --- a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py +++ b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py @@ -6,6 +6,20 @@ Flavor adapters supply local attention and affine summary operations. The shared CUDA operation composes states in document order. + +Adapter contract: +* Positional tensors carry a real-token leading axis for fragment selection; + additional axes, such as GDP's Householder axis, remain intact until FE dispatch. +* Forward hooks return FP32 H [fragments, heads, V, K] and T [fragments, heads, K, K] + satisfying X_end = X_start @ T + H. Reverse hooks provide the analogous + state-gradient recipe, with transitions already in backward orientation. +* Local attention and summaries must use identical gate and normalization semantics. +* Kernel options are configuration, not an autograd path for trainable parameters. + Future non-token parameters need explicit gradient inputs/returns, or a + differentiable token-gate transformation before the CP Function. Their gradient + reduction must match the caller's parameter-ownership contract. + +Only GDN supplies these hooks today; other flavors require adapter validation. """ from collections import OrderedDict From 716f1353c4a9af5d87dfef5ddc8ab8d0c4539657 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Fri, 13 Nov 2026 20:09:27 -0800 Subject: [PATCH 13/34] Reduce idle affine prefix threads Signed-off-by: Kshitij Lakhani --- transformer_engine/common/linear_attention/affine_prefix.cu | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/transformer_engine/common/linear_attention/affine_prefix.cu b/transformer_engine/common/linear_attention/affine_prefix.cu index ae5a6e78163..430fe5c43cd 100644 --- a/transformer_engine/common/linear_attention/affine_prefix.cu +++ b/transformer_engine/common/linear_attention/affine_prefix.cu @@ -128,7 +128,9 @@ void nvte_linear_attention_affine_prefix(const NVTETensor initial_states, const "Inconsistent affine tensor shapes."); if (output_data.shape[0] == 0) return; const size_t value_tiles = output_data.shape[0] * heads * ((value_dim + kRows - 1) / kRows); - affine_prefix_kernel<<>>( + // Extra threads have no key columns to process when K is at most 64. + const int threads = key_dim <= 64 ? 64 : 128; + affine_prefix_kernel<<>>( static_cast(initial_data.dptr), static_cast(additive_data.dptr), static_cast(transition_data.dptr), static_cast(descriptor_data.dptr), static_cast(output_data.dptr), From 61d9b894ab20f05637326713e0669955f39e8dc6 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Fri, 13 Nov 2026 20:35:12 -0800 Subject: [PATCH 14/34] Separate CP metadata caches by inference mode Signed-off-by: Kshitij Lakhani --- .../linear_attention_context_parallel.rst | 10 ++++++---- tests/pytorch/attention/run_gdn_with_cp.py | 16 ++++++++++++---- .../linear_attention/context_parallel.py | 6 +++++- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/docs/features/other_optimizations/linear_attention_context_parallel.rst b/docs/features/other_optimizations/linear_attention_context_parallel.rst index a18f2fb5cc0..013f805376f 100644 --- a/docs/features/other_optimizations/linear_attention_context_parallel.rst +++ b/docs/features/other_optimizations/linear_attention_context_parallel.rst @@ -124,7 +124,9 @@ metadata when it is already available from the input pipeline to avoid that read Each attention module caches its most recent partition. A bounded shared cache retains up to eight partitions per process so layers using the same metadata -on the same CUDA stream can reuse that preparation. Cache keys include tensor -identity and its mutation counter; changing the tensor in place invalidates -the entry. Newly allocated metadata requires preparation again. Tensors created -in inference mode bypass caching because they have no mutation counter. +on the same CUDA stream can reuse that preparation. Inference-mode entries are +kept separate so autograd never saves inference tensors during training. +Cache keys include tensor identity and its mutation counter; tracked in-place +updates invalidate the entry. New ``cu_seqlens`` objects require preparation +again. Input ``cu_seqlens`` tensors created in inference mode bypass caching +because they have no mutation counter. diff --git a/tests/pytorch/attention/run_gdn_with_cp.py b/tests/pytorch/attention/run_gdn_with_cp.py index 8ca227bfba8..cdbc1c7f30b 100644 --- a/tests/pytorch/attention/run_gdn_with_cp.py +++ b/tests/pytorch/attention/run_gdn_with_cp.py @@ -70,6 +70,7 @@ def _case( independent=False, dense_global=False, state_only=False, + inference_warmup=False, ): """Compare each local output/gradient to the corresponding full-sequence result.""" size, rank = dist.get_world_size(group), dist.get_rank(group) @@ -78,7 +79,7 @@ def _case( f"GDN CP case: lengths={lengths}, dtype={dtype}, K={key_dim}, V={value_dim}," f" layout={layout}, norm={norm}, initial={initial}, final={final}," f" checkpoint={checkpoint}, last_loss={last_loss}, fallback={fallback}," - f" independent={independent}", + f" independent={independent}, inference_warmup={inference_warmup}", flush=True, ) device = torch.device("cuda", int(os.environ["LOCAL_RANK"])) @@ -137,14 +138,13 @@ def local(tensor): inputs = tuple(local(t).detach().requires_grad_(True) for t in full[:5]) state = full[5].detach().requires_grad_(True) if initial else None tp_size = 1 if tp_group is None else dist.get_world_size(tp_group) - module = GatedDeltaNetAttention( - heads * tp_size, - (key_dim, value_dim), + module_kwargs = dict( qkv_format=layout, scale=scale, cp_group=group, tp_group=tp_group, ) + module = GatedDeltaNetAttention(heads * tp_size, (key_dim, value_dim), **module_kwargs) kwargs = dict( cu_seqlens=( (global_cu.cpu() if size == 1 else global_cu) @@ -156,6 +156,11 @@ def local(tensor): checkpoint_core_attention=checkpoint, use_qk_l2norm_in_kernel=norm, ) + if inference_warmup: + with torch.inference_mode(): + module(*inputs, **kwargs) + # A second layer must be able to train with the shared metadata. + module = GatedDeltaNetAttention(heads * tp_size, (key_dim, value_dim), **module_kwargs) summary_patch = ( patch.object( module.gdn_attention, "_summary_op", side_effect=ImportError("Missing summary") @@ -427,6 +432,9 @@ def main(): cases += 1 _state_chaining(group) cases += 1 + for layout, lengths in (("thd", [17, 13]), ("bshd", [size * 4] * 2)): + _case(group, lengths, layout=layout, inference_warmup=True, independent=True) + cases += 1 _invalid_metadata(group) cases += 1 # Independent complete documents need no state exchange or prefix work. diff --git a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py index 25a86ad756f..341389293b3 100644 --- a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py +++ b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py @@ -382,6 +382,8 @@ def context_parallel_attention( adapter._validate_inputs(q, k, v, gates, initial_state, qkv_format) cp_size, cp_rank = dist.get_world_size(group), dist.get_rank(group) stream = torch.cuda.current_stream(q.device).cuda_stream + # Autograd cannot save metadata tensors created in inference mode. + inference_mode = torch.is_inference_mode_enabled() tensors = tuple(_to_thd(t, qkv_format).contiguous() for t in (q, k, v, *gates.values())) dense = qkv_format != "thd" batch, length = None, None @@ -404,9 +406,11 @@ def context_parallel_attention( cp_size, cp_rank, stream, + inference_mode, ) else: - key, cacheable = (qkv_format, batch, length, q.device, cp_size, cp_rank, stream), True + key = (qkv_format, batch, length, q.device, cp_size, cp_rank, stream, inference_mode) + cacheable = True cache = getattr(adapter, "_cp_partition_cache", None) if not cacheable: # Inference tensors have no mutation counter, so revalidate their From 519bacfbeef5ea8fe66b6a9f15f852e64ebfc03c Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Tue, 17 Nov 2026 12:35:44 -0800 Subject: [PATCH 15/34] Document affine prefix parameters Signed-off-by: Kshitij Lakhani --- .../transformer_engine/linear_attention.h | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/transformer_engine/common/include/transformer_engine/linear_attention.h b/transformer_engine/common/include/transformer_engine/linear_attention.h index 93e31f55e33..d4ef5e12e7b 100644 --- a/transformer_engine/common/include/transformer_engine/linear_attention.h +++ b/transformer_engine/common/include/transformer_engine/linear_attention.h @@ -5,7 +5,7 @@ ************************************************************************/ /*! \file linear_attention.h - * \brief Framework-independent linear-attention state composition. + * \brief Functions for linear-attention state composition. */ #ifndef TRANSFORMER_ENGINE_LINEAR_ATTENTION_H_ #define TRANSFORMER_ENGINE_LINEAR_ATTENTION_H_ @@ -16,22 +16,30 @@ extern "C" { #endif -/*! \brief Apply ordered affine summaries to selected recurrent states. +/*! \brief Compose recurrent states from ordered affine summaries. * * Each output starts from a selected initial memory and repeatedly evaluates * ``state = state @ transition[record] + additive[record]``. This operation * serves both forward prefixes and backward suffixes; backward callers pass * the backward-oriented transitions and a negative record stride. * - * All tensors are contiguous device tensors. Initial states are FP32 - * [initial_memories, heads, value_dim, key_dim]; additives are FP32 - * [records, heads, value_dim, key_dim]. - * Transitions are FP32 [records, heads, key_dim, key_dim]. Descriptors are - * INT32 [outputs, 4], with rows (initial_memory_index, first_record, count, stride). - * Outputs are FP32 [outputs, heads, value_dim, key_dim]. Referenced initial-memory - * and record indices must be valid; count must be nonnegative. - * An initial_memory_index of -1 means an implicit zero state. Zero-count - * descriptors copy their initial memory (or write zeros). No memory is allocated. + * All tensors must be contiguous device tensors. No memory is allocated. + * + * \param[in] initial_states FP32 initial memories, + * [initial_memories, heads, value_dim, key_dim]. + * \param[in] additive FP32 additive summaries, + * [records, heads, value_dim, key_dim]. + * \param[in] transitions FP32 transition matrices, + * [records, heads, key_dim, key_dim]. + * \param[in] descriptors INT32 [outputs, 4], with rows + * (initial_memory_index, first_record, count, stride). + * An initial_memory_index of -1 selects an implicit zero state. + * Other referenced memory and record indices must be valid. + * Count must be nonnegative; zero copies the initial memory + * (or writes zeros). + * \param[out] output Caller-allocated FP32 composed memories, + * [outputs, heads, value_dim, key_dim]. + * \param[in] stream CUDA stream on which to execute the operation. */ void nvte_linear_attention_affine_prefix(const NVTETensor initial_states, const NVTETensor additive, const NVTETensor transitions, const NVTETensor descriptors, From aafc7215f36306a887f982cf9ceb061b34a48214 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Tue, 17 Nov 2026 17:33:03 -0800 Subject: [PATCH 16/34] Clarify linear attention partition metadata Signed-off-by: Kshitij Lakhani --- .../common/linear_attention/__init__.py | 54 ++++++++++++++----- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/transformer_engine/common/linear_attention/__init__.py b/transformer_engine/common/linear_attention/__init__.py index b9811d1e930..0a859ba6518 100644 --- a/transformer_engine/common/linear_attention/__init__.py +++ b/transformer_engine/common/linear_attention/__init__.py @@ -2,7 +2,7 @@ # # See LICENSE for license information. -"""Framework-independent metadata for contiguous linear-attention partitions.""" +"""Metadata for linear-attention partitions with contiguous token ownership""" from bisect import bisect_right from dataclasses import dataclass @@ -11,7 +11,24 @@ @dataclass(frozen=True) class LinearAttentionFragment: - """Intersection of one independent document with one rank's token interval.""" + """Metadata for a fragment: the intersection of one independent document + with one rank's token interval. + + For example, global cu_seqlens [0, 8, 12, 24] describes documents + A [0, 8), B [8, 12), and C [12, 24). With CP=4, each rank owns six + consecutive tokens. Each row below describes one fragment: + + Rank Rank interval Document start end first_rank last_rank + 0 [0, 6) A (0) 0 6 0 1 + 1 [6, 12) A (0) 6 8 0 1 + 1 [6, 12) B (1) 8 12 1 1 + 2 [12, 18) C (2) 12 18 2 3 + 3 [18, 24) C (2) 18 24 2 3 + + ``start`` is inclusive and ``end`` is exclusive; both are global offsets. + ``first_rank`` and ``last_rank`` identify the first and last ranks owning + tokens of the whole document, not just this fragment. + """ document: int start: int @@ -22,27 +39,38 @@ class LinearAttentionFragment: @dataclass(frozen=True) class LinearAttentionCPPartition: - """Host metadata, independent of tensor frameworks and attention flavors. + """Host metadata for one CP rank. Packed (THD) inputs intersect documents with contiguous rank intervals. Dense (BSHD/SBHD) inputs give each rank a nearly equal slice of every document. Empty documents have no fragments; their state is unchanged. + + Using the fragment example from LinearAttentionFragment, packed rank 1 owns: + + fragments: A [6, 8), B [8, 12) + local_cu_seqlens: (0, 2, 6) + incoming: (0,) # A continues from rank 0. + outgoing: () # Neither document continues to a later rank. + terminal: (0, 1) # Both documents end on this rank. """ - cu_seqlens: Tuple[int, ...] + cu_seqlens: Tuple[int, ...] # Global cumulative sequence lengths. cp_rank: int cp_size: int fragments: Tuple[LinearAttentionFragment, ...] - local_cu_seqlens: Tuple[int, ...] + local_cu_seqlens: Tuple[int, ...] # Cumulative lengths of nonempty local fragments. + # Select preceding summaries to compose each fragment's starting memory. forward_descriptors: Tuple[Tuple[int, int, int, int], ...] + # Select following reverse summaries to compose each fragment's ending-state gradient. backward_descriptors: Tuple[Tuple[int, int, int, int], ...] - outgoing: Tuple[int, ...] - incoming: Tuple[int, ...] - terminal: Tuple[int, ...] - initial_state_owners: Tuple[int, ...] - empty_owned: Tuple[int, ...] + # The following three tuples contain local fragment indices, not document IDs. + outgoing: Tuple[int, ...] # Document continues to a later rank. + incoming: Tuple[int, ...] # Document starts on an earlier rank. + terminal: Tuple[int, ...] # Document ends on this rank. + initial_state_owners: Tuple[int, ...] # Initial-state owner rank for each global document. + empty_owned: Tuple[int, ...] # Empty global document IDs assigned to this rank. slots: int # One per rank for packed inputs; one per document for dense inputs. - communicates: bool + communicates: bool # Any global document spans multiple ranks. def build_linear_attention_partition(cu_seqlens, cp_size, cp_rank, *, dense=False): @@ -58,9 +86,9 @@ def build_linear_attention_partition(cu_seqlens, cp_size, cp_rank, *, dense=Fals if cp_size < 1 or not 0 <= cp_rank < cp_size: raise ValueError("CP size must be positive and rank must lie inside the group.") if len(cu_seqlens) < 2 or cu_seqlens[0] != 0: - raise ValueError("Global cumulative sequence lengths must start at zero.") + raise ValueError("cu_seqlens must start at zero and contain at least two entries.") if any(a > b for a, b in zip(cu_seqlens, cu_seqlens[1:])): - raise ValueError("Global cumulative sequence lengths must be nondecreasing.") + raise ValueError("cu_seqlens must be nondecreasing.") total = cu_seqlens[-1] boundaries = tuple(r * total // cp_size for r in range(cp_size + 1)) slots = len(cu_seqlens) - 1 if dense else 1 From 341f83520c3ea007ce001d651a15c14f1446d994 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Tue, 17 Nov 2026 17:33:08 -0800 Subject: [PATCH 17/34] Explain affine prefix kernel execution in comments Signed-off-by: Kshitij Lakhani --- .../common/linear_attention/affine_prefix.cu | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/transformer_engine/common/linear_attention/affine_prefix.cu b/transformer_engine/common/linear_attention/affine_prefix.cu index 430fe5c43cd..dc3363c97b9 100644 --- a/transformer_engine/common/linear_attention/affine_prefix.cu +++ b/transformer_engine/common/linear_attention/affine_prefix.cu @@ -15,10 +15,12 @@ namespace { // Reuse transition loads across value rows, preserving FP32 accumulation. constexpr int kRows = 4; +// Each block composes one value-row tile of a fragment's recurrent state for one head. __global__ void affine_prefix_kernel(const float *initial_states, const float *additive, const float *transitions, const int *descriptors, float *output, size_t num_initial_states, size_t num_records, size_t heads, size_t value_dim, size_t key_dim) { + // Shared memory bounce buffers to alternate between state and next_state extern __shared__ float storage[]; float *state = storage; float *next_state = storage + kRows * key_dim; @@ -30,6 +32,7 @@ __global__ void affine_prefix_kernel(const float *initial_states, const float *a const int *descriptor = descriptors + fragment * 4; const int initial_state_index = descriptor[0], first_record = descriptor[1], record_count = descriptor[2], stride = descriptor[3]; + // Confirm the descriptor is valid before the block uses it. assert((initial_state_index == -1 || (initial_state_index >= 0 && static_cast(initial_state_index) < num_initial_states)) && @@ -56,11 +59,13 @@ __global__ void affine_prefix_kernel(const float *initial_states, const float *a state[row * key_dim + col] = initial; } } + // Make the initialized state visible to all threads before it is read. __syncthreads(); for (int step = first_step; step < record_count; ++step) { const int record = first_record + step * stride; assert(record >= 0 && static_cast(record) < num_records); const float *transition_matrix = transitions + (record * heads + head) * key_dim * key_dim; + // Each thread accumulates H + state @ T for its key columns in registers. for (size_t col = threadIdx.x; col < key_dim; col += blockDim.x) { float result[kRows]; for (int row = 0; row < kRows; ++row) { @@ -69,6 +74,7 @@ __global__ void affine_prefix_kernel(const float *initial_states, const float *a ? additive[((record * heads + head) * value_dim + value) * key_dim + col] : 0.f; } + // Reuse each transition element across the tile's value rows. for (size_t inner = 0; inner < key_dim; ++inner) { const float transition = transition_matrix[inner * key_dim + col]; #pragma unroll @@ -78,6 +84,7 @@ __global__ void affine_prefix_kernel(const float *initial_states, const float *a } for (int row = 0; row < kRows; ++row) next_state[row * key_dim + col] = result[row]; } + // Complete all shared-memory reads and writes before swapping state buffers. __syncthreads(); float *swap = state; state = next_state; From 94a4d9f17721992cd2bed75ada17609749e9caa3 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Wed, 18 Nov 2026 11:32:28 -0800 Subject: [PATCH 18/34] Rename linear attention CP metadata Signed-off-by: Kshitij Lakhani --- .../linear_attention_context_parallel.rst | 4 +- .../attention/test_gdn_attention_with_cp.py | 40 ++-- .../common/linear_attention/__init__.py | 8 +- .../linear_attention/context_parallel.py | 171 ++++++++++-------- .../pytorch/attention/linear_attention/gdn.py | 2 +- 5 files changed, 119 insertions(+), 106 deletions(-) diff --git a/docs/features/other_optimizations/linear_attention_context_parallel.rst b/docs/features/other_optimizations/linear_attention_context_parallel.rst index 013f805376f..28263632b6b 100644 --- a/docs/features/other_optimizations/linear_attention_context_parallel.rst +++ b/docs/features/other_optimizations/linear_attention_context_parallel.rst @@ -122,8 +122,8 @@ TE prepares partitions on the host and uploads the derived indices to the GPU. CUDA ``cu_seqlens`` requires a synchronized host read on the first use. Pass CPU metadata when it is already available from the input pipeline to avoid that read. -Each attention module caches its most recent partition. A bounded shared cache -retains up to eight partitions per process so layers using the same metadata +Each attention module caches its most recent rank metadata. A bounded shared cache +retains up to eight prepared entries per process so layers using the same metadata on the same CUDA stream can reuse that preparation. Inference-mode entries are kept separate so autograd never saves inference tensors during training. Cache keys include tensor identity and its mutation counter; tracked in-place diff --git a/tests/pytorch/attention/test_gdn_attention_with_cp.py b/tests/pytorch/attention/test_gdn_attention_with_cp.py index 82ea79530c8..9f8bfb01bdd 100644 --- a/tests/pytorch/attention/test_gdn_attention_with_cp.py +++ b/tests/pytorch/attention/test_gdn_attention_with_cp.py @@ -16,7 +16,7 @@ from test_gdn_attention import _GDN_AVAILABLE import transformer_engine -from transformer_engine.common.linear_attention import build_linear_attention_partition +from transformer_engine.common.linear_attention import build_linear_attention_cp_host_metadata import transformer_engine_torch as tex @@ -29,34 +29,36 @@ def test_partition_intersections(cp_size): cu = [0] for length in lengths: cu.append(cu[-1] + length) - partitions = [ - build_linear_attention_partition(cu, cp_size=cp_size, cp_rank=cp_rank) + rank_metadata = [ + build_linear_attention_cp_host_metadata(cu, cp_size=cp_size, cp_rank=cp_rank) for cp_rank in range(cp_size) ] for doc, (start, end) in enumerate(zip(cu, cu[1:])): - fragments = [f for p in partitions for f in p.fragments if f.document == doc] + fragments = [f for p in rank_metadata for f in p.fragments if f.document == doc] assert sum(f.end - f.start for f in fragments) == end - start assert all(start <= f.start < f.end <= end for f in fragments) assert all(a.end == b.start for a, b in zip(fragments, fragments[1:])) - for cp_rank, partition in enumerate(partitions): + for cp_rank, host_metadata in enumerate(rank_metadata): assert ( - partition.local_cu_seqlens[-1] + host_metadata.local_cu_seqlens[-1] == (cp_rank + 1) * cu[-1] // cp_size - cp_rank * cu[-1] // cp_size ) - assert len(partition.incoming) <= 1 - assert len(partition.outgoing) <= 1 + assert len(host_metadata.incoming) <= 1 + assert len(host_metadata.outgoing) <= 1 for f, forward, backward in zip( - partition.fragments, partition.forward_descriptors, partition.backward_descriptors + host_metadata.fragments, + host_metadata.forward_descriptors, + host_metadata.backward_descriptors, ): assert forward[2] == cp_rank - f.first_rank assert backward[2] == f.last_rank - cp_rank # Every referenced recipe belongs to this same document. for predecessor in range(f.first_rank, cp_rank): - p = partitions[predecessor] + p = rank_metadata[predecessor] if p.fragments: assert p.fragments[p.outgoing[0]].document == f.document for successor in range(cp_rank + 1, f.last_rank + 1): - p = partitions[successor] + p = rank_metadata[successor] # An empty rank contributes an identity recipe. if p.fragments: assert p.fragments[p.incoming[0]].document == f.document @@ -66,31 +68,31 @@ def test_partition_intersections(cp_size): def test_invalid_documents(cu_seqlens): """Reject malformed global offsets before computing communication descriptors.""" with pytest.raises(ValueError): - build_linear_attention_partition(cu_seqlens, cp_size=4, cp_rank=0) + build_linear_attention_cp_host_metadata(cu_seqlens, cp_size=4, cp_rank=0) @pytest.mark.parametrize("cp_size", [1, 2, 4, 8]) @pytest.mark.parametrize("length", [0, 1, 5, 9]) def test_dense_partition_intersections(cp_size, length): """Dense rows retain their document identity, including uneven and empty ranks.""" - partitions = [ - build_linear_attention_partition( + rank_metadata = [ + build_linear_attention_cp_host_metadata( [0, length, 2 * length], cp_size=cp_size, cp_rank=cp_rank, dense=True ) for cp_rank in range(cp_size) ] for doc in (0, 1): - spans = [f for p in partitions for f in p.fragments if f.document == doc] + spans = [f for p in rank_metadata for f in p.fragments if f.document == doc] assert sum(f.end - f.start for f in spans) == length assert all(a.end == b.start for a, b in zip(spans, spans[1:])) if length: assert spans[0].start == doc * length assert spans[-1].end == (doc + 1) * length - for cp_rank, partition in enumerate(partitions): + for cp_rank, host_metadata in enumerate(rank_metadata): local_length = (cp_rank + 1) * length // cp_size - cp_rank * length // cp_size - assert partition.local_cu_seqlens[-1] == 2 * local_length - assert partition.slots == 2 - assert len(partition.fragments) == (2 if local_length else 0) + assert host_metadata.local_cu_seqlens[-1] == 2 * local_length + assert host_metadata.slots == 2 + assert len(host_metadata.fragments) == (2 if local_length else 0) @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") diff --git a/transformer_engine/common/linear_attention/__init__.py b/transformer_engine/common/linear_attention/__init__.py index 0a859ba6518..78c13b50bbb 100644 --- a/transformer_engine/common/linear_attention/__init__.py +++ b/transformer_engine/common/linear_attention/__init__.py @@ -38,8 +38,8 @@ class LinearAttentionFragment: @dataclass(frozen=True) -class LinearAttentionCPPartition: - """Host metadata for one CP rank. +class LinearAttentionCPHostMetadata: + """Global document boundaries and local fragment metadata for one CP rank. Packed (THD) inputs intersect documents with contiguous rank intervals. Dense (BSHD/SBHD) inputs give each rank a nearly equal slice of every document. @@ -73,7 +73,7 @@ class LinearAttentionCPPartition: communicates: bool # Any global document spans multiple ranks. -def build_linear_attention_partition(cu_seqlens, cp_size, cp_rank, *, dense=False): +def build_linear_attention_cp_host_metadata(cu_seqlens, cp_size, cp_rank, *, dense=False): """Build one rank's fragment and summary metadata on the host. ``cu_seqlens`` contains full-document boundaries as host integers. For @@ -137,7 +137,7 @@ def build_linear_attention_partition(cu_seqlens, cp_size, cp_rank, *, dense=Fals terminal.append(index) if cp_rank > frag.first_rank: incoming.append(index) - return LinearAttentionCPPartition( + return LinearAttentionCPHostMetadata( cu_seqlens, cp_rank, cp_size, diff --git a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py index 341389293b3..5f49d4b167a 100644 --- a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py +++ b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py @@ -32,8 +32,8 @@ import transformer_engine_torch as tex from transformer_engine.common.linear_attention import ( - LinearAttentionCPPartition, - build_linear_attention_partition, + LinearAttentionCPHostMetadata, + build_linear_attention_cp_host_metadata, ) from .base import _to_thd @@ -44,7 +44,7 @@ def _get_shared_cp_metadata(key): - """Look up a prepared partition shared by attention modules on one stream.""" + """Look up prepared rank metadata shared by attention modules on one stream.""" with _cp_metadata_cache_lock: entry = _cp_metadata_cache.get(key) if entry is not None: @@ -53,7 +53,7 @@ def _get_shared_cp_metadata(key): def _cache_cp_metadata(entry): - """Retain at most eight shared partitions, including their input metadata.""" + """Retain at most eight prepared entries, including their input metadata.""" with _cp_metadata_cache_lock: key = entry[0] _cp_metadata_cache[key] = entry @@ -63,11 +63,11 @@ def _cache_cp_metadata(entry): @dataclass -class _CPMetadata: - """Logical partition and device tensors used to execute linear-attention CP.""" +class _LinearAttentionCPDeviceMetadata: + """Host metadata reference and device tensors for one CP rank.""" - partition: LinearAttentionCPPartition - cu_seqlens: torch.Tensor + host_metadata: LinearAttentionCPHostMetadata + cu_seqlens: torch.Tensor # Local cumulative fragment lengths on the device. forward_descriptors: torch.Tensor backward_descriptors: torch.Tensor forward_zero_descriptors: torch.Tensor @@ -83,8 +83,8 @@ class _CPMetadata: incoming_cu_seqlens: torch.Tensor -def _build_cp_metadata(partition, device): - """Build device metadata for an existing logical partition.""" +def _build_linear_attention_cp_device_metadata(host_metadata, device): + """Build device tensors and retain their associated host metadata.""" def indices(values): return torch.tensor(values, device=device, dtype=torch.int64) @@ -92,16 +92,18 @@ def indices(values): def cumulative(selected): offsets = [0] for index in selected: - frag = partition.fragments[index] + frag = host_metadata.fragments[index] offsets.append(offsets[-1] + frag.end - frag.start) return torch.tensor(offsets, device=device, dtype=torch.int32) - first = [i for i, f in enumerate(partition.fragments) if f.first_rank == partition.cp_rank] + first = [ + i for i, f in enumerate(host_metadata.fragments) if f.first_rank == host_metadata.cp_rank + ] forward_descriptors = torch.tensor( - partition.forward_descriptors, device=device, dtype=torch.int32 + host_metadata.forward_descriptors, device=device, dtype=torch.int32 ).reshape(-1, 4) backward_descriptors = torch.tensor( - partition.backward_descriptors, device=device, dtype=torch.int32 + host_metadata.backward_descriptors, device=device, dtype=torch.int32 ).reshape(-1, 4) forward_zero_descriptors, backward_zero_descriptors = ( forward_descriptors.clone(), @@ -109,22 +111,22 @@ def cumulative(selected): ) forward_zero_descriptors[:, 0] = -1 backward_zero_descriptors[:, 0] = -1 - return _CPMetadata( - partition, - torch.tensor(partition.local_cu_seqlens, device=device, dtype=torch.int32), + return _LinearAttentionCPDeviceMetadata( + host_metadata, + torch.tensor(host_metadata.local_cu_seqlens, device=device, dtype=torch.int32), forward_descriptors, backward_descriptors, forward_zero_descriptors, backward_zero_descriptors, - indices([f.document for f in partition.fragments]), - indices(partition.outgoing), - indices(partition.incoming), - indices(partition.terminal), + indices([f.document for f in host_metadata.fragments]), + indices(host_metadata.outgoing), + indices(host_metadata.incoming), + indices(host_metadata.terminal), indices(first), - indices([partition.fragments[i].document for i in first]), - indices(partition.empty_owned), - cumulative(partition.outgoing), - cumulative(partition.incoming), + indices([host_metadata.fragments[i].document for i in first]), + indices(host_metadata.empty_owned), + cumulative(host_metadata.outgoing), + cumulative(host_metadata.incoming), ) @@ -151,17 +153,17 @@ def _summary_buffers(state_template, slots): return additive, transition.expand(slots, heads, key_dim, key_dim).contiguous() -def _selected_span(metadata, selected): +def _selected_span(device_metadata, selected): """Selected spans are consecutive: all dense spans, or one packed edge span.""" - cu_seqlens = metadata.partition.local_cu_seqlens + cu_seqlens = device_metadata.host_metadata.local_cu_seqlens return slice(cu_seqlens[selected[0]], cu_seqlens[selected[-1] + 1]) -def _slot_indices(metadata, indices): +def _slot_indices(device_metadata, indices): """Dense slots are document IDs; packed partitions have only one edge slot.""" - if metadata.partition.slots == 1: + if device_metadata.host_metadata.slots == 1: return torch.zeros_like(indices) - return metadata.document_indices.index_select(0, indices) + return device_metadata.document_indices.index_select(0, indices) def _local_attention( @@ -190,7 +192,7 @@ class _LinearAttentionContextParallel(torch.autograd.Function): def forward( ctx, adapter, - metadata, + device_metadata, group, options, output_final_state, @@ -201,12 +203,12 @@ def forward( ): """Establish starting states, then retain the local attention graph.""" ctx.set_materialize_grads(False) - partition = metadata.partition + host_metadata = device_metadata.host_metadata q = tensors[0] k = tensors[1] v = tensors[2] state_shape = ( - len(partition.cu_seqlens) - 1, + len(host_metadata.cu_seqlens) - 1, q.shape[1], v.shape[-1], k.shape[-1], @@ -217,29 +219,29 @@ def forward( if initial_state is None else initial_state ) - additive, transition = _summary_buffers(initial_state_buffer, partition.slots) - if partition.outgoing: - span = _selected_span(metadata, partition.outgoing) + additive, transition = _summary_buffers(initial_state_buffer, host_metadata.slots) + if host_metadata.outgoing: + span = _selected_span(device_metadata, host_metadata.outgoing) selected = tuple(t[span] for t in tensors) h, t = adapter._cp_forward_summary( *selected[:3], dict(zip(gate_names, selected[3:])), - metadata.outgoing_cu_seqlens, + device_metadata.outgoing_cu_seqlens, **options, ) - slots = _slot_indices(metadata, metadata.outgoing) + slots = _slot_indices(device_metadata, device_metadata.outgoing) additive.index_copy_(0, slots, h) transition.index_copy_(0, slots, t) - if partition.communicates: + if host_metadata.communicates: additive, transition = _gather_summaries(additive, transition, group) starting_states = tex.linear_attention_affine_prefix( initial_state_buffer.contiguous(), additive, transition, ( - metadata.forward_descriptors + device_metadata.forward_descriptors if initial_state is not None - else metadata.forward_zero_descriptors + else device_metadata.forward_zero_descriptors ), ) # Retain FE's local graph only for training. This Function owns the @@ -248,9 +250,14 @@ def forward( inputs = tuple( t.detach().requires_grad_(track_grad) for t in (*tensors, starting_states) ) - if partition.fragments: + if host_metadata.fragments: output, states = _local_attention( - adapter, inputs[:-1], gate_names, metadata.cu_seqlens, inputs[-1], options + adapter, + inputs[:-1], + gate_names, + device_metadata.cu_seqlens, + inputs[-1], + options, ) else: output = v.new_empty((q.shape[0], q.shape[1], v.shape[-1])) @@ -263,19 +270,23 @@ def forward( if output_final_state: terminal.index_copy_( 0, - metadata.document_indices.index_select(0, metadata.terminal), - states.detach().index_select(0, metadata.terminal), + device_metadata.document_indices.index_select(0, device_metadata.terminal), + states.detach().index_select(0, device_metadata.terminal), ) if initial_state is not None: terminal.index_copy_( 0, - metadata.empty_documents, - initial_state_buffer.index_select(0, metadata.empty_documents), + device_metadata.empty_documents, + initial_state_buffer.index_select(0, device_metadata.empty_documents), ) dist.all_reduce(terminal, group=group) if track_grad: ctx.save_for_backward(*inputs, output, states) - ctx.cp_metadata, ctx.group, ctx.output_final_state = metadata, group, output_final_state + ctx.cp_device_metadata, ctx.group, ctx.output_final_state = ( + device_metadata, + group, + output_final_state, + ) ctx.adapter, ctx.options, ctx.gate_names = adapter, options, gate_names ctx.has_initial_state = initial_state is not None return output.detach(), terminal @@ -287,13 +298,13 @@ def backward(ctx, grad_output, grad_final): tensors = ctx.saved_tensors[:-3] starting_states, output, states = ctx.saved_tensors[-3:] q, k, v = tensors[:3] - metadata, group = ctx.cp_metadata, ctx.group - partition = metadata.partition + device_metadata, group = ctx.cp_device_metadata, ctx.group + host_metadata = device_metadata.host_metadata if grad_output is None: grad_output = torch.zeros_like(output) grad_output = grad_output.contiguous() shape = ( - len(partition.cu_seqlens) - 1, + len(host_metadata.cu_seqlens) - 1, q.shape[1], v.shape[-1], k.shape[-1], @@ -306,33 +317,33 @@ def backward(ctx, grad_output, grad_final): final_state_grad.copy_(grad_final) # Final states are replicated outputs: sum losses consuming any replica. dist.all_reduce(final_state_grad, group=group) - additive, transition = _summary_buffers(final_state_grad, partition.slots) - if partition.incoming: - span = _selected_span(metadata, partition.incoming) + additive, transition = _summary_buffers(final_state_grad, host_metadata.slots) + if host_metadata.incoming: + span = _selected_span(device_metadata, host_metadata.incoming) selected = tuple(t[span] for t in tensors) h, t = ctx.adapter._cp_backward_summary( *selected[:3], dict(zip(ctx.gate_names, selected[3:])), grad_output[span], - metadata.incoming_cu_seqlens, + device_metadata.incoming_cu_seqlens, **ctx.options, ) - slots = _slot_indices(metadata, metadata.incoming) + slots = _slot_indices(device_metadata, device_metadata.incoming) additive.index_copy_(0, slots, h) transition.index_copy_(0, slots, t) - if partition.communicates: + if host_metadata.communicates: additive, transition = _gather_summaries(additive, transition, group) ending_state_grad = tex.linear_attention_affine_prefix( final_state_grad, additive, transition, ( - metadata.backward_descriptors + device_metadata.backward_descriptors if ctx.output_final_state - else metadata.backward_zero_descriptors + else device_metadata.backward_zero_descriptors ), ) - if partition.fragments: + if host_metadata.fragments: with torch.enable_grad(): grads = torch.autograd.grad( (output, states), @@ -348,14 +359,14 @@ def backward(ctx, grad_output, grad_final): initial_state_grad = final_state_grad.new_zeros(shape) initial_state_grad.index_add_( 0, - metadata.initial_state_documents, - grads[-1].index_select(0, metadata.initial_state_fragments), + device_metadata.initial_state_documents, + grads[-1].index_select(0, device_metadata.initial_state_fragments), ) if ctx.output_final_state: initial_state_grad.index_copy_( 0, - metadata.empty_documents, - final_state_grad.index_select(0, metadata.empty_documents), + device_metadata.empty_documents, + final_state_grad.index_select(0, device_metadata.empty_documents), ) # Each document's initial-state gradient belongs to its first rank. Do not # replicate it here: if initial_state came from a previous CP call, @@ -411,15 +422,15 @@ def context_parallel_attention( else: key = (qkv_format, batch, length, q.device, cp_size, cp_rank, stream, inference_mode) cacheable = True - cache = getattr(adapter, "_cp_partition_cache", None) + cache = getattr(adapter, "_cp_metadata_cache", None) if not cacheable: # Inference tensors have no mutation counter, so revalidate their - # metadata instead of silently reusing a stale partition. + # metadata instead of silently reusing a stale entry. cache = None elif cache is None or cache[0] != key: # Layers using the same batch can share metadata without another host round trip. cache = _get_shared_cp_metadata(key) - # Build and upload metadata only when the cached partition no longer matches. + # Build and upload metadata only when the cached entry no longer matches. if cache is None or cache[0] != key: document_cu_seqlens = ( tuple(cu_seqlens.tolist()) @@ -428,23 +439,23 @@ def context_parallel_attention( ) if dense and len(document_cu_seqlens) != batch + 1: raise ValueError("Dense cu_seqlens must describe the full batch.") - partition = build_linear_attention_partition( + host_metadata = build_linear_attention_cp_host_metadata( document_cu_seqlens, cp_size=cp_size, cp_rank=cp_rank, dense=dense ) - metadata = _build_cp_metadata(partition, q.device) + device_metadata = _build_linear_attention_cp_device_metadata(host_metadata, q.device) # Hold the metadata tensor to prevent object-ID reuse in the cache key. - adapter._cp_partition_cache = (key, metadata, cu_seqlens) + adapter._cp_metadata_cache = (key, device_metadata, cu_seqlens) if cacheable: - _cache_cp_metadata(adapter._cp_partition_cache) + _cache_cp_metadata(adapter._cp_metadata_cache) else: - metadata = cache[1] - partition = metadata.partition - adapter._cp_partition_cache = cache - if tensors[0].shape[0] != partition.local_cu_seqlens[-1]: + device_metadata = cache[1] + host_metadata = device_metadata.host_metadata + adapter._cp_metadata_cache = cache + if tensors[0].shape[0] != host_metadata.local_cu_seqlens[-1]: raise ValueError("Local token count does not match this CP rank's token interval.") adapter._validate_initial_state( initial_state, - batch_size=len(partition.cu_seqlens) - 1, + batch_size=len(host_metadata.cu_seqlens) - 1, device=q.device, ) tensors, initial_state = adapter._prepare_cp_inputs(tensors, initial_state) @@ -453,16 +464,16 @@ def context_parallel_attention( adapter._op = adapter._resolve_op() # Without shared documents or explicit states, use ordinary local autograd. if ( - not partition.communicates + not host_metadata.communicates and initial_state is None and not output_final_state - and partition.fragments + and host_metadata.fragments ): output, states = _local_attention( adapter, tensors, tuple(gates), - metadata.cu_seqlens, + device_metadata.cu_seqlens, None, kernel_kwargs, output_final_state=False, @@ -473,7 +484,7 @@ def context_parallel_attention( ) output, states = _LinearAttentionContextParallel.apply( adapter, - metadata, + device_metadata, group, kernel_kwargs, output_final_state, diff --git a/transformer_engine/pytorch/attention/linear_attention/gdn.py b/transformer_engine/pytorch/attention/linear_attention/gdn.py index c47e9fc0996..acf49d40b75 100644 --- a/transformer_engine/pytorch/attention/linear_attention/gdn.py +++ b/transformer_engine/pytorch/attention/linear_attention/gdn.py @@ -351,7 +351,7 @@ def set_context_parallel_group(self, cp_group: Optional[dist.ProcessGroup]) -> N if cp_group is not None and not isinstance(cp_group, dist.ProcessGroup): raise TypeError("GDN cp_group must be one torch.distributed.ProcessGroup or None.") self.cp_group = cp_group - self.gdn_attention._cp_partition_cache = None + self.gdn_attention._cp_metadata_cache = None @no_torch_dynamo(when=_needs_eager_linear_attention) def forward( From f8494514cd0b72fe739a4f05bb05f146833b56d6 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Wed, 18 Nov 2026 12:16:52 -0800 Subject: [PATCH 19/34] Separate host and device CP metadata Signed-off-by: Kshitij Lakhani --- .../linear_attention/context_parallel.py | 56 +++++++++++-------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py index 5f49d4b167a..7ed6902bc16 100644 --- a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py +++ b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py @@ -64,9 +64,8 @@ def _cache_cp_metadata(entry): @dataclass class _LinearAttentionCPDeviceMetadata: - """Host metadata reference and device tensors for one CP rank.""" + """Device metadata tensors for one CP rank.""" - host_metadata: LinearAttentionCPHostMetadata cu_seqlens: torch.Tensor # Local cumulative fragment lengths on the device. forward_descriptors: torch.Tensor backward_descriptors: torch.Tensor @@ -83,8 +82,16 @@ class _LinearAttentionCPDeviceMetadata: incoming_cu_seqlens: torch.Tensor +@dataclass +class _LinearAttentionCPMetadata: + """Host and device metadata prepared for one CP rank.""" + + host_metadata: LinearAttentionCPHostMetadata + device_metadata: _LinearAttentionCPDeviceMetadata + + def _build_linear_attention_cp_device_metadata(host_metadata, device): - """Build device tensors and retain their associated host metadata.""" + """Build device metadata tensors from one rank's host metadata.""" def indices(values): return torch.tensor(values, device=device, dtype=torch.int64) @@ -112,7 +119,6 @@ def cumulative(selected): forward_zero_descriptors[:, 0] = -1 backward_zero_descriptors[:, 0] = -1 return _LinearAttentionCPDeviceMetadata( - host_metadata, torch.tensor(host_metadata.local_cu_seqlens, device=device, dtype=torch.int32), forward_descriptors, backward_descriptors, @@ -153,17 +159,17 @@ def _summary_buffers(state_template, slots): return additive, transition.expand(slots, heads, key_dim, key_dim).contiguous() -def _selected_span(device_metadata, selected): +def _selected_span(host_metadata, selected): """Selected spans are consecutive: all dense spans, or one packed edge span.""" - cu_seqlens = device_metadata.host_metadata.local_cu_seqlens + cu_seqlens = host_metadata.local_cu_seqlens return slice(cu_seqlens[selected[0]], cu_seqlens[selected[-1] + 1]) -def _slot_indices(device_metadata, indices): +def _slot_indices(metadata, indices): """Dense slots are document IDs; packed partitions have only one edge slot.""" - if device_metadata.host_metadata.slots == 1: + if metadata.host_metadata.slots == 1: return torch.zeros_like(indices) - return device_metadata.document_indices.index_select(0, indices) + return metadata.device_metadata.document_indices.index_select(0, indices) def _local_attention( @@ -192,7 +198,7 @@ class _LinearAttentionContextParallel(torch.autograd.Function): def forward( ctx, adapter, - device_metadata, + metadata, group, options, output_final_state, @@ -203,7 +209,8 @@ def forward( ): """Establish starting states, then retain the local attention graph.""" ctx.set_materialize_grads(False) - host_metadata = device_metadata.host_metadata + host_metadata = metadata.host_metadata + device_metadata = metadata.device_metadata q = tensors[0] k = tensors[1] v = tensors[2] @@ -221,7 +228,7 @@ def forward( ) additive, transition = _summary_buffers(initial_state_buffer, host_metadata.slots) if host_metadata.outgoing: - span = _selected_span(device_metadata, host_metadata.outgoing) + span = _selected_span(host_metadata, host_metadata.outgoing) selected = tuple(t[span] for t in tensors) h, t = adapter._cp_forward_summary( *selected[:3], @@ -229,7 +236,7 @@ def forward( device_metadata.outgoing_cu_seqlens, **options, ) - slots = _slot_indices(device_metadata, device_metadata.outgoing) + slots = _slot_indices(metadata, device_metadata.outgoing) additive.index_copy_(0, slots, h) transition.index_copy_(0, slots, t) if host_metadata.communicates: @@ -282,8 +289,8 @@ def forward( dist.all_reduce(terminal, group=group) if track_grad: ctx.save_for_backward(*inputs, output, states) - ctx.cp_device_metadata, ctx.group, ctx.output_final_state = ( - device_metadata, + ctx.cp_metadata, ctx.group, ctx.output_final_state = ( + metadata, group, output_final_state, ) @@ -298,8 +305,9 @@ def backward(ctx, grad_output, grad_final): tensors = ctx.saved_tensors[:-3] starting_states, output, states = ctx.saved_tensors[-3:] q, k, v = tensors[:3] - device_metadata, group = ctx.cp_device_metadata, ctx.group - host_metadata = device_metadata.host_metadata + metadata, group = ctx.cp_metadata, ctx.group + host_metadata = metadata.host_metadata + device_metadata = metadata.device_metadata if grad_output is None: grad_output = torch.zeros_like(output) grad_output = grad_output.contiguous() @@ -319,7 +327,7 @@ def backward(ctx, grad_output, grad_final): dist.all_reduce(final_state_grad, group=group) additive, transition = _summary_buffers(final_state_grad, host_metadata.slots) if host_metadata.incoming: - span = _selected_span(device_metadata, host_metadata.incoming) + span = _selected_span(host_metadata, host_metadata.incoming) selected = tuple(t[span] for t in tensors) h, t = ctx.adapter._cp_backward_summary( *selected[:3], @@ -328,7 +336,7 @@ def backward(ctx, grad_output, grad_final): device_metadata.incoming_cu_seqlens, **ctx.options, ) - slots = _slot_indices(device_metadata, device_metadata.incoming) + slots = _slot_indices(metadata, device_metadata.incoming) additive.index_copy_(0, slots, h) transition.index_copy_(0, slots, t) if host_metadata.communicates: @@ -443,13 +451,15 @@ def context_parallel_attention( document_cu_seqlens, cp_size=cp_size, cp_rank=cp_rank, dense=dense ) device_metadata = _build_linear_attention_cp_device_metadata(host_metadata, q.device) + metadata = _LinearAttentionCPMetadata(host_metadata, device_metadata) # Hold the metadata tensor to prevent object-ID reuse in the cache key. - adapter._cp_metadata_cache = (key, device_metadata, cu_seqlens) + adapter._cp_metadata_cache = (key, metadata, cu_seqlens) if cacheable: _cache_cp_metadata(adapter._cp_metadata_cache) else: - device_metadata = cache[1] - host_metadata = device_metadata.host_metadata + metadata = cache[1] + host_metadata = metadata.host_metadata + device_metadata = metadata.device_metadata adapter._cp_metadata_cache = cache if tensors[0].shape[0] != host_metadata.local_cu_seqlens[-1]: raise ValueError("Local token count does not match this CP rank's token interval.") @@ -484,7 +494,7 @@ def context_parallel_attention( ) output, states = _LinearAttentionContextParallel.apply( adapter, - device_metadata, + metadata, group, kernel_kwargs, output_final_state, From b9ed64939f85359f7d16b26f657f35180cc92c6e Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Wed, 18 Nov 2026 12:16:58 -0800 Subject: [PATCH 20/34] Trim redundant GDN CP checks Signed-off-by: Kshitij Lakhani --- qa/L1_pytorch_distributed_unittest/test.sh | 2 +- tests/pytorch/attention/run_gdn_with_cp.py | 27 +++++++------------ .../attention/test_gdn_attention_with_cp.py | 10 +++---- 3 files changed, 14 insertions(+), 25 deletions(-) diff --git a/qa/L1_pytorch_distributed_unittest/test.sh b/qa/L1_pytorch_distributed_unittest/test.sh index 02a200d9602..e6637cdf9ef 100644 --- a/qa/L1_pytorch_distributed_unittest/test.sh +++ b/qa/L1_pytorch_distributed_unittest/test.sh @@ -43,7 +43,7 @@ else NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_attention_deterministic_with_cp.xml $TE_PATH/tests/pytorch/attention/test_attention_with_cp.py || test_fail "NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 test_attention_with_cp.py" fi -NVTE_GDN_TEST_REQUIRED=1 python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_gdn_attention_with_cp.xml $TE_PATH/tests/pytorch/attention/test_gdn_attention_with_cp.py || test_fail "test_gdn_attention_with_cp.py" +NVTE_GDN_TEST_REQUIRED=1 NVTE_GDN_CP_TEST_REQUIRED=1 python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_gdn_attention_with_cp.xml $TE_PATH/tests/pytorch/attention/test_gdn_attention_with_cp.py || test_fail "test_gdn_attention_with_cp.py" python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_sanity.xml $TE_PATH/tests/pytorch/distributed/test_sanity.py || test_fail "test_sanity.py" python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_parallel_cross_entropy.xml $TE_PATH/tests/pytorch/distributed/test_parallel_cross_entropy.py || test_fail "test_parallel_cross_entropy.py" diff --git a/tests/pytorch/attention/run_gdn_with_cp.py b/tests/pytorch/attention/run_gdn_with_cp.py index cdbc1c7f30b..09064f48e8a 100644 --- a/tests/pytorch/attention/run_gdn_with_cp.py +++ b/tests/pytorch/attention/run_gdn_with_cp.py @@ -16,7 +16,6 @@ import torch.nn.functional as F from transformer_engine.pytorch import GatedDeltaNetAttention -from transformer_engine.pytorch.attention.linear_attention import context_parallel as cp from test_gdn_attention import ( _assert_rms_close, _gdn_reference, @@ -237,7 +236,7 @@ def _inference_and_deferred_group(group): ): with context(): # CP accepts host metadata; inference tensors intentionally bypass - # the partition cache because they have no mutation counter. + # the metadata cache because they have no mutation counter. global_cu = torch.tensor([0, 17, 35], device=metadata_device, dtype=torch.int32) for boundary in (17, 19): global_cu[1] = boundary @@ -355,13 +354,12 @@ def main(): cases = 0 try: for lengths in ( - [8, 4, 12], - [16, 4, 4], - [7, 2, 4, 5, 14], - [6, 6, 6, 6], - [0, 17, 0, 18, 0], - [1, 0, 1], - [0, 0], + [8, 4, 12], # Shared and complete documents on the same rank. + [16, 4, 4], # One document spans three ranks at CP=4. + [6, 6, 6, 6], # Document and rank boundaries coincide. + [0, 17, 0, 18, 0], # Empty documents between nonempty documents. + [1, 0, 1], # Some ranks own no tokens. + [0, 0], # Entirely empty input. ): _case(group, lengths) cases += 1 @@ -437,15 +435,8 @@ def main(): cases += 1 _invalid_metadata(group) cases += 1 - # Independent complete documents need no state exchange or prefix work. - with patch.object( - cp, "_gather_summaries", side_effect=AssertionError("Unexpected exchange") - ), patch.object( - cp.tex, - "linear_attention_affine_prefix", - side_effect=AssertionError("Unexpected prefix"), - ): - _case(group, [8] * size, initial=False, final=False) + # Independent complete documents also match ordinary local attention. + _case(group, [8] * size, initial=False, final=False) cases += 1 # A size-one group follows the ordinary local path, even while WORLD # contains several ranks. Every process creates groups in the same order. diff --git a/tests/pytorch/attention/test_gdn_attention_with_cp.py b/tests/pytorch/attention/test_gdn_attention_with_cp.py index 9f8bfb01bdd..d0729684bbf 100644 --- a/tests/pytorch/attention/test_gdn_attention_with_cp.py +++ b/tests/pytorch/attention/test_gdn_attention_with_cp.py @@ -96,16 +96,14 @@ def test_dense_partition_intersections(cp_size, length): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") -@pytest.mark.parametrize("key_dim", [3, 64, 128, 257]) -def test_common_affine_prefix(key_dim): - """Nonsymmetric transitions, reversed order, zero prefixes, and independent initial states.""" +def test_affine_prefix_binding(): + """Forward summary descriptors and preserve the caller's current CUDA device.""" torch.manual_seed(22) + key_dim = 3 current = torch.cuda.current_device() # Also exercise the binding's device guard: allocating on another GPU must # neither launch on the current GPU nor change the caller's current device. - device = torch.device( - "cuda", (current + 1) % torch.cuda.device_count() if key_dim == 3 else current - ) + device = torch.device("cuda", (current + 1) % torch.cuda.device_count()) initial_states = torch.randn(3, 2, 5, key_dim, device=device) additive = torch.randn(6, 2, 5, key_dim, device=device) transitions = torch.randn(6, 2, key_dim, key_dim, device=device) / key_dim**0.5 From 6f36e0aedeabae532f2c8908f2dd310016742d87 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Wed, 18 Nov 2026 12:17:02 -0800 Subject: [PATCH 21/34] Clarify affine summary indices Signed-off-by: Kshitij Lakhani --- .../include/transformer_engine/linear_attention.h | 10 +++++----- transformer_engine/common/linear_attention/__init__.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/transformer_engine/common/include/transformer_engine/linear_attention.h b/transformer_engine/common/include/transformer_engine/linear_attention.h index d4ef5e12e7b..f9d4f81ecc3 100644 --- a/transformer_engine/common/include/transformer_engine/linear_attention.h +++ b/transformer_engine/common/include/transformer_engine/linear_attention.h @@ -19,7 +19,7 @@ extern "C" { /*! \brief Compose recurrent states from ordered affine summaries. * * Each output starts from a selected initial memory and repeatedly evaluates - * ``state = state @ transition[record] + additive[record]``. This operation + * ``state = state @ transitions[summary_index] + additive[summary_index]``. This operation * serves both forward prefixes and backward suffixes; backward callers pass * the backward-oriented transitions and a negative record stride. * @@ -28,13 +28,13 @@ extern "C" { * \param[in] initial_states FP32 initial memories, * [initial_memories, heads, value_dim, key_dim]. * \param[in] additive FP32 additive summaries, - * [records, heads, value_dim, key_dim]. + * [summaries, heads, value_dim, key_dim]. * \param[in] transitions FP32 transition matrices, - * [records, heads, key_dim, key_dim]. + * [summaries, heads, key_dim, key_dim]. * \param[in] descriptors INT32 [outputs, 4], with rows - * (initial_memory_index, first_record, count, stride). + * (initial_memory_index, first_summary, count, stride). * An initial_memory_index of -1 selects an implicit zero state. - * Other referenced memory and record indices must be valid. + * Other referenced memory and summary indices must be valid. * Count must be nonnegative; zero copies the initial memory * (or writes zeros). * \param[out] output Caller-allocated FP32 composed memories, diff --git a/transformer_engine/common/linear_attention/__init__.py b/transformer_engine/common/linear_attention/__init__.py index 78c13b50bbb..679322b119b 100644 --- a/transformer_engine/common/linear_attention/__init__.py +++ b/transformer_engine/common/linear_attention/__init__.py @@ -79,7 +79,7 @@ def build_linear_attention_cp_host_metadata(cu_seqlens, cp_size, cp_rank, *, den ``cu_seqlens`` contains full-document boundaries as host integers. For P = ``cp_size``, packed rank r owns [r*T//P:(r+1)*T//P] of T total tokens. Dense rank r owns [r*S//P:(r+1)*S//P] of each length-S document. - Descriptor rows are (document, first_record, count, stride). Each selects + Descriptor rows are (document, first_summary, count, stride). Each selects summaries of one document, in forward or reverse order. """ cu_seqlens = tuple(cu_seqlens) From 36ac0a1f5db62b8537ea45c422ad3c6f2a821cb1 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Wed, 18 Nov 2026 14:52:09 -0800 Subject: [PATCH 22/34] Name CP fragment indices and descriptors Signed-off-by: Kshitij Lakhani --- .../attention/test_gdn_attention_with_cp.py | 14 +++---- .../common/linear_attention/__init__.py | 41 ++++++++++--------- .../linear_attention/context_parallel.py | 34 +++++++-------- 3 files changed, 47 insertions(+), 42 deletions(-) diff --git a/tests/pytorch/attention/test_gdn_attention_with_cp.py b/tests/pytorch/attention/test_gdn_attention_with_cp.py index d0729684bbf..c9eb03769cf 100644 --- a/tests/pytorch/attention/test_gdn_attention_with_cp.py +++ b/tests/pytorch/attention/test_gdn_attention_with_cp.py @@ -43,25 +43,25 @@ def test_partition_intersections(cp_size): host_metadata.local_cu_seqlens[-1] == (cp_rank + 1) * cu[-1] // cp_size - cp_rank * cu[-1] // cp_size ) - assert len(host_metadata.incoming) <= 1 - assert len(host_metadata.outgoing) <= 1 - for f, forward, backward in zip( + assert len(host_metadata.incoming_fragment_indices) <= 1 + assert len(host_metadata.outgoing_fragment_indices) <= 1 + for f, forward_desc, backward_desc in zip( host_metadata.fragments, host_metadata.forward_descriptors, host_metadata.backward_descriptors, ): - assert forward[2] == cp_rank - f.first_rank - assert backward[2] == f.last_rank - cp_rank + assert forward_desc[2] == cp_rank - f.first_rank + assert backward_desc[2] == f.last_rank - cp_rank # Every referenced recipe belongs to this same document. for predecessor in range(f.first_rank, cp_rank): p = rank_metadata[predecessor] if p.fragments: - assert p.fragments[p.outgoing[0]].document == f.document + assert p.fragments[p.outgoing_fragment_indices[0]].document == f.document for successor in range(cp_rank + 1, f.last_rank + 1): p = rank_metadata[successor] # An empty rank contributes an identity recipe. if p.fragments: - assert p.fragments[p.incoming[0]].document == f.document + assert p.fragments[p.incoming_fragment_indices[0]].document == f.document @pytest.mark.parametrize("cu_seqlens", [[1, 2], [0], [0, 2, 1]]) diff --git a/transformer_engine/common/linear_attention/__init__.py b/transformer_engine/common/linear_attention/__init__.py index 679322b119b..b42ce7773f3 100644 --- a/transformer_engine/common/linear_attention/__init__.py +++ b/transformer_engine/common/linear_attention/__init__.py @@ -47,11 +47,11 @@ class LinearAttentionCPHostMetadata: Using the fragment example from LinearAttentionFragment, packed rank 1 owns: - fragments: A [6, 8), B [8, 12) - local_cu_seqlens: (0, 2, 6) - incoming: (0,) # A continues from rank 0. - outgoing: () # Neither document continues to a later rank. - terminal: (0, 1) # Both documents end on this rank. + fragments: A [6, 8), B [8, 12) + local_cu_seqlens: (0, 2, 6) + incoming_fragment_indices: (0,) # A continues from rank 0. + outgoing_fragment_indices: () # Neither document continues to a later rank. + terminal_fragment_indices: (0, 1) # Both documents end on this rank. """ cu_seqlens: Tuple[int, ...] # Global cumulative sequence lengths. @@ -64,9 +64,9 @@ class LinearAttentionCPHostMetadata: # Select following reverse summaries to compose each fragment's ending-state gradient. backward_descriptors: Tuple[Tuple[int, int, int, int], ...] # The following three tuples contain local fragment indices, not document IDs. - outgoing: Tuple[int, ...] # Document continues to a later rank. - incoming: Tuple[int, ...] # Document starts on an earlier rank. - terminal: Tuple[int, ...] # Document ends on this rank. + outgoing_fragment_indices: Tuple[int, ...] # Document continues to a later rank. + incoming_fragment_indices: Tuple[int, ...] # Document starts on an earlier rank. + terminal_fragment_indices: Tuple[int, ...] # Document ends on this rank. initial_state_owners: Tuple[int, ...] # Initial-state owner rank for each global document. empty_owned: Tuple[int, ...] # Empty global document IDs assigned to this rank. slots: int # One per rank for packed inputs; one per document for dense inputs. @@ -121,33 +121,36 @@ def build_linear_attention_cp_host_metadata(cu_seqlens, cp_size, cp_rank, *, den LinearAttentionFragment(doc, fragment_start, fragment_end, first_rank, last_rank) ) cu = [0] - forward, backward, outgoing, incoming, terminal = [], [], [], [], [] + forward_desc, backward_desc = [], [] + outgoing_fragment_indices = [] + incoming_fragment_indices = [] + terminal_fragment_indices = [] for index, frag in enumerate(fragments): cu.append(cu[-1] + frag.end - frag.start) slot = frag.document if dense else 0 - forward.append( + forward_desc.append( (frag.document, frag.first_rank * slots + slot, cp_rank - frag.first_rank, slots) ) - backward.append( + backward_desc.append( (frag.document, frag.last_rank * slots + slot, frag.last_rank - cp_rank, -slots) ) if cp_rank < frag.last_rank: - outgoing.append(index) + outgoing_fragment_indices.append(index) else: - terminal.append(index) + terminal_fragment_indices.append(index) if cp_rank > frag.first_rank: - incoming.append(index) + incoming_fragment_indices.append(index) return LinearAttentionCPHostMetadata( cu_seqlens, cp_rank, cp_size, tuple(fragments), tuple(cu), - tuple(forward), - tuple(backward), - tuple(outgoing), - tuple(incoming), - tuple(terminal), + tuple(forward_desc), + tuple(backward_desc), + tuple(outgoing_fragment_indices), + tuple(incoming_fragment_indices), + tuple(terminal_fragment_indices), tuple(initial_state_owners), tuple(empty), slots, diff --git a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py index 7ed6902bc16..7a9f7f7d9c8 100644 --- a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py +++ b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py @@ -72,9 +72,9 @@ class _LinearAttentionCPDeviceMetadata: forward_zero_descriptors: torch.Tensor backward_zero_descriptors: torch.Tensor document_indices: torch.Tensor - outgoing: torch.Tensor - incoming: torch.Tensor - terminal: torch.Tensor + outgoing_fragment_indices: torch.Tensor + incoming_fragment_indices: torch.Tensor + terminal_fragment_indices: torch.Tensor initial_state_fragments: torch.Tensor initial_state_documents: torch.Tensor empty_documents: torch.Tensor @@ -125,14 +125,14 @@ def cumulative(selected): forward_zero_descriptors, backward_zero_descriptors, indices([f.document for f in host_metadata.fragments]), - indices(host_metadata.outgoing), - indices(host_metadata.incoming), - indices(host_metadata.terminal), + indices(host_metadata.outgoing_fragment_indices), + indices(host_metadata.incoming_fragment_indices), + indices(host_metadata.terminal_fragment_indices), indices(first), indices([host_metadata.fragments[i].document for i in first]), indices(host_metadata.empty_owned), - cumulative(host_metadata.outgoing), - cumulative(host_metadata.incoming), + cumulative(host_metadata.outgoing_fragment_indices), + cumulative(host_metadata.incoming_fragment_indices), ) @@ -227,8 +227,8 @@ def forward( else initial_state ) additive, transition = _summary_buffers(initial_state_buffer, host_metadata.slots) - if host_metadata.outgoing: - span = _selected_span(host_metadata, host_metadata.outgoing) + if host_metadata.outgoing_fragment_indices: + span = _selected_span(host_metadata, host_metadata.outgoing_fragment_indices) selected = tuple(t[span] for t in tensors) h, t = adapter._cp_forward_summary( *selected[:3], @@ -236,7 +236,7 @@ def forward( device_metadata.outgoing_cu_seqlens, **options, ) - slots = _slot_indices(metadata, device_metadata.outgoing) + slots = _slot_indices(metadata, device_metadata.outgoing_fragment_indices) additive.index_copy_(0, slots, h) transition.index_copy_(0, slots, t) if host_metadata.communicates: @@ -277,8 +277,10 @@ def forward( if output_final_state: terminal.index_copy_( 0, - device_metadata.document_indices.index_select(0, device_metadata.terminal), - states.detach().index_select(0, device_metadata.terminal), + device_metadata.document_indices.index_select( + 0, device_metadata.terminal_fragment_indices + ), + states.detach().index_select(0, device_metadata.terminal_fragment_indices), ) if initial_state is not None: terminal.index_copy_( @@ -326,8 +328,8 @@ def backward(ctx, grad_output, grad_final): # Final states are replicated outputs: sum losses consuming any replica. dist.all_reduce(final_state_grad, group=group) additive, transition = _summary_buffers(final_state_grad, host_metadata.slots) - if host_metadata.incoming: - span = _selected_span(host_metadata, host_metadata.incoming) + if host_metadata.incoming_fragment_indices: + span = _selected_span(host_metadata, host_metadata.incoming_fragment_indices) selected = tuple(t[span] for t in tensors) h, t = ctx.adapter._cp_backward_summary( *selected[:3], @@ -336,7 +338,7 @@ def backward(ctx, grad_output, grad_final): device_metadata.incoming_cu_seqlens, **ctx.options, ) - slots = _slot_indices(metadata, device_metadata.incoming) + slots = _slot_indices(metadata, device_metadata.incoming_fragment_indices) additive.index_copy_(0, slots, h) transition.index_copy_(0, slots, t) if host_metadata.communicates: From f070366ed2523eb2995ae5194ae6a972803bbe37 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Wed, 18 Nov 2026 14:52:16 -0800 Subject: [PATCH 23/34] Add host and device CP metadata examples Signed-off-by: Kshitij Lakhani --- .../common/linear_attention/__init__.py | 6 ++-- .../linear_attention/context_parallel.py | 36 ++++++++++++++----- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/transformer_engine/common/linear_attention/__init__.py b/transformer_engine/common/linear_attention/__init__.py index b42ce7773f3..efdd90951cb 100644 --- a/transformer_engine/common/linear_attention/__init__.py +++ b/transformer_engine/common/linear_attention/__init__.py @@ -49,6 +49,7 @@ class LinearAttentionCPHostMetadata: fragments: A [6, 8), B [8, 12) local_cu_seqlens: (0, 2, 6) + forward_descriptors: ((0, 0, 1, 1), (1, 1, 0, 1)) incoming_fragment_indices: (0,) # A continues from rank 0. outgoing_fragment_indices: () # Neither document continues to a later rank. terminal_fragment_indices: (0, 1) # Both documents end on this rank. @@ -63,7 +64,6 @@ class LinearAttentionCPHostMetadata: forward_descriptors: Tuple[Tuple[int, int, int, int], ...] # Select following reverse summaries to compose each fragment's ending-state gradient. backward_descriptors: Tuple[Tuple[int, int, int, int], ...] - # The following three tuples contain local fragment indices, not document IDs. outgoing_fragment_indices: Tuple[int, ...] # Document continues to a later rank. incoming_fragment_indices: Tuple[int, ...] # Document starts on an earlier rank. terminal_fragment_indices: Tuple[int, ...] # Document ends on this rank. @@ -79,8 +79,6 @@ def build_linear_attention_cp_host_metadata(cu_seqlens, cp_size, cp_rank, *, den ``cu_seqlens`` contains full-document boundaries as host integers. For P = ``cp_size``, packed rank r owns [r*T//P:(r+1)*T//P] of T total tokens. Dense rank r owns [r*S//P:(r+1)*S//P] of each length-S document. - Descriptor rows are (document, first_summary, count, stride). Each selects - summaries of one document, in forward or reverse order. """ cu_seqlens = tuple(cu_seqlens) if cp_size < 1 or not 0 <= cp_rank < cp_size: @@ -121,6 +119,8 @@ def build_linear_attention_cp_host_metadata(cu_seqlens, cp_size, cp_rank, *, den LinearAttentionFragment(doc, fragment_start, fragment_end, first_rank, last_rank) ) cu = [0] + # Descriptor rows are (document, first_summary, count, stride). Each selects + # summaries of one document, in forward or reverse order. forward_desc, backward_desc = [], [] outgoing_fragment_indices = [] incoming_fragment_indices = [] diff --git a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py index 7a9f7f7d9c8..0732b8d5450 100644 --- a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py +++ b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py @@ -64,20 +64,38 @@ def _cache_cp_metadata(entry): @dataclass class _LinearAttentionCPDeviceMetadata: - """Device metadata tensors for one CP rank.""" + """Device metadata for one CP rank's local attention and state composition. + + For global cu_seqlens [0, 8, 12, 24] and CP=4, rank 1 owns fragments + A [6, 8) and B [8, 12). Its device tensors include:: + + cu_seqlens: [0, 2, 6] + document_indices: [0, 1] + forward_descriptors: [[0, 0, 1, 1], [1, 1, 0, 1]] + incoming_fragment_indices: [0] + incoming_cu_seqlens: [0, 2] + initial_state_fragments: [1] + initial_state_documents: [1] + + Fragment 0 receives memory from rank 0; fragment 1 starts document B + on this rank and maps to global document state 1. + """ cu_seqlens: torch.Tensor # Local cumulative fragment lengths on the device. - forward_descriptors: torch.Tensor - backward_descriptors: torch.Tensor + forward_descriptors: torch.Tensor # Starting-memory composition recipes. + backward_descriptors: torch.Tensor # Ending-state-gradient composition recipes. + # Zero variants set descriptor column 0 to -1 for an implicit zero memory. forward_zero_descriptors: torch.Tensor backward_zero_descriptors: torch.Tensor - document_indices: torch.Tensor - outgoing_fragment_indices: torch.Tensor - incoming_fragment_indices: torch.Tensor - terminal_fragment_indices: torch.Tensor + document_indices: torch.Tensor # Global document ID for each local fragment. + outgoing_fragment_indices: torch.Tensor # Document continues on a later rank. + incoming_fragment_indices: torch.Tensor # Document starts on an earlier rank. + terminal_fragment_indices: torch.Tensor # Document ends on this rank. + # Paired fragment/document indices for documents starting on this rank. initial_state_fragments: torch.Tensor initial_state_documents: torch.Tensor - empty_documents: torch.Tensor + empty_documents: torch.Tensor # Global IDs of empty documents owned by this rank. + # Cumulative lengths within the selected spans passed to cuDNN summary calls. outgoing_cu_seqlens: torch.Tensor incoming_cu_seqlens: torch.Tensor @@ -175,7 +193,7 @@ def _slot_indices(metadata, indices): def _local_attention( adapter, tensors, gate_names, cu_seqlens, state, options, *, output_final_state=True ): - """Call the flavor adapter with packed local inputs.""" + """Call the corresponding linear attention flavor's adapter with packed local inputs.""" return adapter._call_op( *tensors[:3], dict(zip(gate_names, tensors[3:])), From 3ab11588ed1bb0e82e88ac78c8a5d9de388cc5a4 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Wed, 18 Nov 2026 15:27:53 -0800 Subject: [PATCH 24/34] Use variant terminology for linear attention Signed-off-by: Kshitij Lakhani --- .../linear_attention_context_parallel.rst | 2 +- .../pytorch/attention/linear_attention/base.py | 6 +++--- .../attention/linear_attention/context_parallel.py | 9 +++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/features/other_optimizations/linear_attention_context_parallel.rst b/docs/features/other_optimizations/linear_attention_context_parallel.rst index 28263632b6b..416b3794078 100644 --- a/docs/features/other_optimizations/linear_attention_context_parallel.rst +++ b/docs/features/other_optimizations/linear_attention_context_parallel.rst @@ -111,7 +111,7 @@ engine is unavailable, ordinary GDN state passes provide a slower fallback. CP runs eagerly, including inside ``torch.compile`` models, and supports first-order gradients. Full-graph compilation and CUDA graph capture are -outside this implementation's supported contract. Other LA flavors, direct +outside this implementation's supported contract. Other LA variants, direct dual-chunk ownership, headwise CP and whole-document redistribution are separate features. diff --git a/transformer_engine/pytorch/attention/linear_attention/base.py b/transformer_engine/pytorch/attention/linear_attention/base.py index 3aafba5001d..58c4cd6098d 100644 --- a/transformer_engine/pytorch/attention/linear_attention/base.py +++ b/transformer_engine/pytorch/attention/linear_attention/base.py @@ -317,17 +317,17 @@ def _validate_inputs( self._validate_gates(gates, query_layer) def _prepare_cp_inputs(self, tensors, initial_state): - """Normalize a flavor's token-aligned CP inputs before state propagation.""" + """Normalize token-aligned CP inputs before state propagation.""" return tensors, initial_state def _cp_forward_summary(self, query, key, value, gates, cu_seqlens, **kernel_kwargs): - """Return the fragment maps S_out = S_in @ T + H for this flavor.""" + """Return the fragment maps S_out = S_in @ T + H.""" raise NotImplementedError(f"{self.variant} does not implement CP forward summaries.") def _cp_backward_summary( self, query, key, value, gates, grad_output, cu_seqlens, **kernel_kwargs ): - """Return reverse state-gradient maps for this flavor.""" + """Return reverse state-gradient maps for this variant.""" raise NotImplementedError(f"{self.variant} does not implement CP backward summaries.") def _run( diff --git a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py index 0732b8d5450..4fc63fc84ab 100644 --- a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py +++ b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py @@ -4,8 +4,8 @@ """Contiguous linear-attention CP: fragments, collectives and state gradients. -Flavor adapters supply local attention and affine summary operations. The shared -CUDA operation composes states in document order. +Adapters for each linear attention variant supply local attention and affine +summary operations. The shared CUDA operation composes states in document order. Adapter contract: * Positional tensors carry a real-token leading axis for fragment selection; @@ -19,7 +19,8 @@ differentiable token-gate transformation before the CP Function. Their gradient reduction must match the caller's parameter-ownership contract. -Only GDN supplies these hooks today; other flavors require adapter validation. +Only GDN supplies these hooks today; other linear attention variants require +adapter validation. """ from collections import OrderedDict @@ -193,7 +194,7 @@ def _slot_indices(metadata, indices): def _local_attention( adapter, tensors, gate_names, cu_seqlens, state, options, *, output_final_state=True ): - """Call the corresponding linear attention flavor's adapter with packed local inputs.""" + """Call the linear attention variant's adapter with packed local inputs.""" return adapter._call_op( *tensors[:3], dict(zip(gate_names, tensors[3:])), From fb7c9be528311013ad082677b893302244f672f5 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Wed, 18 Nov 2026 15:27:58 -0800 Subject: [PATCH 25/34] Group linear attention CP bindings Signed-off-by: Kshitij Lakhani --- transformer_engine/pytorch/csrc/extensions.h | 8 ++++---- transformer_engine/pytorch/csrc/extensions/pybind.cpp | 8 +++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index fd557620dff..8b19fefb57b 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -27,10 +27,6 @@ class CommOverlapP2P; namespace transformer_engine::pytorch { -at::Tensor linear_attention_affine_prefix(const at::Tensor &initial_states, - const at::Tensor &additive, const at::Tensor &transitions, - const at::Tensor &descriptors); - /*************************************************************************************************** * Router fusion **************************************************************************************************/ @@ -147,6 +143,10 @@ void copy_to_kv_cache(at::Tensor new_k, at::Tensor new_v, at::Tensor k_cache, at NVTE_QKV_Format kv_format, int b, int max_ctx_len, int max_seq_len, int max_pages_per_seq, bool is_non_paged); +at::Tensor linear_attention_affine_prefix(const at::Tensor &initial_states, + const at::Tensor &additive, const at::Tensor &transitions, + const at::Tensor &descriptors); + /*************************************************************************************************** * GEMM **************************************************************************************************/ diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index 2a354713102..4afd02c5b4e 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -646,9 +646,6 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { py::call_guard()); // Support THD format for Context Parallel - m.def("linear_attention_affine_prefix", - &transformer_engine::pytorch::linear_attention_affine_prefix, - "Apply ordered linear-attention state summaries", py::call_guard()); m.def("thd_read_half_tensor", &transformer_engine::pytorch::thd_read_half_tensor, "Read the first half(half_idx=0) or the second half(half_idx=1) of each sequence in a THD " "tensor", @@ -680,6 +677,11 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { "Copy valid THD token entries from a per-split tensor into a rank-local accumulator", py::call_guard()); + // Linear attention context parallelism + m.def("linear_attention_affine_prefix", + &transformer_engine::pytorch::linear_attention_affine_prefix, + "Apply ordered linear-attention state summaries", py::call_guard()); + // nvshmem functions m.def("init_nvshmem_backend", &transformer_engine::pytorch::init_nvshmem_backend, "Initialize nvshmem backend with Pytorch distributed process groups", From ff1a0eae4f3abb8b3578aa0b3ee75ac8a3c8f740 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Wed, 18 Nov 2026 18:51:40 -0800 Subject: [PATCH 26/34] Expand linear attn affine prefix cpp test coverage Signed-off-by: Kshitij Lakhani --- tests/cpp/operator/test_linear_attention.cu | 148 +++++++++++++++----- 1 file changed, 113 insertions(+), 35 deletions(-) diff --git a/tests/cpp/operator/test_linear_attention.cu b/tests/cpp/operator/test_linear_attention.cu index 5b80c0dd4a5..d8e76c1a65a 100644 --- a/tests/cpp/operator/test_linear_attention.cu +++ b/tests/cpp/operator/test_linear_attention.cu @@ -7,73 +7,151 @@ #include #include +#include +#include #include +#include #include +#include +#include #include #include "../test_common.h" namespace { -class LinearAttentionPrefixTest : public ::testing::TestWithParam {}; +constexpr size_t kDescriptorWidth = 4; +using AffineDescriptor = std::array; -TEST_P(LinearAttentionPrefixTest, OrderedAffineRecipes) { +void RunAffinePrefixTest(size_t key_dim, size_t heads, size_t value_dim, size_t num_summaries, + const std::vector &descriptor_rows) { using test::Tensor; using transformer_engine::DType; - const size_t k = GetParam(), heads = 2, values = 5, row_size = heads * values * k; - Tensor initial_states("initial_states", std::vector{3, heads, values, k}, DType::kFloat32); - Tensor additive("additive", std::vector{6, heads, values, k}, DType::kFloat32); - Tensor transitions("transitions", std::vector{6, heads, k, k}, DType::kFloat32); - Tensor descriptors("descriptors", std::vector{6, 4}, DType::kInt32); - Tensor output("output", std::vector{6, heads, values, k}, DType::kFloat32); - std::mt19937 generator(317); + constexpr size_t kNumInitialStates = 3; + constexpr unsigned kSeed = 2026; + constexpr double kTolerance = 2e-5; + const size_t num_outputs = descriptor_rows.size(); + const size_t state_size = heads * value_dim * key_dim; + Tensor initial_states("initial_states", + std::vector{kNumInitialStates, heads, value_dim, key_dim}, + DType::kFloat32); + Tensor additive("additive", std::vector{num_summaries, heads, value_dim, key_dim}, + DType::kFloat32); + Tensor transitions("transitions", std::vector{num_summaries, heads, key_dim, key_dim}, + DType::kFloat32); + Tensor descriptors("descriptors", std::vector{num_outputs, kDescriptorWidth}, + DType::kInt32); + Tensor output("output", std::vector{num_outputs, heads, value_dim, key_dim}, + DType::kFloat32); + std::mt19937 generator(kSeed); std::uniform_real_distribution distribution(-1.f, 1.f); - auto *s = initial_states.rowwise_cpu_dptr(); - auto *h = additive.rowwise_cpu_dptr(); - auto *t = transitions.rowwise_cpu_dptr(); - auto *d = descriptors.rowwise_cpu_dptr(); - for (size_t i = 0; i < 3 * row_size; ++i) s[i] = distribution(generator); - for (size_t i = 0; i < 6 * row_size; ++i) h[i] = distribution(generator); - for (size_t i = 0; i < 6 * heads * k * k; ++i) t[i] = distribution(generator) / sqrt(k); - const int rows[] = {0, 0, 0, 1, 1, 0, 3, 1, 2, 5, 3, -2, - -1, 0, 0, 1, -1, 0, 3, 1, -1, 5, 3, -2}; - std::copy(rows, rows + 24, d); + auto *initial_states_cpu = initial_states.rowwise_cpu_dptr(); + auto *additive_cpu = additive.rowwise_cpu_dptr(); + auto *transitions_cpu = transitions.rowwise_cpu_dptr(); + auto *descriptors_cpu = descriptors.rowwise_cpu_dptr(); + for (size_t i = 0; i < kNumInitialStates * state_size; ++i) { + initial_states_cpu[i] = distribution(generator); + } + for (size_t i = 0; i < num_summaries * state_size; ++i) { + additive_cpu[i] = distribution(generator); + } + for (size_t i = 0; i < num_summaries * heads * key_dim * key_dim; ++i) { + transitions_cpu[i] = distribution(generator) / std::sqrt(key_dim); + } + for (size_t i = 0; i < num_outputs; ++i) { + std::copy(descriptor_rows[i].begin(), descriptor_rows[i].end(), + descriptors_cpu + i * kDescriptorWidth); + } + // Detect unwritten output elements, including outputs that should be zero. + std::fill_n(output.rowwise_cpu_dptr(), num_outputs * state_size, + std::numeric_limits::quiet_NaN()); initial_states.from_cpu(); additive.from_cpu(); transitions.from_cpu(); descriptors.from_cpu(); + output.from_cpu(); nvte_linear_attention_affine_prefix(initial_states.data(), additive.data(), transitions.data(), descriptors.data(), output.data(), nullptr); ASSERT_EQ(cudaDeviceSynchronize(), cudaSuccess); output.to_cpu(); const auto *actual = output.rowwise_cpu_dptr(); - for (size_t fragment = 0; fragment < 6; ++fragment) { + for (size_t output_index = 0; output_index < num_outputs; ++output_index) { + const auto [initial_memory_index, first_summary, count, stride] = descriptor_rows[output_index]; for (size_t head = 0; head < heads; ++head) { - for (size_t value = 0; value < values; ++value) { - const size_t row = head * values + value; - std::vector state(k, 0.), next(k); - if (rows[fragment * 4] >= 0) { - const float *start = s + rows[fragment * 4] * row_size + row * k; - std::copy(start, start + k, state.begin()); + for (size_t value = 0; value < value_dim; ++value) { + const size_t row_offset = (head * value_dim + value) * key_dim; + std::vector state(key_dim, 0.), next(key_dim); + if (initial_memory_index >= 0) { + const float *start = initial_states_cpu + initial_memory_index * state_size + row_offset; + std::copy(start, start + key_dim, state.begin()); } - for (int step = 0; step < rows[fragment * 4 + 2]; ++step) { - const int record = rows[fragment * 4 + 1] + step * rows[fragment * 4 + 3]; - for (size_t col = 0; col < k; ++col) { - next[col] = h[record * row_size + row * k + col]; - for (size_t inner = 0; inner < k; ++inner) { - next[col] += state[inner] * t[(record * heads + head) * k * k + inner * k + col]; + for (int step = 0; step < count; ++step) { + const int summary_index = first_summary + step * stride; + const float *transition_matrix = + transitions_cpu + (summary_index * heads + head) * key_dim * key_dim; + for (size_t col = 0; col < key_dim; ++col) { + next[col] = additive_cpu[summary_index * state_size + row_offset + col]; + for (size_t inner = 0; inner < key_dim; ++inner) { + next[col] += state[inner] * transition_matrix[inner * key_dim + col]; } } state.swap(next); } - for (size_t col = 0; col < k; ++col) { - ASSERT_NEAR(actual[fragment * row_size + row * k + col], state[col], 2e-5); + for (size_t col = 0; col < key_dim; ++col) { + ASSERT_NEAR(actual[output_index * state_size + row_offset + col], state[col], kTolerance) + << "output=" << output_index << ", head=" << head << ", value=" << value + << ", column=" << col; } } } } } -INSTANTIATE_TEST_SUITE_P(Common, LinearAttentionPrefixTest, ::testing::Values(3, 64, 128, 257)); +class LinearAttentionPrefixTest + : public ::testing::TestWithParam> {}; + +TEST_P(LinearAttentionPrefixTest, OrderedAffineRecipes) { + const auto [key_dim, heads, value_dim] = GetParam(); + constexpr size_t kNumSummaries = 7; + // Rows are (initial_memory_index, first_summary, count, stride). + const std::vector descriptor_rows = { + {0, 0, 0, 1}, // Copy initial memory 0. + {1, 0, 3, 1}, // Apply summaries 0, 1, 2 to initial memory 1. + {2, 5, 3, -2}, // Reverse composition using summaries 5, 3, 1. + {-1, 0, 0, 1}, // Zero initial memory, with no summaries. + {-1, 0, 3, 2}, // Zero initial memory, then summaries 0, 2, 4. + {-1, 6, 3, -3}, // Zero initial memory, then summaries 6, 3, 0. + {1, 4, 1, 1}, // One summary with an explicit initial memory. + {-1, 4, 1, 1}, // One summary with zero initial memory: copy H[4]. + }; + RunAffinePrefixTest(key_dim, heads, value_dim, kNumSummaries, descriptor_rows); +} + +TEST(LinearAttentionPrefixLongChainTest, ForwardAndReverse) { + constexpr size_t kKeyDim = 64; + constexpr size_t kHeads = 2; + constexpr size_t kValueDim = 5; + constexpr size_t kNumSummaries = 32; + constexpr int kLastSummary = static_cast(kNumSummaries) - 1; + // CP=32 can require composing 31 preceding or following fragment summaries. + constexpr int kChainLength = 31; + const std::vector descriptor_rows = { + {1, 0, kChainLength, 1}, + {2, kLastSummary, kChainLength, -1}, + {-1, 0, kChainLength, 1}, + {-1, kLastSummary, kChainLength, -1}, + }; + RunAffinePrefixTest(kKeyDim, kHeads, kValueDim, kNumSummaries, descriptor_rows); +} + +INSTANTIATE_TEST_SUITE_P( + Common, LinearAttentionPrefixTest, + ::testing::Combine(::testing::Values(3, 64, 128, 257), ::testing::Values(1, 8), + ::testing::Values(1, 5, 16)), + [](const testing::TestParamInfo &info) { + return "K" + std::to_string(std::get<0>(info.param)) + "H" + + std::to_string(std::get<1>(info.param)) + "V" + + std::to_string(std::get<2>(info.param)); + }); } // namespace From 96ac2fe2c1c35a3ebc2476dc68eb29a61d52b76b Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Wed, 18 Nov 2026 18:59:45 -0800 Subject: [PATCH 27/34] Document affine prefix test choices Signed-off-by: Kshitij Lakhani --- tests/cpp/operator/test_linear_attention.cu | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/cpp/operator/test_linear_attention.cu b/tests/cpp/operator/test_linear_attention.cu index d8e76c1a65a..18ab926cb73 100644 --- a/tests/cpp/operator/test_linear_attention.cu +++ b/tests/cpp/operator/test_linear_attention.cu @@ -28,10 +28,11 @@ void RunAffinePrefixTest(size_t key_dim, size_t heads, size_t value_dim, size_t using test::Tensor; using transformer_engine::DType; constexpr size_t kNumInitialStates = 3; - constexpr unsigned kSeed = 2026; - constexpr double kTolerance = 2e-5; + constexpr unsigned kSeed = 2026; // Keep random inputs reproducible. + constexpr double kTolerance = 2e-5; // Allow FP32 accumulation error against the FP64 reference. const size_t num_outputs = descriptor_rows.size(); const size_t state_size = heads * value_dim * key_dim; + // Create inputs and output storage for the requested state shapes. Tensor initial_states("initial_states", std::vector{kNumInitialStates, heads, value_dim, key_dim}, DType::kFloat32); @@ -55,6 +56,7 @@ void RunAffinePrefixTest(size_t key_dim, size_t heads, size_t value_dim, size_t for (size_t i = 0; i < num_summaries * state_size; ++i) { additive_cpu[i] = distribution(generator); } + // Scale transitions to keep matrix-product variance comparable across key dimensions. for (size_t i = 0; i < num_summaries * heads * key_dim * key_dim; ++i) { transitions_cpu[i] = distribution(generator) / std::sqrt(key_dim); } @@ -74,6 +76,7 @@ void RunAffinePrefixTest(size_t key_dim, size_t heads, size_t value_dim, size_t descriptors.data(), output.data(), nullptr); ASSERT_EQ(cudaDeviceSynchronize(), cudaSuccess); output.to_cpu(); + // Compute the CPU reference in FP64 and compare each output with the kernel. const auto *actual = output.rowwise_cpu_dptr(); for (size_t output_index = 0; output_index < num_outputs; ++output_index) { const auto [initial_memory_index, first_summary, count, stride] = descriptor_rows[output_index]; @@ -81,10 +84,12 @@ void RunAffinePrefixTest(size_t key_dim, size_t heads, size_t value_dim, size_t for (size_t value = 0; value < value_dim; ++value) { const size_t row_offset = (head * value_dim + value) * key_dim; std::vector state(key_dim, 0.), next(key_dim); + // Start from zero unless the descriptor selects an explicit initial memory. if (initial_memory_index >= 0) { const float *start = initial_states_cpu + initial_memory_index * state_size + row_offset; std::copy(start, start + key_dim, state.begin()); } + // Apply state = state @ T + H in the descriptor's summary order. for (int step = 0; step < count; ++step) { const int summary_index = first_summary + step * stride; const float *transition_matrix = @@ -144,6 +149,8 @@ TEST(LinearAttentionPrefixLongChainTest, ForwardAndReverse) { RunAffinePrefixTest(kKeyDim, kHeads, kValueDim, kNumSummaries, descriptor_rows); } +// K covers small/typical states, both block sizes, and multiple columns per thread. +// Heads 1/8 check head indexing; V=1/5/16 covers partial, tail, and multiple full 4-row tiles. INSTANTIATE_TEST_SUITE_P( Common, LinearAttentionPrefixTest, ::testing::Combine(::testing::Values(3, 64, 128, 257), ::testing::Values(1, 8), From b0215f78bd052bd75fd711c9e1c1d4f113452261 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Wed, 18 Nov 2026 19:18:40 -0800 Subject: [PATCH 28/34] Move CP adapter requirements to interface docstrings Signed-off-by: Kshitij Lakhani --- .../attention/linear_attention/base.py | 18 ++++++++++-- .../linear_attention/context_parallel.py | 28 ++++++------------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/transformer_engine/pytorch/attention/linear_attention/base.py b/transformer_engine/pytorch/attention/linear_attention/base.py index 58c4cd6098d..0e339416341 100644 --- a/transformer_engine/pytorch/attention/linear_attention/base.py +++ b/transformer_engine/pytorch/attention/linear_attention/base.py @@ -317,17 +317,29 @@ def _validate_inputs( self._validate_gates(gates, query_layer) def _prepare_cp_inputs(self, tensors, initial_state): - """Normalize token-aligned CP inputs before state propagation.""" + """Normalize token-aligned CP inputs before state propagation. + + Each tensor must retain a real-token leading axis for fragment selection; + additional axes, such as GDP's Householder axis, stay intact until FE dispatch. + """ return tensors, initial_state def _cp_forward_summary(self, query, key, value, gates, cu_seqlens, **kernel_kwargs): - """Return the fragment maps S_out = S_in @ T + H.""" + """Return FP32 (H, T) satisfying S_out = S_in @ T + H for each fragment. + + H has shape [fragments, heads, V, K] and T [fragments, heads, K, K]. + Summaries and local attention must use identical gate and normalization semantics. + """ raise NotImplementedError(f"{self.variant} does not implement CP forward summaries.") def _cp_backward_summary( self, query, key, value, gates, grad_output, cu_seqlens, **kernel_kwargs ): - """Return reverse state-gradient maps for this variant.""" + """Return FP32 reverse state-gradient maps, including local output losses. + + H and T have the forward summary shapes, with T already oriented for backward. + Gate and normalization semantics must match local attention. + """ raise NotImplementedError(f"{self.variant} does not implement CP backward summaries.") def _run( diff --git a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py index 4fc63fc84ab..cd9e27d0119 100644 --- a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py +++ b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py @@ -2,25 +2,11 @@ # # See LICENSE for license information. -"""Contiguous linear-attention CP: fragments, collectives and state gradients. - -Adapters for each linear attention variant supply local attention and affine -summary operations. The shared CUDA operation composes states in document order. - -Adapter contract: -* Positional tensors carry a real-token leading axis for fragment selection; - additional axes, such as GDP's Householder axis, remain intact until FE dispatch. -* Forward hooks return FP32 H [fragments, heads, V, K] and T [fragments, heads, K, K] - satisfying X_end = X_start @ T + H. Reverse hooks provide the analogous - state-gradient recipe, with transitions already in backward orientation. -* Local attention and summaries must use identical gate and normalization semantics. -* Kernel options are configuration, not an autograd path for trainable parameters. - Future non-token parameters need explicit gradient inputs/returns, or a - differentiable token-gate transformation before the CP Function. Their gradient - reduction must match the caller's parameter-ownership contract. - -Only GDN supplies these hooks today; other linear attention variants require -adapter validation. +"""Context parallelism for linear attention with contiguous token ownership. + +Coordinates fragment metadata, affine-summary exchange, state composition, +and distributed backward. Variant adapters provide local attention and summary +operations; currently only GDN is supported. """ from collections import OrderedDict @@ -211,6 +197,10 @@ class _LinearAttentionContextParallel(torch.autograd.Function): Summary APIs intentionally do not have autograd registrations. Differentiating an all-gather of summaries therefore cannot recover cross-rank gradients. The reverse summary exchange supplies the missing final-state adjoints to FE. + + Kernel options carry configuration, not trainable parameters. Non-token + parameters need explicit gradient inputs/returns or differentiable token-gate + preparation before this Function, with reductions matching parameter ownership. """ @staticmethod From 24cf3d2808aa035c70c5641006000bee2d543f3c Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Wed, 18 Nov 2026 19:19:20 -0800 Subject: [PATCH 29/34] Explain dense GDN sequence metadata coverage Signed-off-by: Kshitij Lakhani --- tests/pytorch/attention/test_gdn_attention.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/pytorch/attention/test_gdn_attention.py b/tests/pytorch/attention/test_gdn_attention.py index e99a012a275..6f2496ba534 100644 --- a/tests/pytorch/attention/test_gdn_attention.py +++ b/tests/pytorch/attention/test_gdn_attention.py @@ -325,8 +325,7 @@ def test_gdn_dense_layout(qkv_format, dtype, compile_model): if compile_model: torch._dynamo.reset() attention = torch.compile(attention, fullgraph=True) - # Exercise explicit full-document metadata in eager and compiled dense calls; - # the other dtype retains the shape-inferred path. + # Use BF16 for explicit cu_seqlens and FP16 for inferred offsets to cover both paths. cu_seqlens = ( torch.arange(batch + 1, device=q.device, dtype=torch.int32) * sequence if dtype == torch.bfloat16 From c7ca6ee2524db2eadcf1b3bc0afe4c8b96163f85 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Wed, 18 Nov 2026 19:22:54 -0800 Subject: [PATCH 30/34] Tighten affine prefix test tolerances Signed-off-by: Kshitij Lakhani --- tests/cpp/operator/test_linear_attention.cu | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/cpp/operator/test_linear_attention.cu b/tests/cpp/operator/test_linear_attention.cu index 18ab926cb73..e1a84911ceb 100644 --- a/tests/cpp/operator/test_linear_attention.cu +++ b/tests/cpp/operator/test_linear_attention.cu @@ -28,8 +28,10 @@ void RunAffinePrefixTest(size_t key_dim, size_t heads, size_t value_dim, size_t using test::Tensor; using transformer_engine::DType; constexpr size_t kNumInitialStates = 3; - constexpr unsigned kSeed = 2026; // Keep random inputs reproducible. - constexpr double kTolerance = 2e-5; // Allow FP32 accumulation error against the FP64 reference. + constexpr unsigned kSeed = 2026; // Keep random inputs reproducible. + // Allow FP32 rounding across repeated compositions against the FP64 reference. + constexpr double kAbsoluteTolerance = 5e-6; + constexpr double kRelativeTolerance = 5e-6; const size_t num_outputs = descriptor_rows.size(); const size_t state_size = heads * value_dim * key_dim; // Create inputs and output storage for the requested state shapes. @@ -103,7 +105,10 @@ void RunAffinePrefixTest(size_t key_dim, size_t heads, size_t value_dim, size_t state.swap(next); } for (size_t col = 0; col < key_dim; ++col) { - ASSERT_NEAR(actual[output_index * state_size + row_offset + col], state[col], kTolerance) + // Match the shared test helper's absolute-or-relative acceptance rule. + const double tolerance = + std::max(kAbsoluteTolerance, kRelativeTolerance * std::abs(state[col])); + ASSERT_NEAR(actual[output_index * state_size + row_offset + col], state[col], tolerance) << "output=" << output_index << ", head=" << head << ", value=" << value << ", column=" << col; } From eb6f1fd8a02e52d3a07611598e2ecfa819dea619 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Thu, 19 Nov 2026 17:28:29 -0800 Subject: [PATCH 31/34] Add linear attention tutorial and CI examples Replace the CP design page with a user-facing linear attention guide and runnable dense and packed GDN examples. Include the example source in the notebook and validate it in unit and distributed CI, with independent FP64 reference coverage. Signed-off-by: Kshitij Lakhani --- docs/examples/attention/attention.ipynb | 5 +- .../attention/example_linear_attention.py | 170 ++++++++++++++ .../examples/attention/linear_attention.ipynb | 220 ++++++++++++++++++ docs/features/other_optimizations/index.rst | 1 - .../linear_attention_context_parallel.rst | 132 ----------- docs/index.rst | 1 + qa/L0_pytorch_unittest/test.sh | 3 + qa/L1_pytorch_distributed_unittest/test.sh | 3 + tests/pytorch/attention/test_gdn_attention.py | 44 ++++ 9 files changed, 445 insertions(+), 134 deletions(-) create mode 100644 docs/examples/attention/example_linear_attention.py create mode 100644 docs/examples/attention/linear_attention.ipynb delete mode 100644 docs/features/other_optimizations/linear_attention_context_parallel.rst diff --git a/docs/examples/attention/attention.ipynb b/docs/examples/attention/attention.ipynb index d8b573f6e7a..61a653227c6 100644 --- a/docs/examples/attention/attention.ipynb +++ b/docs/examples/attention/attention.ipynb @@ -17,7 +17,10 @@ "[Transformer Engine](https://github.com/NVIDIA/TransformerEngine.git) supports the calculation of dot product attention in two frameworks, [PyTorch](https://github.com/pytorch/pytorch) and [JAX](https://github.com/google/jax). The API for each framework is\n", "\n", "- [transformer_engine.pytorch.DotProductAttention](../../api/pytorch.rst#transformer_engine.pytorch.DotProductAttention)\n", - "- [transformer_engine.jax.flax.DotProductAttention](../../api/jax.rst#transformer_engine.jax.flax.DotProductAttention)" + "- [transformer_engine.jax.flax.DotProductAttention](../../api/jax.rst#transformer_engine.jax.flax.DotProductAttention)", + "\n", + "\n", + "For linear attention variants and Gated DeltaNet context parallelism, see [Linear Attention](linear_attention.ipynb)." ] }, { diff --git a/docs/examples/attention/example_linear_attention.py b/docs/examples/attention/example_linear_attention.py new file mode 100644 index 00000000000..9c43025b5e0 --- /dev/null +++ b/docs/examples/attention/example_linear_attention.py @@ -0,0 +1,170 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Gated DeltaNet forward and backward, with optional contiguous context parallelism. + +Run from the repository root: + python docs/examples/attention/example_linear_attention.py + python -m torch.distributed.run --standalone --nproc-per-node=2 \ + docs/examples/attention/example_linear_attention.py --cp --qkv-format thd +""" + +import argparse +import os + +import torch +import torch.distributed as dist + +import transformer_engine.pytorch as te + + +def make_inputs(qkv_format): + """Create the same batch on each rank for comparison with a non-CP run.""" + torch.manual_seed(1234) + batch, sequence, heads, head_dim = 16, 128, 8, 64 + if qkv_format == "thd": + # Sample document lengths from 32 to 128 tokens, then pack without padding. + lengths = torch.randint(32, 129, (batch,), device="cpu").tolist() + offsets = [0] + for length in lengths: + offsets.append(offsets[-1] + length) + shape = (offsets[-1], heads, head_dim) + else: + shape = (batch, sequence, heads, head_dim) + offsets = [row * sequence for row in range(batch + 1)] + q, k, v = (torch.randn(shape, device="cuda", dtype=torch.bfloat16) for _ in range(3)) + # Per-token, per-head FP32 gates: g is the log decay; beta is the write strength. + g = -torch.rand(shape[:-1], device="cuda", dtype=torch.float32) * 0.02 + beta = torch.rand_like(g) * 0.7 + inputs = tuple(t.requires_grad_() for t in (q, k, v, g, beta)) + initial_state = ( + torch.randn(batch, heads, head_dim, head_dim, device="cuda", dtype=torch.float32) * 0.1 + ).requires_grad_() + cu_seqlens = torch.tensor(offsets, device="cuda", dtype=torch.int32) + return inputs, initial_state, cu_seqlens + + +def run_single_gpu(inputs, initial_state, cu_seqlens, qkv_format): + """Run on complete documents and differentiate through outputs and final states.""" + q, k, v, g, beta = inputs + attention = te.GatedDeltaNetAttention( + num_attention_heads=q.shape[-2], + kv_channels=q.shape[-1], + qkv_format=qkv_format, + ) + output, final_state = attention( + q, + k, + v, + g=g, + beta=beta, + cu_seqlens=cu_seqlens if qkv_format == "thd" else None, + initial_state=initial_state, + output_final_state=True, + use_qk_l2norm_in_kernel=True, + ) + loss = output.float().square().mean() + final_state.square().mean() + loss.backward() + return output, final_state + + +def local_tokens(tensor, qkv_format, cp_group): + """Select this rank's contiguous interval of the packed buffer or each batch row.""" + rank, size = dist.get_rank(cp_group), dist.get_world_size(cp_group) + if qkv_format == "thd": + length = tensor.shape[0] + return tensor[rank * length // size : (rank + 1) * length // size] + length = tensor.shape[1] + return tensor[:, rank * length // size : (rank + 1) * length // size] + + +def run_context_parallel(inputs, initial_state, cu_seqlens, qkv_format, cp_group): + """Supply local tokens and global document metadata to GDN CP.""" + local_inputs = tuple( + local_tokens(t, qkv_format, cp_group).detach().contiguous().requires_grad_() for t in inputs + ) + q, k, v, g, beta = local_inputs + state = initial_state.detach().clone().requires_grad_() + attention = te.GatedDeltaNetAttention( + num_attention_heads=q.shape[-2], + kv_channels=q.shape[-1], + qkv_format=qkv_format, + cp_group=cp_group, + ) + output, final_state = attention( + q, + k, + v, + g=g, + beta=beta, + cu_seqlens=cu_seqlens, + initial_state=state, + output_final_state=True, + use_qk_l2norm_in_kernel=True, + ) + # Normalize by the global output size, so rank-local losses sum to the + # non-CP loss. Count the replicated final-state loss only once. + cp_size = dist.get_world_size(cp_group) + loss = output.float().square().sum() / inputs[2].numel() + loss = loss + final_state.square().mean() / cp_size + loss.backward() + return output, final_state, local_inputs, state + + +def check_finite(tensors): + """Fail on missing gradients or non-finite results, including in CI.""" + for tensor in tensors: + assert tensor is not None, "Missing gradient" + assert torch.isfinite(tensor).all(), "Non-finite result" + + +def check_close(actual, expected, tolerance): + """Compare BF16 results using relative RMS error, as in the GDN tests.""" + actual, expected = actual.detach().double(), expected.detach().double() + error = (actual - expected).square().mean().sqrt() + reference = expected.square().mean().sqrt().clamp_min(1e-12) + assert error / reference < tolerance, f"Relative RMS error: {(error / reference).item()}" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--cp", action="store_true", help="Run with torch.distributed.run on two or more GPUs" + ) + parser.add_argument("--qkv-format", choices=("bshd", "thd"), default="bshd") + args = parser.parse_args() + torch.cuda.set_device(int(os.environ.get("LOCAL_RANK", "0"))) + if args.cp: + # NCCL is PyTorch's recommended backend for CUDA distributed training. + # CP also accepts other backends that support its CUDA collectives. + dist.init_process_group(backend="nccl") + try: + if args.cp and dist.get_world_size() < 2: + raise ValueError("--cp requires torch.distributed.run with at least two GPUs") + inputs, initial_state, cu_seqlens = make_inputs(args.qkv_format) + output, final_state = run_single_gpu(inputs, initial_state, cu_seqlens, args.qkv_format) + assert output.shape == (*inputs[2].shape[:-2], inputs[2].shape[-2:].numel()) + assert final_state.shape == initial_state.shape + check_finite((output, final_state, initial_state.grad, *(t.grad for t in inputs))) + if args.cp: + cp_group = dist.group.WORLD + local_output, states, local_inputs, state = run_context_parallel( + inputs, initial_state, cu_seqlens, args.qkv_format, cp_group + ) + check_close(local_output, local_tokens(output, args.qkv_format, cp_group), 2e-2) + check_close(states, final_state, 2e-2) + for local, full in zip(local_inputs, inputs): + check_close(local.grad, local_tokens(full.grad, args.qkv_format, cp_group), 4e-2) + # Initial-state gradients belong to each document's first owning rank. + dist.all_reduce(state.grad, group=cp_group) + check_close(state.grad, initial_state.grad, 4e-2) + if not args.cp or dist.get_rank() == 0: + print(f"GDN {args.qkv_format}: forward and backward passed (CP={args.cp})") + finally: + if args.cp: + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/docs/examples/attention/linear_attention.ipynb b/docs/examples/attention/linear_attention.ipynb new file mode 100644 index 00000000000..51a6c50a941 --- /dev/null +++ b/docs/examples/attention/linear_attention.ipynb @@ -0,0 +1,220 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "# Linear Attention\n", + "\n", + "Linear attention maintains a recurrent state instead of forming the pairwise attention scores used by scaled dot product attention (SDPA). Transformer Engine provides PyTorch modules for several linear attention variants through the cuDNN frontend. These APIs are **experimental** and subject to change.\n", + "\n", + "This guide introduces the available variants and shows Gated DeltaNet (GDN) forward and backward on a single GPU and with context parallelism (CP). For SDPA backends, masks, and biases, see [Attention Is All You Need!](attention.ipynb).\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Supported Variants\n", + "\n", + "| Variant | PyTorch API | Gates and updates | Context parallelism |\n", + "| --- | --- | --- | --- |\n", + "| Gated DeltaNet (GDN) | [GatedDeltaNetAttention](../../api/pytorch.rst#transformer_engine.pytorch.GatedDeltaNetAttention) | Per-token, per-head log decay `g` and write strength `beta` | Contiguous CP |\n", + "| Gated DeltaNet v2 (GDN-2) | [GatedDeltaNet2Attention](../../api/pytorch.rst#transformer_engine.pytorch.GatedDeltaNet2Attention) | Per-key-channel `g` and `beta`, per-value-channel `w` | Not supported |\n", + "| Gated DeltaProduct (GDP) | [GatedDeltaProductAttention](../../api/pytorch.rst#transformer_engine.pytorch.GatedDeltaProductAttention) | Multiple delta-rule updates per token | Not supported |\n", + "| Kimi Delta Attention (KDA) | [KimiDeltaAttention](../../api/pytorch.rst#transformer_engine.pytorch.KimiDeltaAttention) | Per-key-channel decay `g`, per-token, per-head `beta` | Not supported |\n", + "\n", + "The modules take Q, K, V and gate tensors produced by the caller; they do not include input projections or an output projection. Q/K/V use FP16 or BF16, and recurrent states use FP32. Gate shapes and dtypes depend on the variant; see the corresponding API reference.\n", + "\n", + "Install Transformer Engine with PyTorch support as described in [Installation](../../installation.rst). The cuDNN frontend must provide the selected linear attention op and its kernel runtime. See the [cuDNN frontend installation guidance](https://github.com/NVIDIA/cudnn-frontend#installation) for its dependencies and installation options. Supported GPU architectures and head dimensions depend on the variant and the available cuDNN frontend engines.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. GDN Inputs and Outputs\n", + "\n", + "GDN is causal; its API does not take SDPA attention masks or bias tensors. It takes two FP32 gates: `g`, the log decay, and `beta`, the write strength. Each gate has one value per token and head. The example uses negative `g` and `beta` between zero and one, and enables Q/K L2 normalization with `use_qk_l2norm_in_kernel=True`.\n", + "\n", + "| `qkv_format` | Q/K shape | V shape | Gate shape | Output shape |\n", + "| --- | --- | --- | --- | --- |\n", + "| `bshd` | `[B, S, H, Dk]` | `[B, S, H, Dv]` | `[B, S, H]` | `[B, S, H * Dv]` |\n", + "| `sbhd` | `[S, B, H, Dk]` | `[S, B, H, Dv]` | `[S, B, H]` | `[S, B, H * Dv]` |\n", + "| `thd` | `[T, H, Dk]` | `[T, H, Dv]` | `[T, H]` | `[T, H * Dv]` |\n", + "\n", + "Here `B` is the number of documents, `S` is the dense sequence length, `T` is the total number of packed tokens, and `H` is the number of heads on this tensor-parallel rank. `kv_channels` sets `Dk` and `Dv`: pass one integer for equal head dimensions or `(Dk, Dv)` otherwise.\n", + "\n", + "For packed inputs, remove padding and pass `cu_seqlens`, the cumulative document lengths, as an `int32` tensor. Documents do not share recurrent state. Without CP, `cu_seqlens` must be on the input device. Dense inputs can omit it; if supplied, it must describe full rows as `[0, S, 2*S, ..., B*S]`.\n", + "\n", + "The optional `initial_state` has shape `[B, H, Dv, Dk]` and dtype FP32. Omitting it starts from zero. Set `output_final_state=True` to return `(output, final_state)`; otherwise the module returns only `output`. Final states have the same shape and dtype as the initial state and can seed a subsequent call.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Single-GPU Example\n", + "\n", + "The runnable example below uses 16 documents and 8 heads with BF16 Q/K/V and FP32 gates and states. Dense sequences have 128 tokens each; packed document lengths are sampled from 32 to 128 tokens." + ] + }, + { + "cell_type": "raw", + "metadata": { + "raw_mimetype": "text/restructuredtext" + }, + "source": [ + "Download the complete :download:`example_linear_attention.py ` script." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Run these commands from the repository root:\n", + "\n", + "```bash\n", + "python docs/examples/attention/example_linear_attention.py\n", + "python docs/examples/attention/example_linear_attention.py --qkv-format thd\n", + "```\n", + "\n", + "Each run checks output and state shapes and finite gradients after forward and backward. Missing kernel dependencies fail the run.\n", + "\n", + "The input preparation covers equal-length dense rows and packed documents of different lengths:" + ] + }, + { + "cell_type": "raw", + "metadata": { + "raw_mimetype": "text/restructuredtext" + }, + "source": [ + ".. literalinclude:: example_linear_attention.py\n", + " :language: python\n", + " :pyobject: make_inputs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The forward call below works with either layout. Backward includes a loss on both the attention output and the final state:" + ] + }, + { + "cell_type": "raw", + "metadata": { + "raw_mimetype": "text/restructuredtext" + }, + "source": [ + ".. literalinclude:: example_linear_attention.py\n", + " :language: python\n", + " :pyobject: run_single_gpu" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. GDN Context Parallelism\n", + "\n", + "GDN supports **contiguous context parallelism**. Create a `torch.distributed` process group and pass it as `cp_group` when constructing the module, or call `set_context_parallel_group` before forward. `None` disables CP; a group of size one follows the local path.\n", + "\n", + "Each rank supplies local Q/K/V and gate tensors. Group rank order defines token order, and all inputs must use the same partition:\n", + "\n", + "- **THD:** rank `r` owns `[r*T//P:(r+1)*T//P]` of the global packed buffer.\n", + "- **BSHD/SBHD:** rank `r` owns `[r*S//P:(r+1)*S//P]` of every batch row.\n", + "\n", + "`P` is the CP group size. These are contiguous intervals, rather than SDPA's dual-chunk partitioning. TE does not redistribute inputs, and outputs stay on the ranks that own their input tokens.\n", + "\n", + "Pass the same **global** `cu_seqlens` on every rank, describing the complete documents before partitioning. For THD it is required. Dense inputs can omit it for equal slices; uneven dense slices require `[0, S, 2*S, ..., B*S]`. With CP, this metadata may be on CPU or CUDA. Prefer CPU metadata when already available from the input pipeline to avoid a synchronized host read.\n", + "\n", + "Document lengths need not be divisible by CP size. Empty documents and ranks with no tokens are supported. All ranks must participate in forward and backward with matching options, including ranks with no tokens or no local loss.\n", + "\n", + "The example selects the local intervals as follows (the `bshd` path slices each batch row):" + ] + }, + { + "cell_type": "raw", + "metadata": { + "raw_mimetype": "text/restructuredtext" + }, + "source": [ + ".. literalinclude:: example_linear_attention.py\n", + " :language: python\n", + " :pyobject: local_tokens" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Launch the dense or packed example on two GPUs:\n", + "\n", + "```bash\n", + "python -m torch.distributed.run --standalone --nproc-per-node=2 \\\n", + " docs/examples/attention/example_linear_attention.py --cp\n", + "\n", + "python -m torch.distributed.run --standalone --nproc-per-node=2 \\\n", + " docs/examples/attention/example_linear_attention.py --cp --qkv-format thd\n", + "```\n", + "\n", + "Each run compares local outputs, replicated final states, and input and initial-state gradients against a non-CP run on the complete documents.\n", + "\n", + "For CP, the script selects NCCL, [PyTorch's recommended backend for CUDA distributed training](https://docs.pytorch.org/docs/stable/distributed.html#which-backend-to-use), and selects each process's GPU using `LOCAL_RANK`. GDN CP also accepts other process-group backends that support its CUDA collectives. For example, a CUDA-capable Gloo build can use `backend=\"gloo\"` in `init_process_group`. For demonstration, each rank generates the same full batch and selects its interval locally. In a training pipeline, supply the local tensors directly.\n", + "\n", + "This is the CP forward and backward:" + ] + }, + { + "cell_type": "raw", + "metadata": { + "raw_mimetype": "text/restructuredtext" + }, + "source": [ + ".. literalinclude:: example_linear_attention.py\n", + " :language: python\n", + " :pyobject: run_context_parallel" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Recurrent States and Losses\n", + "\n", + "With CP, `initial_state` describes **all global documents** and must have identical values on every rank. Requested final states are also replicated for all global documents. Empty documents retain their initial state.\n", + "\n", + "Each rank computes the loss for its own output tokens. Dividing each local sum by the total number of output elements across the CP group makes the combined loss equal to the non-CP mean. Final states are copied to every rank, and their gradients are summed during backward. If every rank applies the same final-state loss, divide it by CP size so it is counted only once.\n", + "\n", + "Each document's initial-state gradient is returned on its first owning rank, with zeros elsewhere. Sum these contributions over CP when inspecting a replicated leaf state's full gradient. Final states can also be chained into subsequent CP calls.\n", + "\n", + "### Supported Execution\n", + "\n", + "GDN CP supports FP16/BF16 inputs, FP32 gates and states, Q/K normalization, activation checkpointing (`checkpoint_core_attention=True`), and tensor parallelism. It supports first-order gradients and executes eagerly, including inside a `torch.compile` model. Full-graph compilation and CUDA graph capture are outside the supported CP execution modes. Without CP, ordinary forward/backward calls support `torch.compile(fullgraph=True)`; activation checkpointing still requires eager attention execution." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.12" + }, + "nbsphinx": { + "execute": "never" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/features/other_optimizations/index.rst b/docs/features/other_optimizations/index.rst index 3946e111c8a..910edc73acf 100644 --- a/docs/features/other_optimizations/index.rst +++ b/docs/features/other_optimizations/index.rst @@ -9,4 +9,3 @@ Other optimizations .. toctree:: cpu_offloading/cpu_offloading.rst - linear_attention_context_parallel.rst diff --git a/docs/features/other_optimizations/linear_attention_context_parallel.rst b/docs/features/other_optimizations/linear_attention_context_parallel.rst deleted file mode 100644 index 416b3794078..00000000000 --- a/docs/features/other_optimizations/linear_attention_context_parallel.rst +++ /dev/null @@ -1,132 +0,0 @@ -.. - Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - - See LICENSE for license information. - -GDN context parallelism -======================= - -PyTorch :class:`~transformer_engine.pytorch.GatedDeltaNetAttention` supports -contiguous context parallelism (CP). Each rank receives local Q, K, V and gate -tensors. TE exchanges recurrent-state summaries so local attention produces -the same result as processing each complete document. Outputs stay on the -input token owners; there is no output all-gather. - -Set ``cp_group`` at construction, or call ``set_context_parallel_group`` before -forward. ``None`` disables CP. A group of size one follows the local path. -Group rank order defines token order within a document. All ranks must use -matching options and participate in forward and backward, including ranks -with no tokens or no local loss. - -Input ownership ---------------- - -For CP size ``P`` and group rank ``r``: - -* **THD:** rank ``r`` owns ``[r*T//P:(r+1)*T//P]`` of the global packed token - buffer. Pass the same global ``int32`` cumulative document lengths as - ``cu_seqlens`` on every rank. This metadata can be on CPU or CUDA; - local fragment boundaries are derived automatically. -* **BSHD/SBHD:** rank ``r`` owns ``[r*S//P:(r+1)*S//P]`` of every batch row. - Here ``B`` is batch size and ``S`` is each row's full sequence length. - When ``S`` is divisible by ``P``, TE can infer ``S`` from the local length. - Otherwise, pass global document offsets ``[0,S,2*S,...,B*S]``. Rank slices - still follow the formula above; arbitrary rank splits are not supported. - -``cu_seqlens`` always describes the complete logical documents, before CP -partitioning. Without CP, those boundaries describe the supplied input directly. -Dense inputs can omit the metadata; if supplied without CP, it must describe -the full rows as ``[0,S,2*S,...,B*S]``. TE keeps the derived local cumulative -lengths private. - -Q, K, V, ``g`` and ``beta`` must follow the same ownership. TE does not -redistribute inputs or interpret SDPA's dual-chunk layout. A local conversion -between BSHD/SBHD and THD does not change which tokens a rank owns. - -Document lengths need not be divisible by CP size. THD sequences are packed -without gaps; the caller removes physical padding before calling LA. Empty -documents are allowed and retain their initial state. - -For example, lengths ``[8,4,12]`` with CP size four give: - -.. code-block:: text - - rank 0: A[0:6] - rank 1: A[6:8], B[0:4] - rank 2: C[0:6] - rank 3: C[6:12] - -Only A and C need cross-rank state propagation. Independent documents never -share memory. The initial collective can still broadcast unused summary slots. - -.. code-block:: python - - import torch - import transformer_engine.pytorch as te - - # cp is an initialized torch.distributed process group. The input pipeline - # supplies this rank's token interval for Q/K/V and both gates. - attn = te.GatedDeltaNetAttention( - num_attention_heads=32, - kv_channels=128, - qkv_format="thd", - cp_group=cp, - ) - global_cu = torch.tensor([0, 8, 12, 24], dtype=torch.int32) - output = attn( - q_local, k_local, v_local, g=g_local, beta=beta_local, - cu_seqlens=global_cu, - ) - -Recurrent states and gradients ------------------------------- - -The optional ``initial_state`` is FP32 with shape -``[global_documents, local_heads, value_dim, key_dim]`` and identical values -on every CP rank. Requested final states have the same global shape and are -replicated using a separate collective. Empty documents return their initial state. - -Each initial-state gradient is returned on its document's first owning rank, -with zeros elsewhere. Sum these contributions over CP when inspecting a -replicated leaf state's full gradient. This ownership also permits state -chaining between CP calls. - -Final-state backward sums gradients from all replicas. Divide a loss applied -identically to replicated final states by CP size to count it once. A loss -over each rank's local attention outputs needs no such adjustment. - -Execution ---------- - -For a fragment, cuDNN produces the affine map ``S_out = S_in @ T + H``. -TE all-gathers the required summaries and composes them in document order to -obtain local starting states. Local cuDNN attention then runs on every rank. -Backward uses a reverse summary exchange to propagate state gradients. -Composition uses FP32 arithmetic in the common library. - -GDN CP supports FP16/BF16 inputs, FP32 gates and states, Q/K normalization, -activation checkpointing and tensor parallelism. Head dimensions and device -support follow the available cuDNN frontend engines. Where a dedicated summary -engine is unavailable, ordinary GDN state passes provide a slower fallback. - -CP runs eagerly, including inside ``torch.compile`` models, and supports -first-order gradients. Full-graph compilation and CUDA graph capture are -outside this implementation's supported contract. Other LA variants, direct -dual-chunk ownership, headwise CP and whole-document redistribution are -separate features. - -Metadata preparation --------------------- - -TE prepares partitions on the host and uploads the derived indices to the GPU. -CUDA ``cu_seqlens`` requires a synchronized host read on the first use. Pass CPU -metadata when it is already available from the input pipeline to avoid that read. - -Each attention module caches its most recent rank metadata. A bounded shared cache -retains up to eight prepared entries per process so layers using the same metadata -on the same CUDA stream can reuse that preparation. Inference-mode entries are -kept separate so autograd never saves inference tensors during training. -Cache keys include tensor identity and its mutation counter; tracked in-place -updates invalidate the entry. New ``cu_seqlens`` objects require preparation -again. Input ``cu_seqlens`` tensors created in inference mode bypass caching -because they have no mutation counter. diff --git a/docs/index.rst b/docs/index.rst index 0d3746fcc2b..50a3c15b0bf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -81,4 +81,5 @@ Transformer Engine documentation debug envvars examples/attention/attention.ipynb + examples/attention/linear_attention.ipynb examples/attention/cp_ag_thd_dpa_jax_deep_dive.ipynb diff --git a/qa/L0_pytorch_unittest/test.sh b/qa/L0_pytorch_unittest/test.sh index bb459dad3b8..4f0fbf67821 100644 --- a/qa/L0_pytorch_unittest/test.sh +++ b/qa/L0_pytorch_unittest/test.sh @@ -69,6 +69,9 @@ NVTE_ALLOW_UNSAFE_PICKLE_EXTRA_STATE=1 python3 -m pytest --tb=auto --junitxml=$X python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_attention_backend_selection.xml $TE_PATH/tests/pytorch/attention/test_attention_backend_selection.py || test_fail "test_attention_backend_selection.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_flex_attention.xml $TE_PATH/tests/pytorch/attention/test_flex_attention.py || test_fail "test_flex_attention.py" NVTE_GDN_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_gdn_attention.xml $TE_PATH/tests/pytorch/attention/test_gdn_attention.py || test_fail "test_gdn_attention.py" +for QKV_FORMAT in bshd thd; do + python3 "$TE_PATH/docs/examples/attention/example_linear_attention.py" --qkv-format "$QKV_FORMAT" || test_fail "example_linear_attention.py ($QKV_FORMAT)" +done NVTE_GDN2_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_gdn2_attention.xml $TE_PATH/tests/pytorch/attention/test_gdn2_attention.py || test_fail "test_gdn2_attention.py" NVTE_GDP_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_gdp_attention.xml $TE_PATH/tests/pytorch/attention/test_gdp_attention.py || test_fail "test_gdp_attention.py" NVTE_KDA_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_kda_attention.xml $TE_PATH/tests/pytorch/attention/test_kda_attention.py || test_fail "test_kda_attention.py" diff --git a/qa/L1_pytorch_distributed_unittest/test.sh b/qa/L1_pytorch_distributed_unittest/test.sh index e6637cdf9ef..9961217bc42 100644 --- a/qa/L1_pytorch_distributed_unittest/test.sh +++ b/qa/L1_pytorch_distributed_unittest/test.sh @@ -44,6 +44,9 @@ else fi NVTE_GDN_TEST_REQUIRED=1 NVTE_GDN_CP_TEST_REQUIRED=1 python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_gdn_attention_with_cp.xml $TE_PATH/tests/pytorch/attention/test_gdn_attention_with_cp.py || test_fail "test_gdn_attention_with_cp.py" +for QKV_FORMAT in bshd thd; do + python3 -m torch.distributed.run --standalone --nproc-per-node=2 "$TE_PATH/docs/examples/attention/example_linear_attention.py" --cp --qkv-format "$QKV_FORMAT" || test_fail "example_linear_attention.py CP ($QKV_FORMAT)" +done python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_sanity.xml $TE_PATH/tests/pytorch/distributed/test_sanity.py || test_fail "test_sanity.py" python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_parallel_cross_entropy.xml $TE_PATH/tests/pytorch/distributed/test_parallel_cross_entropy.py || test_fail "test_parallel_cross_entropy.py" diff --git a/tests/pytorch/attention/test_gdn_attention.py b/tests/pytorch/attention/test_gdn_attention.py index 6f2496ba534..d0ccf527d81 100644 --- a/tests/pytorch/attention/test_gdn_attention.py +++ b/tests/pytorch/attention/test_gdn_attention.py @@ -7,6 +7,7 @@ import importlib.util import math import os +from pathlib import Path import sys from typing import Optional, Tuple @@ -294,6 +295,49 @@ def test_gdn_thd_forward_final_state_and_backward( ) +@requires_gdn +@pytest.mark.parametrize("qkv_format", ["bshd", "thd"]) +def test_gdn_documentation_example(qkv_format): + """Check the tutorial's inputs and forward/backward against the FP64 recurrence.""" + example_path = ( + Path(__file__).resolve().parents[3] / "docs/examples/attention/example_linear_attention.py" + ) + spec = importlib.util.spec_from_file_location("example_linear_attention", example_path) + example = importlib.util.module_from_spec(spec) + spec.loader.exec_module(example) + inputs, initial_state, cu_seqlens = example.make_inputs(qkv_format) + output, final_state = example.run_single_gpu(inputs, initial_state, cu_seqlens, qkv_format) + + reference_inputs = tuple(t.detach().double().requires_grad_() for t in inputs) + initial_state_ref = initial_state.detach().double().requires_grad_() + q, k, v, g, beta = (t.unsqueeze(0) if qkv_format == "thd" else t for t in reference_inputs) + output_ref, final_state_ref = _gdn_reference( + F.normalize(q, dim=-1), + F.normalize(k, dim=-1), + v, + g, + beta, + initial_state=initial_state_ref, + cu_seqlens=cu_seqlens if qkv_format == "thd" else None, + ) + output_ref = output_ref.reshape_as(output) + _assert_rms_close(output, output_ref, _FWD_TOL[inputs[0].dtype], "example output") + _assert_rms_close( + final_state, final_state_ref, _STATE_TOL[inputs[0].dtype], "example final state" + ) + (output_ref.square().mean() + final_state_ref.square().mean()).backward() + for name, actual, reference in zip(("q", "k", "v", "g", "beta"), inputs, reference_inputs): + _assert_rms_close( + actual.grad, reference.grad, _BWD_TOL[inputs[0].dtype], f"example d{name}" + ) + _assert_rms_close( + initial_state.grad, + initial_state_ref.grad, + _BWD_TOL[inputs[0].dtype], + "example dinitial_state", + ) + + @pytest.mark.parametrize("qkv_format", ["bshd", "sbhd"]) @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16], ids=["bf16", "fp16"]) @requires_gdn From 41d46ab4bcb2eb406ec078af0aaa3cfc53421fb0 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Thu, 19 Nov 2026 17:29:10 -0800 Subject: [PATCH 32/34] Streamline GDN context parallel tests Run end-to-end coverage at the largest available CP size among 2, 4 and 8, and remove standalone partition, validation and binding tests. Exercise four-document, eight-head cases and the 256-dimensional summary fallback. Preserve CP/TP subgroup coverage when running on eight GPUs. Signed-off-by: Kshitij Lakhani --- tests/pytorch/attention/run_gdn_with_cp.py | 26 ++-- .../attention/test_gdn_attention_with_cp.py | 130 ++---------------- 2 files changed, 25 insertions(+), 131 deletions(-) diff --git a/tests/pytorch/attention/run_gdn_with_cp.py b/tests/pytorch/attention/run_gdn_with_cp.py index 09064f48e8a..cd3d84a32aa 100644 --- a/tests/pytorch/attention/run_gdn_with_cp.py +++ b/tests/pytorch/attention/run_gdn_with_cp.py @@ -55,6 +55,7 @@ def _case( lengths, *, dtype=torch.bfloat16, + heads=2, key_dim=64, value_dim=64, layout="thd", @@ -75,7 +76,7 @@ def _case( size, rank = dist.get_world_size(group), dist.get_rank(group) if rank == 0: print( - f"GDN CP case: lengths={lengths}, dtype={dtype}, K={key_dim}, V={value_dim}," + f"GDN CP case: lengths={lengths}, dtype={dtype}, H={heads}, K={key_dim}, V={value_dim}," f" layout={layout}, norm={norm}, initial={initial}, final={final}," f" checkpoint={checkpoint}, last_loss={last_loss}, fallback={fallback}," f" independent={independent}, inference_warmup={inference_warmup}", @@ -83,7 +84,6 @@ def _case( ) device = torch.device("cuda", int(os.environ["LOCAL_RANK"])) torch.manual_seed(791 + (dist.get_rank(tp_group) if tp_group is not None else 0)) - heads = 2 total = sum(lengths) cu = [0] for length in lengths: @@ -355,14 +355,19 @@ def main(): try: for lengths in ( [8, 4, 12], # Shared and complete documents on the same rank. - [16, 4, 4], # One document spans three ranks at CP=4. - [6, 6, 6, 6], # Document and rank boundaries coincide. + [16, 4, 4], # One document spans several ranks. + [6] * size, # Document and rank boundaries coincide. [0, 17, 0, 18, 0], # Empty documents between nonempty documents. [1, 0, 1], # Some ranks own no tokens. [0, 0], # Entirely empty input. ): _case(group, lengths) cases += 1 + lengths = torch.randint( + 32, 129, (4,), generator=torch.Generator().manual_seed(1234) + ).tolist() + _case(group, lengths, heads=8, norm=True, independent=True) + cases += 1 for dtype in (torch.float16, torch.bfloat16): for norm in (False, True): _case( @@ -386,15 +391,16 @@ def main(): for layout, dtype in (("bshd", torch.bfloat16), ("sbhd", torch.float16)): _case( group, - [size * 65] * 2, + [size * 65] * 4, dtype=dtype, + heads=8, layout=layout, norm=True, dense_global=layout == "bshd", ) cases += 1 for layout in ("bshd", "sbhd"): - _case(group, [size * 17 + 3] * 2, layout=layout, dense_global=True) + _case(group, [size * 17 + 3] * 4, heads=8, layout=layout, dense_global=True) cases += 1 _case(group, [1, 1], layout="bshd", dense_global=True, checkpoint=True) cases += 1 @@ -424,7 +430,7 @@ def main(): # Larger cuTile geometries have no dedicated FROST summary engine; # exercise the real capability fallback, rather than a mocked error. if importlib.util.find_spec("cuda.tile") is not None: - _case(group, [131, 7], key_dim=192, value_dim=160, independent=True) + _case(group, [131, 7], key_dim=256, value_dim=256, independent=True) cases += 1 _inference_and_deferred_group(group) cases += 1 @@ -445,9 +451,9 @@ def main(): _case(singleton, [17, 13]) _case(singleton, [17, 17], layout="bshd", dense_global=True) cases += 2 - if size == 4: - cp_groups = [dist.new_group(ranks) for ranks in ([0, 1], [2, 3])] - tp_groups = [dist.new_group(ranks) for ranks in ([0, 2], [1, 3])] + if size >= 4: + cp_groups = [dist.new_group([r, r + 1]) for r in range(0, size, 2)] + tp_groups = [dist.new_group(list(range(r, size, 2))) for r in (0, 1)] rank = dist.get_rank() _case(cp_groups[rank // 2], [139, 13], tp_group=tp_groups[rank % 2]) cases += 1 diff --git a/tests/pytorch/attention/test_gdn_attention_with_cp.py b/tests/pytorch/attention/test_gdn_attention_with_cp.py index c9eb03769cf..bba31325f0d 100644 --- a/tests/pytorch/attention/test_gdn_attention_with_cp.py +++ b/tests/pytorch/attention/test_gdn_attention_with_cp.py @@ -2,11 +2,10 @@ # # See LICENSE for license information. -"""Contiguous GDN CP: host boundaries, common arithmetic, and multi-GPU numerics.""" +"""End-to-end numerical tests for contiguous GDN context parallelism.""" import os from pathlib import Path -import random import subprocess import sys import tempfile @@ -16,129 +15,18 @@ from test_gdn_attention import _GDN_AVAILABLE import transformer_engine -from transformer_engine.common.linear_attention import build_linear_attention_cp_host_metadata -import transformer_engine_torch as tex -@pytest.mark.parametrize("cp_size", [1, 2, 4, 8]) -def test_partition_intersections(cp_size): - """Recover all document intervals without dropping tokens or crossing documents.""" - rng = random.Random(301) - for _ in range(100): - lengths = [rng.randrange(0, 20) for _ in range(rng.randrange(1, 10))] - cu = [0] - for length in lengths: - cu.append(cu[-1] + length) - rank_metadata = [ - build_linear_attention_cp_host_metadata(cu, cp_size=cp_size, cp_rank=cp_rank) - for cp_rank in range(cp_size) - ] - for doc, (start, end) in enumerate(zip(cu, cu[1:])): - fragments = [f for p in rank_metadata for f in p.fragments if f.document == doc] - assert sum(f.end - f.start for f in fragments) == end - start - assert all(start <= f.start < f.end <= end for f in fragments) - assert all(a.end == b.start for a, b in zip(fragments, fragments[1:])) - for cp_rank, host_metadata in enumerate(rank_metadata): - assert ( - host_metadata.local_cu_seqlens[-1] - == (cp_rank + 1) * cu[-1] // cp_size - cp_rank * cu[-1] // cp_size - ) - assert len(host_metadata.incoming_fragment_indices) <= 1 - assert len(host_metadata.outgoing_fragment_indices) <= 1 - for f, forward_desc, backward_desc in zip( - host_metadata.fragments, - host_metadata.forward_descriptors, - host_metadata.backward_descriptors, - ): - assert forward_desc[2] == cp_rank - f.first_rank - assert backward_desc[2] == f.last_rank - cp_rank - # Every referenced recipe belongs to this same document. - for predecessor in range(f.first_rank, cp_rank): - p = rank_metadata[predecessor] - if p.fragments: - assert p.fragments[p.outgoing_fragment_indices[0]].document == f.document - for successor in range(cp_rank + 1, f.last_rank + 1): - p = rank_metadata[successor] - # An empty rank contributes an identity recipe. - if p.fragments: - assert p.fragments[p.incoming_fragment_indices[0]].document == f.document - - -@pytest.mark.parametrize("cu_seqlens", [[1, 2], [0], [0, 2, 1]]) -def test_invalid_documents(cu_seqlens): - """Reject malformed global offsets before computing communication descriptors.""" - with pytest.raises(ValueError): - build_linear_attention_cp_host_metadata(cu_seqlens, cp_size=4, cp_rank=0) - - -@pytest.mark.parametrize("cp_size", [1, 2, 4, 8]) -@pytest.mark.parametrize("length", [0, 1, 5, 9]) -def test_dense_partition_intersections(cp_size, length): - """Dense rows retain their document identity, including uneven and empty ranks.""" - rank_metadata = [ - build_linear_attention_cp_host_metadata( - [0, length, 2 * length], cp_size=cp_size, cp_rank=cp_rank, dense=True - ) - for cp_rank in range(cp_size) - ] - for doc in (0, 1): - spans = [f for p in rank_metadata for f in p.fragments if f.document == doc] - assert sum(f.end - f.start for f in spans) == length - assert all(a.end == b.start for a, b in zip(spans, spans[1:])) - if length: - assert spans[0].start == doc * length - assert spans[-1].end == (doc + 1) * length - for cp_rank, host_metadata in enumerate(rank_metadata): - local_length = (cp_rank + 1) * length // cp_size - cp_rank * length // cp_size - assert host_metadata.local_cu_seqlens[-1] == 2 * local_length - assert host_metadata.slots == 2 - assert len(host_metadata.fragments) == (2 if local_length else 0) - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") -def test_affine_prefix_binding(): - """Forward summary descriptors and preserve the caller's current CUDA device.""" - torch.manual_seed(22) - key_dim = 3 - current = torch.cuda.current_device() - # Also exercise the binding's device guard: allocating on another GPU must - # neither launch on the current GPU nor change the caller's current device. - device = torch.device("cuda", (current + 1) % torch.cuda.device_count()) - initial_states = torch.randn(3, 2, 5, key_dim, device=device) - additive = torch.randn(6, 2, 5, key_dim, device=device) - transitions = torch.randn(6, 2, key_dim, key_dim, device=device) / key_dim**0.5 - rows = [ - [0, 0, 0, 1], - [1, 0, 3, 1], - [2, 5, 3, -2], - [-1, 0, 0, 1], - [-1, 0, 3, 1], - [-1, 5, 3, -2], - ] - descriptors = torch.tensor(rows, device=device, dtype=torch.int32) - actual = tex.linear_attention_affine_prefix(initial_states, additive, transitions, descriptors) - assert torch.cuda.current_device() == current - expected = [] - for source, first, count, stride in rows: - x = ( - torch.zeros_like(initial_states[0], dtype=torch.float64) - if source == -1 - else initial_states[source].double() - ) - for step in range(count): - index = first + step * stride - x = x @ transitions[index].double() + additive[index].double() - expected.append(x) - torch.testing.assert_close(actual.double(), torch.stack(expected), atol=2e-5, rtol=2e-5) - - -@pytest.mark.parametrize("size", [2, 4]) -def test_gdn_with_cp(size): +def test_gdn_with_cp(): """Run genuine NCCL CP with all token and state gradients checked numerically.""" - if not _GDN_AVAILABLE or torch.cuda.device_count() < size: + available = torch.cuda.device_count() + size = next((size for size in (8, 4, 2) if available >= size), None) + if size is None: + pytest.skip(f"GDN CP test requires at least 2 GPUs, but found {available}") + if not _GDN_AVAILABLE: if os.getenv("NVTE_GDN_CP_TEST_REQUIRED", "0") == "1": - pytest.fail(f"Required GDN CP test needs FE GDN and {size} CUDA GPUs") - pytest.skip(f"GDN CP test requires FE GDN and {size} CUDA GPUs") + pytest.fail("Required GDN CP test needs the cuDNN frontend GDN runtime") + pytest.skip("GDN CP test requires the cuDNN frontend GDN runtime") runner = Path(__file__).with_name("run_gdn_with_cp.py") env = os.environ.copy() # torchrun starts a script, so its sys.path[0] is the tests directory. From 09cfc7e7882e9629c765792983161ca608654b9a Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Fri, 20 Nov 2026 09:25:34 -0800 Subject: [PATCH 33/34] Remove duplicate GDN example tests Signed-off-by: Kshitij Lakhani --- tests/pytorch/attention/test_gdn_attention.py | 44 ------------------- 1 file changed, 44 deletions(-) diff --git a/tests/pytorch/attention/test_gdn_attention.py b/tests/pytorch/attention/test_gdn_attention.py index d0ccf527d81..6f2496ba534 100644 --- a/tests/pytorch/attention/test_gdn_attention.py +++ b/tests/pytorch/attention/test_gdn_attention.py @@ -7,7 +7,6 @@ import importlib.util import math import os -from pathlib import Path import sys from typing import Optional, Tuple @@ -295,49 +294,6 @@ def test_gdn_thd_forward_final_state_and_backward( ) -@requires_gdn -@pytest.mark.parametrize("qkv_format", ["bshd", "thd"]) -def test_gdn_documentation_example(qkv_format): - """Check the tutorial's inputs and forward/backward against the FP64 recurrence.""" - example_path = ( - Path(__file__).resolve().parents[3] / "docs/examples/attention/example_linear_attention.py" - ) - spec = importlib.util.spec_from_file_location("example_linear_attention", example_path) - example = importlib.util.module_from_spec(spec) - spec.loader.exec_module(example) - inputs, initial_state, cu_seqlens = example.make_inputs(qkv_format) - output, final_state = example.run_single_gpu(inputs, initial_state, cu_seqlens, qkv_format) - - reference_inputs = tuple(t.detach().double().requires_grad_() for t in inputs) - initial_state_ref = initial_state.detach().double().requires_grad_() - q, k, v, g, beta = (t.unsqueeze(0) if qkv_format == "thd" else t for t in reference_inputs) - output_ref, final_state_ref = _gdn_reference( - F.normalize(q, dim=-1), - F.normalize(k, dim=-1), - v, - g, - beta, - initial_state=initial_state_ref, - cu_seqlens=cu_seqlens if qkv_format == "thd" else None, - ) - output_ref = output_ref.reshape_as(output) - _assert_rms_close(output, output_ref, _FWD_TOL[inputs[0].dtype], "example output") - _assert_rms_close( - final_state, final_state_ref, _STATE_TOL[inputs[0].dtype], "example final state" - ) - (output_ref.square().mean() + final_state_ref.square().mean()).backward() - for name, actual, reference in zip(("q", "k", "v", "g", "beta"), inputs, reference_inputs): - _assert_rms_close( - actual.grad, reference.grad, _BWD_TOL[inputs[0].dtype], f"example d{name}" - ) - _assert_rms_close( - initial_state.grad, - initial_state_ref.grad, - _BWD_TOL[inputs[0].dtype], - "example dinitial_state", - ) - - @pytest.mark.parametrize("qkv_format", ["bshd", "sbhd"]) @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16], ids=["bf16", "fp16"]) @requires_gdn From e1d70d4553d3d2d3f1040ee393dbe5956af22804 Mon Sep 17 00:00:00 2001 From: Kshitij Lakhani Date: Fri, 20 Nov 2026 09:30:02 -0800 Subject: [PATCH 34/34] Update linear attention docs and fallback access Select the backward fallback transition directly, explain CP test size selection, and update the tutorial link. Signed-off-by: Kshitij Lakhani --- docs/examples/attention/attention.ipynb | 4 ++-- tests/pytorch/attention/test_gdn_attention_with_cp.py | 1 + transformer_engine/pytorch/attention/linear_attention/gdn.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/examples/attention/attention.ipynb b/docs/examples/attention/attention.ipynb index 61a653227c6..541abcccb9b 100644 --- a/docs/examples/attention/attention.ipynb +++ b/docs/examples/attention/attention.ipynb @@ -17,10 +17,10 @@ "[Transformer Engine](https://github.com/NVIDIA/TransformerEngine.git) supports the calculation of dot product attention in two frameworks, [PyTorch](https://github.com/pytorch/pytorch) and [JAX](https://github.com/google/jax). The API for each framework is\n", "\n", "- [transformer_engine.pytorch.DotProductAttention](../../api/pytorch.rst#transformer_engine.pytorch.DotProductAttention)\n", - "- [transformer_engine.jax.flax.DotProductAttention](../../api/jax.rst#transformer_engine.jax.flax.DotProductAttention)", + "- [transformer_engine.jax.flax.DotProductAttention](../../api/jax.rst#transformer_engine.jax.flax.DotProductAttention)\n", "\n", "\n", - "For linear attention variants and Gated DeltaNet context parallelism, see [Linear Attention](linear_attention.ipynb)." + "For linear attention variants, see [Linear Attention](linear_attention.ipynb)." ] }, { diff --git a/tests/pytorch/attention/test_gdn_attention_with_cp.py b/tests/pytorch/attention/test_gdn_attention_with_cp.py index bba31325f0d..3a5072edbf5 100644 --- a/tests/pytorch/attention/test_gdn_attention_with_cp.py +++ b/tests/pytorch/attention/test_gdn_attention_with_cp.py @@ -20,6 +20,7 @@ def test_gdn_with_cp(): """Run genuine NCCL CP with all token and state gradients checked numerically.""" available = torch.cuda.device_count() + # Test only the largest supported CP size available, rather than all sizes, to reduce test time. size = next((size for size in (8, 4, 2) if available >= size), None) if size is None: pytest.skip(f"GDN CP test requires at least 2 GPUs, but found {available}") diff --git a/transformer_engine/pytorch/attention/linear_attention/gdn.py b/transformer_engine/pytorch/attention/linear_attention/gdn.py index acf49d40b75..a7b53b65b75 100644 --- a/transformer_engine/pytorch/attention/linear_attention/gdn.py +++ b/transformer_engine/pytorch/attention/linear_attention/gdn.py @@ -206,9 +206,9 @@ def _cp_backward_summary( **kernel_kwargs, )[0] (additive,) = torch.autograd.grad(output, initial_state, grad_output) - _, transition = self._cp_forward_summary( + transition = self._cp_forward_summary( q, k, values, {"g": g, "beta": beta}, cu_seqlens, **kernel_kwargs - ) + )[1] return additive, transition.transpose(-1, -2).contiguous() def forward(