diff --git a/docs/api/pytorch.rst b/docs/api/pytorch.rst index 2d8cc87827e..561d3ea3ddd 100644 --- a/docs/api/pytorch.rst +++ b/docs/api/pytorch.rst @@ -26,7 +26,7 @@ PyTorch :members: forward, set_context_parallel_group .. autoapiclass:: transformer_engine.pytorch.GatedDeltaNetAttention(num_attention_heads, kv_channels, **kwargs) - :members: forward + :members: forward, set_context_parallel_group .. autoapiclass:: transformer_engine.pytorch.GatedDeltaNet2Attention(num_attention_heads, kv_channels, **kwargs) :members: forward diff --git a/docs/examples/attention/attention.ipynb b/docs/examples/attention/attention.ipynb index d8b573f6e7a..541abcccb9b 100644 --- a/docs/examples/attention/attention.ipynb +++ b/docs/examples/attention/attention.ipynb @@ -17,7 +17,10 @@ "[Transformer Engine](https://github.com/NVIDIA/TransformerEngine.git) supports the calculation of dot product attention in two frameworks, [PyTorch](https://github.com/pytorch/pytorch) and [JAX](https://github.com/google/jax). The API for each framework is\n", "\n", "- [transformer_engine.pytorch.DotProductAttention](../../api/pytorch.rst#transformer_engine.pytorch.DotProductAttention)\n", - "- [transformer_engine.jax.flax.DotProductAttention](../../api/jax.rst#transformer_engine.jax.flax.DotProductAttention)" + "- [transformer_engine.jax.flax.DotProductAttention](../../api/jax.rst#transformer_engine.jax.flax.DotProductAttention)\n", + "\n", + "\n", + "For linear attention variants, see [Linear Attention](linear_attention.ipynb)." ] }, { diff --git a/docs/examples/attention/example_linear_attention.py b/docs/examples/attention/example_linear_attention.py new file mode 100644 index 00000000000..9c43025b5e0 --- /dev/null +++ b/docs/examples/attention/example_linear_attention.py @@ -0,0 +1,170 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Gated DeltaNet forward and backward, with optional contiguous context parallelism. + +Run from the repository root: + python docs/examples/attention/example_linear_attention.py + python -m torch.distributed.run --standalone --nproc-per-node=2 \ + docs/examples/attention/example_linear_attention.py --cp --qkv-format thd +""" + +import argparse +import os + +import torch +import torch.distributed as dist + +import transformer_engine.pytorch as te + + +def make_inputs(qkv_format): + """Create the same batch on each rank for comparison with a non-CP run.""" + torch.manual_seed(1234) + batch, sequence, heads, head_dim = 16, 128, 8, 64 + if qkv_format == "thd": + # Sample document lengths from 32 to 128 tokens, then pack without padding. + lengths = torch.randint(32, 129, (batch,), device="cpu").tolist() + offsets = [0] + for length in lengths: + offsets.append(offsets[-1] + length) + shape = (offsets[-1], heads, head_dim) + else: + shape = (batch, sequence, heads, head_dim) + offsets = [row * sequence for row in range(batch + 1)] + q, k, v = (torch.randn(shape, device="cuda", dtype=torch.bfloat16) for _ in range(3)) + # Per-token, per-head FP32 gates: g is the log decay; beta is the write strength. + g = -torch.rand(shape[:-1], device="cuda", dtype=torch.float32) * 0.02 + beta = torch.rand_like(g) * 0.7 + inputs = tuple(t.requires_grad_() for t in (q, k, v, g, beta)) + initial_state = ( + torch.randn(batch, heads, head_dim, head_dim, device="cuda", dtype=torch.float32) * 0.1 + ).requires_grad_() + cu_seqlens = torch.tensor(offsets, device="cuda", dtype=torch.int32) + return inputs, initial_state, cu_seqlens + + +def run_single_gpu(inputs, initial_state, cu_seqlens, qkv_format): + """Run on complete documents and differentiate through outputs and final states.""" + q, k, v, g, beta = inputs + attention = te.GatedDeltaNetAttention( + num_attention_heads=q.shape[-2], + kv_channels=q.shape[-1], + qkv_format=qkv_format, + ) + output, final_state = attention( + q, + k, + v, + g=g, + beta=beta, + cu_seqlens=cu_seqlens if qkv_format == "thd" else None, + initial_state=initial_state, + output_final_state=True, + use_qk_l2norm_in_kernel=True, + ) + loss = output.float().square().mean() + final_state.square().mean() + loss.backward() + return output, final_state + + +def local_tokens(tensor, qkv_format, cp_group): + """Select this rank's contiguous interval of the packed buffer or each batch row.""" + rank, size = dist.get_rank(cp_group), dist.get_world_size(cp_group) + if qkv_format == "thd": + length = tensor.shape[0] + return tensor[rank * length // size : (rank + 1) * length // size] + length = tensor.shape[1] + return tensor[:, rank * length // size : (rank + 1) * length // size] + + +def run_context_parallel(inputs, initial_state, cu_seqlens, qkv_format, cp_group): + """Supply local tokens and global document metadata to GDN CP.""" + local_inputs = tuple( + local_tokens(t, qkv_format, cp_group).detach().contiguous().requires_grad_() for t in inputs + ) + q, k, v, g, beta = local_inputs + state = initial_state.detach().clone().requires_grad_() + attention = te.GatedDeltaNetAttention( + num_attention_heads=q.shape[-2], + kv_channels=q.shape[-1], + qkv_format=qkv_format, + cp_group=cp_group, + ) + output, final_state = attention( + q, + k, + v, + g=g, + beta=beta, + cu_seqlens=cu_seqlens, + initial_state=state, + output_final_state=True, + use_qk_l2norm_in_kernel=True, + ) + # Normalize by the global output size, so rank-local losses sum to the + # non-CP loss. Count the replicated final-state loss only once. + cp_size = dist.get_world_size(cp_group) + loss = output.float().square().sum() / inputs[2].numel() + loss = loss + final_state.square().mean() / cp_size + loss.backward() + return output, final_state, local_inputs, state + + +def check_finite(tensors): + """Fail on missing gradients or non-finite results, including in CI.""" + for tensor in tensors: + assert tensor is not None, "Missing gradient" + assert torch.isfinite(tensor).all(), "Non-finite result" + + +def check_close(actual, expected, tolerance): + """Compare BF16 results using relative RMS error, as in the GDN tests.""" + actual, expected = actual.detach().double(), expected.detach().double() + error = (actual - expected).square().mean().sqrt() + reference = expected.square().mean().sqrt().clamp_min(1e-12) + assert error / reference < tolerance, f"Relative RMS error: {(error / reference).item()}" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--cp", action="store_true", help="Run with torch.distributed.run on two or more GPUs" + ) + parser.add_argument("--qkv-format", choices=("bshd", "thd"), default="bshd") + args = parser.parse_args() + torch.cuda.set_device(int(os.environ.get("LOCAL_RANK", "0"))) + if args.cp: + # NCCL is PyTorch's recommended backend for CUDA distributed training. + # CP also accepts other backends that support its CUDA collectives. + dist.init_process_group(backend="nccl") + try: + if args.cp and dist.get_world_size() < 2: + raise ValueError("--cp requires torch.distributed.run with at least two GPUs") + inputs, initial_state, cu_seqlens = make_inputs(args.qkv_format) + output, final_state = run_single_gpu(inputs, initial_state, cu_seqlens, args.qkv_format) + assert output.shape == (*inputs[2].shape[:-2], inputs[2].shape[-2:].numel()) + assert final_state.shape == initial_state.shape + check_finite((output, final_state, initial_state.grad, *(t.grad for t in inputs))) + if args.cp: + cp_group = dist.group.WORLD + local_output, states, local_inputs, state = run_context_parallel( + inputs, initial_state, cu_seqlens, args.qkv_format, cp_group + ) + check_close(local_output, local_tokens(output, args.qkv_format, cp_group), 2e-2) + check_close(states, final_state, 2e-2) + for local, full in zip(local_inputs, inputs): + check_close(local.grad, local_tokens(full.grad, args.qkv_format, cp_group), 4e-2) + # Initial-state gradients belong to each document's first owning rank. + dist.all_reduce(state.grad, group=cp_group) + check_close(state.grad, initial_state.grad, 4e-2) + if not args.cp or dist.get_rank() == 0: + print(f"GDN {args.qkv_format}: forward and backward passed (CP={args.cp})") + finally: + if args.cp: + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/docs/examples/attention/linear_attention.ipynb b/docs/examples/attention/linear_attention.ipynb new file mode 100644 index 00000000000..51a6c50a941 --- /dev/null +++ b/docs/examples/attention/linear_attention.ipynb @@ -0,0 +1,220 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "# Linear Attention\n", + "\n", + "Linear attention maintains a recurrent state instead of forming the pairwise attention scores used by scaled dot product attention (SDPA). Transformer Engine provides PyTorch modules for several linear attention variants through the cuDNN frontend. These APIs are **experimental** and subject to change.\n", + "\n", + "This guide introduces the available variants and shows Gated DeltaNet (GDN) forward and backward on a single GPU and with context parallelism (CP). For SDPA backends, masks, and biases, see [Attention Is All You Need!](attention.ipynb).\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Supported Variants\n", + "\n", + "| Variant | PyTorch API | Gates and updates | Context parallelism |\n", + "| --- | --- | --- | --- |\n", + "| Gated DeltaNet (GDN) | [GatedDeltaNetAttention](../../api/pytorch.rst#transformer_engine.pytorch.GatedDeltaNetAttention) | Per-token, per-head log decay `g` and write strength `beta` | Contiguous CP |\n", + "| Gated DeltaNet v2 (GDN-2) | [GatedDeltaNet2Attention](../../api/pytorch.rst#transformer_engine.pytorch.GatedDeltaNet2Attention) | Per-key-channel `g` and `beta`, per-value-channel `w` | Not supported |\n", + "| Gated DeltaProduct (GDP) | [GatedDeltaProductAttention](../../api/pytorch.rst#transformer_engine.pytorch.GatedDeltaProductAttention) | Multiple delta-rule updates per token | Not supported |\n", + "| Kimi Delta Attention (KDA) | [KimiDeltaAttention](../../api/pytorch.rst#transformer_engine.pytorch.KimiDeltaAttention) | Per-key-channel decay `g`, per-token, per-head `beta` | Not supported |\n", + "\n", + "The modules take Q, K, V and gate tensors produced by the caller; they do not include input projections or an output projection. Q/K/V use FP16 or BF16, and recurrent states use FP32. Gate shapes and dtypes depend on the variant; see the corresponding API reference.\n", + "\n", + "Install Transformer Engine with PyTorch support as described in [Installation](../../installation.rst). The cuDNN frontend must provide the selected linear attention op and its kernel runtime. See the [cuDNN frontend installation guidance](https://github.com/NVIDIA/cudnn-frontend#installation) for its dependencies and installation options. Supported GPU architectures and head dimensions depend on the variant and the available cuDNN frontend engines.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. GDN Inputs and Outputs\n", + "\n", + "GDN is causal; its API does not take SDPA attention masks or bias tensors. It takes two FP32 gates: `g`, the log decay, and `beta`, the write strength. Each gate has one value per token and head. The example uses negative `g` and `beta` between zero and one, and enables Q/K L2 normalization with `use_qk_l2norm_in_kernel=True`.\n", + "\n", + "| `qkv_format` | Q/K shape | V shape | Gate shape | Output shape |\n", + "| --- | --- | --- | --- | --- |\n", + "| `bshd` | `[B, S, H, Dk]` | `[B, S, H, Dv]` | `[B, S, H]` | `[B, S, H * Dv]` |\n", + "| `sbhd` | `[S, B, H, Dk]` | `[S, B, H, Dv]` | `[S, B, H]` | `[S, B, H * Dv]` |\n", + "| `thd` | `[T, H, Dk]` | `[T, H, Dv]` | `[T, H]` | `[T, H * Dv]` |\n", + "\n", + "Here `B` is the number of documents, `S` is the dense sequence length, `T` is the total number of packed tokens, and `H` is the number of heads on this tensor-parallel rank. `kv_channels` sets `Dk` and `Dv`: pass one integer for equal head dimensions or `(Dk, Dv)` otherwise.\n", + "\n", + "For packed inputs, remove padding and pass `cu_seqlens`, the cumulative document lengths, as an `int32` tensor. Documents do not share recurrent state. Without CP, `cu_seqlens` must be on the input device. Dense inputs can omit it; if supplied, it must describe full rows as `[0, S, 2*S, ..., B*S]`.\n", + "\n", + "The optional `initial_state` has shape `[B, H, Dv, Dk]` and dtype FP32. Omitting it starts from zero. Set `output_final_state=True` to return `(output, final_state)`; otherwise the module returns only `output`. Final states have the same shape and dtype as the initial state and can seed a subsequent call.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Single-GPU Example\n", + "\n", + "The runnable example below uses 16 documents and 8 heads with BF16 Q/K/V and FP32 gates and states. Dense sequences have 128 tokens each; packed document lengths are sampled from 32 to 128 tokens." + ] + }, + { + "cell_type": "raw", + "metadata": { + "raw_mimetype": "text/restructuredtext" + }, + "source": [ + "Download the complete :download:`example_linear_attention.py ` script." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Run these commands from the repository root:\n", + "\n", + "```bash\n", + "python docs/examples/attention/example_linear_attention.py\n", + "python docs/examples/attention/example_linear_attention.py --qkv-format thd\n", + "```\n", + "\n", + "Each run checks output and state shapes and finite gradients after forward and backward. Missing kernel dependencies fail the run.\n", + "\n", + "The input preparation covers equal-length dense rows and packed documents of different lengths:" + ] + }, + { + "cell_type": "raw", + "metadata": { + "raw_mimetype": "text/restructuredtext" + }, + "source": [ + ".. literalinclude:: example_linear_attention.py\n", + " :language: python\n", + " :pyobject: make_inputs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The forward call below works with either layout. Backward includes a loss on both the attention output and the final state:" + ] + }, + { + "cell_type": "raw", + "metadata": { + "raw_mimetype": "text/restructuredtext" + }, + "source": [ + ".. literalinclude:: example_linear_attention.py\n", + " :language: python\n", + " :pyobject: run_single_gpu" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. GDN Context Parallelism\n", + "\n", + "GDN supports **contiguous context parallelism**. Create a `torch.distributed` process group and pass it as `cp_group` when constructing the module, or call `set_context_parallel_group` before forward. `None` disables CP; a group of size one follows the local path.\n", + "\n", + "Each rank supplies local Q/K/V and gate tensors. Group rank order defines token order, and all inputs must use the same partition:\n", + "\n", + "- **THD:** rank `r` owns `[r*T//P:(r+1)*T//P]` of the global packed buffer.\n", + "- **BSHD/SBHD:** rank `r` owns `[r*S//P:(r+1)*S//P]` of every batch row.\n", + "\n", + "`P` is the CP group size. These are contiguous intervals, rather than SDPA's dual-chunk partitioning. TE does not redistribute inputs, and outputs stay on the ranks that own their input tokens.\n", + "\n", + "Pass the same **global** `cu_seqlens` on every rank, describing the complete documents before partitioning. For THD it is required. Dense inputs can omit it for equal slices; uneven dense slices require `[0, S, 2*S, ..., B*S]`. With CP, this metadata may be on CPU or CUDA. Prefer CPU metadata when already available from the input pipeline to avoid a synchronized host read.\n", + "\n", + "Document lengths need not be divisible by CP size. Empty documents and ranks with no tokens are supported. All ranks must participate in forward and backward with matching options, including ranks with no tokens or no local loss.\n", + "\n", + "The example selects the local intervals as follows (the `bshd` path slices each batch row):" + ] + }, + { + "cell_type": "raw", + "metadata": { + "raw_mimetype": "text/restructuredtext" + }, + "source": [ + ".. literalinclude:: example_linear_attention.py\n", + " :language: python\n", + " :pyobject: local_tokens" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Launch the dense or packed example on two GPUs:\n", + "\n", + "```bash\n", + "python -m torch.distributed.run --standalone --nproc-per-node=2 \\\n", + " docs/examples/attention/example_linear_attention.py --cp\n", + "\n", + "python -m torch.distributed.run --standalone --nproc-per-node=2 \\\n", + " docs/examples/attention/example_linear_attention.py --cp --qkv-format thd\n", + "```\n", + "\n", + "Each run compares local outputs, replicated final states, and input and initial-state gradients against a non-CP run on the complete documents.\n", + "\n", + "For CP, the script selects NCCL, [PyTorch's recommended backend for CUDA distributed training](https://docs.pytorch.org/docs/stable/distributed.html#which-backend-to-use), and selects each process's GPU using `LOCAL_RANK`. GDN CP also accepts other process-group backends that support its CUDA collectives. For example, a CUDA-capable Gloo build can use `backend=\"gloo\"` in `init_process_group`. For demonstration, each rank generates the same full batch and selects its interval locally. In a training pipeline, supply the local tensors directly.\n", + "\n", + "This is the CP forward and backward:" + ] + }, + { + "cell_type": "raw", + "metadata": { + "raw_mimetype": "text/restructuredtext" + }, + "source": [ + ".. literalinclude:: example_linear_attention.py\n", + " :language: python\n", + " :pyobject: run_context_parallel" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Recurrent States and Losses\n", + "\n", + "With CP, `initial_state` describes **all global documents** and must have identical values on every rank. Requested final states are also replicated for all global documents. Empty documents retain their initial state.\n", + "\n", + "Each rank computes the loss for its own output tokens. Dividing each local sum by the total number of output elements across the CP group makes the combined loss equal to the non-CP mean. Final states are copied to every rank, and their gradients are summed during backward. If every rank applies the same final-state loss, divide it by CP size so it is counted only once.\n", + "\n", + "Each document's initial-state gradient is returned on its first owning rank, with zeros elsewhere. Sum these contributions over CP when inspecting a replicated leaf state's full gradient. Final states can also be chained into subsequent CP calls.\n", + "\n", + "### Supported Execution\n", + "\n", + "GDN CP supports FP16/BF16 inputs, FP32 gates and states, Q/K normalization, activation checkpointing (`checkpoint_core_attention=True`), and tensor parallelism. It supports first-order gradients and executes eagerly, including inside a `torch.compile` model. Full-graph compilation and CUDA graph capture are outside the supported CP execution modes. Without CP, ordinary forward/backward calls support `torch.compile(fullgraph=True)`; activation checkpointing still requires eager attention execution." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.12" + }, + "nbsphinx": { + "execute": "never" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/features/other_optimizations/index.rst b/docs/features/other_optimizations/index.rst index 05e89c4b05d..910edc73acf 100644 --- a/docs/features/other_optimizations/index.rst +++ b/docs/features/other_optimizations/index.rst @@ -9,4 +9,3 @@ Other optimizations .. toctree:: cpu_offloading/cpu_offloading.rst - diff --git a/docs/index.rst b/docs/index.rst index 0d3746fcc2b..50a3c15b0bf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -81,4 +81,5 @@ Transformer Engine documentation debug envvars examples/attention/attention.ipynb + examples/attention/linear_attention.ipynb examples/attention/cp_ag_thd_dpa_jax_deep_dive.ipynb diff --git a/qa/L0_pytorch_unittest/test.sh b/qa/L0_pytorch_unittest/test.sh index 4697880310f..03d457c8054 100644 --- a/qa/L0_pytorch_unittest/test.sh +++ b/qa/L0_pytorch_unittest/test.sh @@ -72,6 +72,9 @@ NVTE_ALLOW_UNSAFE_PICKLE_EXTRA_STATE=1 python3 -m pytest --tb=auto --junitxml=$X python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_attention_backend_selection.xml $TE_PATH/tests/pytorch/attention/test_attention_backend_selection.py || test_fail "test_attention_backend_selection.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_flex_attention.xml $TE_PATH/tests/pytorch/attention/test_flex_attention.py || test_fail "test_flex_attention.py" NVTE_GDN_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_gdn_attention.xml $TE_PATH/tests/pytorch/attention/test_gdn_attention.py || test_fail "test_gdn_attention.py" +for QKV_FORMAT in bshd thd; do + python3 "$TE_PATH/docs/examples/attention/example_linear_attention.py" --qkv-format "$QKV_FORMAT" || test_fail "example_linear_attention.py ($QKV_FORMAT)" +done NVTE_GDN2_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_gdn2_attention.xml $TE_PATH/tests/pytorch/attention/test_gdn2_attention.py || test_fail "test_gdn2_attention.py" NVTE_GDP_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_gdp_attention.xml $TE_PATH/tests/pytorch/attention/test_gdp_attention.py || test_fail "test_gdp_attention.py" NVTE_KDA_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_kda_attention.xml $TE_PATH/tests/pytorch/attention/test_kda_attention.py || test_fail "test_kda_attention.py" diff --git a/qa/L1_pytorch_distributed_unittest/test.sh b/qa/L1_pytorch_distributed_unittest/test.sh index 41ab6aeb444..9961217bc42 100644 --- a/qa/L1_pytorch_distributed_unittest/test.sh +++ b/qa/L1_pytorch_distributed_unittest/test.sh @@ -43,6 +43,11 @@ else NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_attention_deterministic_with_cp.xml $TE_PATH/tests/pytorch/attention/test_attention_with_cp.py || test_fail "NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 test_attention_with_cp.py" fi +NVTE_GDN_TEST_REQUIRED=1 NVTE_GDN_CP_TEST_REQUIRED=1 python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_gdn_attention_with_cp.xml $TE_PATH/tests/pytorch/attention/test_gdn_attention_with_cp.py || test_fail "test_gdn_attention_with_cp.py" +for QKV_FORMAT in bshd thd; do + python3 -m torch.distributed.run --standalone --nproc-per-node=2 "$TE_PATH/docs/examples/attention/example_linear_attention.py" --cp --qkv-format "$QKV_FORMAT" || test_fail "example_linear_attention.py CP ($QKV_FORMAT)" +done + python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_sanity.xml $TE_PATH/tests/pytorch/distributed/test_sanity.py || test_fail "test_sanity.py" python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_parallel_cross_entropy.xml $TE_PATH/tests/pytorch/distributed/test_parallel_cross_entropy.py || test_fail "test_parallel_cross_entropy.py" python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest_test_symmetric_memory.xml $TE_PATH/tests/pytorch/distributed/test_symmetric_memory.py || test_fail "test_symmetric_memory.py" diff --git a/tests/cpp/operator/CMakeLists.txt b/tests/cpp/operator/CMakeLists.txt index ecb746d7b55..4634c7ed52b 100644 --- a/tests/cpp/operator/CMakeLists.txt +++ b/tests/cpp/operator/CMakeLists.txt @@ -48,6 +48,7 @@ add_executable(test_operator test_swap_first_dims.cu test_gemm.cu test_grouped_gemm.cu + test_linear_attention.cu ../test_common.cu) # Find required packages diff --git a/tests/cpp/operator/test_linear_attention.cu b/tests/cpp/operator/test_linear_attention.cu new file mode 100644 index 00000000000..e1a84911ceb --- /dev/null +++ b/tests/cpp/operator/test_linear_attention.cu @@ -0,0 +1,169 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../test_common.h" + +namespace { + +constexpr size_t kDescriptorWidth = 4; +using AffineDescriptor = std::array; + +void RunAffinePrefixTest(size_t key_dim, size_t heads, size_t value_dim, size_t num_summaries, + const std::vector &descriptor_rows) { + using test::Tensor; + using transformer_engine::DType; + constexpr size_t kNumInitialStates = 3; + constexpr unsigned kSeed = 2026; // Keep random inputs reproducible. + // Allow FP32 rounding across repeated compositions against the FP64 reference. + constexpr double kAbsoluteTolerance = 5e-6; + constexpr double kRelativeTolerance = 5e-6; + const size_t num_outputs = descriptor_rows.size(); + const size_t state_size = heads * value_dim * key_dim; + // Create inputs and output storage for the requested state shapes. + Tensor initial_states("initial_states", + std::vector{kNumInitialStates, heads, value_dim, key_dim}, + DType::kFloat32); + Tensor additive("additive", std::vector{num_summaries, heads, value_dim, key_dim}, + DType::kFloat32); + Tensor transitions("transitions", std::vector{num_summaries, heads, key_dim, key_dim}, + DType::kFloat32); + Tensor descriptors("descriptors", std::vector{num_outputs, kDescriptorWidth}, + DType::kInt32); + Tensor output("output", std::vector{num_outputs, heads, value_dim, key_dim}, + DType::kFloat32); + std::mt19937 generator(kSeed); + std::uniform_real_distribution distribution(-1.f, 1.f); + auto *initial_states_cpu = initial_states.rowwise_cpu_dptr(); + auto *additive_cpu = additive.rowwise_cpu_dptr(); + auto *transitions_cpu = transitions.rowwise_cpu_dptr(); + auto *descriptors_cpu = descriptors.rowwise_cpu_dptr(); + for (size_t i = 0; i < kNumInitialStates * state_size; ++i) { + initial_states_cpu[i] = distribution(generator); + } + for (size_t i = 0; i < num_summaries * state_size; ++i) { + additive_cpu[i] = distribution(generator); + } + // Scale transitions to keep matrix-product variance comparable across key dimensions. + for (size_t i = 0; i < num_summaries * heads * key_dim * key_dim; ++i) { + transitions_cpu[i] = distribution(generator) / std::sqrt(key_dim); + } + for (size_t i = 0; i < num_outputs; ++i) { + std::copy(descriptor_rows[i].begin(), descriptor_rows[i].end(), + descriptors_cpu + i * kDescriptorWidth); + } + // Detect unwritten output elements, including outputs that should be zero. + std::fill_n(output.rowwise_cpu_dptr(), num_outputs * state_size, + std::numeric_limits::quiet_NaN()); + initial_states.from_cpu(); + additive.from_cpu(); + transitions.from_cpu(); + descriptors.from_cpu(); + output.from_cpu(); + nvte_linear_attention_affine_prefix(initial_states.data(), additive.data(), transitions.data(), + descriptors.data(), output.data(), nullptr); + ASSERT_EQ(cudaDeviceSynchronize(), cudaSuccess); + output.to_cpu(); + // Compute the CPU reference in FP64 and compare each output with the kernel. + const auto *actual = output.rowwise_cpu_dptr(); + for (size_t output_index = 0; output_index < num_outputs; ++output_index) { + const auto [initial_memory_index, first_summary, count, stride] = descriptor_rows[output_index]; + for (size_t head = 0; head < heads; ++head) { + for (size_t value = 0; value < value_dim; ++value) { + const size_t row_offset = (head * value_dim + value) * key_dim; + std::vector state(key_dim, 0.), next(key_dim); + // Start from zero unless the descriptor selects an explicit initial memory. + if (initial_memory_index >= 0) { + const float *start = initial_states_cpu + initial_memory_index * state_size + row_offset; + std::copy(start, start + key_dim, state.begin()); + } + // Apply state = state @ T + H in the descriptor's summary order. + for (int step = 0; step < count; ++step) { + const int summary_index = first_summary + step * stride; + const float *transition_matrix = + transitions_cpu + (summary_index * heads + head) * key_dim * key_dim; + for (size_t col = 0; col < key_dim; ++col) { + next[col] = additive_cpu[summary_index * state_size + row_offset + col]; + for (size_t inner = 0; inner < key_dim; ++inner) { + next[col] += state[inner] * transition_matrix[inner * key_dim + col]; + } + } + state.swap(next); + } + for (size_t col = 0; col < key_dim; ++col) { + // Match the shared test helper's absolute-or-relative acceptance rule. + const double tolerance = + std::max(kAbsoluteTolerance, kRelativeTolerance * std::abs(state[col])); + ASSERT_NEAR(actual[output_index * state_size + row_offset + col], state[col], tolerance) + << "output=" << output_index << ", head=" << head << ", value=" << value + << ", column=" << col; + } + } + } + } +} + +class LinearAttentionPrefixTest + : public ::testing::TestWithParam> {}; + +TEST_P(LinearAttentionPrefixTest, OrderedAffineRecipes) { + const auto [key_dim, heads, value_dim] = GetParam(); + constexpr size_t kNumSummaries = 7; + // Rows are (initial_memory_index, first_summary, count, stride). + const std::vector descriptor_rows = { + {0, 0, 0, 1}, // Copy initial memory 0. + {1, 0, 3, 1}, // Apply summaries 0, 1, 2 to initial memory 1. + {2, 5, 3, -2}, // Reverse composition using summaries 5, 3, 1. + {-1, 0, 0, 1}, // Zero initial memory, with no summaries. + {-1, 0, 3, 2}, // Zero initial memory, then summaries 0, 2, 4. + {-1, 6, 3, -3}, // Zero initial memory, then summaries 6, 3, 0. + {1, 4, 1, 1}, // One summary with an explicit initial memory. + {-1, 4, 1, 1}, // One summary with zero initial memory: copy H[4]. + }; + RunAffinePrefixTest(key_dim, heads, value_dim, kNumSummaries, descriptor_rows); +} + +TEST(LinearAttentionPrefixLongChainTest, ForwardAndReverse) { + constexpr size_t kKeyDim = 64; + constexpr size_t kHeads = 2; + constexpr size_t kValueDim = 5; + constexpr size_t kNumSummaries = 32; + constexpr int kLastSummary = static_cast(kNumSummaries) - 1; + // CP=32 can require composing 31 preceding or following fragment summaries. + constexpr int kChainLength = 31; + const std::vector descriptor_rows = { + {1, 0, kChainLength, 1}, + {2, kLastSummary, kChainLength, -1}, + {-1, 0, kChainLength, 1}, + {-1, kLastSummary, kChainLength, -1}, + }; + RunAffinePrefixTest(kKeyDim, kHeads, kValueDim, kNumSummaries, descriptor_rows); +} + +// K covers small/typical states, both block sizes, and multiple columns per thread. +// Heads 1/8 check head indexing; V=1/5/16 covers partial, tail, and multiple full 4-row tiles. +INSTANTIATE_TEST_SUITE_P( + Common, LinearAttentionPrefixTest, + ::testing::Combine(::testing::Values(3, 64, 128, 257), ::testing::Values(1, 8), + ::testing::Values(1, 5, 16)), + [](const testing::TestParamInfo &info) { + return "K" + std::to_string(std::get<0>(info.param)) + "H" + + std::to_string(std::get<1>(info.param)) + "V" + + std::to_string(std::get<2>(info.param)); + }); + +} // namespace diff --git a/tests/pytorch/attention/run_gdn_with_cp.py b/tests/pytorch/attention/run_gdn_with_cp.py new file mode 100644 index 00000000000..cd3d84a32aa --- /dev/null +++ b/tests/pytorch/attention/run_gdn_with_cp.py @@ -0,0 +1,475 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Distributed GDN CP numerical checks; launch with torch.distributed.run.""" + +import importlib.util +import os +import traceback +from contextlib import nullcontext +from unittest.mock import patch + +import pytest +import torch +import torch.distributed as dist +import torch.nn.functional as F + +from transformer_engine.pytorch import GatedDeltaNetAttention +from test_gdn_attention import ( + _assert_rms_close, + _gdn_reference, + _FWD_TOL, + _STATE_TOL, + _BWD_TOL, +) + + +def _reference(inputs, cu, scale, norm): + """Independent FP64 recurrence, including unchanged states for empty documents.""" + q, k, v, g, beta, initial = inputs + outputs, states = [], [] + if norm: + q, k = F.normalize(q.double(), dim=-1), F.normalize(k.double(), dim=-1) + for doc, (start, end) in enumerate(zip(cu, cu[1:])): + if start == end: + states.append(initial[doc].double()) + continue + out, state = _gdn_reference( + q[start:end].unsqueeze(0), + k[start:end].unsqueeze(0), + v[start:end].unsqueeze(0), + g[start:end].unsqueeze(0), + beta[start:end].unsqueeze(0), + scale=scale, + initial_state=initial[doc : doc + 1], + cu_seqlens=torch.tensor([0, end - start], device=q.device, dtype=torch.int32), + ) + outputs.append(out[0]) + states.append(state[0]) + return (torch.cat(outputs) if outputs else v.double() * 0), torch.stack(states) + + +def _case( + group, + lengths, + *, + dtype=torch.bfloat16, + heads=2, + key_dim=64, + value_dim=64, + layout="thd", + norm=False, + initial=True, + final=True, + checkpoint=False, + scale=0.2, + last_loss=False, + fallback=False, + tp_group=None, + independent=False, + dense_global=False, + state_only=False, + inference_warmup=False, +): + """Compare each local output/gradient to the corresponding full-sequence result.""" + size, rank = dist.get_world_size(group), dist.get_rank(group) + if rank == 0: + print( + f"GDN CP case: lengths={lengths}, dtype={dtype}, H={heads}, K={key_dim}, V={value_dim}," + f" layout={layout}, norm={norm}, initial={initial}, final={final}," + f" checkpoint={checkpoint}, last_loss={last_loss}, fallback={fallback}," + f" independent={independent}, inference_warmup={inference_warmup}", + flush=True, + ) + device = torch.device("cuda", int(os.environ["LOCAL_RANK"])) + torch.manual_seed(791 + (dist.get_rank(tp_group) if tp_group is not None else 0)) + total = sum(lengths) + cu = [0] + for length in lengths: + cu.append(cu[-1] + length) + q = torch.randn(total, heads, key_dim, device=device, dtype=dtype) + k = torch.randn_like(q) + if not norm: + q, k = F.normalize(q, dim=-1), F.normalize(k, dim=-1) + v = torch.randn(total, heads, value_dim, device=device, dtype=dtype) + g = -torch.rand(total, heads, device=device) * 0.02 + beta = torch.rand(total, heads, device=device) * 0.7 + initial_state = torch.randn(len(lengths), heads, value_dim, key_dim, device=device) * 0.1 + if not initial: + initial_state.zero_() + full = tuple(t.detach().requires_grad_(True) for t in (q, k, v, g, beta, initial_state)) + global_cu = torch.tensor(cu, device=device, dtype=torch.int32) + if independent or total == 0: + ref_output, ref_states = _reference(full, cu, scale, norm) + else: + reference = GatedDeltaNetAttention( + heads, (key_dim, value_dim), qkv_format="thd", scale=scale + ) + ref_output, ref_states = reference( + *full[:5], + cu_seqlens=global_cu, + initial_state=full[5] if initial else None, + output_final_state=True, + use_qk_l2norm_in_kernel=norm, + ) + ref_output = ref_output.reshape(total, heads, value_dim) + grad_output = torch.randn_like(v) + if last_loss: + grad_output[: -max(1, total // size)] = 0 + grad_states = ( + torch.randn_like(initial_state) * 0.1 if final else torch.zeros_like(initial_state) + ) + loss = (ref_states * grad_states).sum() + if not state_only: + loss = loss + (ref_output * grad_output).sum() + ref_grads = torch.autograd.grad(loss, full, allow_unused=True) + ref_grads = tuple(torch.zeros_like(t) if d is None else d for t, d in zip(full, ref_grads)) + + def local(tensor): + if layout == "thd": + return tensor[rank * total // size : (rank + 1) * total // size] + batch, length = len(lengths), lengths[0] + tensor = tensor.reshape(batch, length, *tensor.shape[1:]) + tensor = tensor[:, rank * length // size : (rank + 1) * length // size] + return tensor.transpose(0, 1) if layout == "sbhd" else tensor + + inputs = tuple(local(t).detach().requires_grad_(True) for t in full[:5]) + state = full[5].detach().requires_grad_(True) if initial else None + tp_size = 1 if tp_group is None else dist.get_world_size(tp_group) + module_kwargs = dict( + qkv_format=layout, + scale=scale, + cp_group=group, + tp_group=tp_group, + ) + module = GatedDeltaNetAttention(heads * tp_size, (key_dim, value_dim), **module_kwargs) + kwargs = dict( + cu_seqlens=( + (global_cu.cpu() if size == 1 else global_cu) + if layout == "thd" or dense_global + else None + ), + initial_state=state, + output_final_state=final, + checkpoint_core_attention=checkpoint, + use_qk_l2norm_in_kernel=norm, + ) + if inference_warmup: + with torch.inference_mode(): + module(*inputs, **kwargs) + # A second layer must be able to train with the shared metadata. + module = GatedDeltaNetAttention(heads * tp_size, (key_dim, value_dim), **module_kwargs) + summary_patch = ( + patch.object( + module.gdn_attention, "_summary_op", side_effect=ImportError("Missing summary") + ) + if fallback + else nullcontext() + ) + with summary_patch: + result = module(*inputs, **kwargs) + output, states = result if final else (result, None) + expected = local(ref_output).reshape_as(output) + if output.numel(): + _assert_rms_close(output, expected, _FWD_TOL[dtype], "CP output") + if final: + _assert_rms_close(states, ref_states, _STATE_TOL[dtype], "CP final states") + local_loss = 0 if state_only else (output * local(grad_output).reshape_as(output)).sum() + if final: + local_loss = local_loss + (states * grad_states).sum() / size + local_loss.backward() + for name, tensor, grad in zip(("q", "k", "v", "g", "beta"), inputs, ref_grads[:5]): + if tensor.numel(): + expected = local(grad) + if expected.count_nonzero() == 0: + torch.testing.assert_close(tensor.grad, expected, atol=1e-6, rtol=0) + else: + _assert_rms_close(tensor.grad, expected, _BWD_TOL[dtype], f"CP d{name}") + if initial: + initial_grad = state.grad.detach().clone() + dist.all_reduce(initial_grad, group=group) + _assert_rms_close(initial_grad, ref_grads[5], _BWD_TOL[dtype], "CP initial-state gradient") + dist.barrier(group=group) + + +def _invalid_metadata(group): + """Reject incompatible packed metadata before entering a collective.""" + device = torch.device("cuda", int(os.environ["LOCAL_RANK"])) + size = dist.get_world_size(group) + q = torch.zeros(4, 2, 64, device=device, dtype=torch.bfloat16) + gate = torch.zeros(4, 2, device=device) + module = GatedDeltaNetAttention(2, 64, qkv_format="thd", cp_group=group) + invalid = ( + {}, + {"cu_seqlens": torch.tensor([0, 4 * size], dtype=torch.int64)}, + {"cu_seqlens": torch.tensor([0, 4 * size, 2 * size], dtype=torch.int32)}, + {"cu_seqlens": torch.tensor([0, 5 * size], dtype=torch.int32)}, + {"cu_seqlens": torch.tensor([0, 4], dtype=torch.int32, device=device)}, + ) + for kwargs in invalid: + with pytest.raises(ValueError): + module(q, q, q, g=gate, beta=gate, **kwargs) + dist.barrier(group=group) + + +def _inference_and_deferred_group(group): + """Default scale, deferred configuration, and metadata created in inference mode.""" + size, rank = dist.get_world_size(group), dist.get_rank(group) + device = torch.device("cuda", int(os.environ["LOCAL_RANK"])) + torch.manual_seed(998) + q, k = ( + F.normalize(torch.randn(35, 2, 64, device=device, dtype=torch.bfloat16), dim=-1) + for _ in range(2) + ) + v = torch.randn_like(q) + g, beta = -torch.rand(35, 2, device=device) * 0.02, torch.rand(35, 2, device=device) * 0.7 + reference = GatedDeltaNetAttention(2, 64, qkv_format="thd") + modules = [GatedDeltaNetAttention(2, 64, qkv_format="thd") for _ in range(2)] + for module in modules: + module.set_context_parallel_group(group) + q, k, v, g, beta = (t.requires_grad_() for t in (q, k, v, g, beta)) + for context, metadata_device in ( + (torch.no_grad, "cpu"), + (torch.no_grad, device), + (torch.inference_mode, device), + ): + with context(): + # CP accepts host metadata; inference tensors intentionally bypass + # the metadata cache because they have no mutation counter. + global_cu = torch.tensor([0, 17, 35], device=metadata_device, dtype=torch.int32) + for boundary in (17, 19): + global_cu[1] = boundary + expected, expected_states = reference( + q, k, v, g, beta, cu_seqlens=global_cu.to(device), output_final_state=True + ) + lo, hi = rank * 35 // size, (rank + 1) * 35 // size + # Reuse one batch across layers, then invalidate it in place. + for module in modules: + output, states = module( + q[lo:hi], + k[lo:hi], + v[lo:hi], + g[lo:hi], + beta[lo:hi], + cu_seqlens=global_cu, + output_final_state=True, + ) + assert not output.requires_grad and not states.requires_grad + _assert_rms_close( + output, expected[lo:hi], _FWD_TOL[q.dtype], "inference output" + ) + _assert_rms_close( + states, expected_states, _STATE_TOL[q.dtype], "inference states" + ) + # Prepare a separate entry when the same modules run on another stream. + stream = torch.cuda.Stream(device=device) + stream.wait_stream(torch.cuda.current_stream(device)) + with torch.no_grad(), torch.cuda.stream(stream): + global_cu = torch.tensor([0, 19, 35], dtype=torch.int32) + results = [ + module( + q[lo:hi], + k[lo:hi], + v[lo:hi], + g[lo:hi], + beta[lo:hi], + cu_seqlens=global_cu, + output_final_state=True, + ) + for module in modules + ] + stream.synchronize() + for output, states in results: + _assert_rms_close(output, expected[lo:hi], _FWD_TOL[q.dtype], "stream output") + _assert_rms_close(states, expected_states, _STATE_TOL[q.dtype], "stream states") + dist.barrier(group=group) + + +def _state_chaining(group): + """Backpropagate through two CP calls joined by their replicated final states.""" + size, rank = dist.get_world_size(group), dist.get_rank(group) + device = torch.device("cuda", int(os.environ["LOCAL_RANK"])) + torch.manual_seed(932) + tokens, heads, dim = 35, 2, 64 + full = [] + for _ in range(2): + q, k = ( + F.normalize( + torch.randn(tokens, heads, dim, device=device, dtype=torch.bfloat16), dim=-1 + ) + for _ in range(2) + ) + v = torch.randn_like(q) + g = -torch.rand(tokens, heads, device=device) * 0.02 + beta = torch.rand(tokens, heads, device=device) * 0.7 + full.extend(t.detach().requires_grad_(True) for t in (q, k, v, g, beta)) + initial_state = (torch.randn(2, heads, dim, dim, device=device) * 0.1).requires_grad_(True) + cu = torch.tensor([0, 17, 35], device=device, dtype=torch.int32) + reference = GatedDeltaNetAttention(heads, dim, qkv_format="thd") + first, state = reference( + *full[:5], cu_seqlens=cu, initial_state=initial_state, output_final_state=True + ) + second = reference(*full[5:], cu_seqlens=cu, initial_state=state) + grad_first, grad_second = torch.randn_like(first), torch.randn_like(second) + ((first * grad_first).sum() * 0.3 + (second * grad_second).sum()).backward() + lo, hi = rank * tokens // size, (rank + 1) * tokens // size + inputs = tuple(t[lo:hi].detach().requires_grad_(True) for t in full) + local_initial_state = initial_state.detach().requires_grad_(True) + module = GatedDeltaNetAttention(heads, dim, qkv_format="thd", cp_group=group) + local_first, state = module( + *inputs[:5], cu_seqlens=cu, initial_state=local_initial_state, output_final_state=True + ) + local_second = module(*inputs[5:], cu_seqlens=cu, initial_state=state) + loss = (local_first * grad_first[lo:hi]).sum() * 0.3 + loss = loss + (local_second * grad_second[lo:hi]).sum() + for retain in (True, False): + loss.backward(retain_graph=retain) + for actual, expected in zip(inputs, full): + _assert_rms_close( + actual.grad, + expected.grad[lo:hi], + _BWD_TOL[torch.bfloat16], + "chained token gradient", + ) + initial_grad = local_initial_state.grad.clone() + dist.all_reduce(initial_grad, group=group) + _assert_rms_close( + initial_grad, + initial_state.grad, + _BWD_TOL[torch.bfloat16], + "chained initial-state gradient", + ) + for tensor in (*inputs, local_initial_state): + tensor.grad = None + dist.barrier(group=group) + + +def main(): + """Run one persistent process group to amortize FE compilation across cases.""" + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + dist.init_process_group("nccl") + group = dist.group.WORLD + size = dist.get_world_size(group) + cases = 0 + try: + for lengths in ( + [8, 4, 12], # Shared and complete documents on the same rank. + [16, 4, 4], # One document spans several ranks. + [6] * size, # Document and rank boundaries coincide. + [0, 17, 0, 18, 0], # Empty documents between nonempty documents. + [1, 0, 1], # Some ranks own no tokens. + [0, 0], # Entirely empty input. + ): + _case(group, lengths) + cases += 1 + lengths = torch.randint( + 32, 129, (4,), generator=torch.Generator().manual_seed(1234) + ).tolist() + _case(group, lengths, heads=8, norm=True, independent=True) + cases += 1 + for dtype in (torch.float16, torch.bfloat16): + for norm in (False, True): + _case( + group, + [size * 65 + 3, 7], + dtype=dtype, + value_dim=128, + norm=norm, + independent=True, + ) + cases += 1 + _case( + group, + [size * 65 + 3, 7], + dtype=torch.float16, + key_dim=128, + value_dim=64, + independent=True, + ) + cases += 1 + for layout, dtype in (("bshd", torch.bfloat16), ("sbhd", torch.float16)): + _case( + group, + [size * 65] * 4, + dtype=dtype, + heads=8, + layout=layout, + norm=True, + dense_global=layout == "bshd", + ) + cases += 1 + for layout in ("bshd", "sbhd"): + _case(group, [size * 17 + 3] * 4, heads=8, layout=layout, dense_global=True) + cases += 1 + _case(group, [1, 1], layout="bshd", dense_global=True, checkpoint=True) + cases += 1 + for initial, final, checkpoint, last_loss in ( + (False, False, False, True), + (True, False, False, True), + (False, True, True, True), + (True, True, True, False), + ): + _case( + group, + [size * 67], + initial=initial, + final=final, + checkpoint=checkpoint, + last_loss=last_loss, + ) + cases += 1 + _case(group, [size * 65 + 1, 7], fallback=True, norm=True) + cases += 1 + _case(group, [size * 65 + 1, 7], state_only=True) + cases += 1 + # Irregular features are zero-extended internally because ordinary FE + # state gradients are incorrect for some unpadded cuTile geometries. + _case(group, [size * 65 + 1, 7], key_dim=32, value_dim=96, norm=True, independent=True) + cases += 1 + # Larger cuTile geometries have no dedicated FROST summary engine; + # exercise the real capability fallback, rather than a mocked error. + if importlib.util.find_spec("cuda.tile") is not None: + _case(group, [131, 7], key_dim=256, value_dim=256, independent=True) + cases += 1 + _inference_and_deferred_group(group) + cases += 1 + _state_chaining(group) + cases += 1 + for layout, lengths in (("thd", [17, 13]), ("bshd", [size * 4] * 2)): + _case(group, lengths, layout=layout, inference_warmup=True, independent=True) + cases += 1 + _invalid_metadata(group) + cases += 1 + # Independent complete documents also match ordinary local attention. + _case(group, [8] * size, initial=False, final=False) + cases += 1 + # A size-one group follows the ordinary local path, even while WORLD + # contains several ranks. Every process creates groups in the same order. + singletons = [dist.new_group([r]) for r in range(size)] + singleton = singletons[dist.get_rank(group)] + _case(singleton, [17, 13]) + _case(singleton, [17, 17], layout="bshd", dense_global=True) + cases += 2 + if size >= 4: + cp_groups = [dist.new_group([r, r + 1]) for r in range(0, size, 2)] + tp_groups = [dist.new_group(list(range(r, size, 2))) for r in (0, 1)] + rank = dist.get_rank() + _case(cp_groups[rank // 2], [139, 13], tp_group=tp_groups[rank % 2]) + cases += 1 + if dist.get_rank() == 0: + print( + f"GDN CP: {cases} correctness cases passed with CP={size}", + flush=True, + ) + except BaseException: # pylint: disable=broad-exception-caught + # A failed rank must exit before a peer waits forever for its next + # collective. NCCL's orderly shutdown can wait on that missing peer. + traceback.print_exc() + os._exit(1) + else: + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/tests/pytorch/attention/test_gdn_attention.py b/tests/pytorch/attention/test_gdn_attention.py index 8b0b3898956..6f2496ba534 100644 --- a/tests/pytorch/attention/test_gdn_attention.py +++ b/tests/pytorch/attention/test_gdn_attention.py @@ -325,7 +325,13 @@ def test_gdn_dense_layout(qkv_format, dtype, compile_model): if compile_model: torch._dynamo.reset() attention = torch.compile(attention, fullgraph=True) - output = attention(q, k, v, g=g, beta=beta, use_qk_l2norm_in_kernel=True) + # Use BF16 for explicit cu_seqlens and FP16 for inferred offsets to cover both paths. + cu_seqlens = ( + torch.arange(batch + 1, device=q.device, dtype=torch.int32) * sequence + if dtype == torch.bfloat16 + else None + ) + output = attention(q, k, v, g=g, beta=beta, cu_seqlens=cu_seqlens, use_qk_l2norm_in_kernel=True) assert output.shape == expected.shape _assert_rms_close(output, expected, _FWD_TOL[dtype], "output") diff --git a/tests/pytorch/attention/test_gdn_attention_with_cp.py b/tests/pytorch/attention/test_gdn_attention_with_cp.py new file mode 100644 index 00000000000..3a5072edbf5 --- /dev/null +++ b/tests/pytorch/attention/test_gdn_attention_with_cp.py @@ -0,0 +1,59 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""End-to-end numerical tests for contiguous GDN context parallelism.""" + +import os +from pathlib import Path +import subprocess +import sys +import tempfile + +import pytest +import torch + +from test_gdn_attention import _GDN_AVAILABLE +import transformer_engine + + +def test_gdn_with_cp(): + """Run genuine NCCL CP with all token and state gradients checked numerically.""" + available = torch.cuda.device_count() + # Test only the largest supported CP size available, rather than all sizes, to reduce test time. + size = next((size for size in (8, 4, 2) if available >= size), None) + if size is None: + pytest.skip(f"GDN CP test requires at least 2 GPUs, but found {available}") + if not _GDN_AVAILABLE: + if os.getenv("NVTE_GDN_CP_TEST_REQUIRED", "0") == "1": + pytest.fail("Required GDN CP test needs the cuDNN frontend GDN runtime") + pytest.skip("GDN CP test requires the cuDNN frontend GDN runtime") + runner = Path(__file__).with_name("run_gdn_with_cp.py") + env = os.environ.copy() + # torchrun starts a script, so its sys.path[0] is the tests directory. + # Preserve the TE package that pytest imported, including source worktrees. + package_root = str(Path(transformer_engine.__file__).resolve().parents[1]) + env["PYTHONPATH"] = os.pathsep.join((package_root, env.get("PYTHONPATH", ""))) + with tempfile.TemporaryDirectory(prefix="te-gdn-cp-") as log_dir: + log_path = Path(log_dir) / "worker.log" + print(f"CP={size} progress log: {log_path}", flush=True) + with log_path.open("w", encoding="utf-8") as log: + result = subprocess.run( + [ + sys.executable, + "-m", + "torch.distributed.run", + "--standalone", + f"--nproc_per_node={size}", + str(runner), + ], + check=False, + text=True, + stdout=log, + stderr=subprocess.STDOUT, + timeout=1200, + env=env, + ) + output = log_path.read_text() + assert result.returncode == 0, output + print(output) diff --git a/transformer_engine/common/CMakeLists.txt b/transformer_engine/common/CMakeLists.txt index f90f74449c5..c0faeed65ba 100644 --- a/transformer_engine/common/CMakeLists.txt +++ b/transformer_engine/common/CMakeLists.txt @@ -300,6 +300,7 @@ list(APPEND transformer_engine_cuda_sources transpose/swap_first_dims.cu dropout/dropout.cu fused_attn/context_parallel.cu + linear_attention/affine_prefix.cu fused_attn/kv_cache.cu fused_attn/fused_attn_f16_arbitrary_seqlen.cu fused_attn/fused_attn_fp8.cu diff --git a/transformer_engine/common/include/transformer_engine/linear_attention.h b/transformer_engine/common/include/transformer_engine/linear_attention.h new file mode 100644 index 00000000000..f9d4f81ecc3 --- /dev/null +++ b/transformer_engine/common/include/transformer_engine/linear_attention.h @@ -0,0 +1,52 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +/*! \file linear_attention.h + * \brief Functions for linear-attention state composition. + */ +#ifndef TRANSFORMER_ENGINE_LINEAR_ATTENTION_H_ +#define TRANSFORMER_ENGINE_LINEAR_ATTENTION_H_ + +#include "transformer_engine.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/*! \brief Compose recurrent states from ordered affine summaries. + * + * Each output starts from a selected initial memory and repeatedly evaluates + * ``state = state @ transitions[summary_index] + additive[summary_index]``. This operation + * serves both forward prefixes and backward suffixes; backward callers pass + * the backward-oriented transitions and a negative record stride. + * + * All tensors must be contiguous device tensors. No memory is allocated. + * + * \param[in] initial_states FP32 initial memories, + * [initial_memories, heads, value_dim, key_dim]. + * \param[in] additive FP32 additive summaries, + * [summaries, heads, value_dim, key_dim]. + * \param[in] transitions FP32 transition matrices, + * [summaries, heads, key_dim, key_dim]. + * \param[in] descriptors INT32 [outputs, 4], with rows + * (initial_memory_index, first_summary, count, stride). + * An initial_memory_index of -1 selects an implicit zero state. + * Other referenced memory and summary indices must be valid. + * Count must be nonnegative; zero copies the initial memory + * (or writes zeros). + * \param[out] output Caller-allocated FP32 composed memories, + * [outputs, heads, value_dim, key_dim]. + * \param[in] stream CUDA stream on which to execute the operation. + */ +void nvte_linear_attention_affine_prefix(const NVTETensor initial_states, const NVTETensor additive, + const NVTETensor transitions, const NVTETensor descriptors, + NVTETensor output, cudaStream_t stream); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TRANSFORMER_ENGINE_LINEAR_ATTENTION_H_ diff --git a/transformer_engine/common/linear_attention/__init__.py b/transformer_engine/common/linear_attention/__init__.py new file mode 100644 index 00000000000..efdd90951cb --- /dev/null +++ b/transformer_engine/common/linear_attention/__init__.py @@ -0,0 +1,158 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Metadata for linear-attention partitions with contiguous token ownership""" + +from bisect import bisect_right +from dataclasses import dataclass +from typing import Tuple + + +@dataclass(frozen=True) +class LinearAttentionFragment: + """Metadata for a fragment: the intersection of one independent document + with one rank's token interval. + + For example, global cu_seqlens [0, 8, 12, 24] describes documents + A [0, 8), B [8, 12), and C [12, 24). With CP=4, each rank owns six + consecutive tokens. Each row below describes one fragment: + + Rank Rank interval Document start end first_rank last_rank + 0 [0, 6) A (0) 0 6 0 1 + 1 [6, 12) A (0) 6 8 0 1 + 1 [6, 12) B (1) 8 12 1 1 + 2 [12, 18) C (2) 12 18 2 3 + 3 [18, 24) C (2) 18 24 2 3 + + ``start`` is inclusive and ``end`` is exclusive; both are global offsets. + ``first_rank`` and ``last_rank`` identify the first and last ranks owning + tokens of the whole document, not just this fragment. + """ + + document: int + start: int + end: int + first_rank: int + last_rank: int + + +@dataclass(frozen=True) +class LinearAttentionCPHostMetadata: + """Global document boundaries and local fragment metadata for one CP rank. + + Packed (THD) inputs intersect documents with contiguous rank intervals. + Dense (BSHD/SBHD) inputs give each rank a nearly equal slice of every document. + Empty documents have no fragments; their state is unchanged. + + Using the fragment example from LinearAttentionFragment, packed rank 1 owns: + + fragments: A [6, 8), B [8, 12) + local_cu_seqlens: (0, 2, 6) + forward_descriptors: ((0, 0, 1, 1), (1, 1, 0, 1)) + incoming_fragment_indices: (0,) # A continues from rank 0. + outgoing_fragment_indices: () # Neither document continues to a later rank. + terminal_fragment_indices: (0, 1) # Both documents end on this rank. + """ + + cu_seqlens: Tuple[int, ...] # Global cumulative sequence lengths. + cp_rank: int + cp_size: int + fragments: Tuple[LinearAttentionFragment, ...] + local_cu_seqlens: Tuple[int, ...] # Cumulative lengths of nonempty local fragments. + # Select preceding summaries to compose each fragment's starting memory. + forward_descriptors: Tuple[Tuple[int, int, int, int], ...] + # Select following reverse summaries to compose each fragment's ending-state gradient. + backward_descriptors: Tuple[Tuple[int, int, int, int], ...] + outgoing_fragment_indices: Tuple[int, ...] # Document continues to a later rank. + incoming_fragment_indices: Tuple[int, ...] # Document starts on an earlier rank. + terminal_fragment_indices: Tuple[int, ...] # Document ends on this rank. + initial_state_owners: Tuple[int, ...] # Initial-state owner rank for each global document. + empty_owned: Tuple[int, ...] # Empty global document IDs assigned to this rank. + slots: int # One per rank for packed inputs; one per document for dense inputs. + communicates: bool # Any global document spans multiple ranks. + + +def build_linear_attention_cp_host_metadata(cu_seqlens, cp_size, cp_rank, *, dense=False): + """Build one rank's fragment and summary metadata on the host. + + ``cu_seqlens`` contains full-document boundaries as host integers. For + P = ``cp_size``, packed rank r owns [r*T//P:(r+1)*T//P] of T total tokens. + Dense rank r owns [r*S//P:(r+1)*S//P] of each length-S document. + """ + cu_seqlens = tuple(cu_seqlens) + if cp_size < 1 or not 0 <= cp_rank < cp_size: + raise ValueError("CP size must be positive and rank must lie inside the group.") + if len(cu_seqlens) < 2 or cu_seqlens[0] != 0: + raise ValueError("cu_seqlens must start at zero and contain at least two entries.") + if any(a > b for a, b in zip(cu_seqlens, cu_seqlens[1:])): + raise ValueError("cu_seqlens must be nondecreasing.") + total = cu_seqlens[-1] + boundaries = tuple(r * total // cp_size for r in range(cp_size + 1)) + slots = len(cu_seqlens) - 1 if dense else 1 + fragments, initial_state_owners, empty = [], [], [] + communicates = False + for doc, (start, end) in enumerate(zip(cu_seqlens, cu_seqlens[1:])): + if dense: + length = end - start + if length != cu_seqlens[1]: + raise ValueError("Dense CP needs equal global document lengths.") + cuts = tuple(r * length // cp_size for r in range(cp_size + 1)) + first_rank = min(cp_size - 1, bisect_right(cuts, 0) - 1) if length else 0 + last_rank = min(cp_size - 1, bisect_right(cuts, length - 1) - 1) if length else 0 + fragment_start, fragment_end = start + cuts[cp_rank], start + cuts[cp_rank + 1] + else: + first_rank = min(cp_size - 1, bisect_right(boundaries, start) - 1) + last_rank = ( + min(cp_size - 1, bisect_right(boundaries, end - 1) - 1) + if end > start + else first_rank + ) + fragment_start = max(start, boundaries[cp_rank]) + fragment_end = min(end, boundaries[cp_rank + 1]) + initial_state_owners.append(first_rank) + communicates |= first_rank != last_rank + if start == end and first_rank == cp_rank: + empty.append(doc) + if fragment_start < fragment_end: + fragments.append( + LinearAttentionFragment(doc, fragment_start, fragment_end, first_rank, last_rank) + ) + cu = [0] + # Descriptor rows are (document, first_summary, count, stride). Each selects + # summaries of one document, in forward or reverse order. + forward_desc, backward_desc = [], [] + outgoing_fragment_indices = [] + incoming_fragment_indices = [] + terminal_fragment_indices = [] + for index, frag in enumerate(fragments): + cu.append(cu[-1] + frag.end - frag.start) + slot = frag.document if dense else 0 + forward_desc.append( + (frag.document, frag.first_rank * slots + slot, cp_rank - frag.first_rank, slots) + ) + backward_desc.append( + (frag.document, frag.last_rank * slots + slot, frag.last_rank - cp_rank, -slots) + ) + if cp_rank < frag.last_rank: + outgoing_fragment_indices.append(index) + else: + terminal_fragment_indices.append(index) + if cp_rank > frag.first_rank: + incoming_fragment_indices.append(index) + return LinearAttentionCPHostMetadata( + cu_seqlens, + cp_rank, + cp_size, + tuple(fragments), + tuple(cu), + tuple(forward_desc), + tuple(backward_desc), + tuple(outgoing_fragment_indices), + tuple(incoming_fragment_indices), + tuple(terminal_fragment_indices), + tuple(initial_state_owners), + tuple(empty), + slots, + communicates, + ) diff --git a/transformer_engine/common/linear_attention/affine_prefix.cu b/transformer_engine/common/linear_attention/affine_prefix.cu new file mode 100644 index 00000000000..dc3363c97b9 --- /dev/null +++ b/transformer_engine/common/linear_attention/affine_prefix.cu @@ -0,0 +1,146 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +#include + +#include "../common.h" +#include "transformer_engine/linear_attention.h" + +namespace transformer_engine { +namespace { + +// Reuse transition loads across value rows, preserving FP32 accumulation. +constexpr int kRows = 4; + +// Each block composes one value-row tile of a fragment's recurrent state for one head. +__global__ void affine_prefix_kernel(const float *initial_states, const float *additive, + const float *transitions, const int *descriptors, + float *output, size_t num_initial_states, size_t num_records, + size_t heads, size_t value_dim, size_t key_dim) { + // Shared memory bounce buffers to alternate between state and next_state + extern __shared__ float storage[]; + float *state = storage; + float *next_state = storage + kRows * key_dim; + const size_t value_tiles = (value_dim + kRows - 1) / kRows; + // Each fragment/head pair owns one block per tile of value rows. + const size_t fragment = blockIdx.x / (heads * value_tiles); + const size_t head = (blockIdx.x / value_tiles) % heads; + const size_t first_value_row = (blockIdx.x % value_tiles) * kRows; + const int *descriptor = descriptors + fragment * 4; + const int initial_state_index = descriptor[0], first_record = descriptor[1], + record_count = descriptor[2], stride = descriptor[3]; + // Confirm the descriptor is valid before the block uses it. + assert((initial_state_index == -1 || + (initial_state_index >= 0 && + static_cast(initial_state_index) < num_initial_states)) && + record_count >= 0); + // An implicit zero initial state makes the first matrix product unnecessary. + const int first_step = initial_state_index == -1 && record_count > 0 ? 1 : 0; + for (size_t col = threadIdx.x; col < key_dim; col += blockDim.x) { + for (int row = 0; row < kRows; ++row) { + const size_t value = first_value_row + row; + float initial = 0.f; + if (value < value_dim) { + if (first_step) { + assert(first_record >= 0 && static_cast(first_record) < num_records); + // H[first_record, head, value, col]: summary record, head, value row, key column. + // Flat offset = first_record*(heads*value_dim*key_dim) + head*(value_dim*key_dim) + // + value*key_dim + col. + initial = additive[((first_record * heads + head) * value_dim + value) * key_dim + col]; + } else if (initial_state_index >= 0) { + initial = + initial_states[((initial_state_index * heads + head) * value_dim + value) * key_dim + + col]; + } + } + state[row * key_dim + col] = initial; + } + } + // Make the initialized state visible to all threads before it is read. + __syncthreads(); + for (int step = first_step; step < record_count; ++step) { + const int record = first_record + step * stride; + assert(record >= 0 && static_cast(record) < num_records); + const float *transition_matrix = transitions + (record * heads + head) * key_dim * key_dim; + // Each thread accumulates H + state @ T for its key columns in registers. + for (size_t col = threadIdx.x; col < key_dim; col += blockDim.x) { + float result[kRows]; + for (int row = 0; row < kRows; ++row) { + const size_t value = first_value_row + row; + result[row] = value < value_dim + ? additive[((record * heads + head) * value_dim + value) * key_dim + col] + : 0.f; + } + // Reuse each transition element across the tile's value rows. + for (size_t inner = 0; inner < key_dim; ++inner) { + const float transition = transition_matrix[inner * key_dim + col]; +#pragma unroll + for (int row = 0; row < kRows; ++row) { + result[row] = fmaf(state[row * key_dim + inner], transition, result[row]); + } + } + for (int row = 0; row < kRows; ++row) next_state[row * key_dim + col] = result[row]; + } + // Complete all shared-memory reads and writes before swapping state buffers. + __syncthreads(); + float *swap = state; + state = next_state; + next_state = swap; + } + for (size_t col = threadIdx.x; col < key_dim; col += blockDim.x) { + for (int row = 0; row < kRows; ++row) { + const size_t value = first_value_row + row; + if (value < value_dim) { + output[((fragment * heads + head) * value_dim + value) * key_dim + col] = + state[row * key_dim + col]; + } + } + } +} + +} // namespace +} // namespace transformer_engine + +void nvte_linear_attention_affine_prefix(const NVTETensor initial_states, const NVTETensor additive, + const NVTETensor transitions, const NVTETensor descriptors, + NVTETensor output, cudaStream_t stream) { + NVTE_API_CALL(nvte_linear_attention_affine_prefix); + using namespace transformer_engine; + const auto &initial_data = convertNVTETensorCheck(initial_states)->data; + const auto &additive_data = convertNVTETensorCheck(additive)->data; + const auto &transition_data = convertNVTETensorCheck(transitions)->data; + const auto &descriptor_data = convertNVTETensorCheck(descriptors)->data; + auto &output_data = convertNVTETensorCheck(output)->data; + NVTE_CHECK(initial_data.dtype == DType::kFloat32 && additive_data.dtype == DType::kFloat32 && + transition_data.dtype == DType::kFloat32 && output_data.dtype == DType::kFloat32, + "Affine states and summaries must be FP32."); + NVTE_CHECK(descriptor_data.dtype == DType::kInt32, "Affine descriptors must be INT32."); + NVTE_CHECK(initial_data.shape.size() == 4 && additive_data.shape.size() == 4 && + transition_data.shape.size() == 4 && output_data.shape.size() == 4 && + descriptor_data.shape.size() == 2, + "Invalid affine tensor ranks."); + const size_t heads = initial_data.shape[1], value_dim = initial_data.shape[2], + key_dim = initial_data.shape[3]; + NVTE_CHECK(heads > 0 && value_dim > 0 && key_dim > 0, "State dimensions must be positive."); + NVTE_CHECK((additive_data.shape == + std::vector{transition_data.shape[0], heads, value_dim, key_dim}) && + (transition_data.shape == + std::vector{additive_data.shape[0], heads, key_dim, key_dim}) && + (output_data.shape == + std::vector{descriptor_data.shape[0], heads, value_dim, key_dim}) && + descriptor_data.shape[1] == 4, + "Inconsistent affine tensor shapes."); + if (output_data.shape[0] == 0) return; + const size_t value_tiles = output_data.shape[0] * heads * ((value_dim + kRows - 1) / kRows); + // Extra threads have no key columns to process when K is at most 64. + const int threads = key_dim <= 64 ? 64 : 128; + affine_prefix_kernel<<>>( + static_cast(initial_data.dptr), static_cast(additive_data.dptr), + static_cast(transition_data.dptr), + static_cast(descriptor_data.dptr), static_cast(output_data.dptr), + initial_data.shape[0], additive_data.shape[0], heads, value_dim, key_dim); + NVTE_CHECK_CUDA(cudaGetLastError()); +} diff --git a/transformer_engine/pytorch/attention/linear_attention/base.py b/transformer_engine/pytorch/attention/linear_attention/base.py index f1dfe199eb4..0e339416341 100644 --- a/transformer_engine/pytorch/attention/linear_attention/base.py +++ b/transformer_engine/pytorch/attention/linear_attention/base.py @@ -65,6 +65,8 @@ def _needs_eager_linear_attention(call: Dict[str, Any]) -> Optional[str]: `call` maps the module's `forward` parameter names to the arguments this call passed, including `self`. """ + if getattr(call.get("self"), "cp_group", None) is not None: + return "linear-attention context parallelism" if call.get("checkpoint_core_attention", False): return "activation checkpointing of the attention" return None @@ -269,20 +271,16 @@ def _validate_initial_state( f"got {tuple(initial_state.shape)}." ) - def _run( + def _validate_inputs( self, query_layer: torch.Tensor, key_layer: torch.Tensor, value_layer: torch.Tensor, gates: Dict[str, torch.Tensor], - initial_state: Optional[torch.Tensor] = None, - *, + initial_state: Optional[torch.Tensor], qkv_format: str, - cu_seqlens: Optional[torch.Tensor] = None, - output_final_state: bool = False, - **kernel_kwargs: Any, - ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: - """Validate the inputs, run the variant's op, and restore the TE layout.""" + ) -> None: + """Validate tensor/layout contracts for local and context-parallel adapters.""" if qkv_format not in {"thd", "bshd", "sbhd"}: raise ValueError( f"{self.variant} attention only supports qkv_format={{'thd', 'bshd', 'sbhd'}}, " @@ -318,6 +316,49 @@ def _run( ) self._validate_gates(gates, query_layer) + def _prepare_cp_inputs(self, tensors, initial_state): + """Normalize token-aligned CP inputs before state propagation. + + Each tensor must retain a real-token leading axis for fragment selection; + additional axes, such as GDP's Householder axis, stay intact until FE dispatch. + """ + return tensors, initial_state + + def _cp_forward_summary(self, query, key, value, gates, cu_seqlens, **kernel_kwargs): + """Return FP32 (H, T) satisfying S_out = S_in @ T + H for each fragment. + + H has shape [fragments, heads, V, K] and T [fragments, heads, K, K]. + Summaries and local attention must use identical gate and normalization semantics. + """ + raise NotImplementedError(f"{self.variant} does not implement CP forward summaries.") + + def _cp_backward_summary( + self, query, key, value, gates, grad_output, cu_seqlens, **kernel_kwargs + ): + """Return FP32 reverse state-gradient maps, including local output losses. + + H and T have the forward summary shapes, with T already oriented for backward. + Gate and normalization semantics must match local attention. + """ + raise NotImplementedError(f"{self.variant} does not implement CP backward summaries.") + + def _run( + self, + query_layer: torch.Tensor, + key_layer: torch.Tensor, + value_layer: torch.Tensor, + gates: Dict[str, torch.Tensor], + initial_state: Optional[torch.Tensor] = None, + *, + qkv_format: str, + cu_seqlens: Optional[torch.Tensor] = None, + output_final_state: bool = False, + **kernel_kwargs: Any, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + """Validate the inputs, run the variant's op, and restore the TE layout.""" + self._validate_inputs(query_layer, key_layer, value_layer, gates, initial_state, qkv_format) + device = query_layer.device + cu_seqlens, sequence_length = self._resolve_cu_seqlens(query_layer, qkv_format, cu_seqlens) batch_size = cu_seqlens.shape[0] - 1 self._validate_initial_state(initial_state, batch_size=batch_size, device=device) diff --git a/transformer_engine/pytorch/attention/linear_attention/context_parallel.py b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py new file mode 100644 index 00000000000..cd9e27d0119 --- /dev/null +++ b/transformer_engine/pytorch/attention/linear_attention/context_parallel.py @@ -0,0 +1,527 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Context parallelism for linear attention with contiguous token ownership. + +Coordinates fragment metadata, affine-summary exchange, state composition, +and distributed backward. Variant adapters provide local attention and summary +operations; currently only GDN is supported. +""" + +from collections import OrderedDict +from dataclasses import dataclass +from threading import Lock + +import torch +import torch.distributed as dist +from torch.autograd.function import once_differentiable + +import transformer_engine_torch as tex +from transformer_engine.common.linear_attention import ( + LinearAttentionCPHostMetadata, + build_linear_attention_cp_host_metadata, +) +from .base import _to_thd + + +_CP_METADATA_CACHE_SIZE = 8 +_cp_metadata_cache = OrderedDict() +_cp_metadata_cache_lock = Lock() + + +def _get_shared_cp_metadata(key): + """Look up prepared rank metadata shared by attention modules on one stream.""" + with _cp_metadata_cache_lock: + entry = _cp_metadata_cache.get(key) + if entry is not None: + _cp_metadata_cache.move_to_end(key) + return entry + + +def _cache_cp_metadata(entry): + """Retain at most eight prepared entries, including their input metadata.""" + with _cp_metadata_cache_lock: + key = entry[0] + _cp_metadata_cache[key] = entry + _cp_metadata_cache.move_to_end(key) + if len(_cp_metadata_cache) > _CP_METADATA_CACHE_SIZE: + _cp_metadata_cache.popitem(last=False) + + +@dataclass +class _LinearAttentionCPDeviceMetadata: + """Device metadata for one CP rank's local attention and state composition. + + For global cu_seqlens [0, 8, 12, 24] and CP=4, rank 1 owns fragments + A [6, 8) and B [8, 12). Its device tensors include:: + + cu_seqlens: [0, 2, 6] + document_indices: [0, 1] + forward_descriptors: [[0, 0, 1, 1], [1, 1, 0, 1]] + incoming_fragment_indices: [0] + incoming_cu_seqlens: [0, 2] + initial_state_fragments: [1] + initial_state_documents: [1] + + Fragment 0 receives memory from rank 0; fragment 1 starts document B + on this rank and maps to global document state 1. + """ + + cu_seqlens: torch.Tensor # Local cumulative fragment lengths on the device. + forward_descriptors: torch.Tensor # Starting-memory composition recipes. + backward_descriptors: torch.Tensor # Ending-state-gradient composition recipes. + # Zero variants set descriptor column 0 to -1 for an implicit zero memory. + forward_zero_descriptors: torch.Tensor + backward_zero_descriptors: torch.Tensor + document_indices: torch.Tensor # Global document ID for each local fragment. + outgoing_fragment_indices: torch.Tensor # Document continues on a later rank. + incoming_fragment_indices: torch.Tensor # Document starts on an earlier rank. + terminal_fragment_indices: torch.Tensor # Document ends on this rank. + # Paired fragment/document indices for documents starting on this rank. + initial_state_fragments: torch.Tensor + initial_state_documents: torch.Tensor + empty_documents: torch.Tensor # Global IDs of empty documents owned by this rank. + # Cumulative lengths within the selected spans passed to cuDNN summary calls. + outgoing_cu_seqlens: torch.Tensor + incoming_cu_seqlens: torch.Tensor + + +@dataclass +class _LinearAttentionCPMetadata: + """Host and device metadata prepared for one CP rank.""" + + host_metadata: LinearAttentionCPHostMetadata + device_metadata: _LinearAttentionCPDeviceMetadata + + +def _build_linear_attention_cp_device_metadata(host_metadata, device): + """Build device metadata tensors from one rank's host metadata.""" + + def indices(values): + return torch.tensor(values, device=device, dtype=torch.int64) + + def cumulative(selected): + offsets = [0] + for index in selected: + frag = host_metadata.fragments[index] + offsets.append(offsets[-1] + frag.end - frag.start) + return torch.tensor(offsets, device=device, dtype=torch.int32) + + first = [ + i for i, f in enumerate(host_metadata.fragments) if f.first_rank == host_metadata.cp_rank + ] + forward_descriptors = torch.tensor( + host_metadata.forward_descriptors, device=device, dtype=torch.int32 + ).reshape(-1, 4) + backward_descriptors = torch.tensor( + host_metadata.backward_descriptors, device=device, dtype=torch.int32 + ).reshape(-1, 4) + forward_zero_descriptors, backward_zero_descriptors = ( + forward_descriptors.clone(), + backward_descriptors.clone(), + ) + forward_zero_descriptors[:, 0] = -1 + backward_zero_descriptors[:, 0] = -1 + return _LinearAttentionCPDeviceMetadata( + torch.tensor(host_metadata.local_cu_seqlens, device=device, dtype=torch.int32), + forward_descriptors, + backward_descriptors, + forward_zero_descriptors, + backward_zero_descriptors, + indices([f.document for f in host_metadata.fragments]), + indices(host_metadata.outgoing_fragment_indices), + indices(host_metadata.incoming_fragment_indices), + indices(host_metadata.terminal_fragment_indices), + indices(first), + indices([host_metadata.fragments[i].document for i in first]), + indices(host_metadata.empty_owned), + cumulative(host_metadata.outgoing_fragment_indices), + cumulative(host_metadata.incoming_fragment_indices), + ) + + +def _gather_summaries(additive, transition, group): + """One fixed-size all-gather for both summary tensors, in CP group rank order.""" + slots, heads, value_dim, key_dim = additive.shape + cp_size = dist.get_world_size(group) + packed = torch.cat((additive.flatten(), transition.flatten())) + gathered = torch.empty(cp_size * packed.numel(), device=packed.device, dtype=packed.dtype) + dist.all_gather_into_tensor(gathered, packed, group=group) + gathered = gathered.view(cp_size, -1) + split = additive.numel() + return ( + gathered[:, :split].contiguous().view(cp_size * slots, heads, value_dim, key_dim), + gathered[:, split:].contiguous().view(cp_size * slots, heads, key_dim, key_dim), + ) + + +def _summary_buffers(state_template, slots): + """Unused slots are identity recipes, including ranks owning zero tokens.""" + _, heads, value_dim, key_dim = state_template.shape + additive = state_template.new_zeros((slots, heads, value_dim, key_dim)) + transition = torch.eye(key_dim, device=state_template.device, dtype=state_template.dtype) + return additive, transition.expand(slots, heads, key_dim, key_dim).contiguous() + + +def _selected_span(host_metadata, selected): + """Selected spans are consecutive: all dense spans, or one packed edge span.""" + cu_seqlens = host_metadata.local_cu_seqlens + return slice(cu_seqlens[selected[0]], cu_seqlens[selected[-1] + 1]) + + +def _slot_indices(metadata, indices): + """Dense slots are document IDs; packed partitions have only one edge slot.""" + if metadata.host_metadata.slots == 1: + return torch.zeros_like(indices) + return metadata.device_metadata.document_indices.index_select(0, indices) + + +def _local_attention( + adapter, tensors, gate_names, cu_seqlens, state, options, *, output_final_state=True +): + """Call the linear attention variant's adapter with packed local inputs.""" + return adapter._call_op( + *tensors[:3], + dict(zip(gate_names, tensors[3:])), + cu_seqlens, + initial_state=state, + output_final_state=output_final_state, + **options, + ) + + +class _LinearAttentionContextParallel(torch.autograd.Function): + """A distributed first-order VJP, with full local attention retained for FE backward. + + Summary APIs intentionally do not have autograd registrations. Differentiating + an all-gather of summaries therefore cannot recover cross-rank gradients. + The reverse summary exchange supplies the missing final-state adjoints to FE. + + Kernel options carry configuration, not trainable parameters. Non-token + parameters need explicit gradient inputs/returns or differentiable token-gate + preparation before this Function, with reductions matching parameter ownership. + """ + + @staticmethod + def forward( + ctx, + adapter, + metadata, + group, + options, + output_final_state, + gate_names, + track_grad, + initial_state, + *tensors, + ): + """Establish starting states, then retain the local attention graph.""" + ctx.set_materialize_grads(False) + host_metadata = metadata.host_metadata + device_metadata = metadata.device_metadata + q = tensors[0] + k = tensors[1] + v = tensors[2] + state_shape = ( + len(host_metadata.cu_seqlens) - 1, + q.shape[1], + v.shape[-1], + k.shape[-1], + ) + # Zero-prefix descriptors need only one placeholder when no initial state is supplied. + initial_state_buffer = ( + q.new_zeros((1, *state_shape[1:]), dtype=torch.float32) + if initial_state is None + else initial_state + ) + additive, transition = _summary_buffers(initial_state_buffer, host_metadata.slots) + if host_metadata.outgoing_fragment_indices: + span = _selected_span(host_metadata, host_metadata.outgoing_fragment_indices) + selected = tuple(t[span] for t in tensors) + h, t = adapter._cp_forward_summary( + *selected[:3], + dict(zip(gate_names, selected[3:])), + device_metadata.outgoing_cu_seqlens, + **options, + ) + slots = _slot_indices(metadata, device_metadata.outgoing_fragment_indices) + additive.index_copy_(0, slots, h) + transition.index_copy_(0, slots, t) + if host_metadata.communicates: + additive, transition = _gather_summaries(additive, transition, group) + starting_states = tex.linear_attention_affine_prefix( + initial_state_buffer.contiguous(), + additive, + transition, + ( + device_metadata.forward_descriptors + if initial_state is not None + else device_metadata.forward_zero_descriptors + ), + ) + # Retain FE's local graph only for training. This Function owns the + # distributed VJP and the outer state gradients. + with torch.set_grad_enabled(track_grad): + inputs = tuple( + t.detach().requires_grad_(track_grad) for t in (*tensors, starting_states) + ) + if host_metadata.fragments: + output, states = _local_attention( + adapter, + inputs[:-1], + gate_names, + device_metadata.cu_seqlens, + inputs[-1], + options, + ) + else: + output = v.new_empty((q.shape[0], q.shape[1], v.shape[-1])) + states = starting_states + terminal = ( + initial_state_buffer.new_zeros(state_shape) + if output_final_state + else initial_state_buffer.new_empty((0,)) + ) + if output_final_state: + terminal.index_copy_( + 0, + device_metadata.document_indices.index_select( + 0, device_metadata.terminal_fragment_indices + ), + states.detach().index_select(0, device_metadata.terminal_fragment_indices), + ) + if initial_state is not None: + terminal.index_copy_( + 0, + device_metadata.empty_documents, + initial_state_buffer.index_select(0, device_metadata.empty_documents), + ) + dist.all_reduce(terminal, group=group) + if track_grad: + ctx.save_for_backward(*inputs, output, states) + ctx.cp_metadata, ctx.group, ctx.output_final_state = ( + metadata, + group, + output_final_state, + ) + ctx.adapter, ctx.options, ctx.gate_names = adapter, options, gate_names + ctx.has_initial_state = initial_state is not None + return output.detach(), terminal + + @staticmethod + @once_differentiable + def backward(ctx, grad_output, grad_final): + """Exchange reverse recipes and pass outgoing state adjoints to FE's VJP.""" + tensors = ctx.saved_tensors[:-3] + starting_states, output, states = ctx.saved_tensors[-3:] + q, k, v = tensors[:3] + metadata, group = ctx.cp_metadata, ctx.group + host_metadata = metadata.host_metadata + device_metadata = metadata.device_metadata + if grad_output is None: + grad_output = torch.zeros_like(output) + grad_output = grad_output.contiguous() + shape = ( + len(host_metadata.cu_seqlens) - 1, + q.shape[1], + v.shape[-1], + k.shape[-1], + ) + final_state_grad = starting_states.new_zeros( + shape if ctx.output_final_state else (1, *shape[1:]) + ) + if ctx.output_final_state: + if grad_final is not None: + final_state_grad.copy_(grad_final) + # Final states are replicated outputs: sum losses consuming any replica. + dist.all_reduce(final_state_grad, group=group) + additive, transition = _summary_buffers(final_state_grad, host_metadata.slots) + if host_metadata.incoming_fragment_indices: + span = _selected_span(host_metadata, host_metadata.incoming_fragment_indices) + selected = tuple(t[span] for t in tensors) + h, t = ctx.adapter._cp_backward_summary( + *selected[:3], + dict(zip(ctx.gate_names, selected[3:])), + grad_output[span], + device_metadata.incoming_cu_seqlens, + **ctx.options, + ) + slots = _slot_indices(metadata, device_metadata.incoming_fragment_indices) + additive.index_copy_(0, slots, h) + transition.index_copy_(0, slots, t) + if host_metadata.communicates: + additive, transition = _gather_summaries(additive, transition, group) + ending_state_grad = tex.linear_attention_affine_prefix( + final_state_grad, + additive, + transition, + ( + device_metadata.backward_descriptors + if ctx.output_final_state + else device_metadata.backward_zero_descriptors + ), + ) + if host_metadata.fragments: + with torch.enable_grad(): + grads = torch.autograd.grad( + (output, states), + (*tensors, starting_states), + (grad_output, ending_state_grad), + # The outer graph controls lifetime, including retain_graph=True. + retain_graph=True, + ) + else: + grads = tuple(torch.zeros_like(t) for t in (*tensors, starting_states)) + initial_state_grad = None + if ctx.has_initial_state: + initial_state_grad = final_state_grad.new_zeros(shape) + initial_state_grad.index_add_( + 0, + device_metadata.initial_state_documents, + grads[-1].index_select(0, device_metadata.initial_state_fragments), + ) + if ctx.output_final_state: + initial_state_grad.index_copy_( + 0, + device_metadata.empty_documents, + final_state_grad.index_select(0, device_metadata.empty_documents), + ) + # Each document's initial-state gradient belongs to its first rank. Do not + # replicate it here: if initial_state came from a previous CP call, + # that call's replicated-final-state VJP sums these owner gradients. + # Replicating now would count chained state gradients CP-size times. + return (None, None, None, None, None, None, None, initial_state_grad, *grads[:-1]) + + +def context_parallel_attention( + adapter, + q, + k, + v, + gates, + initial_state, + *, + group, + qkv_format, + cu_seqlens, + output_final_state, + **kernel_kwargs, +): + """Derive local fragment boundaries from full-document metadata and run CP.""" + adapter._validate_inputs(q, k, v, gates, initial_state, qkv_format) + cp_size, cp_rank = dist.get_world_size(group), dist.get_rank(group) + stream = torch.cuda.current_stream(q.device).cuda_stream + # Autograd cannot save metadata tensors created in inference mode. + inference_mode = torch.is_inference_mode_enabled() + tensors = tuple(_to_thd(t, qkv_format).contiguous() for t in (q, k, v, *gates.values())) + dense = qkv_format != "thd" + batch, length = None, None + if dense: + batch, length = q.shape[:2] if qkv_format == "bshd" else (q.shape[1], q.shape[0]) + elif cu_seqlens is None: + raise ValueError("Linear-attention CP with THD requires cu_seqlens for all documents.") + if cu_seqlens is not None: + if not isinstance(cu_seqlens, torch.Tensor): + raise TypeError("cu_seqlens must be a torch.Tensor.") + if cu_seqlens.dtype != torch.int32 or cu_seqlens.dim() != 1: + raise ValueError("cu_seqlens must be a one-dimensional int32 tensor.") + cacheable = not torch.is_inference(cu_seqlens) + key = ( + qkv_format, + tuple(q.shape[:2]), + id(cu_seqlens), + cu_seqlens._version if cacheable else None, + q.device, + cp_size, + cp_rank, + stream, + inference_mode, + ) + else: + key = (qkv_format, batch, length, q.device, cp_size, cp_rank, stream, inference_mode) + cacheable = True + cache = getattr(adapter, "_cp_metadata_cache", None) + if not cacheable: + # Inference tensors have no mutation counter, so revalidate their + # metadata instead of silently reusing a stale entry. + cache = None + elif cache is None or cache[0] != key: + # Layers using the same batch can share metadata without another host round trip. + cache = _get_shared_cp_metadata(key) + # Build and upload metadata only when the cached entry no longer matches. + if cache is None or cache[0] != key: + document_cu_seqlens = ( + tuple(cu_seqlens.tolist()) + if cu_seqlens is not None + else tuple(i * length * cp_size for i in range(batch + 1)) + ) + if dense and len(document_cu_seqlens) != batch + 1: + raise ValueError("Dense cu_seqlens must describe the full batch.") + host_metadata = build_linear_attention_cp_host_metadata( + document_cu_seqlens, cp_size=cp_size, cp_rank=cp_rank, dense=dense + ) + device_metadata = _build_linear_attention_cp_device_metadata(host_metadata, q.device) + metadata = _LinearAttentionCPMetadata(host_metadata, device_metadata) + # Hold the metadata tensor to prevent object-ID reuse in the cache key. + adapter._cp_metadata_cache = (key, metadata, cu_seqlens) + if cacheable: + _cache_cp_metadata(adapter._cp_metadata_cache) + else: + metadata = cache[1] + host_metadata = metadata.host_metadata + device_metadata = metadata.device_metadata + adapter._cp_metadata_cache = cache + if tensors[0].shape[0] != host_metadata.local_cu_seqlens[-1]: + raise ValueError("Local token count does not match this CP rank's token interval.") + adapter._validate_initial_state( + initial_state, + batch_size=len(host_metadata.cu_seqlens) - 1, + device=q.device, + ) + tensors, initial_state = adapter._prepare_cp_inputs(tensors, initial_state) + # Resolve the cuDNN operation once and reuse it on later calls. + if adapter._op is None: + adapter._op = adapter._resolve_op() + # Without shared documents or explicit states, use ordinary local autograd. + if ( + not host_metadata.communicates + and initial_state is None + and not output_final_state + and host_metadata.fragments + ): + output, states = _local_attention( + adapter, + tensors, + tuple(gates), + device_metadata.cu_seqlens, + None, + kernel_kwargs, + output_final_state=False, + ) + else: + track_grad = torch.is_grad_enabled() and any( + t is not None and t.requires_grad for t in (*tensors, initial_state) + ) + output, states = _LinearAttentionContextParallel.apply( + adapter, + metadata, + group, + kernel_kwargs, + output_final_state, + tuple(gates), + track_grad, + initial_state, + *tensors, + ) + output = output[..., : v.shape[-1]].contiguous() + if output_final_state: + states = states[..., : v.shape[-1], : k.shape[-1]].contiguous() + output = ( + output.reshape(*q.shape[:-2], q.shape[-2] * v.shape[-1]) + if qkv_format == "thd" + else output.reshape(batch, length, q.shape[-2] * v.shape[-1]) + ) + if qkv_format == "sbhd": + output = output.transpose(0, 1).contiguous() + return (output, states) if output_final_state else output diff --git a/transformer_engine/pytorch/attention/linear_attention/gdn.py b/transformer_engine/pytorch/attention/linear_attention/gdn.py index a9875f5e6b8..a7b53b65b75 100644 --- a/transformer_engine/pytorch/attention/linear_attention/gdn.py +++ b/transformer_engine/pytorch/attention/linear_attention/gdn.py @@ -10,6 +10,8 @@ from typing import Dict, Optional, Tuple, Union import torch +import torch.distributed as dist +import torch.nn.functional as F from transformer_engine.pytorch.constants import dist_group_type from transformer_engine.pytorch.jit import no_torch_dynamo @@ -18,9 +20,16 @@ AlignedTimelineKernelAdapter, LinearAttentionBase, _needs_eager_linear_attention, + _validate_cu_seqlens, ) +def _aligned(tensor): + """Clone offset views when FE needs a 16-byte-aligned address.""" + tensor = tensor.contiguous() + return tensor.clone() if tensor.data_ptr() % 16 else tensor + + class _GDNKernelAdapter(AlignedTimelineKernelAdapter): """Adapter from TransformerEngine attention layouts to cuDNN frontend GDN. @@ -49,6 +58,20 @@ def _validate_gates( f"{expected_gate_shape}; got {tuple(g.shape)} and {tuple(beta.shape)}." ) + def _resolve_cu_seqlens(self, query_layer, qkv_format, cu_seqlens): + """Accept full-document metadata for dense inputs as well as packed inputs.""" + if qkv_format == "thd" or cu_seqlens is None: + return super()._resolve_cu_seqlens(query_layer, qkv_format, cu_seqlens) + _validate_cu_seqlens(cu_seqlens, device=query_layer.device, name="cu_seqlens") + batch, length = ( + query_layer.shape[:2] + if qkv_format == "bshd" + else (query_layer.shape[1], query_layer.shape[0]) + ) + if cu_seqlens.numel() != batch + 1: + raise ValueError("Dense cu_seqlens must have shape [batch_size + 1].") + return cu_seqlens, length + def _call_op( self, query_layer: torch.Tensor, @@ -74,6 +97,120 @@ def _call_op( use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, ) + def _prepare_cp_inputs(self, tensors, initial_state): + """Pad partial feature tiles to keep FE's state gradients correct.""" + # Some FE engines mishandle irregular tiles, such as K=32, V=96. + # Zero extension preserves the recurrence and the original scale. + q, k, v, g, beta = (_aligned(t) for t in tensors) + key_dim, value_dim = k.shape[-1], v.shape[-1] + key_pad = max(64, 1 << (key_dim - 1).bit_length()) - key_dim + value_pad = max(64, 1 << (value_dim - 1).bit_length()) - value_dim + if key_pad: + q, k = F.pad(q, (0, key_pad)), F.pad(k, (0, key_pad)) + if value_pad: + v = F.pad(v, (0, value_pad)) + if initial_state is not None and (key_pad or value_pad): + initial_state = F.pad(initial_state, (0, key_pad, 0, value_pad)) + return (q, k, v, g, beta), initial_state + + @staticmethod + def _summary_op(name): + """Resolve optional FE summary entry points without requiring them at import.""" + try: + from cudnn.linear_attention import ops + + return getattr(ops, name) + except (AttributeError, ImportError) as exc: + raise ImportError(f"cuDNN frontend does not provide {name}.") from exc + + def _cp_forward_summary(self, query, key, value, gates, cu_seqlens, **kernel_kwargs): + """Return H,T; use ordinary state passes if no summary engine supports this shape.""" + del query + k, v, g, beta = (_aligned(t) for t in (key, value, gates["g"], gates["beta"])) + try: + return self._summary_op("gated_delta_net_summary")( + k, v, g, beta, cu_seqlens, **kernel_kwargs + ) + except ImportError: + pass + except Exception as exc: # pylint: disable=broad-exception-caught + if type(exc).__name__ != "cudnnGraphNotSupportedError": + raise + # A zero initial state gives H; an identity initial state with zero values gives T. + additive = self._call_op( + k, + k, + v, + {"g": g, "beta": beta}, + cu_seqlens, + initial_state=None, + output_final_state=True, + **kernel_kwargs, + )[1] + count, heads, _, key_dim = additive.shape + identity = torch.eye(key_dim, device=k.device, dtype=torch.float32) + identity = identity.expand(count, heads, key_dim, key_dim).contiguous() + transition = self._call_op( + k, + k, + torch.zeros_like(k), + {"g": g, "beta": beta}, + cu_seqlens, + initial_state=identity, + output_final_state=True, + **kernel_kwargs, + )[1] + return additive, transition + + def _cp_backward_summary( + self, query, key, value, gates, grad_output, cu_seqlens, **kernel_kwargs + ): + """Return the reverse state-gradient recipe, including local output losses.""" + del value + q, k, g, beta, grad_output = ( + _aligned(t) for t in (query, key, gates["g"], gates["beta"], grad_output) + ) + try: + return self._summary_op("gated_delta_net_summary_bwd")( + q, + k, + g, + beta, + grad_output, + cu_seqlens, + scale=self.scale, + output_transition=True, + **kernel_kwargs, + ) + except ImportError: + pass + except Exception as exc: # pylint: disable=broad-exception-caught + if type(exc).__name__ != "cudnnGraphNotSupportedError": + raise + values = torch.zeros_like(grad_output) + with torch.enable_grad(): + initial_state = torch.zeros( + (cu_seqlens.numel() - 1, q.shape[1], grad_output.shape[-1], k.shape[-1]), + device=q.device, + dtype=torch.float32, + requires_grad=True, + ) + output = self._call_op( + q.detach(), + k.detach(), + values, + {"g": g.detach(), "beta": beta.detach()}, + cu_seqlens, + initial_state=initial_state, + output_final_state=False, + **kernel_kwargs, + )[0] + (additive,) = torch.autograd.grad(output, initial_state, grad_output) + transition = self._cp_forward_summary( + q, k, values, {"g": g, "beta": beta}, cu_seqlens, **kernel_kwargs + )[1] + return additive, transition.transpose(-1, -2).contiguous() + def forward( self, query_layer: torch.Tensor, @@ -87,8 +224,41 @@ def forward( cu_seqlens: Optional[torch.Tensor] = None, output_final_state: bool = False, use_qk_l2norm_in_kernel: bool = False, + cp_group: Optional[dist.ProcessGroup] = None, ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: """Run GDN and return a TE-layout output, optionally with the final state.""" + if cp_group is not None and dist.get_world_size(cp_group) > 1: + from .context_parallel import context_parallel_attention + + return context_parallel_attention( + self, + query_layer, + key_layer, + value_layer, + {"g": g, "beta": beta}, + initial_state, + group=cp_group, + qkv_format=qkv_format, + cu_seqlens=cu_seqlens, + output_final_state=output_final_state, + use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, + ) + if cp_group is not None and cu_seqlens is not None: + if not isinstance(cu_seqlens, torch.Tensor): + raise TypeError("cu_seqlens must be a torch.Tensor.") + if cu_seqlens.dtype != torch.int32 or cu_seqlens.dim() != 1: + raise ValueError("cu_seqlens must be a one-dimensional int32 tensor.") + if qkv_format == "thd": + cu_seqlens = cu_seqlens.to(query_layer.device) + elif qkv_format in {"bshd", "sbhd"}: + batch, length = ( + query_layer.shape[:2] + if qkv_format == "bshd" + else (query_layer.shape[1], query_layer.shape[0]) + ) + if cu_seqlens.tolist() != [i * length for i in range(batch + 1)]: + raise ValueError("Dense cu_seqlens does not match the size-one CP input.") + cu_seqlens = None return self._run( query_layer, key_layer, @@ -121,9 +291,8 @@ class GatedDeltaNetAttention(LinearAttentionBase): `b` batch size, `h` the number of heads, `d` head size, and `t` the total number of tokens in a batch, with ``t = sum(s_i)`` for all sequences in the batch. Gated DeltaNet is - inherently causal; padded batches must use `qkv_format='thd'` with - `cu_seqlens` passed to `forward` to exclude padding tokens from the - recurrence. + inherently causal. Pack valid tokens from padded batches into + `qkv_format='thd'` and pass their `cu_seqlens` to `forward`. tp_size : int, default = 1 tensor parallel world size. tp_group : ProcessGroup, default = `None` @@ -135,6 +304,12 @@ class GatedDeltaNetAttention(LinearAttentionBase): scale for the Gated DeltaNet recurrence. Defaults to ``1.0 / sqrt(kv_channels)`` (or the query/key head size, if ``kv_channels`` is a tuple). + cp_group : ProcessGroup, default = `None` + Context parallel process group. ``None`` disables CP. Group rank + order must follow document token order. THD uses contiguous pieces + of the global packed buffer; BSHD/SBHD use contiguous pieces + of each sequence. Q/K/V and gates must already be local tensors. + Summary communication uses all-gather. No output gather is performed. """ def __init__( @@ -146,6 +321,8 @@ def __init__( tp_group: Optional[dist_group_type] = None, layer_number: Optional[int] = None, scale: Optional[float] = None, + *, + cp_group: Optional[dist.ProcessGroup] = None, ) -> None: super().__init__( num_attention_heads, @@ -163,6 +340,18 @@ def __init__( self.qk_head_dim, self.v_head_dim, ) + self.set_context_parallel_group(cp_group) + + def set_context_parallel_group(self, cp_group: Optional[dist.ProcessGroup]) -> None: + """Set or disable CP before forward, including after distributed initialization. + + A single process group is supported. CP uses the current CUDA stream + and contiguous token ordering. + """ + if cp_group is not None and not isinstance(cp_group, dist.ProcessGroup): + raise TypeError("GDN cp_group must be one torch.distributed.ProcessGroup or None.") + self.cp_group = cp_group + self.gdn_attention._cp_metadata_cache = None @no_torch_dynamo(when=_needs_eager_linear_attention) def forward( @@ -196,19 +385,52 @@ def forward( qkv_format : Optional[str], default = `None` Overrides the module's configured `qkv_format` for this call. cu_seqlens : Optional[torch.Tensor], default = `None` - Cumulative sequence lengths for packed (`thd`) inputs, of shape - `[batch_size + 1]` and dtype `torch.int32`. + Cumulative document lengths, of shape `[batch_size + 1]` and dtype + `torch.int32`. Required for THD. With CP, describe the complete + documents across the CP group, identically on all ranks; TE derives + the local fragment boundaries. For BSHD/SBHD CP, this metadata is + optional for equal slices and required for uneven slices; see Notes. + Without CP, dense metadata is optional and must describe the + full rows as `[0, S, 2*S, ..., B*S]`. checkpoint_core_attention : bool, default = `False` If true, forward activations for this module are recomputed during the backward pass instead of saved. initial_state : Optional[torch.Tensor], default = `None` - Recurrent state to seed the Gated DeltaNet recurrence with, of shape + Initial recurrent state for Gated DeltaNet, of shape `[batch_size, num_attention_heads // tp_size, v_head_dim, qk_head_dim]` and dtype `torch.float32`. output_final_state : bool, default = `False` If true, also return the final recurrent state. use_qk_l2norm_in_kernel : bool, default = `False` If true, L2-normalize Q and K inside the kernel before the recurrence. + + Notes + ----- + With THD CP, rank ``r`` supplies tokens ``[r*T//P:(r+1)*T//P]`` from + the global packed buffer, where ``T = cu_seqlens[-1]`` and ``P`` is CP + size. Document lengths need not be divisible by CP size; pack tokens + without gaps. + + For BSHD/SBHD CP, rank ``r`` supplies ``[r*S//P:(r+1)*S//P]`` of every + batch row. Here ``S`` is the full sequence length and ``B`` is batch size. + Supply ``cu_seqlens=[0, S, 2*S, ..., B*S]`` when ``S`` is not divisible + by ``P``; otherwise TE can infer ``S = local_sequence_length * P``. + Arbitrary rank splits are not supported. + + With CP, ``initial_state`` describes all global documents and must have + identical values on all ranks. Each document's initial-state gradient is + returned on its first owning rank, with zeros elsewhere; sum these + contributions over CP to inspect a replicated leaf state's full gradient. + This ownership also permits chaining final states into the next call. + Final states are replicated for all global documents. Their backward + sums gradients from every replica; divide a replicated final-state loss + by CP size to count it once. All ranks must call forward and backward with matching + options, even when their local loss or token count is zero. Only + first-order gradients are supported, matching the underlying FE path. + CP executes eagerly, including inside a ``torch.compile`` model. + Irregular feature sizes are internally extended with zeros and cropped + back, preserving the recurrence and original attention scale. No token + padding is introduced. """ if g is None or beta is None: raise ValueError( @@ -222,6 +444,7 @@ def forward( "cu_seqlens": cu_seqlens, "output_final_state": output_final_state, "use_qk_l2norm_in_kernel": use_qk_l2norm_in_kernel, + "cp_group": self.cp_group, } with self.prepare_forward_ctx(query_layer, allow_non_contiguous=True) as query_layer: return self._dispatch_attention( diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index 02bf6773442..ff585e82615 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -143,6 +143,10 @@ void copy_to_kv_cache(at::Tensor new_k, at::Tensor new_v, at::Tensor k_cache, at NVTE_QKV_Format kv_format, int b, int max_ctx_len, int max_seq_len, int max_pages_per_seq, bool is_non_paged); +at::Tensor linear_attention_affine_prefix(const at::Tensor &initial_states, + const at::Tensor &additive, const at::Tensor &transitions, + const at::Tensor &descriptors); + /*************************************************************************************************** * GEMM **************************************************************************************************/ diff --git a/transformer_engine/pytorch/csrc/extensions/attention.cpp b/transformer_engine/pytorch/csrc/extensions/attention.cpp index 98d55d2b8b5..9c22ef3c4ea 100644 --- a/transformer_engine/pytorch/csrc/extensions/attention.cpp +++ b/transformer_engine/pytorch/csrc/extensions/attention.cpp @@ -7,9 +7,34 @@ #include "../extensions.h" #include "common.h" #include "pybind.h" +#include "transformer_engine/linear_attention.h" namespace transformer_engine::pytorch { +at::Tensor linear_attention_affine_prefix(const at::Tensor &initial_states, + const at::Tensor &additive, const at::Tensor &transitions, + const at::Tensor &descriptors) { + for (const auto &tensor : {initial_states, additive, transitions, descriptors}) { + NVTE_CHECK(tensor.is_cuda() && tensor.is_contiguous(), + "Affine prefix tensors must be contiguous CUDA tensors."); + NVTE_CHECK(tensor.device() == initial_states.device(), "Affine prefix devices must agree."); + } + NVTE_CHECK(initial_states.dim() == 4 && descriptors.dim() == 2, "Invalid affine prefix ranks."); + at::cuda::CUDAGuard device_guard(initial_states.device()); + at::Tensor output = at::empty( + {descriptors.size(0), initial_states.size(1), initial_states.size(2), initial_states.size(3)}, + initial_states.options()); + auto initial_tensor = makeTransformerEngineTensor(initial_states); + auto additive_tensor = makeTransformerEngineTensor(additive); + auto transition_tensor = makeTransformerEngineTensor(transitions); + auto descriptor_tensor = makeTransformerEngineTensor(descriptors); + auto output_tensor = makeTransformerEngineTensor(output); + nvte_linear_attention_affine_prefix(initial_tensor.data(), additive_tensor.data(), + transition_tensor.data(), descriptor_tensor.data(), + output_tensor.data(), at::cuda::getCurrentCUDAStream()); + return output; +} + // get the fused attention backend std::tuple get_fused_attn_backend(const py::object &p) { FusedAttnConfigWrapper cfg; diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index df6d16782d2..2a883d8792b 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -676,6 +676,11 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { "Copy valid THD token entries from a per-split tensor into a rank-local accumulator", py::call_guard()); + // Linear attention context parallelism + m.def("linear_attention_affine_prefix", + &transformer_engine::pytorch::linear_attention_affine_prefix, + "Apply ordered linear-attention state summaries", py::call_guard()); + // nvshmem functions m.def("init_nvshmem_backend", &transformer_engine::pytorch::init_nvshmem_backend, "Initialize nvshmem backend with Pytorch distributed process groups",