Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
6e8155c
Add shared linear attention CP utilities
KshitijLakhani Nov 6, 2026
bf36a95
Add contiguous CP for PyTorch GDN
KshitijLakhani Nov 6, 2026
bfd296a
Test GDN context parallelism
KshitijLakhani Nov 6, 2026
61edd9b
Document GDN context parallelism
KshitijLakhani Nov 6, 2026
392ba75
Document GDN context parallel metadata local and global cu seqlens
KshitijLakhani Nov 11, 2026
72ce1a6
Unify GDN CP sequence metadata
KshitijLakhani Nov 11, 2026
19b4ffe
Use descriptive linear attention CP metadata names
KshitijLakhani Nov 11, 2026
a3e16ed
Rename affine prefix seed terminology to initial memory
KshitijLakhani Nov 12, 2026
be43173
Rename CP metadata and state names
KshitijLakhani Nov 14, 2026
6c4d7cf
Clarify linear attention CP state names
KshitijLakhani Nov 14, 2026
0b299b6
Share CP metadata across attention modules
KshitijLakhani Nov 14, 2026
87f8422
Document the linear attention CP adapter contract
KshitijLakhani Nov 14, 2026
716f135
Reduce idle affine prefix threads
KshitijLakhani Nov 14, 2026
61d9b89
Separate CP metadata caches by inference mode
KshitijLakhani Nov 14, 2026
519bacf
Document affine prefix parameters
KshitijLakhani Nov 17, 2026
aafc721
Clarify linear attention partition metadata
KshitijLakhani Nov 18, 2026
341f835
Explain affine prefix kernel execution in comments
KshitijLakhani Nov 18, 2026
94a4d9f
Rename linear attention CP metadata
KshitijLakhani Nov 18, 2026
f849451
Separate host and device CP metadata
KshitijLakhani Nov 18, 2026
b9ed649
Trim redundant GDN CP checks
KshitijLakhani Nov 18, 2026
6f36e0a
Clarify affine summary indices
KshitijLakhani Nov 18, 2026
36ac0a1
Name CP fragment indices and descriptors
KshitijLakhani Nov 18, 2026
f070366
Add host and device CP metadata examples
KshitijLakhani Nov 18, 2026
3ab1158
Use variant terminology for linear attention
KshitijLakhani Nov 18, 2026
fb7c9be
Group linear attention CP bindings
KshitijLakhani Nov 18, 2026
ff1a0ea
Expand linear attn affine prefix cpp test coverage
KshitijLakhani Nov 19, 2026
96ac2fe
Document affine prefix test choices
KshitijLakhani Nov 19, 2026
b0215f7
Move CP adapter requirements to interface docstrings
KshitijLakhani Nov 19, 2026
24cf3d2
Explain dense GDN sequence metadata coverage
KshitijLakhani Nov 19, 2026
c7ca6ee
Tighten affine prefix test tolerances
KshitijLakhani Nov 19, 2026
eb6f1fd
Add linear attention tutorial and CI examples
KshitijLakhani Nov 20, 2026
41d46ab
Streamline GDN context parallel tests
KshitijLakhani Nov 20, 2026
f4e8be4
Merge upstream main into linear attention CP
KshitijLakhani Nov 20, 2026
09cfc7e
Remove duplicate GDN example tests
KshitijLakhani Nov 20, 2026
e1d70d4
Update linear attention docs and fallback access
KshitijLakhani Nov 20, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/api/pytorch.rst
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ PyTorch
:members: forward, set_context_parallel_group

.. autoapiclass:: transformer_engine.pytorch.GatedDeltaNetAttention(num_attention_heads, kv_channels, **kwargs)
:members: forward
:members: forward, set_context_parallel_group

.. autoapiclass:: transformer_engine.pytorch.GatedDeltaNet2Attention(num_attention_heads, kv_channels, **kwargs)
:members: forward
Expand Down
5 changes: 4 additions & 1 deletion docs/examples/attention/attention.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@
"[Transformer Engine](https://github.com/NVIDIA/TransformerEngine.git) supports the calculation of dot product attention in two frameworks, [PyTorch](https://github.com/pytorch/pytorch) and [JAX](https://github.com/google/jax). The API for each framework is\n",
"\n",
"- [transformer_engine.pytorch.DotProductAttention](../../api/pytorch.rst#transformer_engine.pytorch.DotProductAttention)\n",
"- [transformer_engine.jax.flax.DotProductAttention](../../api/jax.rst#transformer_engine.jax.flax.DotProductAttention)"
"- [transformer_engine.jax.flax.DotProductAttention](../../api/jax.rst#transformer_engine.jax.flax.DotProductAttention)\n",
"\n",
"\n",
"For linear attention variants, see [Linear Attention](linear_attention.ipynb)."
]
},
{
Expand Down
170 changes: 170 additions & 0 deletions docs/examples/attention/example_linear_attention.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# See LICENSE for license information.

"""Gated DeltaNet forward and backward, with optional contiguous context parallelism.

Run from the repository root:
python docs/examples/attention/example_linear_attention.py
python -m torch.distributed.run --standalone --nproc-per-node=2 \
docs/examples/attention/example_linear_attention.py --cp --qkv-format thd
"""

import argparse
import os

import torch
import torch.distributed as dist

import transformer_engine.pytorch as te


def make_inputs(qkv_format):
"""Create the same batch on each rank for comparison with a non-CP run."""
torch.manual_seed(1234)
batch, sequence, heads, head_dim = 16, 128, 8, 64
if qkv_format == "thd":
# Sample document lengths from 32 to 128 tokens, then pack without padding.
lengths = torch.randint(32, 129, (batch,), device="cpu").tolist()
offsets = [0]
for length in lengths:
offsets.append(offsets[-1] + length)
shape = (offsets[-1], heads, head_dim)
else:
shape = (batch, sequence, heads, head_dim)
offsets = [row * sequence for row in range(batch + 1)]
q, k, v = (torch.randn(shape, device="cuda", dtype=torch.bfloat16) for _ in range(3))
# Per-token, per-head FP32 gates: g is the log decay; beta is the write strength.
g = -torch.rand(shape[:-1], device="cuda", dtype=torch.float32) * 0.02
beta = torch.rand_like(g) * 0.7
inputs = tuple(t.requires_grad_() for t in (q, k, v, g, beta))
initial_state = (
torch.randn(batch, heads, head_dim, head_dim, device="cuda", dtype=torch.float32) * 0.1
).requires_grad_()
cu_seqlens = torch.tensor(offsets, device="cuda", dtype=torch.int32)
return inputs, initial_state, cu_seqlens


def run_single_gpu(inputs, initial_state, cu_seqlens, qkv_format):
"""Run on complete documents and differentiate through outputs and final states."""
q, k, v, g, beta = inputs
attention = te.GatedDeltaNetAttention(
num_attention_heads=q.shape[-2],
kv_channels=q.shape[-1],
qkv_format=qkv_format,
)
output, final_state = attention(
q,
k,
v,
g=g,
beta=beta,
cu_seqlens=cu_seqlens if qkv_format == "thd" else None,
initial_state=initial_state,
output_final_state=True,
use_qk_l2norm_in_kernel=True,
)
loss = output.float().square().mean() + final_state.square().mean()
loss.backward()
return output, final_state


def local_tokens(tensor, qkv_format, cp_group):
"""Select this rank's contiguous interval of the packed buffer or each batch row."""
rank, size = dist.get_rank(cp_group), dist.get_world_size(cp_group)
if qkv_format == "thd":
length = tensor.shape[0]
return tensor[rank * length // size : (rank + 1) * length // size]
length = tensor.shape[1]
return tensor[:, rank * length // size : (rank + 1) * length // size]


def run_context_parallel(inputs, initial_state, cu_seqlens, qkv_format, cp_group):
"""Supply local tokens and global document metadata to GDN CP."""
local_inputs = tuple(
local_tokens(t, qkv_format, cp_group).detach().contiguous().requires_grad_() for t in inputs
)
q, k, v, g, beta = local_inputs
state = initial_state.detach().clone().requires_grad_()
attention = te.GatedDeltaNetAttention(
num_attention_heads=q.shape[-2],
kv_channels=q.shape[-1],
qkv_format=qkv_format,
cp_group=cp_group,
)
output, final_state = attention(
q,
k,
v,
g=g,
beta=beta,
cu_seqlens=cu_seqlens,
initial_state=state,
output_final_state=True,
use_qk_l2norm_in_kernel=True,
)
# Normalize by the global output size, so rank-local losses sum to the
# non-CP loss. Count the replicated final-state loss only once.
cp_size = dist.get_world_size(cp_group)
loss = output.float().square().sum() / inputs[2].numel()
loss = loss + final_state.square().mean() / cp_size
loss.backward()
return output, final_state, local_inputs, state


def check_finite(tensors):
"""Fail on missing gradients or non-finite results, including in CI."""
for tensor in tensors:
assert tensor is not None, "Missing gradient"
assert torch.isfinite(tensor).all(), "Non-finite result"


def check_close(actual, expected, tolerance):
"""Compare BF16 results using relative RMS error, as in the GDN tests."""
actual, expected = actual.detach().double(), expected.detach().double()
error = (actual - expected).square().mean().sqrt()
reference = expected.square().mean().sqrt().clamp_min(1e-12)
assert error / reference < tolerance, f"Relative RMS error: {(error / reference).item()}"


def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--cp", action="store_true", help="Run with torch.distributed.run on two or more GPUs"
)
parser.add_argument("--qkv-format", choices=("bshd", "thd"), default="bshd")
args = parser.parse_args()
torch.cuda.set_device(int(os.environ.get("LOCAL_RANK", "0")))
if args.cp:
# NCCL is PyTorch's recommended backend for CUDA distributed training.
# CP also accepts other backends that support its CUDA collectives.
dist.init_process_group(backend="nccl")
try:
if args.cp and dist.get_world_size() < 2:
raise ValueError("--cp requires torch.distributed.run with at least two GPUs")
inputs, initial_state, cu_seqlens = make_inputs(args.qkv_format)
output, final_state = run_single_gpu(inputs, initial_state, cu_seqlens, args.qkv_format)
assert output.shape == (*inputs[2].shape[:-2], inputs[2].shape[-2:].numel())
assert final_state.shape == initial_state.shape
check_finite((output, final_state, initial_state.grad, *(t.grad for t in inputs)))
if args.cp:
cp_group = dist.group.WORLD
local_output, states, local_inputs, state = run_context_parallel(
inputs, initial_state, cu_seqlens, args.qkv_format, cp_group
)
check_close(local_output, local_tokens(output, args.qkv_format, cp_group), 2e-2)
check_close(states, final_state, 2e-2)
for local, full in zip(local_inputs, inputs):
check_close(local.grad, local_tokens(full.grad, args.qkv_format, cp_group), 4e-2)
# Initial-state gradients belong to each document's first owning rank.
dist.all_reduce(state.grad, group=cp_group)
check_close(state.grad, initial_state.grad, 4e-2)
if not args.cp or dist.get_rank() == 0:
print(f"GDN {args.qkv_format}: forward and backward passed (CP={args.cp})")
finally:
if args.cp:
dist.destroy_process_group()


if __name__ == "__main__":
main()
Loading
Loading