From 151f2b2e981213fae4058797fec8c81ae45b37c6 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 10 Sep 2026 13:23:34 +0200 Subject: [PATCH 01/31] [PyTorch] Add torch.compile custom ops for GroupedLinear Reuse eager computation for both grouped paths and add fake implementations, registrations, saved-state serialization and compile dispatch. Extend argument packing and autograd routing for tensor lists and SymInt lists, including empty lists, with compile regression coverage. Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 546 +++++++++++ .../pytorch/dynamo/custom_op.py | 176 +++- .../pytorch/module/grouped_linear.py | 853 ++++++++++++++++-- transformer_engine/pytorch/utils.py | 20 + 4 files changed, 1521 insertions(+), 74 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index e1c6eaf5991..242ff3c1212 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -2339,3 +2339,549 @@ def fn(inp): "Unexpected recompilation(s) across different batch sizes: " f"{unique_graphs_after - unique_graphs_baseline} extra graph(s) compiled" ) + + +# --------------------------------------------------------------------------- +# te.GroupedLinear +# --------------------------------------------------------------------------- + +_GROUPED_M_SPLITS = [32, 16, 48, 32] # FP8-legal splits (each divisible by 8) +_GROUPED_NUM_GEMMS = len(_GROUPED_M_SPLITS) +_GROUPED_IN, _GROUPED_OUT = 64, 32 + + +def _grouped_input(dtype, device, requires_grad=False): + return torch.randn( + sum(_GROUPED_M_SPLITS), + _GROUPED_IN, + dtype=dtype, + device=device, + requires_grad=requires_grad, + ) + + +def _assert_close_grouped(fn, compiled, model, base, ref_model=None): + """Run ``fn`` eagerly and ``compiled`` on identical inputs; assert the + forward output, the input gradient and every expert's weight / bias + gradients match.""" + ref_model = model if ref_model is None else ref_model + inp_eager = base.detach().clone().requires_grad_(True) + ref_model.zero_grad(set_to_none=True) + out_eager = fn(inp_eager) + out_eager.sum().backward() + ref_out = out_eager.detach().clone() + ref_igrad = inp_eager.grad.detach().clone() + ref_grads = [param.grad.detach().clone() for param in ref_model.parameters()] + + inp_compiled = base.detach().clone().requires_grad_(True) + model.zero_grad(set_to_none=True) + # Clone before a later cuda-graph replay overwrites the static output buffer. + out_compiled = compiled(inp_compiled).clone() + out_compiled.sum().backward() + + torch.testing.assert_close(out_compiled, ref_out, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) + torch.testing.assert_close(inp_compiled.grad, ref_igrad, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) + for (name, param), ref_grad in zip(model.named_parameters(), ref_grads): + torch.testing.assert_close( + param.grad, + ref_grad, + atol=_EAGER_ATOL, + rtol=_EAGER_RTOL, + msg=f"gradient mismatch for {name}", + ) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("compile_mode", _compile_modes) +@pytest.mark.parametrize( + "fp8_recipe", + [None, *_all_recipes], + ids=lambda r: "bf16" if r is None else type(r).__name__, +) +def test_te_grouped_linear_compiles(fp8_recipe, compile_mode): + """ + torch.compile(fullgraph=True) of ``te.GroupedLinear`` under every built-in + recipe (plus the bf16-only baseline), for both the default backend and + ``mode="reduce-overhead"`` (CUDA-graph trees). + """ + dtype = torch.bfloat16 + device = "cuda" + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, _GROUPED_IN, _GROUPED_OUT, params_dtype=dtype, device=device + ) + + def fn(inp): + if fp8_recipe is None: + return model(inp, _GROUPED_M_SPLITS) + with te.autocast(recipe=fp8_recipe): + return model(inp, _GROUPED_M_SPLITS) + + torch._dynamo.reset() + if compile_mode == "reduce-overhead": + _cudagraph_warmup(fn, _grouped_input(dtype, device, requires_grad=True), backward=True) + model.zero_grad(set_to_none=True) + compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) + + n_iters = 3 if compile_mode == "reduce-overhead" else 1 + with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): + for _ in range(n_iters): + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +@pytest.mark.parametrize("compile_mode", _compile_modes) +def test_te_grouped_linear_compile_with_quantized_fp8_weight(compile_mode): + """torch.compile of GroupedLinear with the weights initialized as FP8 + tensors (exercises the wrapper op's input flattening inside a ``Tensor[]`` + slot group).""" + dtype = torch.bfloat16 + device = "cuda" + fp8_recipe = recipe.Float8CurrentScaling() + + with te.quantized_model_init(enabled=True, recipe=fp8_recipe): + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, + _GROUPED_IN, + _GROUPED_OUT, + params_dtype=dtype, + device=device, + ) + assert isinstance(model.weight0, te.Float8Tensor) + + def fn(inp): + with te.autocast(recipe=fp8_recipe): + return model(inp, _GROUPED_M_SPLITS) + + torch._dynamo.reset() + if compile_mode == "reduce-overhead": + _cudagraph_warmup(fn, _grouped_input(dtype, device, requires_grad=True), backward=True) + model.zero_grad(set_to_none=True) + compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) + + n_iters = 3 if compile_mode == "reduce-overhead" else 1 + with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): + for _ in range(n_iters): + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +@pytest.mark.parametrize("compile_mode", _compile_modes) +def test_te_grouped_linear_compile_is_first_microbatch(compile_mode): + """torch.compile of ``te.GroupedLinear`` across a microbatch schedule: + ``is_first_microbatch=True`` caches every expert's FP8 weight, later steps + must reuse the caches and stay numerically aligned with eager.""" + dtype = torch.bfloat16 + device = "cuda" + fp8_recipe = recipe.Float8CurrentScaling() + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, _GROUPED_IN, _GROUPED_OUT, params_dtype=dtype, device=device + ) + ref_model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, _GROUPED_IN, _GROUPED_OUT, params_dtype=dtype, device=device + ) + with torch.no_grad(): + for i in range(_GROUPED_NUM_GEMMS): + getattr(ref_model, f"weight{i}").copy_(getattr(model, f"weight{i}")) + getattr(ref_model, f"bias{i}").copy_(getattr(model, f"bias{i}")) + + schedule = [True, False, False] + is_first = schedule[0] # rebound each step; closed over by the fns. + + def fn(inp): + with te.autocast(recipe=fp8_recipe): + return model(inp, _GROUPED_M_SPLITS, is_first_microbatch=is_first) + + def ref_fn(inp): + with te.autocast(recipe=fp8_recipe): + return ref_model(inp, _GROUPED_M_SPLITS, is_first_microbatch=is_first) + + # Eager priming: FP8 state must exist before tracing (creating quantizers + # in-graph breaks later recompiles; upstream Dynamo bug). + is_first = None + fn(_grouped_input(dtype, device, requires_grad=True)) + is_first = schedule[0] + + torch._dynamo.reset() + if compile_mode == "reduce-overhead": + _cudagraph_warmup(fn, _grouped_input(dtype, device, requires_grad=True), backward=True) + model.zero_grad(set_to_none=True) + compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) + + cached_workspaces = None + with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): + for step, is_first in enumerate(schedule): + base = _grouped_input(dtype, device) + + _assert_close_grouped(ref_fn, compiled, model, base, ref_model=ref_model) + + workspaces = [ + model._fp8_workspaces.get(f"weight{i}") for i in range(_GROUPED_NUM_GEMMS) + ] + assert all( + ws is not None for ws in workspaces + ), f"missing cached FP8 weight(s) after step {step}" + if step == 0: + cached_workspaces = workspaces + else: + for i in range(_GROUPED_NUM_GEMMS): + assert ( + workspaces[i] is cached_workspaces[i] + ), f"cache for expert {i} rebuilt at step {step}" + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +def test_te_grouped_linear_m_splits_change(): + """Unchanged ``m_splits`` reuse the graph. A changed distribution makes + Dynamo mark the values dynamic; they cross the op boundary as a + ``SymInt[]`` slot, so the (recompiled, now symbolic) graph still runs the + compiled path with eager numerics -- for any later distribution too.""" + dtype = torch.bfloat16 + device = "cuda" + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, _GROUPED_IN, _GROUPED_OUT, params_dtype=dtype, device=device + ) + m_splits = list(_GROUPED_M_SPLITS) + + def fn(inp): + return model(inp, m_splits) + + torch._dynamo.reset() + compiled = torch.compile(fn, fullgraph=True) + + # Two warmup calls: the second absorbs the one-time recompile from module + # attributes lazily created during call one. + for _ in range(2): + compiled(_grouped_input(dtype, device, requires_grad=True)).sum().backward() + model.zero_grad(set_to_none=True) + baseline = _dynamo_counter("stats", "unique_graphs") + + # Same splits: no recompile. + compiled(_grouped_input(dtype, device, requires_grad=True)).sum().backward() + model.zero_grad(set_to_none=True) + if baseline: + assert _dynamo_counter("stats", "unique_graphs") == baseline + + # New split distributions (same total): compiled path, same numerics. + m_splits[:2] = [16, 32] + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + m_splits[2:] = [40, 40] + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +@pytest.mark.xfail(reason="waiting for a PyTorch fix", strict=False) +def test_te_grouped_linear_compile_train_eval_switch(): + """train -> eval -> train on the same compiled ``te.GroupedLinear``, vs eager.""" + dtype = torch.bfloat16 + device = "cuda" + fp8_recipe = recipe.Float8CurrentScaling() + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, _GROUPED_IN, _GROUPED_OUT, params_dtype=dtype, device=device + ) + + def fn(inp): + with te.autocast(recipe=fp8_recipe): + return model(inp, _GROUPED_M_SPLITS, is_first_microbatch=True) + + torch._dynamo.reset() + compiled = torch.compile(fn, fullgraph=True) + + def train_step(): + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + model.zero_grad(set_to_none=True) + + train_step() + + model.eval() + x = _grouped_input(dtype, device) + with torch.no_grad(): + out_eval = compiled(x) + ref_eval = fn(x) + torch.testing.assert_close(out_eval, ref_eval, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) + + model.train() + train_step() + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +def test_te_grouped_linear_compile_m_splits_tensor_falls_back(): + """A device-tensor ``m_splits`` inside the compiled region cannot cross the + op boundary (its values would be unbacked); the module must warn and fall + back to eager (a graph break).""" + dtype = torch.bfloat16 + device = "cuda" + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, _GROUPED_IN, _GROUPED_OUT, params_dtype=dtype, device=device + ) + m_splits_tensor = torch.tensor(_GROUPED_M_SPLITS, dtype=torch.int64, device="cpu") + + def fn(inp): + return model(inp, m_splits_tensor) + + torch._dynamo.reset() + # Graph-break fallback: numerics must match eager. + compiled = torch.compile(fn) + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + + # fullgraph=True: the explicit graph break surfaces the reason. + torch._dynamo.reset() + strict = torch.compile(fn, fullgraph=True) + with pytest.raises(Exception, match="falling back to eager"): + strict(_grouped_input(dtype, device, requires_grad=True)) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +def test_te_grouped_linear_compile_delayed_scaling_falls_back(): + """Delayed-scaling quantizers are not value-opaque; under torch.compile the + module must fall back to eager (a graph break) with eager numerics.""" + dtype = torch.bfloat16 + device = "cuda" + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, _GROUPED_IN, _GROUPED_OUT, params_dtype=dtype, device=device + ) + fp8_recipe = recipe.DelayedScaling() + + def fn(inp): + with te.autocast(recipe=fp8_recipe): + return model(inp, _GROUPED_M_SPLITS) + + torch._dynamo.reset() + compiled = torch.compile(fn) # no fullgraph: the fallback graph-breaks + # Functional smoke only: delayed scaling advances amax history every call, + # so eager-vs-compiled outputs are not bit-comparable across calls. + try: + inp = _grouped_input(dtype, device, requires_grad=True) + out = compiled(inp) + out.sum().backward() + assert out.shape == (sum(_GROUPED_M_SPLITS), _GROUPED_OUT) + assert inp.grad is not None + for i in range(_GROUPED_NUM_GEMMS): + assert getattr(model, f"weight{i}").grad is not None + finally: + # Drop the delayed-scaling amax-reduction registrations, or every later + # autocast exit would run the (untraceable) fused amax reduction. + te.quantization.FP8GlobalStateManager.reset() + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +def test_te_grouped_linear_compile_save_original_input(): + """torch.compile with ``save_original_input=True``: the op saves the input + by alias and re-splits/re-quantizes it in backward.""" + dtype = torch.bfloat16 + device = "cuda" + fp8_recipe = recipe.Float8CurrentScaling() + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, + _GROUPED_IN, + _GROUPED_OUT, + params_dtype=dtype, + device=device, + save_original_input=True, + ) + + def fn(inp): + with te.autocast(recipe=fp8_recipe): + return model(inp, _GROUPED_M_SPLITS) + + torch._dynamo.reset() + compiled = torch.compile(fn, fullgraph=True) + for _ in range(2): + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + + +_fused_grouped_cc_ok = torch.cuda.is_available() and ( + (9, 0) <= torch.cuda.get_device_capability() <= (11, 0) +) +_fused_grouped_cublas_ok = _fused_grouped_cc_ok and tex.get_cublasLt_version() >= ( + 130400 if torch.cuda.get_device_capability() < (10, 0) else 130300 +) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif( + not _fused_grouped_cublas_ok, + reason="fused grouped GEMM needs Hopper/Blackwell and a recent cuBLASLt", +) +@pytest.mark.parametrize("compile_mode", _compile_modes) +@pytest.mark.parametrize( + "fp8_recipe", + [None] + ([recipe.Float8CurrentScaling()] if fp8_available else []), + ids=lambda r: "bf16" if r is None else type(r).__name__, +) +def test_te_grouped_linear_fused_compiles(fp8_recipe, compile_mode): + """torch.compile(fullgraph=True) of the fused GroupedTensor path + (``use_grouped_tensor=True``): ``m_splits`` stays a + device tensor, so changing the split distribution in place reuses the very + same graph -- no recompiles, no host sync.""" + if fp8_recipe is not None and torch.cuda.get_device_capability() < (10, 0): + if tex.get_cublasLt_version() < 130500: + pytest.skip("FP8 current-scaling fused grouped GEMM on Hopper needs cuBLASLt 13.5+") + dtype = torch.bfloat16 + device = "cuda" + model = te.GroupedLinear( + _GROUPED_NUM_GEMMS, + _GROUPED_IN, + _GROUPED_OUT, + params_dtype=dtype, + device=device, + use_grouped_tensor=True, + ) + m_splits_dev = torch.tensor(_GROUPED_M_SPLITS, dtype=torch.int64, device=device) + + def fn(inp): + if fp8_recipe is None: + return model(inp, m_splits_dev) + with te.autocast(recipe=fp8_recipe): + return model(inp, m_splits_dev) + + torch._dynamo.reset() + if compile_mode == "reduce-overhead": + _cudagraph_warmup(fn, _grouped_input(dtype, device, requires_grad=True), backward=True) + model.zero_grad(set_to_none=True) + compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) + + with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): + # Two warmup calls absorb the one-time recompile from lazily created + # module attributes. + for _ in range(2): + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + baseline = _dynamo_counter("stats", "unique_graphs") + + # Change the split distribution IN PLACE: same graph, no recompile. + m_splits_dev.copy_(torch.tensor([16, 32, 40, 40], dtype=torch.int64, device=device)) + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + m_splits_dev.copy_(torch.tensor([64, 8, 32, 24], dtype=torch.int64, device=device)) + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + if baseline: + assert ( + _dynamo_counter("stats", "unique_graphs") == baseline + ), "changing device-tensor m_splits values must not recompile" + + +# --------------------------------------------------------------------------- +# Custom-op list adapters (framework unit test via a toy op) +# --------------------------------------------------------------------------- + + +if _opaque_available: + from dataclasses import dataclass + from typing import List, Optional, Tuple + + from transformer_engine.pytorch.dynamo import register_custom_op + + @dataclass + class _ToyListFwdArgs: + inp: torch.Tensor + weights: List[Optional[torch.Tensor]] + scales: Tuple[float, ...] + split_metadata: List[int] + + @dataclass + class _ToyListBwdArgs: + grad_output: Optional[torch.Tensor] = None + inp: Optional[torch.Tensor] = None + weights: List[Optional[torch.Tensor]] = None + scales: Tuple[float, ...] = () + + def _toy_list_fwd(args): + out = torch.zeros_like(args.inp) + for w, s in zip(args.weights, args.scales): + if w is not None: + out = out + args.inp * w * s + return (out, None, None) + + def _toy_list_fwd_fake(args): + out = TensorSpec( + shape=tuple(args.inp.shape), + dtype=args.inp.dtype, + requires_grad=True, + device=args.inp.device, + ) + return (out, None, None) + + def _toy_list_setup_ctx(bwd_args, fwd_args, outputs, ctx_attrs, saved): + del outputs, ctx_attrs, saved + bwd_args.inp = fwd_args.inp + bwd_args.weights = fwd_args.weights + bwd_args.scales = fwd_args.scales + return () + + def _toy_list_bwd(args): + dgrad = torch.zeros_like(args.inp) + wgrads = [] + for w, s in zip(args.weights, args.scales): + if w is None: + wgrads.append(None) + continue + dgrad = dgrad + args.grad_output * w * s + wgrads.append(args.grad_output * args.inp * s) + return (dgrad, wgrads) + + def _toy_list_bwd_fake(args): + spec = lambda t: TensorSpec(shape=tuple(t.shape), dtype=t.dtype, device=t.device) + dgrad = spec(args.inp) + wgrads = [None if w is None else spec(w) for w in args.weights] + return (dgrad, wgrads) + + _toy_list_op = register_custom_op( + op_name="toy_list_op_test", + input_tensors_for_grad=["inp", "weights"], + fwd_arg_type=_ToyListFwdArgs, + fwd_impl=_toy_list_fwd, + fwd_fake_impl=_toy_list_fwd_fake, + setup_context=_toy_list_setup_ctx, + bwd_arg_type=_ToyListBwdArgs, + bwd_impl=_toy_list_bwd, + bwd_fake_impl=_toy_list_bwd_fake, + ) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("split_metadata", [[], [4, 4]]) +def test_custom_op_tensor_list_grads(split_metadata): + """A toy op with a ``List[Optional[Tensor]]`` input (including a ``None`` + entry riding the sentinel) compiles fullgraph and routes one gradient per + list element.""" + assert _toy_list_op is not None + device = "cuda" + scales = (2.0, 3.0, 0.5) + + def run(fn): + torch.manual_seed(7) + inp = torch.randn(8, 4, device=device, requires_grad=True) + w0 = torch.randn(8, 4, device=device, requires_grad=True) + w2 = torch.randn(8, 4, device=device, requires_grad=True) + out = fn(inp, w0, w2) + out.sum().backward() + return out.detach().clone(), inp.grad, w0.grad, w2.grad + + def fn(inp, w0, w2): + args = _ToyListFwdArgs( + inp=inp, weights=[w0, None, w2], scales=scales, split_metadata=split_metadata + ) + return _toy_list_op(args) + + def ref_fn(inp, w0, w2): + return inp * w0 * scales[0] + inp * w2 * scales[2] + + torch._dynamo.reset() + compiled = torch.compile(fn, fullgraph=True) + + out_ref, igrad_ref, w0grad_ref, w2grad_ref = run(ref_fn) + out, igrad, w0grad, w2grad = run(compiled) + torch.testing.assert_close(out, out_ref) + torch.testing.assert_close(igrad, igrad_ref) + torch.testing.assert_close(w0grad, w0grad_ref) + torch.testing.assert_close(w2grad, w2grad_ref) + + # Eager through the same op (no compile) must agree too. + out_e, igrad_e, w0grad_e, w2grad_e = run(fn) + torch.testing.assert_close(out_e, out_ref) + torch.testing.assert_close(igrad_e, igrad_ref) + torch.testing.assert_close(w0grad_e, w0grad_ref) + torch.testing.assert_close(w2grad_e, w2grad_ref) diff --git a/transformer_engine/pytorch/dynamo/custom_op.py b/transformer_engine/pytorch/dynamo/custom_op.py index f887a48d95a..6b28f89d64a 100644 --- a/transformer_engine/pytorch/dynamo/custom_op.py +++ b/transformer_engine/pytorch/dynamo/custom_op.py @@ -32,6 +32,8 @@ * ``TENSOR_OR_QUANTIZED`` -- a field that may be a plain tensor, a bare quantized storage, or ``None``: three slots (the tensor, its flat inner buffers, and a ``__kind__`` tag) so a quantized tensor crosses as its buffers. + * Tensor kinds also support lists, with per-element gradients and metadata. + * ``SYMINT_LIST`` -- ``List[int]`` in a ``SymInt[]`` slot for dynamic splits. * ``SIMPLE`` -- every remaining simple value (scalars, enums, sizes, quantizers -- value-opaque constants baked into the graph -- and nested collections of them), gathered into one shared ``OpaqueValueBundle`` slot. @@ -390,6 +392,7 @@ class _FieldKind(Enum): TENSOR = "tensor" # one ``Tensor`` / ``Tensor?`` slot TENSOR_OR_QUANTIZED = "tensor_or_quantized" # 3 slots: tensor / inner / meta + SYMINT_LIST = "symint_list" PROCESS_GROUP = "process_group" # c10d group name inside the shared bundle SIMPLE = "simple" # value carried verbatim inside the shared bundle UNSUPPORTED = "unsupported" # no slots; only a trivial value may cross @@ -430,6 +433,7 @@ class _FieldPlan: name: str kind: _FieldKind slots: Tuple[_SlotSpec, ...] + is_list: bool = False def _is_tensor_storage_union(annot: Any) -> bool: @@ -473,6 +477,22 @@ def _is_simple_annot(annot: Any) -> bool: def _parse_field(name: str, annot: Any) -> _FieldPlan: """Parse one field's annotation into its :class:`_FieldPlan`.""" + stripped, _ = _strip_optional(annot) + if get_origin(stripped) is list and len(get_args(stripped)) == 1: + element_annot = get_args(stripped)[0] + if element_annot is int: + return _FieldPlan(name, _FieldKind.SYMINT_LIST, (_SlotSpec(name, "SymInt[]"),)) + element = _parse_field(name, element_annot) + if element.is_list: + raise TypeError(f"field {name!r}: nested tensor lists are not supported") + if element.kind in (_FieldKind.TENSOR, _FieldKind.TENSOR_OR_QUANTIZED): + slots = tuple( + _SlotSpec( + slot.name, "Tensor[]" if slot.type_str.startswith("Tensor") else slot.type_str + ) + for slot in element.slots + ) + return _FieldPlan(name, element.kind, slots, is_list=True) if _is_tensor_storage_union(annot): slots = ( _SlotSpec(name, "Tensor?"), @@ -551,6 +571,49 @@ def _unpack_tensor_or_quantized(field: _FieldPlan, slots: Dict[str, Any]) -> Any return _storage_unflatten(meta, slots[inner_slot]) +def _pack_tensor_list(field: _FieldPlan, values: Any, slots: Dict[str, Any]) -> None: + """Pack a tensor field's list using the scalar representation per element.""" + if field.kind is _FieldKind.TENSOR: + slots[field.slots[0].name] = [_encode_none(value) for value in values] + return + outer, inner, items, counts = [], [], [], [] + tensor_slot, inner_slot, meta_slot = (s.name for s in field.slots) + for value in values: + element_slots = {} + _pack_tensor_or_quantized(field, value, element_slots) + outer.append(_encode_none(element_slots[tensor_slot])) + tensors = element_slots[inner_slot] + inner.extend(tensors) + counts.append(len(tensors)) + items.append(element_slots[meta_slot]) + slots[tensor_slot] = outer + slots[inner_slot] = inner + slots[meta_slot] = OpaqueValueBundle({"items": items, "counts": counts}) + + +def _unpack_tensor_list(field: _FieldPlan, slots: Dict[str, Any]) -> List[Any]: + """Inverse of :func:`_pack_tensor_list`.""" + if field.kind is _FieldKind.TENSOR: + return [_decode_none(value) for value in slots[field.slots[0].name]] + tensor_slot, inner_slot, meta_slot = (s.name for s in field.slots) + bundle = slots[meta_slot] + values = [] + cursor = 0 + for i, (meta, count) in enumerate(zip(bundle["items"], bundle["counts"])): + values.append( + _unpack_tensor_or_quantized( + field, + { + tensor_slot: _decode_none(slots[tensor_slot][i]), + inner_slot: slots[inner_slot][cursor : cursor + count], + meta_slot: meta, + }, + ) + ) + cursor += count + return values + + @dataclasses.dataclass(frozen=True) class _ArgPlan: """The parsed plan for one args dataclass: the single source of truth for @@ -578,17 +641,17 @@ def tensor_field_names(self) -> Tuple[str, ...]: if f.kind in (_FieldKind.TENSOR, _FieldKind.TENSOR_OR_QUANTIZED) ) - def tensor_or_quantized_offsets(self) -> List[int]: + def tensor_or_quantized_offsets(self) -> List[Tuple[int, bool]]: """Start offset of each tensor-or-quantized slot group. Derived from ``fields`` on demand -- used once per registration, for the subclass-flattening dispatch wiring. """ - offsets: List[int] = [] + offsets: List[Tuple[int, bool]] = [] offset = 0 for field in self.fields: if field.kind is _FieldKind.TENSOR_OR_QUANTIZED: - offsets.append(offset) + offsets.append((offset, field.is_list)) offset += len(field.slots) return offsets @@ -623,7 +686,12 @@ def pack(self, obj: Any) -> Dict[str, Any]: simple: Dict[str, Any] = {} for field in self.fields: value = getattr(obj, field.name, None) + if field.is_list: + _pack_tensor_list(field, value, slots) + continue match field.kind: + case _FieldKind.SYMINT_LIST: + slots[field.slots[0].name] = list(value) case _FieldKind.TENSOR: slots[field.slots[0].name] = value case _FieldKind.TENSOR_OR_QUANTIZED: @@ -657,7 +725,12 @@ def unpack(self, slots: Dict[str, Any]) -> Any: kwargs: Dict[str, Any] = {} bundle = slots.get(_SIMPLE_META_SLOT) for field in self.fields: + if field.is_list: + kwargs[field.name] = _unpack_tensor_list(field, slots) + continue match field.kind: + case _FieldKind.SYMINT_LIST: + kwargs[field.name] = list(slots[field.slots[0].name]) case _FieldKind.TENSOR: kwargs[field.name] = slots[field.slots[0].name] case _FieldKind.TENSOR_OR_QUANTIZED: @@ -734,7 +807,12 @@ def _spec_view(obj: Any, tensor_field_names: Sequence[str]) -> Any: overrides: Dict[str, Any] = {} for name in tensor_field_names: value = getattr(obj, name, None) - if value is not None and not isinstance(value, TensorSpec): + if isinstance(value, (list, tuple)): + overrides[name] = [ + to_tensor_spec(v) if v is not None and not isinstance(v, TensorSpec) else v + for v in value + ] + elif value is not None and not isinstance(value, TensorSpec): overrides[name] = to_tensor_spec(value) if not overrides: return obj @@ -827,8 +905,7 @@ def _pack_fwd_result(result: Any) -> List[torch.Tensor]: def _pack_bwd_result(grads: Any, num_grad_inputs: int, op_qualname: str) -> List[torch.Tensor]: """Pack a backward-impl return tuple into the op's ``Tensor[]`` payload. - Each grad occupies exactly one slot (validated against ``num_grad_inputs``); - a :class:`TensorSpec` grad is materialized into a single tensor. + List gradients occupy one slot per element; TensorSpec grads are materialized. """ grads = list(grads) if len(grads) != num_grad_inputs: @@ -837,11 +914,12 @@ def _pack_bwd_result(grads: Any, num_grad_inputs: int, op_qualname: str) -> List f"(one per input_tensors_for_grad entry), got {len(grads)}" ) out: List[torch.Tensor] = [] - for g in grads: - if isinstance(g, TensorSpec): - out.append(_encode_none(g.create_tensor())) - else: - out.append(_encode_none(g)) + for grad in grads: + for g in grad if isinstance(grad, (list, tuple)) else (grad,): + if isinstance(g, TensorSpec): + out.append(_encode_none(g.create_tensor())) + else: + out.append(_encode_none(g)) return out @@ -1009,11 +1087,19 @@ def _register_autograd_for_op( the saved tuple + ``ctx_attrs`` to the module's ``setup_context`` and stashes the plan on ``ctx`` so backward can slice its grads per user output. """ + tensor_list_slots = tuple( + i + for i, slot in enumerate(s for f in fwd_plan.fields for s in f.slots) + if slot.type_str == "Tensor[]" + ) bwd_takes_grad_tuple = any(f.name == "grad_outputs" for f in bwd_plan.fields) def _setup_context(ctx, inputs, output): + # Autograd treats empty SymInt[] inputs as tensor lists too. ctx.fwd_tensor_list_lengths = { - i: len(value) for i, value in enumerate(inputs) if isinstance(value, list) + i: len(value) + for i, value in enumerate(inputs) + if i in tensor_list_slots or (isinstance(value, list) and not value) } fwd_obj = fwd_plan.unpack(dict(zip(fwd_plan.slot_names, inputs))) spec_obj = _spec_view(fwd_obj, fwd_plan.tensor_field_names()) @@ -1039,7 +1125,13 @@ def _setup_context(ctx, inputs, output): # rederive shapes lossily (e.g. rank-1 inputs come back rank-2), so the # returned grads are viewed back to the true input shapes below. ctx.grad_input_shapes = { - pos: inputs[pos].shape for pos in grad_targets if isinstance(inputs[pos], torch.Tensor) + pos: ( + [value.shape for value in inputs[pos]] + if pos in tensor_list_slots + else inputs[pos].shape + ) + for pos in grad_targets + if pos in tensor_list_slots or isinstance(inputs[pos], torch.Tensor) } def _autograd_backward(ctx, *grad_outputs): @@ -1063,25 +1155,63 @@ def _autograd_backward(ctx, *grad_outputs): out: List[Any] = [None] * len(fwd_plan.slot_names) for pos, length in ctx.fwd_tensor_list_lengths.items(): out[pos] = [None] * length - for pos, g in zip(grad_targets, grads): - if g is not None: - shape = ctx.grad_input_shapes.get(pos) - if shape is not None and g.shape != shape: - g = g.view(shape) - out[pos] = g + cursor = 0 + for pos in grad_targets: + is_list = pos in ctx.fwd_tensor_list_lengths + length = ctx.fwd_tensor_list_lengths[pos] if is_list else 1 + if cursor + length > len(grads): + raise RuntimeError("bwd op returned too few flat grads for its input fields") + shapes = ctx.grad_input_shapes.get(pos) + values = list(grads[cursor : cursor + length]) + for i, g in enumerate(values): + shape = shapes[i] if is_list else shapes + if g is not None and shape is not None and g.shape != shape: + values[i] = g.view(shape) + out[pos] = values if is_list else values[0] + cursor += length + if cursor != len(grads): + raise RuntimeError(f"bwd op returned {len(grads)} flat grads, expected {cursor}") ctx.grad_input_shapes = None return tuple(out) fwd_op.register_autograd(_autograd_backward, setup_context=_setup_context) +def _flatten_list_subclasses(new_args: List[Any], offset: int, subclass: type) -> None: + """Flatten subclass entries inside a tensor-or-quantized list slot.""" + outer = list(new_args[offset]) + if not any(isinstance(value, subclass) for value in outer): + return + bundle = new_args[offset + 2] + items = list(bundle["items"]) + counts = list(bundle["counts"]) + inner = [] + cursor = 0 + for i, value in enumerate(outer): + tensors = new_args[offset + 1][cursor : cursor + counts[i]] + cursor += counts[i] + if isinstance(value, subclass): + items[i], tensors = _storage_flatten( + value, {_TQ_KIND_KEY: _TensorOrQuantizedKind.STORAGE} + ) + outer[i] = _encode_none(None) + counts[i] = len(tensors) + inner.extend(tensors) + new_args[offset] = outer + new_args[offset + 1] = inner + new_args[offset + 2] = OpaqueValueBundle({"items": items, "counts": counts}) + + def _flatten_subclass_into_slots( - new_args: List[Any], slot_offsets: Sequence[int], subclass: type + new_args: List[Any], slot_offsets: Sequence[Tuple[int, bool]], subclass: type ) -> None: """Rewrite each tensor-or-quantized slot group whose ``Tensor?`` slot holds an instance of ``subclass`` into the storage layout (3 slots: name / tensors / meta). """ - for offset in slot_offsets: + for offset, is_list in slot_offsets: + if is_list: + _flatten_list_subclasses(new_args, offset, subclass) + continue val = new_args[offset] if not isinstance(val, subclass): continue @@ -1092,7 +1222,7 @@ def _flatten_subclass_into_slots( def _make_slot_forwarder( - base_op: Any, slot_offsets: Sequence[int], subclasses: Sequence[type] + base_op: Any, slot_offsets: Sequence[Tuple[int, bool]], subclasses: Sequence[type] ) -> Callable[[Sequence[Any]], List[torch.Tensor]]: """Return ``call(args)`` forwarding to ``base_op``, first flattening any ``subclasses`` instance sitting in the tensor-or-quantized slot groups at @@ -1133,7 +1263,7 @@ def _register_wrapper_op( wrapper_op_name: str, schema_str: str, base_op: Any, - slot_offsets: Sequence[int] = (), + slot_offsets: Sequence[Tuple[int, bool]] = (), subclasses: Sequence[type] = (), ) -> Any: """Define the wrapper op via ``torch.library.custom_op``: forward to the base diff --git a/transformer_engine/pytorch/module/grouped_linear.py b/transformer_engine/pytorch/module/grouped_linear.py index 4521461c3d4..416161eca41 100644 --- a/transformer_engine/pytorch/module/grouped_linear.py +++ b/transformer_engine/pytorch/module/grouped_linear.py @@ -5,8 +5,9 @@ """GroupedLinear API""" from dataclasses import dataclass -from typing import Any, Union, Optional, Callable, Tuple, List +from typing import Any, Union, Optional, Callable, Dict, Tuple, List from itertools import chain +import math import os import warnings import weakref @@ -44,6 +45,10 @@ requires_grad, resolve_grouped_linear_single_param_flags, get_nvtx_range_context, + warn_compile_eager_fallback, + warn_if_compile_disabled, + check_grouped_gemm_dims, + is_non_tn_fp8_gemm_supported, ) from ..distributed import ( set_tensor_model_parallel_attributes, @@ -61,6 +66,13 @@ general_grouped_gemm, general_grouped_gemm_for_grouped_tensor, ) +from ..cpp_extensions.gemm import get_cublas_workspace +from ..dynamo import ( + TensorSpec, + register_custom_op, + is_value_opaque_quantizer, +) +from .linear import _fake_workspace_valid from ..constants import GemmParallelModes, dist_group_type from ..jit import no_torch_dynamo from ..cpu_offload import is_cpu_offload_enabled, mark_not_offload, start_offload @@ -124,7 +136,7 @@ def is_module_grouped_tensor_path_supported( device_capability = get_device_compute_capability() if not (9, 0) <= device_capability <= (11, 0): return False - cublaslt_version = tex.get_cublasLt_version() + cublaslt_version = _get_cublaslt_version() if cublaslt_version < 130300: return False if device_capability < (10, 0) and cublaslt_version < 130400: @@ -152,6 +164,13 @@ def is_module_grouped_tensor_path_supported( return False +@torch.compiler.assume_constant_result +@functools.lru_cache(maxsize=None) +def _get_cublaslt_version() -> int: + """Cached, Dynamo-constant cuBLASLt version (the pybind call is untraceable).""" + return tex.get_cublasLt_version() + + @dataclass(slots=True) class GroupedLinearFwdArgs: """Forward configuration and operands.""" @@ -168,6 +187,7 @@ class GroupedLinearFwdArgs: input_requires_grad: bool weights_requires_grad: bool + bias_requires_grad: bool input_quantizers: List[Quantizer] weight_quantizers: List[Quantizer] @@ -193,6 +213,7 @@ class GroupedLinearFwdArgs: cache_weight: bool use_grouped_tensor_path: bool + single_grouped_param: bool single_grouped_weight: bool single_grouped_bias: bool @@ -201,6 +222,50 @@ class GroupedLinearFwdArgs: wgrad_store: Optional[Any] cpu_offloading: bool is_grad_enabled: bool + compiled_op: bool = False + + def compile_unsupported_reason(self) -> Optional[str]: + """Reason this config can't use the torch.compile custom-op path (else None).""" + if self.debug: + return "debug instrumentation (nvidia-dlfw-inspect)" + if is_distributed_weight(self.weights[0]): + return "a DistributedWeight (custom weight parallelism, e.g. GTP)" + if self.out is not None or self.dgrad_out is not None: + return "a user-provided out/dgrad_out buffer (the op would return an input alias)" + if self.single_grouped_param: + return "single_grouped_weight/single_grouped_bias parameter views" + if self.cpu_offloading: + return "CPU activation offloading" + if self.wgrad_store is not None: + return "delayed wgrad compute (wgrad_store)" + if self.fuse_wgrad_accumulation: + return "fuse_wgrad_accumulation (main_grad)" + if self.fp8_calibration: + return "fp8_calibration" + if self.use_grouped_tensor_path: + if self.save_original_input: + return "save_original_input on the fused grouped-tensor path" + if self.fp8 and not isinstance(self.input_quantizers[0], Float8CurrentScalingQuantizer): + return ( + "a fused-path FP8 recipe other than per-tensor current scaling " + "(grouped scale offsets are per-split host metadata)" + ) + elif self.m_splits is None: + return "m_splits passed as a device tensor inside the compiled region" + if any(w.requires_grad != self.weights_requires_grad for w in self.weights): + return "mixed requires_grad across the weights list" + for quantizer_list in ( + self.input_quantizers, + self.weight_quantizers, + self.output_quantizers, + self.grad_input_quantizers, + self.grad_weight_quantizers, + self.grad_output_quantizers, + ): + for quantizer in quantizer_list: + if quantizer is not None and not is_value_opaque_quantizer(quantizer): + return "a quantizer not registered as a torch.compile value-opaque type" + return None @dataclass(slots=True) @@ -208,6 +273,7 @@ class GroupedLinearBwdArgs: """Backward configuration and saved operands.""" grad_output: Optional[torch.Tensor] = None + inputmat_full: Optional[torch.Tensor] = None inputmats: List[Union[torch.Tensor, QuantizedTensorStorage]] = None weights_fp8: List[Union[torch.Tensor, QuantizedTensorStorage]] = None saved_weights: List[Union[torch.Tensor, QuantizedTensorStorage]] = None @@ -246,21 +312,29 @@ class GroupedLinearBwdArgs: reduce_and_update_bwd_fp8_tensors: bool = False cpu_offloading: bool = False + compiled_op: bool = False def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: """Pull saved tensors from ``ctx`` into the fields backward consumes.""" saved = restore_from_func_ctx(ctx) n = self.num_gemms - self.inputmats = list(saved[:n]) - self.weights_fp8 = list(saved[n : 2 * n]) - self.saved_weights = list(saved[2 * n : 3 * n]) - self.biases = list(saved[3 * n : 4 * n]) + self.inputmat_full = saved[0] + self.inputmats = list(saved[1 : 1 + n]) + self.weights_fp8 = list(saved[1 + n : 1 + 2 * n]) + self.saved_weights = list(saved[1 + 2 * n : 1 + 3 * n]) + self.biases = list(saved[1 + 3 * n : 1 + 4 * n]) def _grouped_linear_forward_impl( args: GroupedLinearFwdArgs, ) -> Tuple[Any, ...]: - """Compute the legacy forward and collect tensors needed by backward.""" + """Forward implementation for the grouped linear layer (legacy, non-fused path). + + Returns ``(out, *new_workspaces, tensors_to_save, ctx_attrs)``. + ``new_workspaces`` are the freshly produced FP8 weight workspaces (returned + alongside ``out`` so the caller can refresh its cache). The trailing two are + ``None`` when gradients are disabled. + """ inp = args.inp weights = list(args.weights) biases = list(args.biases) @@ -326,14 +400,30 @@ def _grouped_linear_forward_impl( ) inp_view = inp.reshape(-1, in_features) - inputmats, _ = _split_quantization._split_quantize( - inp_view, - m_splits, - input_quantizers, - activation_dtype, - with_quantized_output=fp8 or debug, - disable_bulk_allocation=cpu_offloading, - ) + fp8_or_debug = fp8 or debug + inputmat_full = None + if fp8_or_debug: + # Disable bulk allocation when CPU offloading is active: offloading skips small + # tensors (like scales), but bulk allocation shares storage across all tensors, + # so if scales can't be offloaded, nothing in the group can be offloaded. + # The compiled op also disables it: bulk-allocated storages alias each + # other, and the op returns them as saved tensors. + inputmats, _ = _split_quantization._split_quantize( + inp_view, + m_splits, + with_quantized_output=True, + quantizers=input_quantizers, + activation_dtype=activation_dtype, + disable_bulk_allocation=cpu_offloading or args.compiled_op, + ) + else: + # Plain path: split views of one (possibly cast) buffer. Save that + # buffer once (backward re-splits it) instead of N aliasing views -- + # except under CPU offloading, which marks the individual views. + inputmat_full = cast_if_needed(inp_view, activation_dtype) + inputmats = torch.split(inputmat_full, m_splits) + if cpu_offloading: + inputmat_full = None if cpu_offloading: start_offload(*inputmats) @@ -341,7 +431,7 @@ def _grouped_linear_forward_impl( # Initialize weights weights_fp8: list new_workspaces = [None] * num_gemms - if fp8 or debug: + if fp8_or_debug: weights_fp8 = [] update_ws = args.is_first_microbatch is None or args.is_first_microbatch for i in range(num_gemms): @@ -395,12 +485,21 @@ def _grouped_linear_forward_impl( mark_not_offload(*weights_fp8, *weights) tensors_to_save = None + ctx_attrs = None if is_grad_enabled: + # Saved-tensor layout: ``(inputmat_full, *inputmats, *weights_fp8, + # *saved_weights, *biases)`` -- 1 + 4N slots. Slots that alias a forward + # input or another op return are deduped through name-based alias tags + # (rebuilt in ``_grouped_linear_setup_ctx``): a custom op may not return + # aliasing tensors. + aliases: List[Optional[Tuple]] = [None] * (1 + 4 * num_gemms) + # TODO: update after #1638 is merged. # pylint: disable=fixme if weight_requires_grad: if save_original_input: + inputmat_full = None inputmats = [None] * num_gemms - inputmats[0] = inp + aliases[0] = ("inp",) else: for inputmat in inputmats: if isinstance(inputmat, QuantizedTensorStorage): @@ -410,32 +509,269 @@ def _grouped_linear_forward_impl( inputmat.update_usage(rowwise_usage=True, columnwise_usage=False) else: inputmat.update_usage(rowwise_usage=False, columnwise_usage=True) + if inputmat_full is not None: + # Plain path: the views in ``inputmats`` are not saved. + inputmats = [None] * num_gemms + if inputmat_full is inp_view: + # No-op cast: inp_view is inp itself or a view of it. + inputmat_full = None + aliases[0] = ("inp",) else: + inputmat_full = None inputmats = [None] * num_gemms # Original weights are only needed by high_precision dgrad. The weakrefs # used for fused wgrad accumulation serve a different purpose: restoring # Python parameter attributes without keeping the parameter alive here. - saved_weights = ( - weights - if backward_override == "high_precision" and args.input_requires_grad - else [None] * num_gemms - ) + save_origin_weights = backward_override == "high_precision" and args.input_requires_grad + saved_weights = [None] * num_gemms + wt_saves = list(weights_fp8) + for i in range(num_gemms): + slot = 1 + num_gemms + i + if wt_saves[i] is weights[i]: + aliases[slot] = ("weights", i) + wt_saves[i] = None + elif new_workspaces[i] is not None and wt_saves[i] is new_workspaces[i]: + aliases[slot] = ("new_weight_workspaces", i) + wt_saves[i] = None + elif ( + args.weight_workspaces + and args.weight_workspaces[i] is not None + and wt_saves[i] is args.weight_workspaces[i] + ): + aliases[slot] = ("weight_workspaces", i) + wt_saves[i] = None + if save_origin_weights: + aliases[1 + 2 * num_gemms + i] = ("weights", i) if is_dist_weight: # GTP: gathered workspace is transient (re-gathered in backward), don't save it. - weights_fp8 = [None] * num_gemms - saved_weights = args.weights - tensors_to_save = (*inputmats, *weights_fp8, *saved_weights, *biases) + wt_saves = [None] * num_gemms + for i in range(num_gemms): + aliases[1 + num_gemms + i] = None + aliases[1 + 2 * num_gemms + i] = ("weights", i) - return out.view(-1, *inp.shape[1:-1], out.shape[-1]), new_workspaces, tensors_to_save + saved_biases = list(biases) + for i in range(num_gemms): + if saved_biases[i] is not None and saved_biases[i] is args.biases[i]: + aliases[1 + 3 * num_gemms + i] = ("biases", i) + saved_biases[i] = None + + tensors_to_save = ( + inputmat_full, + *inputmats, + *wt_saves, + *saved_weights, + *saved_biases, + ) + ctx_attrs = {"saved_tensor_aliases": tuple(aliases)} + + # [*, in_features] -> [*, out_features] + out = out.view(-1, *inp.shape[1:-1], out.shape[-1]) + return (out, *new_workspaces, tensors_to_save, ctx_attrs) + + +def _fake_grouped_weights(args: GroupedLinearFwdArgs, weight_quantizers): + """Describe weight casts, quantization and workspace aliases for either path.""" + new_workspaces = [None] * args.num_gemms + saved_weights = [None] * args.num_gemms + aliases = [None] * args.num_gemms + for i, weight in enumerate(args.weights): + if (args.fp8 and weight.is_quantized) or ( + not args.fp8 and weight.dtype == args.activation_dtype + ): + aliases[i] = ("weights", i) + continue + quantizer = weight_quantizers[i] if args.fp8 else None + workspace = args.weight_workspaces[i] if args.fp8 else None + if workspace is not None and _fake_workspace_valid(workspace, quantizer): + aliases[i] = ("weight_workspaces", i) + continue + weightmat = TensorSpec( + shape=tuple(weight.shape), + dtype=args.activation_dtype, + quantizer=quantizer, + device=weight.device, + ) + if args.fp8 and args.cache_weight: + if weightmat.quantizer is not None: + weightmat.quantizer.internal = False + new_workspaces[i] = weightmat + aliases[i] = ("new_weight_workspaces", i) + else: + saved_weights[i] = weightmat + return new_workspaces, saved_weights, aliases + + +def _grouped_linear_forward_fake( + args: GroupedLinearFwdArgs, +) -> Tuple[Any, ...]: + """Shape/metadata-only twin of :func:`_grouped_linear_forward_impl` for + torch.compile. Only mirrors configs the compiled path admits (see + ``compile_unsupported_reason``): no debug, offloading, distributed weights, + calibration, or fused GroupedTensor path. + """ + inp = args.inp + weights = list(args.weights) + num_gemms = args.num_gemms + m_splits = list(args.m_splits) + input_quantizers = args.input_quantizers + weight_quantizers = args.weight_quantizers + output_quantizers = args.output_quantizers + activation_dtype = args.activation_dtype + fp8 = args.fp8 + is_grad_enabled = args.is_grad_enabled + save_original_input = args.save_original_input + backward_override = args.backward_override + weight_requires_grad = args.weights_requires_grad + in_features = weights[0].shape[-1] + out_features = weights[0].shape[0] + + # Mirror the impl's quantizer usage setup exactly (buffer layouts must agree). + if input_quantizers[0] is not None: + for input_quantizer in input_quantizers: + input_quantizer.set_usage( + rowwise=True, + columnwise=( + is_grad_enabled + and weight_requires_grad + and not save_original_input + and backward_override is None + ), + ) + columnwise_usage = is_grad_enabled and args.input_requires_grad + if backward_override is not None: + columnwise_usage = False + if not columnwise_usage: + columnwise_usage = ( + is_fp8_activation_recompute_enabled() and not in_fp8_activation_recompute_phase() + ) + if weight_quantizers[0] is not None and not weights[0].is_quantized: + for weight_quantizer in weight_quantizers: + weight_quantizer.set_usage(rowwise=True, columnwise=columnwise_usage) + elif weights[0].is_quantized: + weight_quantizers = [weight.quantizer for weight in weights] + if output_quantizers[0] is not None: + for output_quantizer in output_quantizers: + output_quantizer.set_usage(rowwise=True, columnwise=False) + + # Input pipeline: quantized per-split storages (fp8) or one full cast buffer. + inputmat_full = None + inputmats: List[Optional[TensorSpec]] = [None] * num_gemms + inputmat_full_aliases_inp = False + if fp8: + inputmats = [ + TensorSpec( + shape=(m_splits[i], in_features), + dtype=activation_dtype, + quantizer=input_quantizers[i], + device=inp.device, + ) + for i in range(num_gemms) + ] + else: + inputmat_full_aliases_inp = inp.dtype == activation_dtype + inputmat_full = TensorSpec( + shape=(sum(m_splits), in_features), + dtype=activation_dtype, + device=inp.device, + ) + + new_workspaces, weights_fp8, weight_aliases = _fake_grouped_weights(args, weight_quantizers) + + # Bias pipeline. + bias_dtype = activation_dtype + if fp8 and activation_dtype == torch.float32: + bias_dtype = torch.bfloat16 + saved_biases: List[Optional[TensorSpec]] = [None] * num_gemms + bias_aliases: List[Optional[Tuple]] = [None] * num_gemms + for i in range(num_gemms): + bias = args.biases[i] + if bias is None: + continue + if not args.use_bias or bias.dtype == bias_dtype: + bias_aliases[i] = ("biases", i) + else: + saved_biases[i] = TensorSpec( + shape=tuple(bias.shape), dtype=bias_dtype, device=bias.device + ) + + out = TensorSpec( + shape=(*tuple(inp.shape[:-1]), out_features), + dtype=activation_dtype, + quantizer=None, + requires_grad=is_grad_enabled + and (args.input_requires_grad or weight_requires_grad or args.bias_requires_grad), + device=inp.device, + ) + + tensors_to_save = None + ctx_attrs = None + if is_grad_enabled: + aliases: List[Optional[Tuple]] = [None] * (1 + 4 * num_gemms) + if weight_requires_grad: + if save_original_input: + inputmat_full = None + inputmats = [None] * num_gemms + aliases[0] = ("inp",) + else: + if fp8: + for inputmat in inputmats: + if backward_override is not None: + inputmat.update_usage(rowwise_usage=True, columnwise_usage=False) + else: + inputmat.update_usage(rowwise_usage=False, columnwise_usage=True) + elif inputmat_full_aliases_inp: + inputmat_full = None + aliases[0] = ("inp",) + else: + inputmat_full = None + inputmats = [None] * num_gemms + + saved_weights = [None] * num_gemms + save_origin_weights = backward_override == "high_precision" and args.input_requires_grad + for i in range(num_gemms): + aliases[1 + num_gemms + i] = weight_aliases[i] + if save_origin_weights: + aliases[1 + 2 * num_gemms + i] = ("weights", i) + aliases[1 + 3 * num_gemms + i] = bias_aliases[i] + + tensors_to_save = ( + inputmat_full, + *inputmats, + *weights_fp8, + *saved_weights, + *saved_biases, + ) + ctx_attrs = {"saved_tensor_aliases": tuple(aliases)} + + return (out, *new_workspaces, tensors_to_save, ctx_attrs) + + +def _restore_saved_aliases(fwd_args, fwd_outputs, ctx_attrs, tensors_to_save): + """Restore op inputs and workspaces omitted from the saved output payload.""" + saved = list(tensors_to_save) + for slot, alias in enumerate(ctx_attrs["saved_tensor_aliases"]): + if alias is not None: + value = ( + fwd_outputs[1:] + if alias[0] == "new_weight_workspaces" + else getattr(fwd_args, alias[0]) + ) + saved[slot] = value[alias[1]] if len(alias) == 2 else value + return tuple(saved) def _grouped_linear_setup_ctx( bwd_args: GroupedLinearBwdArgs, fwd_args: GroupedLinearFwdArgs, -) -> None: - """Populate backward arguments from the forward configuration.""" + fwd_outputs: Tuple[Any, ...], + ctx_attrs: Dict, + tensors_to_save_from_forward: Tuple[Any, ...], +) -> Tuple[Any, ...]: + """Populate ``bwd_args`` from forward state and return the tensors to persist + (alias-tagged slots rebuilt from ``fwd_args`` / ``fwd_outputs``).""" num_gemms = fwd_args.num_gemms + weights = fwd_args.weights weight_quantizers = fwd_args.weight_quantizers if isinstance(weights[0], QuantizedTensorStorage) and not fwd_args.debug: @@ -468,6 +804,7 @@ def _grouped_linear_setup_ctx( bwd_args.wgrad_store = fwd_args.wgrad_store bwd_args.cpu_offloading = fwd_args.cpu_offloading bwd_args.dgrad_out = fwd_args.dgrad_out + bwd_args.compiled_op = fwd_args.compiled_op if fwd_args.fuse_wgrad_accumulation and fwd_args.weights_requires_grad: # Keep weakrefs to weights to preserve attributes like main_grad @@ -491,6 +828,8 @@ def _grouped_linear_setup_ctx( bwd_args.grad_weight_quantizers = [None] * num_gemms bwd_args.grad_output_quantizers = [None] * num_gemms + return _restore_saved_aliases(fwd_args, fwd_outputs, ctx_attrs, tensors_to_save_from_forward) + def _finish_wgrad(weight, main_grad, wgrad, fuse_wgrad_accumulation): """Return the gradient expected by autograd or MCore's main-grad hooks.""" @@ -524,6 +863,10 @@ def _grouped_linear_backward_impl( in_features = args.weights_shape_1 dgrad = None + # Plain (non-quantized) inputs are saved as one full buffer; re-split it. + if args.inputmat_full is not None and not args.save_original_input: + inputmats = list(torch.split(args.inputmat_full.reshape(-1, in_features), m_splits)) + # Restore from weakrefs to get original weight python objects # (preserves attributes like main_grad, grad_added_to_main_grad, etc.) # Only needed when fuse_wgrad_accumulation is enabled. @@ -623,6 +966,16 @@ def _grouped_linear_backward_impl( if args.weights_requires_grad: if args.fuse_wgrad_accumulation: wgrad_list = main_grads + elif args.compiled_op: + # Packed allocation would make the returned wgrads alias each other. + wgrad_list = [ + torch.empty( + (out_features, in_features), + dtype=args.activation_dtype, + device=device, + ) + for _ in range(num_gemms) + ] else: wgrad_packed = torch.empty( num_gemms, @@ -637,7 +990,7 @@ def _grouped_linear_backward_impl( del weights if args.save_original_input: - inp = inputmats[0] + inp = args.inputmat_full inp_view = inp.reshape(-1, in_features) if args.input_quantizers[0] is not None: for input_quantizer in args.input_quantizers: @@ -688,7 +1041,10 @@ def _grouped_linear_backward_impl( grad_biases[i] = grad_biases_[i] del grad_biases_ - clear_tensor_data(*inputmats) + # Deallocate input tensor (in-place storage resize: not allowed on + # the compiled path, where the inputs belong to the op caller). + if not args.compiled_op: + clear_tensor_data(*inputmats) if is_dist_weight: wgrad_list = finalize_weight_grads(origin_weights, wgrad_list) @@ -712,6 +1068,76 @@ def _grouped_linear_backward_impl( return (dgrad_out, wgrad_list, list(grad_biases)) +def _fake_grouped_grads(args, in_features, compute_bias): + """Describe gradient allocations shared by both compiled paths.""" + grad_output = args.grad_output + out_features = grad_output.shape[-1] + dgrad = None + if args.requires_dgrad: + dgrad = TensorSpec( + shape=(*tuple(grad_output.shape[:-1]), in_features), + dtype=args.activation_dtype, + device=grad_output.device, + ) + + wgrad_list: List[Optional[TensorSpec]] = [None] * args.num_gemms + if args.weights_requires_grad and not args.fuse_wgrad_accumulation: + wgrad_list = [ + TensorSpec( + shape=(out_features, in_features), + dtype=args.activation_dtype, + device=grad_output.device, + ) + for _ in range(args.num_gemms) + ] + + grad_biases: List[Optional[TensorSpec]] = [None] * args.num_gemms + if compute_bias: + grad_biases = [ + TensorSpec( + shape=(out_features,), + dtype=args.activation_dtype, + device=grad_output.device, + ) + for _ in range(args.num_gemms) + ] + + return (dgrad, wgrad_list, grad_biases) + + +def _grouped_linear_backward_fake( + args: GroupedLinearBwdArgs, +) -> Tuple[Optional[TensorSpec], List[Optional[TensorSpec]], List[Optional[TensorSpec]]]: + """Allocation-free fake of :func:`_grouped_linear_backward_impl` on ``TensorSpec``.""" + + # Mirror the impl's hybrid grad-output usage retarget. + grad_output_reference = args.grad_output_quantizers[0] + if args.fp8 and isinstance(grad_output_reference, HybridQuantizer): + for grad_output_quantizer in args.grad_output_quantizers: + grad_output_quantizer.set_usage( + rowwise=args.requires_dgrad, + columnwise=args.weights_requires_grad, + ) + + return _fake_grouped_grads( + args, args.weights_shape_1, args.use_bias and (args.weights_requires_grad or args.fp8) + ) + + +# Custom op used under ``torch.compile``. +_grouped_linear_op = register_custom_op( + op_name="grouped_linear", + input_tensors_for_grad=["inp", "weights", "biases"], + fwd_arg_type=GroupedLinearFwdArgs, + fwd_impl=_grouped_linear_forward_impl, + fwd_fake_impl=_grouped_linear_forward_fake, + setup_context=_grouped_linear_setup_ctx, + bwd_arg_type=GroupedLinearBwdArgs, + bwd_impl=_grouped_linear_backward_impl, + bwd_fake_impl=_grouped_linear_backward_fake, +) + + @dataclass(slots=True) class GroupedLinearFusedBwdArgs: """Backward configuration and saved grouped operands.""" @@ -747,9 +1173,19 @@ class GroupedLinearFusedBwdArgs: wgrad_store: Optional[WeightGradStore] = None reduce_and_update_bwd_fp8_tensors: bool = False + gx_payload: List[torch.Tensor] = None + gx_present: bool = False + gx_swizzled: bool = False + compiled_op: bool = False + def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: """Restore the grouped operands and split metadata.""" saved = restore_from_func_ctx(ctx) + if self.compiled_op: + n_payload = len(_GX_PAYLOAD_KEYS) + self.gx_payload = list(saved[:n_payload]) + self.weights_fp8 = list(saved[n_payload : n_payload + self.num_gemms]) + return n = 1 if self.single_grouped_weight else self.num_gemms self.inputmat = saved[0] self.weights_fp8 = list(saved[1 : 1 + n]) @@ -1043,7 +1479,10 @@ def _grouped_linear_fused_backward( if args.single_grouped_bias: grad_bias_args = [dbias_packed.to(dtype=args.activation_dtype)] else: - grad_bias_args = [dbias_packed[i].to(dtype=args.activation_dtype) for i in range(N)] + grad_bias_args = [ + dbias_packed[i].to(dtype=args.activation_dtype, copy=args.compiled_op) + for i in range(N) + ] else: num_bias_args = 1 if args.single_grouped_bias else N grad_bias_args = [None] * num_bias_args @@ -1112,14 +1551,24 @@ def _grouped_linear_fused_backward( grouped_wgrad.rowwise_data.view(N, args.out_features, args.in_features) ] else: - wgrad_packed = torch.empty( - N, - args.out_features, - args.in_features, - dtype=args.activation_dtype, - device=grad_output.device, - ) - wgrad_output = [wgrad_packed[i] for i in range(N)] + if args.compiled_op: + wgrad_output = [ + torch.empty( + (args.out_features, args.in_features), + dtype=args.activation_dtype, + device=grad_output.device, + ) + for _ in range(N) + ] + else: + wgrad_packed = torch.empty( + N, + args.out_features, + args.in_features, + dtype=args.activation_dtype, + device=grad_output.device, + ) + wgrad_output = [wgrad_packed[i] for i in range(N)] wgrad_list = wgrad_output accumulate = ( @@ -1156,6 +1605,254 @@ def grouped_gemm_wgrad(inputmats, grad_output_mats, grad_weights): return dgrad, wgrad_list, grad_bias_args +# --------------------------------------------------------------------------- # +# Fused GroupedTensor path under torch.compile +# --------------------------------------------------------------------------- # + +# GroupedTensorStorage payload slots, in ``prepare_for_saving`` order. +_GX_PAYLOAD_KEYS = ( + "data", + "columnwise_data", + "scale_inv", + "columnwise_scale_inv", + "amax", + "columnwise_amax", + "scale", + "first_dims", + "last_dims", + "tensor_offsets", +) + + +def _rebuild_grouped_input(args: GroupedLinearFusedBwdArgs) -> Optional[GroupedTensorStorage]: + """Rebuild the saved grouped input storage from its flat payload.""" + if not args.gx_present: + return None + payload = dict(zip(_GX_PAYLOAD_KEYS, args.gx_payload)) + if payload["data"] is not None: + payload["data"] = payload["data"].reshape(-1) + data = payload["data"] if payload["data"] is not None else payload["columnwise_data"] + payload["first_dims"] = args.m_splits_tensor + payload["tensor_offsets"] = args.input_tensor_offsets + scale_offsets = list(range(args.num_gemms + 1)) if args.fp8 else None + return GroupedTensorStorage( + shape=(data.numel() // args.in_features, args.in_features), + dtype=args.activation_dtype, + num_tensors=args.num_gemms, + quantizer=args.input_quantizers[0] if args.fp8 else None, + scale_inv_offsets=scale_offsets if payload["scale_inv"] is not None else None, + columnwise_scale_inv_offsets=( + scale_offsets if payload["columnwise_scale_inv"] is not None else None + ), + with_gemm_swizzled_scales=args.gx_swizzled, + **payload, + ) + + +def _grouped_linear_fused_forward_impl(args: GroupedLinearFwdArgs) -> Tuple[Any, ...]: + """Encode the shared grouped forward's saved tensors without aliases.""" + out, new_workspaces, saved = _grouped_linear_fused_forward(args) + num_gemms = args.num_gemms + grouped_x = saved[0] if saved is not None else None + weights_to_save = list(saved[1 : 1 + num_gemms]) if saved is not None else [] + tensors_to_save = None + ctx_attrs = None + if args.is_grad_enabled: + n_payload = len(_GX_PAYLOAD_KEYS) + aliases: List[Optional[Tuple]] = [None] * (n_payload + num_gemms) + gx_payload: List[Optional[torch.Tensor]] = [None] * n_payload + gx_present = args.weights_requires_grad + if gx_present: + gx_payload = list(grouped_x.get_data_tensors()) + # first_dims is the split tensor itself and tensor_offsets derives + # from it: both rebuilt in backward from m_splits_tensor instead of + # being saved (they may alias the op input). + gx_payload[7] = None + gx_payload[9] = None + if ( + not args.fp8 + and gx_payload[0] is not None + and args.inp.dtype == args.activation_dtype + ): + # No-op cast: the packed data aliases the op input. + gx_payload[0] = None + aliases[0] = ("inp",) + + for i in range(num_gemms): + slot = n_payload + i + if weights_to_save[i] is None: + continue + if weights_to_save[i] is args.weights[i]: + aliases[slot] = ("weights", i) + weights_to_save[i] = None + elif new_workspaces[i] is not None and weights_to_save[i] is new_workspaces[i]: + aliases[slot] = ("new_weight_workspaces", i) + weights_to_save[i] = None + elif ( + args.weight_workspaces + and args.weight_workspaces[i] is not None + and weights_to_save[i] is args.weight_workspaces[i] + ): + aliases[slot] = ("weight_workspaces", i) + weights_to_save[i] = None + + tensors_to_save = (*gx_payload, *weights_to_save) + ctx_attrs = { + "saved_tensor_aliases": tuple(aliases), + "gx_present": gx_present, + "gx_swizzled": bool(getattr(grouped_x, "_with_gemm_swizzled_scales", False)), + } + + return (out, *new_workspaces, tensors_to_save, ctx_attrs) + + +def _grouped_linear_fused_forward_fake(args: GroupedLinearFwdArgs) -> Tuple[Any, ...]: + """Shape/metadata-only twin of :func:`_grouped_linear_fused_forward_impl`. + + Only mirrors configs the fused compiled gate admits: bf16/fp16 or FP8 + per-tensor current scaling. + """ + inp = args.inp + num_gemms = args.num_gemms + in_features = args.weights[0].shape[-1] + out_features = args.weights[0].shape[-2] + activation_dtype = args.activation_dtype + fp8 = args.fp8 + is_grad_enabled = args.is_grad_enabled + weight_requires_grad = args.weights_requires_grad + tokens = math.prod(inp.shape[:-1]) + device = inp.device + + if fp8: + input_quantizer = args.input_quantizers[0] + input_quantizer.set_usage( + rowwise=True, + columnwise=is_grad_enabled and weight_requires_grad, + ) + input_quantizer.optimize_for_gemm = True + + if fp8: + for quantizer in args.weight_quantizers: + quantizer.set_usage( + rowwise=True, columnwise=is_grad_enabled and args.input_requires_grad + ) + new_workspaces, weights_saved, weight_aliases = _fake_grouped_weights( + args, args.weight_quantizers + ) + + out = TensorSpec( + shape=(*tuple(inp.shape[:-1]), out_features), + dtype=activation_dtype, + requires_grad=is_grad_enabled + and (args.input_requires_grad or weight_requires_grad or args.bias_requires_grad), + device=device, + ) + + tensors_to_save = None + ctx_attrs = None + if is_grad_enabled: + n_payload = len(_GX_PAYLOAD_KEYS) + aliases: List[Optional[Tuple]] = [None] * (n_payload + num_gemms) + gx_payload: List[Optional[TensorSpec]] = [None] * n_payload + gx_present = weight_requires_grad + if gx_present: + total = tokens * in_features + + def _spec(numel, dtype): + return TensorSpec(shape=(numel,), dtype=dtype, device=device) + + if not fp8: + if inp.dtype == activation_dtype: + aliases[0] = ("inp",) + else: + gx_payload[0] = _spec(total, activation_dtype) + else: + # FP8 per-tensor current scaling; on Hopper the rowwise data is + # freed after the fprop GEMM when a columnwise copy exists. + has_columnwise = is_grad_enabled and weight_requires_grad + keep_rowwise = not has_columnwise or is_non_tn_fp8_gemm_supported() + if keep_rowwise: + gx_payload[0] = _spec(total, torch.uint8) # data + gx_payload[2] = _spec(num_gemms, torch.float32) # scale_inv + if has_columnwise and not is_non_tn_fp8_gemm_supported(): + gx_payload[1] = _spec(total, torch.uint8) # columnwise_data + gx_payload[3] = _spec(num_gemms, torch.float32) # columnwise_scale_inv + gx_payload[4] = _spec(num_gemms, torch.float32) # amax + gx_payload[6] = _spec(num_gemms, torch.float32) # scale + + for i in range(num_gemms): + if args.input_requires_grad: + aliases[n_payload + i] = weight_aliases[i] + else: + weights_saved[i] = None + + tensors_to_save = (*gx_payload, *weights_saved) + ctx_attrs = { + "saved_tensor_aliases": tuple(aliases), + "gx_present": gx_present, + "gx_swizzled": False, + } + + return (out, *new_workspaces, tensors_to_save, ctx_attrs) + + +def _grouped_linear_fused_setup_ctx( + bwd_args: GroupedLinearFusedBwdArgs, + fwd_args: GroupedLinearFwdArgs, + fwd_outputs: Tuple[Any, ...], + ctx_attrs: Dict, + tensors_to_save_from_forward: Tuple[Any, ...], +) -> Tuple[Any, ...]: + """Populate the fused backward args and rebuild alias-deduped save slots.""" + _grouped_linear_fused_setup(bwd_args, fwd_args) + bwd_args.gx_present = ctx_attrs["gx_present"] + bwd_args.gx_swizzled = ctx_attrs["gx_swizzled"] + bwd_args.compiled_op = fwd_args.compiled_op + + return _restore_saved_aliases(fwd_args, fwd_outputs, ctx_attrs, tensors_to_save_from_forward) + + +def _grouped_linear_fused_backward_impl( + args: GroupedLinearFusedBwdArgs, +) -> Tuple[Optional[torch.Tensor], List[Optional[torch.Tensor]], List[Optional[torch.Tensor]]]: + """Rebuild the op's saved grouped input and run shared backward.""" + args.m_splits_tensor = args.m_splits_tensor.to(device=args.grad_output.device) + args.base_split_offsets = tex.splits_to_offsets(args.m_splits_tensor, 1) + args.input_tensor_offsets = args.base_split_offsets * args.in_features + args.output_tensor_offsets = args.base_split_offsets * args.out_features + args.inputmat = _rebuild_grouped_input(args) + return _grouped_linear_fused_backward(args) + + +def _grouped_linear_fused_backward_fake( + args: GroupedLinearFusedBwdArgs, +) -> Tuple[Optional[TensorSpec], List[Optional[TensorSpec]], List[Optional[TensorSpec]]]: + """Allocation-free fake of :func:`_grouped_linear_fused_backward_impl`.""" + + if args.fp8: + grad_output_quantizer = args.grad_output_quantizers[0] + grad_output_quantizer.set_usage( + rowwise=args.requires_dgrad, + columnwise=args.weights_requires_grad, + ) + + return _fake_grouped_grads(args, args.in_features, args.use_bias) + + +# Custom op for the fused GroupedTensor path under ``torch.compile``. +_grouped_linear_fused_op = register_custom_op( + op_name="grouped_linear_fused", + input_tensors_for_grad=["inp", "weights", "biases"], + fwd_arg_type=GroupedLinearFwdArgs, + fwd_impl=_grouped_linear_fused_forward_impl, + fwd_fake_impl=_grouped_linear_fused_forward_fake, + setup_context=_grouped_linear_fused_setup_ctx, + bwd_arg_type=GroupedLinearFusedBwdArgs, + bwd_impl=_grouped_linear_fused_backward_impl, + bwd_fake_impl=_grouped_linear_fused_backward_fake, +) + + @no_torch_dynamo() def _grouped_linear_eager( inp: torch.Tensor, @@ -1467,12 +2164,22 @@ def forward( if fwd_args.use_grouped_tensor_path: return _GroupedLinear._forward_grouped_tensor(ctx, fwd_args) - out, new_workspaces, saved = _grouped_linear_forward_impl(fwd_args) + outputs = _grouped_linear_forward_impl(fwd_args) + out = outputs[0] + new_workspaces = list(outputs[1 : 1 + num_gemms]) + tensors_to_save_from_forward, ctx_attrs = outputs[-2], outputs[-1] + if ctx is not None: ctx.use_grouped_tensor_path = False bwd_args = GroupedLinearBwdArgs() - _grouped_linear_setup_ctx(bwd_args, fwd_args) - tensors_to_save, tensor_objects = prepare_for_saving(*saved) + tensors_to_save_from_setup = _grouped_linear_setup_ctx( + bwd_args, + fwd_args, + (out, *new_workspaces), + ctx_attrs, + tensors_to_save_from_forward, + ) + tensors_to_save, tensor_objects = prepare_for_saving(*tensors_to_save_from_setup) ctx.save_for_backward(*tensors_to_save) ctx.tensor_objects = tensor_objects ctx.backward_objects = bwd_args @@ -1964,6 +2671,16 @@ def reset_parameters(self, defer_init=False): self.make_grouped_weights(defer_init=defer_init) elif self.single_grouped_bias: self._make_grouped_biases() + if not defer_init: + # Allocate the process-global grouped cuBLAS workspace eagerly: + # under torch.compile the first grouped GEMM can run inside + # CUDA-graph capture, and a workspace first allocated there would + # live in the graph pool. + weight = getattr(self, "weight0", None) + if weight is None: + weight = getattr(self, "weight", None) + if weight is not None and weight.device.type == "cuda": + get_cublas_workspace(weight.device.index, False, True) def set_tensor_parallel_attributes(self, defer_init=False) -> None: """Set attributes needed for TP""" @@ -2128,7 +2845,6 @@ def _load_from_state_dict( error_msgs, ) - @no_torch_dynamo() def forward( self, inp: torch.Tensor, @@ -2184,7 +2900,9 @@ def forward( "use_grouped_tensor=True; the split-quantize path only supports discrete " "parameters." ) + m_splits_host: Optional[Tuple[int, ...]] = None if not isinstance(m_splits, torch.Tensor): + m_splits_host = tuple(int(s) for s in m_splits) # Convert list of ints to tensor for backward compatibility m_splits = torch.tensor(m_splits, dtype=torch.int64, device="cpu") elif m_splits.dtype != torch.int64: @@ -2244,6 +2962,10 @@ def forward( for q in weight_quantizers: q.optimize_for_gemm = optimize_for_gemm + use_compiled_op = torch.compiler.is_compiling() and _grouped_linear_op is not None + if _grouped_linear_op is None and torch.compiler.is_compiling(): + warn_if_compile_disabled() + cache_weight = is_first_microbatch is not None if self.single_grouped_weight: weight_workspaces = [self._fp8_workspaces.get("weight")] if cache_weight else [None] @@ -2365,6 +3087,7 @@ def forward( # requires_grad flags input_requires_grad=inp.requires_grad, weights_requires_grad=weight_requires_grad, + bias_requires_grad=(bias_tensors[0].requires_grad if self.apply_bias else False), # quantizers input_quantizers=input_quantizers, weight_quantizers=weight_quantizers, @@ -2373,7 +3096,7 @@ def forward( grad_weight_quantizers=grad_weight_quantizers, grad_output_quantizers=grad_output_quantizers, # split geometry - m_splits=None, + m_splits=m_splits_host, num_gemms=num_gemms, # numerical / dtype config activation_dtype=self.activation_dtype, @@ -2390,6 +3113,7 @@ def forward( cache_weight=cache_weight, # fused GroupedTensor path use_grouped_tensor_path=use_grouped_tensor_path, + single_grouped_param=self.single_grouped_weight or self.single_grouped_bias, single_grouped_weight=self.single_grouped_weight, single_grouped_bias=use_grouped_bias, # misc @@ -2400,13 +3124,40 @@ def forward( is_grad_enabled=is_grad_enabled, ) - out, new_workspaces = _grouped_linear_eager( - inp, - m_splits, - fwd_args, - (*weight_tensors, *bias_tensors), - is_grad_enabled, - ) + if use_compiled_op: + compiled_op = ( + _grouped_linear_fused_op if use_grouped_tensor_path else _grouped_linear_op + ) + fallback_reason = ( + fwd_args.compile_unsupported_reason() + if compiled_op is not None + else "custom-op registration unavailable" + ) + if fallback_reason is not None: + torch._dynamo.graph_break( + msg=f"te.GroupedLinear falling back to eager: {fallback_reason}" + ) + warn_compile_eager_fallback(fallback_reason) + use_compiled_op = False + + if use_compiled_op: + fwd_args.compiled_op = True + if use_grouped_tensor_path: + fwd_args.m_splits = [] + else: + fwd_args.m_splits_tensor = None + check_grouped_gemm_dims(inp, weight_tensors[0], fwd_args.m_splits, self.fp8) + outputs = compiled_op(fwd_args) + out = outputs[0] + new_workspaces = list(outputs[1:]) + else: + out, new_workspaces = _grouped_linear_eager( + inp, + m_splits, + fwd_args, + (*weight_tensors, *bias_tensors), + is_grad_enabled, + ) if cache_weight: for i, ws in enumerate(new_workspaces): diff --git a/transformer_engine/pytorch/utils.py b/transformer_engine/pytorch/utils.py index c4c6024f47c..af1a3f7ecde 100644 --- a/transformer_engine/pytorch/utils.py +++ b/transformer_engine/pytorch/utils.py @@ -717,6 +717,26 @@ def check_gemm_dims(inp: torch.Tensor, weight: torch.Tensor, fp8: bool) -> None: ) +def check_grouped_gemm_dims( + inp: torch.Tensor, + weight: torch.Tensor, + m_splits: Sequence[int], + fp8: bool, +) -> None: + """Check compiled grouped GEMM dimensions and host split sizes.""" + # pylint: disable=protected-access + check_gemm_dims(inp, weight, fp8) + torch._check( + math.prod(inp.shape[:-1]) == sum(m_splits), + lambda: "GEMM not possible: m_splits must sum to the input's token count", + ) + if fp8: + torch._check( + all(m % 8 == 0 for m in m_splits), + lambda: "FP8 execution requires every m_splits entry to be divisible by 8", + ) + + def is_bf16_compatible() -> bool: """Replaces torch.cuda.is_bf16_compatible() with an explicit check on device compute capability to enforce sm_80 or higher. From e5529472a6345c71bec5e2f12dccb08b1bcdedd2 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 1 Oct 2026 12:56:43 +0200 Subject: [PATCH 02/31] [PyTorch] Fix GroupedLinear compile metadata and extend fused input saving Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 85 ++++++++++++++++--- .../pytorch/dynamo/tensor_spec.py | 11 ++- transformer_engine/pytorch/module/_common.py | 25 +++++- .../pytorch/module/grouped_linear.py | 22 +++-- transformer_engine/pytorch/utils.py | 2 + 5 files changed, 127 insertions(+), 18 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 05d4605bd55..55be31be7c6 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -2407,7 +2407,8 @@ def _assert_close_grouped(fn, compiled, model, base, ref_model=None): inp_eager = base.detach().clone().requires_grad_(True) ref_model.zero_grad(set_to_none=True) out_eager = fn(inp_eager) - out_eager.sum().backward() + grad_output = torch.randn_like(out_eager) + out_eager.backward(grad_output) ref_out = out_eager.detach().clone() ref_igrad = inp_eager.grad.detach().clone() ref_grads = [param.grad.detach().clone() for param in ref_model.parameters()] @@ -2416,7 +2417,7 @@ def _assert_close_grouped(fn, compiled, model, base, ref_model=None): model.zero_grad(set_to_none=True) # Clone before a later cuda-graph replay overwrites the static output buffer. out_compiled = compiled(inp_compiled).clone() - out_compiled.sum().backward() + out_compiled.backward(grad_output) torch.testing.assert_close(out_compiled, ref_out, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) torch.testing.assert_close(inp_compiled.grad, ref_igrad, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) @@ -2426,7 +2427,7 @@ def _assert_close_grouped(fn, compiled, model, base, ref_model=None): ref_grad, atol=_EAGER_ATOL, rtol=_EAGER_RTOL, - msg=f"gradient mismatch for {name}", + msg=lambda msg: f"gradient mismatch for {name}: {msg}", ) @@ -2445,26 +2446,36 @@ def test_te_grouped_linear_compiles(fp8_recipe, compile_mode): """ dtype = torch.bfloat16 device = "cuda" + if fp8_recipe is not None and fp8_recipe.nvfp4(): + fp8_recipe = dataclasses.replace(fp8_recipe, disable_stochastic_rounding=True) + block_scaled = fp8_recipe is not None and not fp8_recipe.float8_current_scaling() + m_splits = [128, 256, 128, 256] if block_scaled else _GROUPED_M_SPLITS + in_features, out_features = (128, 128) if block_scaled else (_GROUPED_IN, _GROUPED_OUT) model = te.GroupedLinear( - _GROUPED_NUM_GEMMS, _GROUPED_IN, _GROUPED_OUT, params_dtype=dtype, device=device + _GROUPED_NUM_GEMMS, in_features, out_features, params_dtype=dtype, device=device ) + def make_input(): + return torch.randn( + sum(m_splits), in_features, dtype=dtype, device=device, requires_grad=True + ) + def fn(inp): if fp8_recipe is None: - return model(inp, _GROUPED_M_SPLITS) + return model(inp, m_splits) with te.autocast(recipe=fp8_recipe): - return model(inp, _GROUPED_M_SPLITS) + return model(inp, m_splits) torch._dynamo.reset() if compile_mode == "reduce-overhead": - _cudagraph_warmup(fn, _grouped_input(dtype, device, requires_grad=True), backward=True) + _cudagraph_warmup(fn, make_input(), backward=True) model.zero_grad(set_to_none=True) compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) n_iters = 3 if compile_mode == "reduce-overhead" else 1 with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): for _ in range(n_iters): - _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + _assert_close_grouped(fn, compiled, model, make_input()) @pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") @@ -2608,10 +2619,17 @@ def fn(inp): m_splits[2:] = [40, 40] _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + m_splits[:] = [24, 40, 32, 32] + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + baseline = _dynamo_counter("stats", "unique_graphs") + m_splits[:] = [40, 24, 48, 16] + _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + if baseline: + assert _dynamo_counter("stats", "unique_graphs") == baseline + @pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") @pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) -@pytest.mark.xfail(reason="waiting for a PyTorch fix", strict=False) def test_te_grouped_linear_compile_train_eval_switch(): """train -> eval -> train on the same compiled ``te.GroupedLinear``, vs eager.""" dtype = torch.bfloat16 @@ -2733,6 +2751,51 @@ def fn(inp): _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("layout", ["flat", "rank4", "strided"]) +@pytest.mark.parametrize("input_grad,weight_grad", [(True, True), (False, True), (True, False)]) +def test_te_grouped_linear_compile_empty_experts(layout, input_grad, weight_grad): + """Empty experts and non-2D inputs must match native PyTorch gradients.""" + torch._dynamo.reset() + splits = [32, 0, 64, 32] + model = te.GroupedLinear(4, 64, 32, bias=False, params_dtype=torch.bfloat16, device="cuda") + model.requires_grad_(weight_grad) + # Keep the native reference exact across GEMM implementations. + with torch.no_grad(): + for weight in model.parameters(): + weight.copy_(torch.randint_like(weight, -1, 2) / 8) + x = torch.randint(-1, 2, (128, 64), device="cuda").to(torch.bfloat16) / 8 + if layout == "rank4": + x = x.reshape(2, 4, 16, 64) + elif layout == "strided": + x = x.t().contiguous().t() + x.requires_grad_(input_grad) + weights = tuple(model.parameters()) + ref_x = x.detach().clone().requires_grad_(input_grad) + ref_weights = tuple(w.detach().clone().requires_grad_(weight_grad) for w in weights) + ref = torch.cat( + [ + torch.nn.functional.linear(part, weight) + for part, weight in zip(ref_x.reshape(-1, 64).split(splits), ref_weights) + ] + ).reshape(*x.shape[:-1], 32) + out = torch.compile(lambda inp: model(inp, splits), fullgraph=True)(x) + dy = torch.randint_like(out, -1, 2) / 8 + params = ((x,) if input_grad else ()) + (weights if weight_grad else ()) + ref_params = ((ref_x,) if input_grad else ()) + (ref_weights if weight_grad else ()) + actual = out, torch.autograd.grad(out, params, dy) + expected = ref, torch.autograd.grad(ref, ref_params, dy) + torch.testing.assert_close(actual, expected, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +def test_te_grouped_linear_compile_rejects_negative_splits(): + model = te.GroupedLinear(4, 64, 32, params_dtype=torch.bfloat16, device="cuda") + compiled = torch.compile(lambda inp: model(inp, [-8, 136, 0, 0]), fullgraph=True) + with pytest.raises(Exception, match="m_splits entries must be non-negative"): + compiled(_grouped_input(torch.bfloat16, "cuda")) + + _fused_grouped_cc_ok = torch.cuda.is_available() and ( (9, 0) <= torch.cuda.get_device_capability() <= (11, 0) ) @@ -2752,7 +2815,8 @@ def fn(inp): [None] + ([recipe.Float8CurrentScaling()] if fp8_available else []), ids=lambda r: "bf16" if r is None else type(r).__name__, ) -def test_te_grouped_linear_fused_compiles(fp8_recipe, compile_mode): +@pytest.mark.parametrize("save_original_input", [False, True]) +def test_te_grouped_linear_fused_compiles(fp8_recipe, compile_mode, save_original_input): """torch.compile(fullgraph=True) of the fused GroupedTensor path (``use_grouped_tensor=True``): ``m_splits`` stays a device tensor, so changing the split distribution in place reuses the very @@ -2769,6 +2833,7 @@ def test_te_grouped_linear_fused_compiles(fp8_recipe, compile_mode): params_dtype=dtype, device=device, use_grouped_tensor=True, + save_original_input=save_original_input, ) m_splits_dev = torch.tensor(_GROUPED_M_SPLITS, dtype=torch.int64, device=device) diff --git a/transformer_engine/pytorch/dynamo/tensor_spec.py b/transformer_engine/pytorch/dynamo/tensor_spec.py index 4cfe225952a..4a0886b23b9 100644 --- a/transformer_engine/pytorch/dynamo/tensor_spec.py +++ b/transformer_engine/pytorch/dynamo/tensor_spec.py @@ -29,6 +29,7 @@ class TensorSpec: quantizer: Optional[Any] = None requires_grad: bool = False device: Optional[torch.device] = field(default=None) + with_gemm_swizzled_scales: Optional[bool] = None def __post_init__(self) -> None: # Own a private copy of the quantizer so usage changes (update_usage) @@ -37,6 +38,10 @@ def __post_init__(self) -> None: if self.quantizer is not None: q = self.quantizer self.quantizer = q.copy() if hasattr(q, "copy") else _copy.copy(q) + if self.with_gemm_swizzled_scales is None: + self.with_gemm_swizzled_scales = self.quantizer.storage_metadata(self.dtype)[ + "nontensor_kwargs" + ].get("with_gemm_swizzled_scales") @property def is_quantized(self) -> bool: @@ -94,9 +99,12 @@ def create_metadata(self) -> Dict[str, Any]: "dtype": self.dtype, "requires_grad": self.requires_grad, } - return self.quantizer.create_metadata( + meta = self.quantizer.create_metadata( tuple(self.shape), dtype=self.dtype, requires_grad=self.requires_grad ) + if self.with_gemm_swizzled_scales is not None: + meta["nontensor_kwargs"]["with_gemm_swizzled_scales"] = self.with_gemm_swizzled_scales + return meta def create_inner_tensors(self) -> List[torch.Tensor]: """Materialize the flat inner tensors (in :meth:`inner_names` order). @@ -162,4 +170,5 @@ def to_tensor_spec(tensor: Any) -> TensorSpec: quantizer=getattr(tensor, "_quantizer", None), requires_grad=requires_grad, device=tensor.device, + with_gemm_swizzled_scales=getattr(tensor, "_with_gemm_swizzled_scales", None), ) diff --git a/transformer_engine/pytorch/module/_common.py b/transformer_engine/pytorch/module/_common.py index 3d04882698a..2c3dc3bf267 100644 --- a/transformer_engine/pytorch/module/_common.py +++ b/transformer_engine/pytorch/module/_common.py @@ -5,6 +5,7 @@ """Internal function used by multiple modules.""" import dataclasses +import math import queue from typing import Any, Callable, List, Optional, Tuple, Union @@ -13,10 +14,32 @@ from .. import cpp_extensions as tex from ..constants import TE_DType from ..distributed import in_fp8_activation_recompute_phase +from ..dynamo import TensorSpec from ..export import is_in_onnx_export_mode from ..quantization import FP8GlobalStateManager from ..tensor.hybrid_tensor import HybridQuantizer -from ..utils import get_default_init_method +from ..tensor.nvfp4_tensor import NVFP4Quantizer +from ..utils import get_default_init_method, get_device_compute_capability + + +def update_nvfp4_direct_output_spec(spec: TensorSpec) -> None: + """Match the NVFP4 layout emitted without a post-quantize swizzle.""" + quantizer = spec.quantizer + if not isinstance(quantizer, NVFP4Quantizer) or not quantizer.optimize_for_gemm: + return + rows, cols = math.prod(spec.shape[:-1]), spec.shape[-1] + if not (10, 0) <= get_device_compute_capability() <= (11, 0): + spec.with_gemm_swizzled_scales = False + elif quantizer.with_rht: + spec.with_gemm_swizzled_scales = bool(rows % 64 == 0 and cols % 128 == 0) + else: + spec.with_gemm_swizzled_scales = bool( + quantizer.with_2d_quantization + and not quantizer.row_scaled_nvfp4 + and not quantizer.nvfp4_use_4over6 + and rows % 128 == 0 + and cols % 128 == 0 + ) def sum_bias_grad(tensor: torch.Tensor) -> torch.Tensor: diff --git a/transformer_engine/pytorch/module/grouped_linear.py b/transformer_engine/pytorch/module/grouped_linear.py index 20036c97d9e..b82cd0e3162 100644 --- a/transformer_engine/pytorch/module/grouped_linear.py +++ b/transformer_engine/pytorch/module/grouped_linear.py @@ -74,6 +74,7 @@ is_value_opaque_quantizer, ) from .linear import _fake_workspace_valid +from ._common import update_nvfp4_direct_output_spec from ..constants import GemmParallelModes, dist_group_type from ..jit import no_torch_dynamo from ..cpu_offload import is_cpu_offload_enabled, mark_not_offload, start_offload @@ -165,8 +166,8 @@ def is_module_grouped_tensor_path_supported( return False -@torch.compiler.assume_constant_result @functools.lru_cache(maxsize=None) +@torch.compiler.assume_constant_result def _get_cublaslt_version() -> int: """Cached, Dynamo-constant cuBLASLt version (the pybind call is untraceable).""" return tex.get_cublasLt_version() @@ -244,8 +245,6 @@ def compile_unsupported_reason(self) -> Optional[str]: if self.fp8_calibration: return "fp8_calibration" if self.use_grouped_tensor_path: - if self.save_original_input: - return "save_original_input on the fused grouped-tensor path" if self.fp8 and not isinstance(self.input_quantizers[0], Float8CurrentScalingQuantizer): return ( "a fused-path FP8 recipe other than per-tensor current scaling " @@ -669,6 +668,8 @@ def _grouped_linear_forward_fake( ) for i in range(num_gemms) ] + for inputmat in inputmats: + update_nvfp4_direct_output_spec(inputmat) else: inputmat_full_aliases_inp = inp.dtype == activation_dtype inputmat_full = TensorSpec( @@ -1729,7 +1730,9 @@ def _grouped_linear_fused_forward_impl(args: GroupedLinearFwdArgs) -> Tuple[Any, aliases: List[Optional[Tuple]] = [None] * (n_payload + num_gemms) gx_payload: List[Optional[torch.Tensor]] = [None] * n_payload gx_present = args.weights_requires_grad - if gx_present: + if gx_present and args.save_original_input: + aliases[0] = ("inp",) + elif gx_present: gx_payload = list(grouped_x.get_data_tensors()) # first_dims is the split tensor itself and tensor_offsets derives # from it: both rebuilt in backward from m_splits_tensor instead of @@ -1822,7 +1825,9 @@ def _grouped_linear_fused_forward_fake(args: GroupedLinearFwdArgs) -> Tuple[Any, aliases: List[Optional[Tuple]] = [None] * (n_payload + num_gemms) gx_payload: List[Optional[TensorSpec]] = [None] * n_payload gx_present = weight_requires_grad - if gx_present: + if gx_present and args.save_original_input: + aliases[0] = ("inp",) + elif gx_present: total = tokens * in_features def _spec(numel, dtype): @@ -1887,7 +1892,7 @@ def _grouped_linear_fused_backward_impl( args.base_split_offsets = tex.splits_to_offsets(args.m_splits_tensor, 1) args.input_tensor_offsets = args.base_split_offsets * args.in_features args.output_tensor_offsets = args.base_split_offsets * args.out_features - args.inputmat = _rebuild_grouped_input(args) + args.inputmat = args.gx_payload[0] if args.save_original_input else _rebuild_grouped_input(args) return _grouped_linear_fused_backward(args) @@ -2288,6 +2293,11 @@ class GroupedLinear(TransformerEngineBaseModule): """Applies linear transformations to the incoming data list :math:`y_i = x_iA_i^T + b_i` in a grouped way. + ``torch.compile(fullgraph=True)`` supports discrete expert parameters with host + split lists. The native grouped-tensor path supports CUDA split tensors with + BF16/FP16 or FP8 current scaling, including ``save_original_input=True``. + Unsupported configurations fall back to eager when graph breaks are allowed. + Parameters ---------- num_gemms : int diff --git a/transformer_engine/pytorch/utils.py b/transformer_engine/pytorch/utils.py index fcef7e36223..65edb8f110c 100644 --- a/transformer_engine/pytorch/utils.py +++ b/transformer_engine/pytorch/utils.py @@ -751,6 +751,8 @@ def check_grouped_gemm_dims( math.prod(inp.shape[:-1]) == sum(m_splits), lambda: "GEMM not possible: m_splits must sum to the input's token count", ) + for split in m_splits: + torch._check(split >= 0, lambda: "m_splits entries must be non-negative") if fp8: torch._check( all(m % 8 == 0 for m in m_splits), From a6d62a5054d0bc65b7e78c7215bb94bf8ffef499 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:10:17 +0000 Subject: [PATCH 03/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../pytorch/module/grouped_linear.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/transformer_engine/pytorch/module/grouped_linear.py b/transformer_engine/pytorch/module/grouped_linear.py index b82cd0e3162..6f3bbd0da52 100644 --- a/transformer_engine/pytorch/module/grouped_linear.py +++ b/transformer_engine/pytorch/module/grouped_linear.py @@ -3118,11 +3118,11 @@ def forward( and (10, 0) <= get_device_compute_capability() <= (11, 0) ): raise RuntimeError( - "use_grouped_tensor=True does not support the FP8 block-scaling recipe on " - "Blackwell GPUs: the native grouped FP8 block-scaling path is Hopper-only. " - "Set use_grouped_tensor=False, or unset " - "NVTE_GROUPED_LINEAR_USE_FUSED_GROUPED_GEMM if it enabled this path, to use " - "the MXFP8-emulated path on Blackwell." + "use_grouped_tensor=True does not support the FP8 block-scaling recipe on" + " Blackwell GPUs: the native grouped FP8 block-scaling path is Hopper-only." + " Set use_grouped_tensor=False, or unset" + " NVTE_GROUPED_LINEAR_USE_FUSED_GROUPED_GEMM if it enabled this path, to" + " use the MXFP8-emulated path on Blackwell." ) use_grouped_tensor_path = is_module_grouped_tensor_path_supported( _recipe, @@ -3134,10 +3134,10 @@ def forward( and (self.single_grouped_weight or use_grouped_bias) ): raise RuntimeError( - "Single grouped parameters require the native grouped-tensor path, but the active " - "device, cuBLASLt version, quantization recipe, or GroupedLinear feature " - "configuration does not support it. Disable single_grouped_weight and " - "single_grouped_bias to allow the split-quantize fallback." + "Single grouped parameters require the native grouped-tensor path, but the" + " active device, cuBLASLt version, quantization recipe, or GroupedLinear" + " feature configuration does not support it. Disable single_grouped_weight and" + " single_grouped_bias to allow the split-quantize fallback." ) if use_grouped_tensor_path: if m_splits.device.type != "cuda": From 7a85879eb43d63c593866a700240c929aa95d768 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Fri, 2 Oct 2026 13:44:36 +0200 Subject: [PATCH 04/31] [PyTorch] Compile fused GroupedLinear with MXFP8 and NVFP4 Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 35 ++++++++++----- .../pytorch/module/grouped_linear.py | 43 ++++++++++--------- 2 files changed, 48 insertions(+), 30 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 55be31be7c6..85fe92d610b 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -2812,7 +2812,10 @@ def test_te_grouped_linear_compile_rejects_negative_splits(): @pytest.mark.parametrize("compile_mode", _compile_modes) @pytest.mark.parametrize( "fp8_recipe", - [None] + ([recipe.Float8CurrentScaling()] if fp8_available else []), + [None] + + ([recipe.Float8CurrentScaling()] if fp8_available else []) + + ([recipe.MXFP8BlockScaling()] if mxfp8_available else []) + + ([recipe.NVFP4BlockScaling(disable_stochastic_rounding=True)] if nvfp4_available else []), ids=lambda r: "bf16" if r is None else type(r).__name__, ) @pytest.mark.parametrize("save_original_input", [False, True]) @@ -2826,16 +2829,24 @@ def test_te_grouped_linear_fused_compiles(fp8_recipe, compile_mode, save_origina pytest.skip("FP8 current-scaling fused grouped GEMM on Hopper needs cuBLASLt 13.5+") dtype = torch.bfloat16 device = "cuda" + block_scaled = fp8_recipe is not None and not fp8_recipe.float8_current_scaling() + m_splits = [128, 256, 128, 256] if block_scaled else _GROUPED_M_SPLITS + in_features, out_features = (128, 128) if block_scaled else (_GROUPED_IN, _GROUPED_OUT) model = te.GroupedLinear( _GROUPED_NUM_GEMMS, - _GROUPED_IN, - _GROUPED_OUT, + in_features, + out_features, params_dtype=dtype, device=device, use_grouped_tensor=True, save_original_input=save_original_input, ) - m_splits_dev = torch.tensor(_GROUPED_M_SPLITS, dtype=torch.int64, device=device) + m_splits_dev = torch.tensor(m_splits, dtype=torch.int64, device=device) + + def make_input(): + return torch.randn( + sum(m_splits), in_features, dtype=dtype, device=device, requires_grad=True + ) def fn(inp): if fp8_recipe is None: @@ -2845,7 +2856,7 @@ def fn(inp): torch._dynamo.reset() if compile_mode == "reduce-overhead": - _cudagraph_warmup(fn, _grouped_input(dtype, device, requires_grad=True), backward=True) + _cudagraph_warmup(fn, make_input(), backward=True) model.zero_grad(set_to_none=True) compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) @@ -2853,14 +2864,18 @@ def fn(inp): # Two warmup calls absorb the one-time recompile from lazily created # module attributes. for _ in range(2): - _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + _assert_close_grouped(fn, compiled, model, make_input()) baseline = _dynamo_counter("stats", "unique_graphs") # Change the split distribution IN PLACE: same graph, no recompile. - m_splits_dev.copy_(torch.tensor([16, 32, 40, 40], dtype=torch.int64, device=device)) - _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) - m_splits_dev.copy_(torch.tensor([64, 8, 32, 24], dtype=torch.int64, device=device)) - _assert_close_grouped(fn, compiled, model, _grouped_input(dtype, device)) + changed_splits = ( + [[256, 128, 256, 128], [128, 128, 128, 384]] + if block_scaled + else [[16, 32, 40, 40], [64, 8, 32, 24]] + ) + for splits in changed_splits: + m_splits_dev.copy_(torch.tensor(splits, dtype=torch.int64, device=device)) + _assert_close_grouped(fn, compiled, model, make_input()) if baseline: assert ( _dynamo_counter("stats", "unique_graphs") == baseline diff --git a/transformer_engine/pytorch/module/grouped_linear.py b/transformer_engine/pytorch/module/grouped_linear.py index 6f3bbd0da52..19ab93c8160 100644 --- a/transformer_engine/pytorch/module/grouped_linear.py +++ b/transformer_engine/pytorch/module/grouped_linear.py @@ -87,6 +87,7 @@ HybridQuantizer, IdentityQuantizer, MXFP8Quantizer, + NVFP4Quantizer, ) from ..quantized_tensor import ( QuantizedTensorStorage, @@ -245,11 +246,11 @@ def compile_unsupported_reason(self) -> Optional[str]: if self.fp8_calibration: return "fp8_calibration" if self.use_grouped_tensor_path: - if self.fp8 and not isinstance(self.input_quantizers[0], Float8CurrentScalingQuantizer): - return ( - "a fused-path FP8 recipe other than per-tensor current scaling " - "(grouped scale offsets are per-split host metadata)" - ) + if self.fp8 and not isinstance( + self.input_quantizers[0], + (Float8CurrentScalingQuantizer, MXFP8Quantizer, NVFP4Quantizer), + ): + return "a fused-path recipe other than current scaling, MXFP8, or NVFP4" elif self.m_splits is None: return "m_splits passed as a device tensor inside the compiled region" if any(w.requires_grad != self.weights_requires_grad for w in self.weights): @@ -1699,19 +1700,13 @@ def _rebuild_grouped_input(args: GroupedLinearFusedBwdArgs) -> Optional[GroupedT payload = dict(zip(_GX_PAYLOAD_KEYS, args.gx_payload)) if payload["data"] is not None: payload["data"] = payload["data"].reshape(-1) - data = payload["data"] if payload["data"] is not None else payload["columnwise_data"] payload["first_dims"] = args.m_splits_tensor payload["tensor_offsets"] = args.input_tensor_offsets - scale_offsets = list(range(args.num_gemms + 1)) if args.fp8 else None return GroupedTensorStorage( - shape=(data.numel() // args.in_features, args.in_features), + shape=(math.prod(args.grad_output.shape[:-1]), args.in_features), dtype=args.activation_dtype, num_tensors=args.num_gemms, quantizer=args.input_quantizers[0] if args.fp8 else None, - scale_inv_offsets=scale_offsets if payload["scale_inv"] is not None else None, - columnwise_scale_inv_offsets=( - scale_offsets if payload["columnwise_scale_inv"] is not None else None - ), with_gemm_swizzled_scales=args.gx_swizzled, **payload, ) @@ -1777,11 +1772,7 @@ def _grouped_linear_fused_forward_impl(args: GroupedLinearFwdArgs) -> Tuple[Any, def _grouped_linear_fused_forward_fake(args: GroupedLinearFwdArgs) -> Tuple[Any, ...]: - """Shape/metadata-only twin of :func:`_grouped_linear_fused_forward_impl`. - - Only mirrors configs the fused compiled gate admits: bf16/fp16 or FP8 - per-tensor current scaling. - """ + """Shape/metadata-only twin of :func:`_grouped_linear_fused_forward_impl`.""" inp = args.inp num_gemms = args.num_gemms in_features = args.weights[0].shape[-1] @@ -1797,7 +1788,7 @@ def _grouped_linear_fused_forward_fake(args: GroupedLinearFwdArgs) -> Tuple[Any, input_quantizer = args.input_quantizers[0] input_quantizer.set_usage( rowwise=True, - columnwise=is_grad_enabled and weight_requires_grad, + columnwise=is_grad_enabled and weight_requires_grad and not args.save_original_input, ) input_quantizer.optimize_for_gemm = True @@ -1838,6 +1829,16 @@ def _spec(numel, dtype): aliases[0] = ("inp",) else: gx_payload[0] = _spec(total, activation_dtype) + elif isinstance(input_quantizer, (MXFP8Quantizer, NVFP4Quantizer)): + is_nvfp4 = isinstance(input_quantizer, NVFP4Quantizer) + gx_payload[1] = _spec(total // 2 if is_nvfp4 else total, torch.uint8) + gx_payload[3] = _spec( + math.prod(input_quantizer.get_scale_shape((tokens, in_features), True)), + torch.uint8, + ) + if is_nvfp4: + gx_payload[4] = _spec(num_gemms, torch.float32) + gx_payload[5] = _spec(num_gemms, torch.float32) else: # FP8 per-tensor current scaling; on Hopper the rowwise data is # freed after the fprop GEMM when a columnwise copy exists. @@ -1862,7 +1863,9 @@ def _spec(numel, dtype): ctx_attrs = { "saved_tensor_aliases": tuple(aliases), "gx_present": gx_present, - "gx_swizzled": False, + "gx_swizzled": gx_present + and not args.save_original_input + and isinstance(args.input_quantizers[0], (MXFP8Quantizer, NVFP4Quantizer)), } return (out, *new_workspaces, tensors_to_save, ctx_attrs) @@ -2295,7 +2298,7 @@ class GroupedLinear(TransformerEngineBaseModule): ``torch.compile(fullgraph=True)`` supports discrete expert parameters with host split lists. The native grouped-tensor path supports CUDA split tensors with - BF16/FP16 or FP8 current scaling, including ``save_original_input=True``. + BF16/FP16, FP8 current scaling, MXFP8, or NVFP4, including ``save_original_input=True``. Unsupported configurations fall back to eager when graph breaks are allowed. Parameters From 524ea34811d1801ebd917dac3782e1137d85b24a Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Fri, 2 Oct 2026 14:12:56 +0200 Subject: [PATCH 05/31] [PyTorch] Compile single grouped weights and biases Expose grouped buffers through the existing custom-op field plan and preserve their logical metadata when saving for backward. Describe grouped weight allocations and packed gradients in fake implementations, and reuse cached workspaces across compiled microbatches. Validate fused recipes and parameter layouts under fullgraph and CUDA graphs, primary quantized weights, cache reuse, and gradients and optimizer updates against independent linear operations. Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 147 ++++++++++++++- .../pytorch/dynamo/custom_op.py | 19 +- .../pytorch/dynamo/tensor_spec.py | 43 +++++ .../pytorch/module/grouped_linear.py | 169 ++++++++++++++---- .../pytorch/quantized_tensor.py | 2 +- .../pytorch/tensor/grouped_tensor.py | 5 +- .../tensor/storage/grouped_tensor_storage.py | 56 +++++- 7 files changed, 398 insertions(+), 43 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 85fe92d610b..96aa50cebec 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -2819,11 +2819,21 @@ def test_te_grouped_linear_compile_rejects_negative_splits(): ids=lambda r: "bf16" if r is None else type(r).__name__, ) @pytest.mark.parametrize("save_original_input", [False, True]) -def test_te_grouped_linear_fused_compiles(fp8_recipe, compile_mode, save_original_input): +@pytest.mark.parametrize( + "single_weight,single_bias", + [(False, False), (True, False), (False, True), (True, True)], + ids=["discrete", "grouped-weight", "grouped-bias", "grouped-both"], +) +def test_te_grouped_linear_fused_compiles( + monkeypatch, fp8_recipe, compile_mode, save_original_input, single_weight, single_bias +): """torch.compile(fullgraph=True) of the fused GroupedTensor path (``use_grouped_tensor=True``): ``m_splits`` stays a device tensor, so changing the split distribution in place reuses the very same graph -- no recompiles, no host sync.""" + monkeypatch.setenv("NVTE_GROUPED_LINEAR_SINGLE_PARAM", "1") + if single_weight and fp8_recipe is not None and fp8_recipe.nvfp4(): + pytest.skip("Native NVFP4 grouped GEMM does not support single grouped weights.") if fp8_recipe is not None and torch.cuda.get_device_capability() < (10, 0): if tex.get_cublasLt_version() < 130500: pytest.skip("FP8 current-scaling fused grouped GEMM on Hopper needs cuBLASLt 13.5+") @@ -2840,6 +2850,8 @@ def test_te_grouped_linear_fused_compiles(fp8_recipe, compile_mode, save_origina device=device, use_grouped_tensor=True, save_original_input=save_original_input, + single_grouped_weight=single_weight, + single_grouped_bias=single_bias, ) m_splits_dev = torch.tensor(m_splits, dtype=torch.int64, device=device) @@ -2882,6 +2894,139 @@ def fn(inp): ), "changing device-tensor m_splits values must not recompile" +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not _fused_grouped_cublas_ok, reason="native grouped GEMM unavailable") +@pytest.mark.parametrize("compile_mode", _compile_modes) +@pytest.mark.parametrize("primary_quantized", [False, True]) +@pytest.mark.parametrize( + "fp8_recipe", + ([recipe.Float8CurrentScaling()] if fp8_available else []) + + ([recipe.MXFP8BlockScaling()] if mxfp8_available else []), + ids=lambda r: type(r).__name__, +) +def test_te_grouped_linear_grouped_weight_cache( + monkeypatch, compile_mode, primary_quantized, fp8_recipe +): + """Packed primary weights and cached grouped workspaces retain their gradients.""" + torch._dynamo.reset() + monkeypatch.setenv("NVTE_GROUPED_LINEAR_SINGLE_PARAM", "1") + if torch.cuda.get_device_capability() < (10, 0) and tex.get_cublasLt_version() < 130500: + pytest.skip("Current scaling grouped GEMM on Hopper needs cuBLASLt 13.5+") + models = [] + for _ in range(2): + torch.manual_seed(1234) + with te.quantized_model_init(enabled=primary_quantized, recipe=fp8_recipe): + models.append( + te.GroupedLinear( + 2, + 256, + 256, + params_dtype=torch.bfloat16, + device="cuda", + use_grouped_tensor=True, + single_grouped_weight=True, + single_grouped_bias=True, + ) + ) + model, reference = models + splits = torch.tensor([256, 256], device="cuda", dtype=torch.int64) + is_first = None + + def fn(inp): + with te.autocast(recipe=fp8_recipe): + return model(inp, splits, is_first_microbatch=is_first) + + def ref_fn(inp): + with te.autocast(recipe=fp8_recipe): + return reference(inp, splits, is_first_microbatch=is_first) + + def make_input(): + return torch.randn(512, 256, dtype=torch.bfloat16, device="cuda", requires_grad=True) + + fn(make_input()).sum().backward() + model.zero_grad(set_to_none=True) + is_first = True + if compile_mode == "reduce-overhead": + _cudagraph_warmup(fn, make_input(), backward=True) + model.zero_grad(set_to_none=True) + compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) + cached = None + with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): + for is_first in [True, False, False, True]: + _assert_close_grouped(ref_fn, compiled, model, make_input(), ref_model=reference) + if not primary_quantized: + workspace = model._fp8_workspaces["weight"] + if cached is not None: + assert workspace is cached + cached = workspace + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not _fused_grouped_cublas_ok, reason="native grouped GEMM unavailable") +@pytest.mark.parametrize("input_grad,weight_grad", [(True, True), (False, True), (True, False)]) +@pytest.mark.parametrize("layout", ["rank4", "strided"]) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +def test_te_grouped_linear_grouped_parameters_native_reference( + monkeypatch, input_grad, weight_grad, layout, dtype +): + """Packed parameter gradients and optimizer updates match independent linears.""" + monkeypatch.setenv("NVTE_GROUPED_LINEAR_SINGLE_PARAM", "1") + torch._dynamo.reset() + splits = [128, 256, 128] + model = te.GroupedLinear( + 3, + 128, + 64, + params_dtype=dtype, + device="cuda", + use_grouped_tensor=True, + single_grouped_weight=True, + single_grouped_bias=True, + ) + model.requires_grad_(weight_grad) + with torch.no_grad(): + for parameter in model.parameters(): + data = parameter.rowwise_data + data.copy_(torch.randint_like(data, -1, 2) / 8) + ref_weight = model.weight.rowwise_data.detach().clone().reshape(3, 64, 128) + ref_bias = model.bias.rowwise_data.detach().clone().reshape(3, 64) + ref_weight.requires_grad_(weight_grad) + ref_bias.requires_grad_(weight_grad) + x = torch.randint(-1, 2, (512, 128), device="cuda").to(dtype) / 8 + x = x.reshape(2, 4, 64, 128) if layout == "rank4" else x.t().contiguous().t() + x.requires_grad_(input_grad) + ref_x = x.detach().clone().requires_grad_(input_grad) + reference = torch.cat( + [ + torch.nn.functional.linear(part, weight, bias) + for part, weight, bias in zip( + ref_x.reshape(-1, 128).split(splits), ref_weight, ref_bias + ) + ] + ).reshape(*x.shape[:-1], 64) + device_splits = torch.tensor(splits, dtype=torch.int64, device="cuda") + output = torch.compile(lambda inp: model(inp, device_splits), fullgraph=True)(x) + dy = torch.randint_like(output, -1, 2) / 8 + output.backward(dy) + reference.backward(dy) + torch.testing.assert_close(output, reference, rtol=0, atol=0) + if input_grad: + torch.testing.assert_close(x.grad, ref_x.grad, rtol=0, atol=0) + if weight_grad: + for param, ref_param in zip(model.parameters(), (ref_weight, ref_bias)): + torch.testing.assert_close(param.grad, ref_param.grad, rtol=0, atol=0) + torch.optim.SGD(model.parameters(), lr=0.125).step() + with torch.no_grad(): + ref_weight.add_(ref_weight.grad, alpha=-0.125) + ref_bias.add_(ref_bias.grad, alpha=-0.125) + torch.testing.assert_close( + model.weight.rowwise_data.reshape_as(ref_weight), ref_weight, rtol=0, atol=0 + ) + torch.testing.assert_close( + model.bias.rowwise_data.reshape_as(ref_bias), ref_bias, rtol=0, atol=0 + ) + + # --------------------------------------------------------------------------- # Custom-op list adapters (framework unit test via a toy op) # --------------------------------------------------------------------------- diff --git a/transformer_engine/pytorch/dynamo/custom_op.py b/transformer_engine/pytorch/dynamo/custom_op.py index e3e48fab953..9a7285e619e 100644 --- a/transformer_engine/pytorch/dynamo/custom_op.py +++ b/transformer_engine/pytorch/dynamo/custom_op.py @@ -362,7 +362,7 @@ def _storage_unflatten(meta: "OpaqueValueBundle", tensors: List[torch.Tensor]) - inner = dict(zip(inner_names, tensors)) outer_shape = meta_dict.get("_outer_shape") stride = make_contiguous_strides_for(tuple(outer_shape)) if outer_shape is not None else None - return QuantizedTensorStorage.__tensor_unflatten__(inner, meta_dict, outer_shape, stride) + return meta_dict["cls"].__tensor_unflatten__(inner, meta_dict, outer_shape, stride) # --------------------------------------------------------------------------- # @@ -542,6 +542,10 @@ def _is_trivial(value: Any) -> bool: def _pack_tensor_or_quantized(field: _FieldPlan, value: Any, slots: Dict[str, Any]) -> None: """Fill a tensor-or-quantized field's three slots from its runtime value.""" + from ..tensor.storage.grouped_tensor_storage import ( + GroupedTensorStorage, + ) # pylint: disable=import-outside-toplevel + tensor_slot, inner_slot, meta_slot = (s.name for s in field.slots) if value is None: slots[tensor_slot] = None @@ -554,7 +558,7 @@ def _pack_tensor_or_quantized(field: _FieldPlan, value: Any, slots: Dict[str, An slots[tensor_slot] = value slots[inner_slot] = [] slots[meta_slot] = OpaqueValueBundle({_TQ_KIND_KEY: _TensorOrQuantizedKind.TENSOR}) - elif isinstance(value, QuantizedTensorStorage): + elif isinstance(value, (QuantizedTensorStorage, GroupedTensorStorage)): meta, tensors = _storage_flatten(value, {_TQ_KIND_KEY: _TensorOrQuantizedKind.STORAGE}) slots[tensor_slot] = None slots[inner_slot] = tensors @@ -562,7 +566,7 @@ def _pack_tensor_or_quantized(field: _FieldPlan, value: Any, slots: Dict[str, An else: raise TypeError( f"field {field.name!r} expected None, torch.Tensor, or " - f"QuantizedTensorStorage, got {type(value).__name__}" + f"QuantizedTensorStorage / GroupedTensorStorage, got {type(value).__name__}" ) @@ -1292,12 +1296,13 @@ def _forward(*flat: Any) -> List[torch.Tensor]: return op_def -def _all_quantized_tensor_subclasses() -> List[type]: - """Return every imported ``QuantizedTensor`` wrapper subclass.""" +def _all_tensor_subclasses() -> List[type]: + """Return grouped and imported quantized wrapper subclasses.""" import transformer_engine.pytorch.tensor # noqa: F401 pylint: disable=import-outside-toplevel,unused-import + from ..tensor.grouped_tensor import GroupedTensor # pylint: disable=import-outside-toplevel found: List[type] = [] - stack = list(QuantizedTensor.__subclasses__()) + stack = [GroupedTensor, *QuantizedTensor.__subclasses__()] while stack: cls = stack.pop() if cls not in found: @@ -1335,7 +1340,7 @@ def _register_op( """ plan = _parse_arg_type(arg_type) schema = f"{plan.schema_str} -> Tensor[]" - subclasses = _all_quantized_tensor_subclasses() + subclasses = _all_tensor_subclasses() slot_offsets = plan.tensor_or_quantized_offsets() namespace = getattr(torch.ops, _TE_OP_NAMESPACE) diff --git a/transformer_engine/pytorch/dynamo/tensor_spec.py b/transformer_engine/pytorch/dynamo/tensor_spec.py index 4a0886b23b9..5ad9c8000b1 100644 --- a/transformer_engine/pytorch/dynamo/tensor_spec.py +++ b/transformer_engine/pytorch/dynamo/tensor_spec.py @@ -153,6 +153,32 @@ def create_tensor(self) -> torch.Tensor: return self.assemble(self.create_inner_tensors()) +@dataclass +class GroupedTensorSpec(TensorSpec): + """Grouped buffers and metadata, including unquantized grouped parameters.""" + + metadata: Dict[str, Any] = field(default_factory=dict) + buffers: Dict[str, TensorSpec] = field(default_factory=dict) + + def inner_names(self) -> Tuple[str, ...]: + return tuple(self.buffers) + + def create_metadata(self) -> Dict[str, Any]: + return self.metadata + + def create_inner_tensors(self) -> List[torch.Tensor]: + return [spec.create_tensor() for spec in self.buffers.values()] + + def assemble(self, inner_tensors: List[torch.Tensor]) -> Any: + inner = dict(zip(self.inner_names(), inner_tensors)) + return self.metadata["cls"].__tensor_unflatten__( + inner, self.metadata, self.shape, make_contiguous_strides_for(self.shape) + ) + + def create_tensor(self) -> Any: + return self.assemble(self.create_inner_tensors()) + + def to_tensor_spec(tensor: Any) -> TensorSpec: """Build a :class:`TensorSpec` describing ``tensor``. @@ -160,7 +186,24 @@ def to_tensor_spec(tensor: Any) -> TensorSpec: ``QuantizedTensor``. A *bare* storage exposes its (fake) dtype via ``_dtype`` rather than ``.dtype``. """ + from ..tensor.storage.grouped_tensor_storage import ( + GroupedTensorStorage, + ) # pylint: disable=import-outside-toplevel + requires_grad = bool(getattr(tensor, "requires_grad", False)) + if isinstance(tensor, GroupedTensorStorage): + names, metadata = GroupedTensorStorage.__tensor_flatten__(tensor) + buffers = {name: to_tensor_spec(getattr(tensor, name)) for name in names} + return GroupedTensorSpec( + shape=tuple(tensor.shape if isinstance(tensor, torch.Tensor) else tensor.logical_shape), + dtype=tensor.fake_dtype, + quantizer=tensor.quantizer, + requires_grad=requires_grad, + device=next(iter(buffers.values())).device, + with_gemm_swizzled_scales=getattr(tensor, "_with_gemm_swizzled_scales"), + metadata=metadata, + buffers=buffers, + ) dtype = getattr(tensor, "dtype", None) if dtype is None: dtype = getattr(tensor, "_dtype", None) diff --git a/transformer_engine/pytorch/module/grouped_linear.py b/transformer_engine/pytorch/module/grouped_linear.py index 19ab93c8160..98b7ef5bc34 100644 --- a/transformer_engine/pytorch/module/grouped_linear.py +++ b/transformer_engine/pytorch/module/grouped_linear.py @@ -73,6 +73,7 @@ register_custom_op_with_autograd, is_value_opaque_quantizer, ) +from ..dynamo.tensor_spec import GroupedTensorSpec from .linear import _fake_workspace_valid from ._common import update_nvfp4_direct_output_spec from ..constants import GemmParallelModes, dist_group_type @@ -180,7 +181,7 @@ class GroupedLinearFwdArgs: inp: torch.Tensor weights: List[Union[torch.Tensor, QuantizedTensorStorage]] - biases: List[torch.Tensor] + biases: List[Union[torch.Tensor, QuantizedTensorStorage]] weight_workspaces: List[Union[torch.Tensor, QuantizedTensorStorage]] out: Optional[torch.Tensor] @@ -216,7 +217,6 @@ class GroupedLinearFwdArgs: cache_weight: bool use_grouped_tensor_path: bool - single_grouped_param: bool single_grouped_weight: bool single_grouped_bias: bool @@ -235,8 +235,6 @@ def compile_unsupported_reason(self) -> Optional[str]: return "a DistributedWeight (custom weight parallelism, e.g. GTP)" if self.out is not None or self.dgrad_out is not None: return "a user-provided out/dgrad_out buffer (the op would return an input alias)" - if self.single_grouped_param: - return "single_grouped_weight/single_grouped_bias parameter views" if self.cpu_offloading: return "CPU activation offloading" if self.wgrad_store is not None: @@ -571,11 +569,82 @@ def _grouped_linear_forward_impl( return (out, *new_workspaces, tensors_to_save, ctx_attrs) +def _fake_single_grouped_weight(weight, quantizer, dtype, num_gemms, cache_weight): + """Describe the grouped weight allocation made by the native quantizer.""" + shape = tuple(weight.metadata["nontensor_kwargs"]["shape"]) + total = math.prod(shape) + metadata = dict(weight.metadata) + kwargs = dict(metadata["nontensor_kwargs"]) + kwargs.update(dtype=dtype, quantizer=quantizer) + metadata.update(nontensor_kwargs=kwargs, requires_grad=False) + + def spec(size, buffer_dtype): + return TensorSpec(shape=(size,), dtype=buffer_dtype, device=weight.device) + + if quantizer is None: + metadata.update(cls=GroupedTensorStorage, is_tensor=False) + buffers = {"rowwise_data": spec(total, dtype)} + else: + metadata.update( + cls=GroupedTensor if cache_weight or not quantizer.internal else GroupedTensorStorage, + is_tensor=cache_weight or not quantizer.internal, + ) + kwargs.update( + shapes=None, offsets=None, scale_inv_offsets=None, columnwise_scale_inv_offsets=None + ) + buffers = {} + if isinstance(quantizer, Float8CurrentScalingQuantizer): + non_tn = is_non_tn_fp8_gemm_supported() + if quantizer.rowwise_usage or non_tn: + buffers["rowwise_data"] = spec(total, torch.uint8) + buffers["scale_inv"] = spec(num_gemms, torch.float32) + if quantizer.columnwise_usage and not non_tn: + buffers["columnwise_data"] = spec(total, torch.uint8) + buffers["columnwise_scale_inv"] = spec(num_gemms, torch.float32) + buffers["amax"] = spec(num_gemms, torch.float32) + buffers["scale"] = spec(num_gemms, torch.float32) + kwargs["with_gemm_swizzled_scales"] = False + else: + for columnwise, usage in ( + (False, quantizer.rowwise_usage), + (True, quantizer.columnwise_usage), + ): + if usage: + prefix = "columnwise_" if columnwise else "" + buffers["columnwise_data" if columnwise else "rowwise_data"] = spec( + total, torch.uint8 + ) + buffers[prefix + "scale_inv"] = spec( + math.prod(quantizer.get_scale_shape(shape, columnwise)), torch.uint8 + ) + kwargs["with_gemm_swizzled_scales"] = quantizer.optimize_for_gemm + metadata["shared_scale"] = ( + "scale_inv" in buffers + and isinstance(quantizer, Float8CurrentScalingQuantizer) + and "columnwise_scale_inv" in buffers + ) + if metadata["shared_scale"]: + del buffers["columnwise_scale_inv"] + buffers = { + name: buffers[name] for name, _ in GroupedTensorStorage._INNER_TENSORS if name in buffers + } + outer_shape = (num_gemms, *weight.shape[-2:]) if kwargs["shapes"] is not None else shape + return GroupedTensorSpec( + shape=outer_shape, + dtype=dtype, + quantizer=quantizer, + device=weight.device, + with_gemm_swizzled_scales=kwargs["with_gemm_swizzled_scales"], + metadata=metadata, + buffers=buffers, + ) + + def _fake_grouped_weights(args: GroupedLinearFwdArgs, weight_quantizers): """Describe weight casts, quantization and workspace aliases for either path.""" - new_workspaces = [None] * args.num_gemms - saved_weights = [None] * args.num_gemms - aliases = [None] * args.num_gemms + new_workspaces = [None] * len(args.weights) + saved_weights = [None] * len(args.weights) + aliases = [None] * len(args.weights) for i, weight in enumerate(args.weights): if (args.fp8 and weight.is_quantized) or ( not args.fp8 and weight.dtype == args.activation_dtype @@ -584,15 +653,34 @@ def _fake_grouped_weights(args: GroupedLinearFwdArgs, weight_quantizers): continue quantizer = weight_quantizers[i] if args.fp8 else None workspace = args.weight_workspaces[i] if args.fp8 else None - if workspace is not None and _fake_workspace_valid(workspace, quantizer): + if args.single_grouped_weight: + workspace_valid = workspace is not None and ( + workspace.metadata["nontensor_kwargs"]["quantizer"] == quantizer + and ( + not quantizer.columnwise_usage + or ( + isinstance(quantizer, Float8CurrentScalingQuantizer) + and is_non_tn_fp8_gemm_supported() + ) + or "columnwise_data" in workspace.buffers + ) + ) + else: + workspace_valid = workspace is not None and _fake_workspace_valid(workspace, quantizer) + if workspace_valid: aliases[i] = ("weight_workspaces", i) continue - weightmat = TensorSpec( - shape=tuple(weight.shape), - dtype=args.activation_dtype, - quantizer=quantizer, - device=weight.device, - ) + if args.single_grouped_weight: + weightmat = _fake_single_grouped_weight( + weight, quantizer, args.activation_dtype, args.num_gemms, args.cache_weight + ) + else: + weightmat = TensorSpec( + shape=tuple(weight.shape), + dtype=args.activation_dtype, + quantizer=quantizer, + device=weight.device, + ) if args.fp8 and args.cache_weight: if weightmat.quantizer is not None: weightmat.quantizer.internal = False @@ -1099,26 +1187,34 @@ def _fake_grouped_grads(args, in_features, compute_bias): device=grad_output.device, ) - wgrad_list: List[Optional[TensorSpec]] = [None] * args.num_gemms + single_weight = getattr(args, "single_grouped_weight", False) + single_bias = getattr(args, "single_grouped_bias", False) + num_weights = 1 if single_weight else args.num_gemms + num_biases = 1 if single_bias else args.num_gemms + wgrad_list: List[Optional[TensorSpec]] = [None] * num_weights if args.weights_requires_grad and not args.fuse_wgrad_accumulation: wgrad_list = [ TensorSpec( - shape=(out_features, in_features), + shape=( + (args.num_gemms, out_features, in_features) + if single_weight + else (out_features, in_features) + ), dtype=args.activation_dtype, device=grad_output.device, ) - for _ in range(args.num_gemms) + for _ in range(num_weights) ] - grad_biases: List[Optional[TensorSpec]] = [None] * args.num_gemms + grad_biases: List[Optional[TensorSpec]] = [None] * num_biases if compute_bias: grad_biases = [ TensorSpec( - shape=(out_features,), + shape=(args.num_gemms, out_features) if single_bias else (out_features,), dtype=args.activation_dtype, device=grad_output.device, ) - for _ in range(args.num_gemms) + for _ in range(num_biases) ] return (dgrad, wgrad_list, grad_biases) @@ -1202,12 +1298,12 @@ class GroupedLinearFusedBwdArgs: def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: """Restore the grouped operands and split metadata.""" saved = restore_from_func_ctx(ctx) + n = 1 if self.single_grouped_weight else self.num_gemms if self.compiled_op: n_payload = len(_GX_PAYLOAD_KEYS) self.gx_payload = list(saved[:n_payload]) - self.weights_fp8 = list(saved[n_payload : n_payload + self.num_gemms]) + self.weights_fp8 = list(saved[n_payload : n_payload + n]) return - n = 1 if self.single_grouped_weight else self.num_gemms self.inputmat = saved[0] self.weights_fp8 = list(saved[1 : 1 + n]) ( @@ -1715,14 +1811,14 @@ def _rebuild_grouped_input(args: GroupedLinearFusedBwdArgs) -> Optional[GroupedT def _grouped_linear_fused_forward_impl(args: GroupedLinearFwdArgs) -> Tuple[Any, ...]: """Encode the shared grouped forward's saved tensors without aliases.""" out, new_workspaces, saved = _grouped_linear_fused_forward(args) - num_gemms = args.num_gemms + num_weights = len(args.weights) grouped_x = saved[0] if saved is not None else None - weights_to_save = list(saved[1 : 1 + num_gemms]) if saved is not None else [] + weights_to_save = list(saved[1 : 1 + num_weights]) if saved is not None else [] tensors_to_save = None ctx_attrs = None if args.is_grad_enabled: n_payload = len(_GX_PAYLOAD_KEYS) - aliases: List[Optional[Tuple]] = [None] * (n_payload + num_gemms) + aliases: List[Optional[Tuple]] = [None] * (n_payload + num_weights) gx_payload: List[Optional[torch.Tensor]] = [None] * n_payload gx_present = args.weights_requires_grad if gx_present and args.save_original_input: @@ -1743,7 +1839,7 @@ def _grouped_linear_fused_forward_impl(args: GroupedLinearFwdArgs) -> Tuple[Any, gx_payload[0] = None aliases[0] = ("inp",) - for i in range(num_gemms): + for i in range(num_weights): slot = n_payload + i if weights_to_save[i] is None: continue @@ -1813,7 +1909,7 @@ def _grouped_linear_fused_forward_fake(args: GroupedLinearFwdArgs) -> Tuple[Any, ctx_attrs = None if is_grad_enabled: n_payload = len(_GX_PAYLOAD_KEYS) - aliases: List[Optional[Tuple]] = [None] * (n_payload + num_gemms) + aliases: List[Optional[Tuple]] = [None] * (n_payload + len(args.weights)) gx_payload: List[Optional[TensorSpec]] = [None] * n_payload gx_present = weight_requires_grad if gx_present and args.save_original_input: @@ -1853,7 +1949,7 @@ def _spec(numel, dtype): gx_payload[4] = _spec(num_gemms, torch.float32) # amax gx_payload[6] = _spec(num_gemms, torch.float32) # scale - for i in range(num_gemms): + for i in range(len(args.weights)): if args.input_requires_grad: aliases[n_payload + i] = weight_aliases[i] else: @@ -2064,8 +2160,15 @@ def _prepare_weights_for_grouped_tensor_gemm( workspace = weight_workspaces[0] if weight_workspaces else None if workspace is not None and ( - workspace.quantizer is not weight_quantizer - or (columnwise_usage and workspace.columnwise_data is None) + workspace.quantizer != weight_quantizer + or ( + columnwise_usage + and workspace.columnwise_data is None + and not ( + isinstance(weight_quantizer, Float8CurrentScalingQuantizer) + and is_non_tn_fp8_gemm_supported() + ) + ) ): workspace = None @@ -2073,6 +2176,7 @@ def _prepare_weights_for_grouped_tensor_gemm( raise RuntimeError("Single grouped weight has no rowwise storage to quantize.") source = weight.rowwise_data.view(weight.logical_shape) update_workspace = is_first_microbatch is None or is_first_microbatch + new_workspace = workspace is None if workspace is None: if cache_weight: # Match quantize_weight(): persistent workspaces must be Tensor subclasses @@ -2088,6 +2192,7 @@ def _prepare_weights_for_grouped_tensor_gemm( if cache_weight: weight_quantizer.internal = saved_internal elif skip_fp8_weight_update is not None or update_workspace: + workspace.quantizer = weight_quantizer grouped_weight = tex.group_quantize( source, weight_quantizer, @@ -2099,7 +2204,7 @@ def _prepare_weights_for_grouped_tensor_gemm( else: grouped_weight = workspace - if cache_weight: + if cache_weight and new_workspace: new_workspaces[0] = grouped_weight return grouped_weight, new_workspaces @@ -2299,6 +2404,7 @@ class GroupedLinear(TransformerEngineBaseModule): ``torch.compile(fullgraph=True)`` supports discrete expert parameters with host split lists. The native grouped-tensor path supports CUDA split tensors with BF16/FP16, FP8 current scaling, MXFP8, or NVFP4, including ``save_original_input=True``. + Single grouped weights and biases are supported for the native path's compatible recipes. Unsupported configurations fall back to eager when graph breaks are allowed. Parameters @@ -3193,7 +3299,6 @@ def forward( cache_weight=cache_weight, # fused GroupedTensor path use_grouped_tensor_path=use_grouped_tensor_path, - single_grouped_param=self.single_grouped_weight or self.single_grouped_bias, single_grouped_weight=self.single_grouped_weight, single_grouped_bias=use_grouped_bias, # misc diff --git a/transformer_engine/pytorch/quantized_tensor.py b/transformer_engine/pytorch/quantized_tensor.py index 4d64bc43c7a..a661638abdc 100644 --- a/transformer_engine/pytorch/quantized_tensor.py +++ b/transformer_engine/pytorch/quantized_tensor.py @@ -337,7 +337,7 @@ def restore_from_saved( count = len(tensor.inner_names) inner = dict(zip(tensor.inner_names, saved_tensors[:count])) tensor_objects.append( - QuantizedTensorStorage.__tensor_unflatten__(inner, tensor.metadata, None, None) + tensor.metadata["cls"].__tensor_unflatten__(inner, tensor.metadata, None, None) ) saved_tensors = saved_tensors[count:] else: diff --git a/transformer_engine/pytorch/tensor/grouped_tensor.py b/transformer_engine/pytorch/tensor/grouped_tensor.py index 0cc03602a1f..06eb5120ff6 100644 --- a/transformer_engine/pytorch/tensor/grouped_tensor.py +++ b/transformer_engine/pytorch/tensor/grouped_tensor.py @@ -10,7 +10,7 @@ import torch from torch.utils._pytree import tree_map -from ..quantized_tensor import QuantizedTensorStorage, Quantizer +from ..quantized_tensor import QuantizedTensorStorage, Quantizer, _quantized_tensor_passthrough_ops from .storage.grouped_tensor_storage import GroupedTensorStorage @@ -179,6 +179,9 @@ def __torch_dispatch__(cls, func, types, args, kwargs=None): if kwargs is None: kwargs = {} + if func in _quantized_tensor_passthrough_ops: + return super().__torch_dispatch__(func, types, args, kwargs) + def copy_grouped_storage_metadata(dst: GroupedTensor, src: GroupedTensor) -> None: """Shallow-copy grouped-storage metadata onto wrapper outputs.""" dst.num_tensors = src.num_tensors diff --git a/transformer_engine/pytorch/tensor/storage/grouped_tensor_storage.py b/transformer_engine/pytorch/tensor/storage/grouped_tensor_storage.py index 3473024c03f..d47630b442f 100644 --- a/transformer_engine/pytorch/tensor/storage/grouped_tensor_storage.py +++ b/transformer_engine/pytorch/tensor/storage/grouped_tensor_storage.py @@ -4,7 +4,7 @@ """Grouped tensor storage class for handling collections of tensors with different shapes""" from __future__ import annotations -from typing import Optional, Tuple, List, Union +from typing import Any, Dict, Optional, Tuple, List, Union import math import torch @@ -343,6 +343,60 @@ def nvfp4_e4m3_max(self) -> int: def nvfp4_e4m3_max(self, nvfp4_e4m3_max: int) -> None: self._nvfp4_e4m3_max = nvfp4_e4m3_max + _INNER_TENSORS = ( + ("rowwise_data", "data"), + ("columnwise_data", "columnwise_data"), + ("scale_inv", "scale_inv"), + ("columnwise_scale_inv", "columnwise_scale_inv"), + ("amax", "amax"), + ("columnwise_amax", "columnwise_amax"), + ("scale", "scale"), + ("first_dims", "first_dims"), + ("last_dims", "last_dims"), + ("tensor_offsets", "tensor_offsets"), + ) + + def __tensor_flatten__(self) -> Tuple[List[str], Dict[str, Any]]: + """Separate grouped buffers from their reconstruction metadata.""" + names = [name for name, _ in self._INNER_TENSORS if getattr(self, name) is not None] + shared_scale = self.scale_inv is not None and self.columnwise_scale_inv is self.scale_inv + if shared_scale: + names.remove("columnwise_scale_inv") + return names, { + "cls": type(self), + "is_tensor": isinstance(self, torch.Tensor), + "requires_grad": getattr(self, "requires_grad", False), + "shared_scale": shared_scale, + "nontensor_kwargs": { + "shape": self.logical_shape, + "dtype": self.fake_dtype, + "num_tensors": self.num_tensors, + "shapes": self.tensor_shapes, + "quantizer": self.quantizer, + "offsets": self.offsets, + "scale_inv_offsets": self.scale_inv_offsets, + "columnwise_scale_inv_offsets": self.columnwise_scale_inv_offsets, + "with_gemm_swizzled_scales": self._with_gemm_swizzled_scales, + "row_scaled_nvfp4": self.row_scaled_nvfp4, + "nvfp4_use_4over6": self.nvfp4_use_4over6, + "nvfp4_e4m3_max": self.nvfp4_e4m3_max, + }, + } + + @staticmethod + def __tensor_unflatten__(inner_tensors, metadata, outer_size, outer_stride): + """Rebuild a grouped storage or tensor without copying its buffers.""" + del outer_size + cls = metadata["cls"] + kwargs = dict(metadata["nontensor_kwargs"]) + kwargs.update({kwarg: inner_tensors.get(name) for name, kwarg in cls._INNER_TENSORS}) + if metadata.get("shared_scale", False): + kwargs["columnwise_scale_inv"] = kwargs["scale_inv"] + if metadata["is_tensor"]: + kwargs["requires_grad"] = metadata["requires_grad"] + kwargs["stride"] = outer_stride + return cls(**kwargs) + def prepare_for_saving( self, ) -> Tuple[list[Optional[torch.Tensor]], "GroupedTensorStorage"]: From d4b9c2d9df8c5d03bed6b8a9fd6913fdd40b8772 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Fri, 2 Oct 2026 16:21:19 +0200 Subject: [PATCH 06/31] [PyTorch] Simplify GroupedLinear saved tensor state Resolve saved input and output references in the custom-op framework and use grouped storage metadata for fused inputs. Share real and fake saved-weight handling and cover dynamic shapes, saved-tensor hooks, and autograd version checks. Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 103 ++++- .../pytorch/dynamo/custom_op.py | 36 +- .../pytorch/module/grouped_linear.py | 394 +++++++----------- 3 files changed, 256 insertions(+), 277 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 96aa50cebec..617256b6ce3 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -2894,6 +2894,44 @@ def fn(inp): ), "changing device-tensor m_splits values must not recompile" +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not _fused_grouped_cublas_ok, reason="native grouped GEMM unavailable") +@pytest.mark.parametrize( + "fp8_recipe", [None] + ([recipe.MXFP8BlockScaling()] if mxfp8_available else []) +) +@pytest.mark.parametrize("save_original_input", [False, True]) +def test_te_grouped_linear_fused_dynamic_input(fp8_recipe, save_original_input): + model = te.GroupedLinear( + 2, + 128, + 128, + params_dtype=torch.bfloat16, + device="cuda", + use_grouped_tensor=True, + single_grouped_weight=False, + single_grouped_bias=False, + save_original_input=save_original_input, + ) + splits = torch.tensor([128, 128], dtype=torch.int64, device="cuda") + + def fn(inp): + if fp8_recipe is None: + return model(inp, splits) + with te.autocast(recipe=fp8_recipe): + return model(inp, splits) + + torch._dynamo.reset() + compiled = torch.compile(fn, fullgraph=True, dynamic=True) + for rows in (256, 256, 512, 768): + splits.fill_(rows // 2) + inp = torch.randn(rows, 128, device="cuda", dtype=torch.bfloat16) + _assert_close_grouped(fn, compiled, model, inp) + if rows == 256: + baseline = _dynamo_counter("stats", "unique_graphs") + if baseline: + assert _dynamo_counter("stats", "unique_graphs") == baseline + + @pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") @pytest.mark.skipif(not _fused_grouped_cublas_ok, reason="native grouped GEMM unavailable") @pytest.mark.parametrize("compile_mode", _compile_modes) @@ -3037,6 +3075,8 @@ def test_te_grouped_linear_grouped_parameters_native_reference( from typing import List, Optional, Tuple from transformer_engine.pytorch.dynamo import register_custom_op_with_autograd + from transformer_engine.pytorch.dynamo.custom_op import SavedTensorRef + from transformer_engine.pytorch.quantized_tensor import restore_from_func_ctx @dataclass class _ToyListFwdArgs: @@ -3049,15 +3089,29 @@ class _ToyListFwdArgs: class _ToyListBwdArgs: grad_output: Optional[torch.Tensor] = None inp: Optional[torch.Tensor] = None + output: Optional[torch.Tensor] = None weights: List[Optional[torch.Tensor]] = None scales: Tuple[float, ...] = () + def setup_saved_tensors(self, ctx): + self.inp, self.output, *self.weights = restore_from_func_ctx(ctx) + + def _toy_list_saved(args): + return ( + SavedTensorRef("inp"), + SavedTensorRef(None, 0), + *( + SavedTensorRef("weights", i) if w is not None else None + for i, w in enumerate(args.weights) + ), + ) + def _toy_list_fwd(args): out = torch.zeros_like(args.inp) for w, s in zip(args.weights, args.scales): if w is not None: out = out + args.inp * w * s - return (out, None, None) + return (out.exp(), _toy_list_saved(args), None) def _toy_list_fwd_fake(args): out = TensorSpec( @@ -3066,24 +3120,23 @@ def _toy_list_fwd_fake(args): requires_grad=True, device=args.inp.device, ) - return (out, None, None) + return (out, _toy_list_saved(args), None) def _toy_list_setup_ctx(bwd_args, fwd_args, outputs, ctx_attrs, saved): - del outputs, ctx_attrs, saved - bwd_args.inp = fwd_args.inp - bwd_args.weights = fwd_args.weights + del outputs, ctx_attrs bwd_args.scales = fwd_args.scales - return () + return saved def _toy_list_bwd(args): + grad = args.grad_output * args.output dgrad = torch.zeros_like(args.inp) wgrads = [] for w, s in zip(args.weights, args.scales): if w is None: wgrads.append(None) continue - dgrad = dgrad + args.grad_output * w * s - wgrads.append(args.grad_output * args.inp * s) + dgrad = dgrad + grad * w * s + wgrads.append(grad * args.inp * s) return (dgrad, wgrads) def _toy_list_bwd_fake(args): @@ -3107,7 +3160,8 @@ def _toy_list_bwd_fake(args): @pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") @pytest.mark.parametrize("split_metadata", [[], [4, 4]]) -def test_custom_op_tensor_list_grads(split_metadata): +@pytest.mark.parametrize("saved_hooks", [False, True]) +def test_custom_op_tensor_list_grads(split_metadata, saved_hooks): """A toy op with a ``List[Optional[Tensor]]`` input (including a ``None`` entry riding the sentinel) compiles fullgraph and routes one gradient per list element.""" @@ -3117,11 +3171,17 @@ def test_custom_op_tensor_list_grads(split_metadata): def run(fn): torch.manual_seed(7) - inp = torch.randn(8, 4, device=device, requires_grad=True) - w0 = torch.randn(8, 4, device=device, requires_grad=True) - w2 = torch.randn(8, 4, device=device, requires_grad=True) - out = fn(inp, w0, w2) - out.sum().backward() + inp = (0.1 * torch.randn(8, 4, device=device)).requires_grad_() + w0 = (0.1 * torch.randn(8, 4, device=device)).requires_grad_() + w2 = (0.1 * torch.randn(8, 4, device=device)).requires_grad_() + hooks = ( + torch.autograd.graph.saved_tensors_hooks(lambda t: t.clone(), lambda t: t) + if saved_hooks + else contextlib.nullcontext() + ) + with hooks: + out = fn(inp, w0, w2) + out.sum().backward() return out.detach().clone(), inp.grad, w0.grad, w2.grad def fn(inp, w0, w2): @@ -3131,7 +3191,7 @@ def fn(inp, w0, w2): return _toy_list_op(args) def ref_fn(inp, w0, w2): - return inp * w0 * scales[0] + inp * w2 * scales[2] + return (inp * w0 * scales[0] + inp * w2 * scales[2]).exp() torch._dynamo.reset() compiled = torch.compile(fn, fullgraph=True) @@ -3151,6 +3211,19 @@ def ref_fn(inp, w0, w2): torch.testing.assert_close(w2grad_e, w2grad_ref) +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("changed_tensor", ["inp", "weight", "output"]) +def test_custom_op_saved_refs_version_check(changed_tensor): + inp = torch.randn(4, device="cuda", requires_grad=True) + weight = torch.randn_like(inp, requires_grad=True) + out = _toy_list_op(_ToyListFwdArgs(inp, [weight], (1.0,), [])) + tensor = {"inp": inp, "weight": weight, "output": out}[changed_tensor] + with torch.no_grad(): + tensor.add_(1) + with pytest.raises(RuntimeError, match="modified by an inplace operation"): + out.sum().backward() + + # --------------------------------------------------------------------------- # # transformer_engine.pytorch.ops under torch.compile # --------------------------------------------------------------------------- # diff --git a/transformer_engine/pytorch/dynamo/custom_op.py b/transformer_engine/pytorch/dynamo/custom_op.py index 9a7285e619e..d16a0d7573c 100644 --- a/transformer_engine/pytorch/dynamo/custom_op.py +++ b/transformer_engine/pytorch/dynamo/custom_op.py @@ -838,22 +838,35 @@ def _spec_view(obj: Any, tensor_field_names: Sequence[str]) -> Any: # --------------------------------------------------------------------------- # -def _spec_slot_count(spec: Optional[TensorSpec]) -> int: +@dataclasses.dataclass(frozen=True) +class SavedTensorRef: + """Saved input field, or user output when ``source`` is None.""" + + source: Optional[str] + index: Optional[int] = None + + def resolve(self, args, outputs): + """Resolve the reference before saving through autograd.""" + value = outputs if self.source is None else getattr(args, self.source) + return value if self.index is None else value[self.index] + + +def _spec_slot_count(spec: Optional[Union[TensorSpec, SavedTensorRef]]) -> int: """Flat ``Tensor[]`` slots the value for ``spec`` occupies.""" - if spec is None: + if spec is None or isinstance(spec, SavedTensorRef): return 1 return len(spec.inner_names()) def _flatten_value( - value: Optional[Union[torch.Tensor, QuantizedTensorStorage, TensorSpec]], + value: Optional[Union[torch.Tensor, QuantizedTensorStorage, TensorSpec, SavedTensorRef]], ) -> List[torch.Tensor]: """Return the flat ``Tensor[]`` slots that represent one op output ``value``. Pack-side inverse of :meth:`_OutputPlan.user_outputs`; the slot count matches :func:`_spec_slot_count`. """ - if value is None: + if value is None or isinstance(value, SavedTensorRef): return [_encode_none(None)] if isinstance(value, TensorSpec): return [_encode_none(t) for t in value.create_inner_tensors()] @@ -950,7 +963,7 @@ class _OutputPlan: """ user_specs: Tuple[Optional[TensorSpec], ...] - saved_specs: Tuple[Optional[TensorSpec], ...] + saved_specs: Tuple[Optional[Union[TensorSpec, SavedTensorRef]], ...] ctx_attrs: Dict[str, Any] user_ranges: Tuple[Tuple[int, int], ...] saved_start: int @@ -992,13 +1005,17 @@ def user_outputs(self, flat: Sequence[Optional[torch.Tensor]]) -> List[Any]: for spec, (start, stop) in zip(self.user_specs, self.user_ranges) ] - def saved_tensors(self, flat: Sequence[Optional[torch.Tensor]]) -> List[Any]: + def saved_tensors(self, flat: Sequence[Optional[torch.Tensor]], args, outputs) -> List[Any]: """Rebuild the saved-for-backward tensors from the op's flat return.""" values: List[Any] = [] cursor = self.saved_start for spec in self.saved_specs: n = _spec_slot_count(spec) - values.append(self._assemble(spec, flat, cursor, cursor + n)) + values.append( + spec.resolve(args, outputs) + if isinstance(spec, SavedTensorRef) + else self._assemble(spec, flat, cursor, cursor + n) + ) cursor += n return values @@ -1120,7 +1137,7 @@ def _setup_context(ctx, inputs, output): out_plan = _OutputPlan.parse(fwd_fake_impl(spec_obj)) user_outputs = out_plan.user_outputs(output) - saved_list = out_plan.saved_tensors(output) + saved_list = out_plan.saved_tensors(output, fwd_obj, user_outputs) bwd_obj = bwd_plan.arg_type() tensors_to_save_from_setup = setup_context_user( @@ -1458,7 +1475,8 @@ def register_custom_op_with_autograd( * ``fwd_impl(fwd_args) -> (*user_outputs, tensors_to_save, ctx_attrs)`` -- the real forward. ``user_outputs``: op outputs (tensor / quantized / ``None``); - ``tensors_to_save``: list/tuple (or ``None``) of tensors for backward; + ``tensors_to_save``: list/tuple (or ``None``) of tensors or + :class:`SavedTensorRef` entries, resolved before ``setup_context``; ``ctx_attrs``: dict (or ``None``) of plain metadata for ``setup_context``. The trailing two slots are fixed (``_FWD_TRAILING_SLOTS``); everything before them is a user output. diff --git a/transformer_engine/pytorch/module/grouped_linear.py b/transformer_engine/pytorch/module/grouped_linear.py index 98b7ef5bc34..23c4de4df5e 100644 --- a/transformer_engine/pytorch/module/grouped_linear.py +++ b/transformer_engine/pytorch/module/grouped_linear.py @@ -73,7 +73,8 @@ register_custom_op_with_autograd, is_value_opaque_quantizer, ) -from ..dynamo.tensor_spec import GroupedTensorSpec +from ..dynamo.tensor_spec import GroupedTensorSpec, to_tensor_spec +from ..dynamo.custom_op import SavedTensorRef from .linear import _fake_workspace_valid from ._common import update_nvfp4_direct_output_spec from ..constants import GemmParallelModes, dist_group_type @@ -484,21 +485,12 @@ def _grouped_linear_forward_impl( mark_not_offload(*weights_fp8, *weights) tensors_to_save = None - ctx_attrs = None if is_grad_enabled: - # Saved-tensor layout: ``(inputmat_full, *inputmats, *weights_fp8, - # *saved_weights, *biases)`` -- 1 + 4N slots. Slots that alias a forward - # input or another op return are deduped through name-based alias tags - # (rebuilt in ``_grouped_linear_setup_ctx``): a custom op may not return - # aliasing tensors. - aliases: List[Optional[Tuple]] = [None] * (1 + 4 * num_gemms) - # TODO: update after #1638 is merged. # pylint: disable=fixme if weight_requires_grad: if save_original_input: - inputmat_full = None + inputmat_full = SavedTensorRef("inp") if args.compiled_op else inp inputmats = [None] * num_gemms - aliases[0] = ("inp",) else: for inputmat in inputmats: if isinstance(inputmat, QuantizedTensorStorage): @@ -513,8 +505,7 @@ def _grouped_linear_forward_impl( inputmats = [None] * num_gemms if inputmat_full is inp_view: # No-op cast: inp_view is inp itself or a view of it. - inputmat_full = None - aliases[0] = ("inp",) + inputmat_full = SavedTensorRef("inp") if args.compiled_op else inp else: inputmat_full = None inputmats = [None] * num_gemms @@ -523,50 +514,30 @@ def _grouped_linear_forward_impl( # used for fused wgrad accumulation serve a different purpose: restoring # Python parameter attributes without keeping the parameter alive here. save_origin_weights = backward_override == "high_precision" and args.input_requires_grad - saved_weights = [None] * num_gemms - wt_saves = list(weights_fp8) - for i in range(num_gemms): - slot = 1 + num_gemms + i - if wt_saves[i] is weights[i]: - aliases[slot] = ("weights", i) - wt_saves[i] = None - elif new_workspaces[i] is not None and wt_saves[i] is new_workspaces[i]: - aliases[slot] = ("new_weight_workspaces", i) - wt_saves[i] = None - elif ( - args.weight_workspaces - and args.weight_workspaces[i] is not None - and wt_saves[i] is args.weight_workspaces[i] - ): - aliases[slot] = ("weight_workspaces", i) - wt_saves[i] = None - if save_origin_weights: - aliases[1 + 2 * num_gemms + i] = ("weights", i) + saved_weights = list(args.weights) if save_origin_weights else [None] * num_gemms if is_dist_weight: - # GTP: gathered workspace is transient (re-gathered in backward), don't save it. - wt_saves = [None] * num_gemms - for i in range(num_gemms): - aliases[1 + num_gemms + i] = None - aliases[1 + 2 * num_gemms + i] = ("weights", i) - - saved_biases = list(biases) - for i in range(num_gemms): - if saved_biases[i] is not None and saved_biases[i] is args.biases[i]: - aliases[1 + 3 * num_gemms + i] = ("biases", i) - saved_biases[i] = None + weights_fp8 = [None] * num_gemms + saved_weights = list(args.weights) + if args.compiled_op: + weights_fp8 = _saved_grouped_weights(args, new_workspaces, weights_fp8) + if save_origin_weights: + saved_weights = [SavedTensorRef("weights", i) for i in range(num_gemms)] + biases = [ + SavedTensorRef("biases", i) if bias is not None and bias is args.biases[i] else bias + for i, bias in enumerate(biases) + ] tensors_to_save = ( inputmat_full, *inputmats, - *wt_saves, + *weights_fp8, *saved_weights, - *saved_biases, + *biases, ) - ctx_attrs = {"saved_tensor_aliases": tuple(aliases)} # [*, in_features] -> [*, out_features] out = out.view(-1, *inp.shape[1:-1], out.shape[-1]) - return (out, *new_workspaces, tensors_to_save, ctx_attrs) + return (out, *new_workspaces, tensors_to_save, None) def _fake_single_grouped_weight(weight, quantizer, dtype, num_gemms, cache_weight): @@ -644,12 +615,11 @@ def _fake_grouped_weights(args: GroupedLinearFwdArgs, weight_quantizers): """Describe weight casts, quantization and workspace aliases for either path.""" new_workspaces = [None] * len(args.weights) saved_weights = [None] * len(args.weights) - aliases = [None] * len(args.weights) for i, weight in enumerate(args.weights): if (args.fp8 and weight.is_quantized) or ( not args.fp8 and weight.dtype == args.activation_dtype ): - aliases[i] = ("weights", i) + saved_weights[i] = weight continue quantizer = weight_quantizers[i] if args.fp8 else None workspace = args.weight_workspaces[i] if args.fp8 else None @@ -668,7 +638,7 @@ def _fake_grouped_weights(args: GroupedLinearFwdArgs, weight_quantizers): else: workspace_valid = workspace is not None and _fake_workspace_valid(workspace, quantizer) if workspace_valid: - aliases[i] = ("weight_workspaces", i) + saved_weights[i] = workspace continue if args.single_grouped_weight: weightmat = _fake_single_grouped_weight( @@ -685,10 +655,8 @@ def _fake_grouped_weights(args: GroupedLinearFwdArgs, weight_quantizers): if weightmat.quantizer is not None: weightmat.quantizer.internal = False new_workspaces[i] = weightmat - aliases[i] = ("new_weight_workspaces", i) - else: - saved_weights[i] = weightmat - return new_workspaces, saved_weights, aliases + saved_weights[i] = weightmat + return new_workspaces, _saved_grouped_weights(args, new_workspaces, saved_weights) def _grouped_linear_forward_fake( @@ -767,20 +735,19 @@ def _grouped_linear_forward_fake( device=inp.device, ) - new_workspaces, weights_fp8, weight_aliases = _fake_grouped_weights(args, weight_quantizers) + new_workspaces, weights_fp8 = _fake_grouped_weights(args, weight_quantizers) # Bias pipeline. bias_dtype = activation_dtype if fp8 and activation_dtype == torch.float32: bias_dtype = torch.bfloat16 - saved_biases: List[Optional[TensorSpec]] = [None] * num_gemms - bias_aliases: List[Optional[Tuple]] = [None] * num_gemms + saved_biases = [None] * num_gemms for i in range(num_gemms): bias = args.biases[i] if bias is None: continue if not args.use_bias or bias.dtype == bias_dtype: - bias_aliases[i] = ("biases", i) + saved_biases[i] = SavedTensorRef("biases", i) else: saved_biases[i] = TensorSpec( shape=tuple(bias.shape), dtype=bias_dtype, device=bias.device @@ -796,14 +763,11 @@ def _grouped_linear_forward_fake( ) tensors_to_save = None - ctx_attrs = None if is_grad_enabled: - aliases: List[Optional[Tuple]] = [None] * (1 + 4 * num_gemms) if weight_requires_grad: if save_original_input: - inputmat_full = None + inputmat_full = SavedTensorRef("inp") inputmats = [None] * num_gemms - aliases[0] = ("inp",) else: if fp8: for inputmat in inputmats: @@ -812,19 +776,15 @@ def _grouped_linear_forward_fake( else: inputmat.update_usage(rowwise_usage=False, columnwise_usage=True) elif inputmat_full_aliases_inp: - inputmat_full = None - aliases[0] = ("inp",) + inputmat_full = SavedTensorRef("inp") else: inputmat_full = None inputmats = [None] * num_gemms - saved_weights = [None] * num_gemms save_origin_weights = backward_override == "high_precision" and args.input_requires_grad - for i in range(num_gemms): - aliases[1 + num_gemms + i] = weight_aliases[i] - if save_origin_weights: - aliases[1 + 2 * num_gemms + i] = ("weights", i) - aliases[1 + 3 * num_gemms + i] = bias_aliases[i] + saved_weights = [ + SavedTensorRef("weights", i) if save_origin_weights else None for i in range(num_gemms) + ] tensors_to_save = ( inputmat_full, @@ -833,23 +793,25 @@ def _grouped_linear_forward_fake( *saved_weights, *saved_biases, ) - ctx_attrs = {"saved_tensor_aliases": tuple(aliases)} - return (out, *new_workspaces, tensors_to_save, ctx_attrs) - - -def _restore_saved_aliases(fwd_args, fwd_outputs, ctx_attrs, tensors_to_save): - """Restore op inputs and workspaces omitted from the saved output payload.""" - saved = list(tensors_to_save) - for slot, alias in enumerate(ctx_attrs["saved_tensor_aliases"]): - if alias is not None: - value = ( - fwd_outputs[1:] - if alias[0] == "new_weight_workspaces" - else getattr(fwd_args, alias[0]) - ) - saved[slot] = value[alias[1]] if len(alias) == 2 else value - return tuple(saved) + return (out, *new_workspaces, tensors_to_save, None) + + +def _saved_grouped_weights(args, new_workspaces, weights): + """Return fresh weights once and refer to existing inputs or cache outputs.""" + saved = [] + for i, weight in enumerate(weights): + if weight is None: + saved.append(None) + elif weight is args.weights[i]: + saved.append(SavedTensorRef("weights", i)) + elif weight is new_workspaces[i]: + saved.append(SavedTensorRef(None, 1 + i)) + elif args.weight_workspaces and weight is args.weight_workspaces[i]: + saved.append(SavedTensorRef("weight_workspaces", i)) + else: + saved.append(weight) + return saved def _grouped_linear_setup_ctx( @@ -859,8 +821,8 @@ def _grouped_linear_setup_ctx( ctx_attrs: Dict, tensors_to_save_from_forward: Tuple[Any, ...], ) -> Tuple[Any, ...]: - """Populate ``bwd_args`` from forward state and return the tensors to persist - (alias-tagged slots rebuilt from ``fwd_args`` / ``fwd_outputs``).""" + """Populate backward configuration and pass through its saved tensors.""" + del fwd_outputs, ctx_attrs num_gemms = fwd_args.num_gemms weights = fwd_args.weights @@ -919,7 +881,7 @@ def _grouped_linear_setup_ctx( bwd_args.grad_weight_quantizers = [None] * num_gemms bwd_args.grad_output_quantizers = [None] * num_gemms - return _restore_saved_aliases(fwd_args, fwd_outputs, ctx_attrs, tensors_to_save_from_forward) + return tensors_to_save_from_forward def _finish_wgrad(weight, main_grad, wgrad, fuse_wgrad_accumulation): @@ -1258,7 +1220,7 @@ class GroupedLinearFusedBwdArgs: """Backward configuration and saved grouped operands.""" grad_output: Optional[torch.Tensor] = None - inputmat: Any = None + inputmat: Union[torch.Tensor, QuantizedTensorStorage] = None weights_fp8: List[Union[torch.Tensor, QuantizedTensorStorage]] = None m_splits_tensor: Optional[torch.Tensor] = None base_split_offsets: Optional[torch.Tensor] = None @@ -1290,22 +1252,16 @@ class GroupedLinearFusedBwdArgs: wgrad_store: Optional[WeightGradStore] = None reduce_and_update_bwd_fp8_tensors: bool = False - gx_payload: List[torch.Tensor] = None - gx_present: bool = False - gx_swizzled: bool = False compiled_op: bool = False def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: """Restore the grouped operands and split metadata.""" saved = restore_from_func_ctx(ctx) n = 1 if self.single_grouped_weight else self.num_gemms - if self.compiled_op: - n_payload = len(_GX_PAYLOAD_KEYS) - self.gx_payload = list(saved[:n_payload]) - self.weights_fp8 = list(saved[n_payload : n_payload + n]) - return self.inputmat = saved[0] self.weights_fp8 = list(saved[1 : 1 + n]) + if self.compiled_op: + return ( self.m_splits_tensor, self.base_split_offsets, @@ -1774,110 +1730,87 @@ def grouped_gemm_wgrad(inputmats, grad_output_mats, grad_weights): # Fused GroupedTensor path under torch.compile # --------------------------------------------------------------------------- # -# GroupedTensorStorage payload slots, in ``prepare_for_saving`` order. -_GX_PAYLOAD_KEYS = ( - "data", - "columnwise_data", - "scale_inv", - "columnwise_scale_inv", - "amax", - "columnwise_amax", - "scale", - "first_dims", - "last_dims", - "tensor_offsets", -) - - -def _rebuild_grouped_input(args: GroupedLinearFusedBwdArgs) -> Optional[GroupedTensorStorage]: - """Rebuild the saved grouped input storage from its flat payload.""" - if not args.gx_present: - return None - payload = dict(zip(_GX_PAYLOAD_KEYS, args.gx_payload)) - if payload["data"] is not None: - payload["data"] = payload["data"].reshape(-1) - payload["first_dims"] = args.m_splits_tensor - payload["tensor_offsets"] = args.input_tensor_offsets - return GroupedTensorStorage( - shape=(math.prod(args.grad_output.shape[:-1]), args.in_features), - dtype=args.activation_dtype, - num_tensors=args.num_gemms, - quantizer=args.input_quantizers[0] if args.fp8 else None, - with_gemm_swizzled_scales=args.gx_swizzled, - **payload, - ) - def _grouped_linear_fused_forward_impl(args: GroupedLinearFwdArgs) -> Tuple[Any, ...]: """Encode the shared grouped forward's saved tensors without aliases.""" out, new_workspaces, saved = _grouped_linear_fused_forward(args) - num_weights = len(args.weights) - grouped_x = saved[0] if saved is not None else None - weights_to_save = list(saved[1 : 1 + num_weights]) if saved is not None else [] tensors_to_save = None - ctx_attrs = None - if args.is_grad_enabled: - n_payload = len(_GX_PAYLOAD_KEYS) - aliases: List[Optional[Tuple]] = [None] * (n_payload + num_weights) - gx_payload: List[Optional[torch.Tensor]] = [None] * n_payload - gx_present = args.weights_requires_grad - if gx_present and args.save_original_input: - aliases[0] = ("inp",) - elif gx_present: - gx_payload = list(grouped_x.get_data_tensors()) - # first_dims is the split tensor itself and tensor_offsets derives - # from it: both rebuilt in backward from m_splits_tensor instead of - # being saved (they may alias the op input). - gx_payload[7] = None - gx_payload[9] = None - if ( - not args.fp8 - and gx_payload[0] is not None - and args.inp.dtype == args.activation_dtype - ): - # No-op cast: the packed data aliases the op input. - gx_payload[0] = None - aliases[0] = ("inp",) - - for i in range(num_weights): - slot = n_payload + i - if weights_to_save[i] is None: - continue - if weights_to_save[i] is args.weights[i]: - aliases[slot] = ("weights", i) - weights_to_save[i] = None - elif new_workspaces[i] is not None and weights_to_save[i] is new_workspaces[i]: - aliases[slot] = ("new_weight_workspaces", i) - weights_to_save[i] = None - elif ( - args.weight_workspaces - and args.weight_workspaces[i] is not None - and weights_to_save[i] is args.weight_workspaces[i] + if saved is not None: + inputmat = saved[0] + if inputmat is not None: + if args.save_original_input or ( + not args.fp8 and args.inp.dtype == args.activation_dtype ): - aliases[slot] = ("weight_workspaces", i) - weights_to_save[i] = None - - tensors_to_save = (*gx_payload, *weights_to_save) - ctx_attrs = { - "saved_tensor_aliases": tuple(aliases), - "gx_present": gx_present, - "gx_swizzled": bool(getattr(grouped_x, "_with_gemm_swizzled_scales", False)), - } - - return (out, *new_workspaces, tensors_to_save, ctx_attrs) + inputmat = SavedTensorRef("inp") + elif not args.fp8: + inputmat = inputmat.rowwise_data + else: + # Dynamic rows and split metadata are restored inside backward. + inputmat.logical_shape = (0, args.weights[0].shape[-1]) + inputmat.first_dims = None + inputmat.tensor_offsets = None + weights = _saved_grouped_weights(args, new_workspaces, saved[1 : 1 + len(args.weights)]) + tensors_to_save = (inputmat, *weights) + return (out, *new_workspaces, tensors_to_save, None) + + +def _fake_grouped_input(args): + """Describe the saved grouped activation using the storage's own layout.""" + if not args.weights_requires_grad: + return None + if args.save_original_input or (not args.fp8 and args.inp.dtype == args.activation_dtype): + return SavedTensorRef("inp") + tokens = math.prod(args.inp.shape[:-1]) + in_features = args.weights[0].shape[-1] + total = tokens * in_features + + def spec(numel, dtype): + return TensorSpec(shape=(numel,), dtype=dtype, device=args.inp.device) + + if not args.fp8: + return spec(total, args.activation_dtype) + quantizer = args.input_quantizers[0] + block_scaled = isinstance(quantizer, (MXFP8Quantizer, NVFP4Quantizer)) + buffers = {} + if block_scaled: + is_nvfp4 = isinstance(quantizer, NVFP4Quantizer) + buffers["columnwise_data"] = spec(total // 2 if is_nvfp4 else total, torch.uint8) + buffers["columnwise_scale_inv"] = spec( + math.prod(quantizer.get_scale_shape((tokens, in_features), True)), torch.uint8 + ) + if is_nvfp4: + buffers["amax"] = spec(args.num_gemms, torch.float32) + buffers["columnwise_amax"] = spec(args.num_gemms, torch.float32) + else: + # Hopper retains the columnwise operand; Blackwell reuses rowwise data. + if is_non_tn_fp8_gemm_supported(): + buffers["data"] = spec(total, torch.uint8) + buffers["scale_inv"] = spec(args.num_gemms, torch.float32) + else: + buffers["columnwise_data"] = spec(total, torch.uint8) + buffers["columnwise_scale_inv"] = spec(args.num_gemms, torch.float32) + buffers["amax"] = spec(args.num_gemms, torch.float32) + buffers["scale"] = spec(args.num_gemms, torch.float32) + return to_tensor_spec( + GroupedTensorStorage( + shape=(0, in_features), + dtype=args.activation_dtype, + num_tensors=args.num_gemms, + quantizer=quantizer, + with_gemm_swizzled_scales=block_scaled, + **buffers, + ) + ) def _grouped_linear_fused_forward_fake(args: GroupedLinearFwdArgs) -> Tuple[Any, ...]: """Shape/metadata-only twin of :func:`_grouped_linear_fused_forward_impl`.""" inp = args.inp - num_gemms = args.num_gemms - in_features = args.weights[0].shape[-1] out_features = args.weights[0].shape[-2] activation_dtype = args.activation_dtype fp8 = args.fp8 is_grad_enabled = args.is_grad_enabled weight_requires_grad = args.weights_requires_grad - tokens = math.prod(inp.shape[:-1]) device = inp.device if fp8: @@ -1893,9 +1826,7 @@ def _grouped_linear_fused_forward_fake(args: GroupedLinearFwdArgs) -> Tuple[Any, quantizer.set_usage( rowwise=True, columnwise=is_grad_enabled and args.input_requires_grad ) - new_workspaces, weights_saved, weight_aliases = _fake_grouped_weights( - args, args.weight_quantizers - ) + new_workspaces, weights_saved = _fake_grouped_weights(args, args.weight_quantizers) out = TensorSpec( shape=(*tuple(inp.shape[:-1]), out_features), @@ -1906,65 +1837,11 @@ def _grouped_linear_fused_forward_fake(args: GroupedLinearFwdArgs) -> Tuple[Any, ) tensors_to_save = None - ctx_attrs = None if is_grad_enabled: - n_payload = len(_GX_PAYLOAD_KEYS) - aliases: List[Optional[Tuple]] = [None] * (n_payload + len(args.weights)) - gx_payload: List[Optional[TensorSpec]] = [None] * n_payload - gx_present = weight_requires_grad - if gx_present and args.save_original_input: - aliases[0] = ("inp",) - elif gx_present: - total = tokens * in_features - - def _spec(numel, dtype): - return TensorSpec(shape=(numel,), dtype=dtype, device=device) - - if not fp8: - if inp.dtype == activation_dtype: - aliases[0] = ("inp",) - else: - gx_payload[0] = _spec(total, activation_dtype) - elif isinstance(input_quantizer, (MXFP8Quantizer, NVFP4Quantizer)): - is_nvfp4 = isinstance(input_quantizer, NVFP4Quantizer) - gx_payload[1] = _spec(total // 2 if is_nvfp4 else total, torch.uint8) - gx_payload[3] = _spec( - math.prod(input_quantizer.get_scale_shape((tokens, in_features), True)), - torch.uint8, - ) - if is_nvfp4: - gx_payload[4] = _spec(num_gemms, torch.float32) - gx_payload[5] = _spec(num_gemms, torch.float32) - else: - # FP8 per-tensor current scaling; on Hopper the rowwise data is - # freed after the fprop GEMM when a columnwise copy exists. - has_columnwise = is_grad_enabled and weight_requires_grad - keep_rowwise = not has_columnwise or is_non_tn_fp8_gemm_supported() - if keep_rowwise: - gx_payload[0] = _spec(total, torch.uint8) # data - gx_payload[2] = _spec(num_gemms, torch.float32) # scale_inv - if has_columnwise and not is_non_tn_fp8_gemm_supported(): - gx_payload[1] = _spec(total, torch.uint8) # columnwise_data - gx_payload[3] = _spec(num_gemms, torch.float32) # columnwise_scale_inv - gx_payload[4] = _spec(num_gemms, torch.float32) # amax - gx_payload[6] = _spec(num_gemms, torch.float32) # scale - - for i in range(len(args.weights)): - if args.input_requires_grad: - aliases[n_payload + i] = weight_aliases[i] - else: - weights_saved[i] = None - - tensors_to_save = (*gx_payload, *weights_saved) - ctx_attrs = { - "saved_tensor_aliases": tuple(aliases), - "gx_present": gx_present, - "gx_swizzled": gx_present - and not args.save_original_input - and isinstance(args.input_quantizers[0], (MXFP8Quantizer, NVFP4Quantizer)), - } - - return (out, *new_workspaces, tensors_to_save, ctx_attrs) + if not args.input_requires_grad: + weights_saved = [None] * len(args.weights) + tensors_to_save = (_fake_grouped_input(args), *weights_saved) + return (out, *new_workspaces, tensors_to_save, None) def _grouped_linear_fused_setup_ctx( @@ -1974,13 +1851,11 @@ def _grouped_linear_fused_setup_ctx( ctx_attrs: Dict, tensors_to_save_from_forward: Tuple[Any, ...], ) -> Tuple[Any, ...]: - """Populate the fused backward args and rebuild alias-deduped save slots.""" + """Populate the fused backward args and pass through its saved tensors.""" + del fwd_outputs, ctx_attrs _grouped_linear_fused_setup(bwd_args, fwd_args) - bwd_args.gx_present = ctx_attrs["gx_present"] - bwd_args.gx_swizzled = ctx_attrs["gx_swizzled"] bwd_args.compiled_op = fwd_args.compiled_op - - return _restore_saved_aliases(fwd_args, fwd_outputs, ctx_attrs, tensors_to_save_from_forward) + return tensors_to_save_from_forward def _grouped_linear_fused_backward_impl( @@ -1991,7 +1866,20 @@ def _grouped_linear_fused_backward_impl( args.base_split_offsets = tex.splits_to_offsets(args.m_splits_tensor, 1) args.input_tensor_offsets = args.base_split_offsets * args.in_features args.output_tensor_offsets = args.base_split_offsets * args.out_features - args.inputmat = args.gx_payload[0] if args.save_original_input else _rebuild_grouped_input(args) + if args.inputmat is not None and not args.save_original_input: + if args.fp8: + args.inputmat.logical_shape = (math.prod(args.grad_output.shape[:-1]), args.in_features) + args.inputmat.first_dims = args.m_splits_tensor + args.inputmat.tensor_offsets = args.input_tensor_offsets + else: + args.inputmat = _GroupedLinear._make_grouped_tensor( + args.inputmat.reshape(-1, args.in_features), + num_gemms=args.num_gemms, + split_sizes=args.m_splits_tensor, + tensor_offsets=args.input_tensor_offsets, + last_dim=args.in_features, + dtype=args.activation_dtype, + ) return _grouped_linear_fused_backward(args) From 3bfcf5c99d01eedb2828585df61794c1a116b418 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Fri, 2 Oct 2026 19:30:59 +0200 Subject: [PATCH 07/31] [PyTorch] Preserve gradient structure in custom-op returns Derive backward return schemas from differentiable forward fields, retaining scalar and list gradients across the operator boundary. Remove flat-gradient slicing and cursor bookkeeping while preserving input-shape restoration. Cover empty gradient lists, missing gradients, and zero or one gradient field through eager and compiled custom-op calls. Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 53 +++++++++- .../pytorch/dynamo/custom_op.py | 97 ++++++++++--------- 2 files changed, 101 insertions(+), 49 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 617256b6ce3..606ded6d3b0 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -3157,11 +3157,29 @@ def _toy_list_bwd_fake(args): bwd_fake_impl=_toy_list_bwd_fake, ) + _toy_grad_ops = {} + for _grad_fields in [("inp",), ("weights",), ()]: + _indices = tuple(("inp", "weights").index(name) for name in _grad_fields) + _toy_grad_ops[_grad_fields] = register_custom_op_with_autograd( + op_name="toy_grad_returns_" + ("_".join(_grad_fields) or "none"), + input_tensors_for_grad=list(_grad_fields), + fwd_arg_type=_ToyListFwdArgs, + fwd_impl=_toy_list_fwd, + fwd_fake_impl=_toy_list_fwd_fake, + setup_context=_toy_list_setup_ctx, + bwd_arg_type=_ToyListBwdArgs, + bwd_impl=lambda args, indices=_indices: tuple(_toy_list_bwd(args)[i] for i in indices), + bwd_fake_impl=lambda args, indices=_indices: tuple( + _toy_list_bwd_fake(args)[i] for i in indices + ), + ) + @pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") @pytest.mark.parametrize("split_metadata", [[], [4, 4]]) @pytest.mark.parametrize("saved_hooks", [False, True]) -def test_custom_op_tensor_list_grads(split_metadata, saved_hooks): +@pytest.mark.parametrize("empty_weights", [False, True]) +def test_custom_op_tensor_list_grads(split_metadata, saved_hooks, empty_weights): """A toy op with a ``List[Optional[Tensor]]`` input (including a ``None`` entry riding the sentinel) compiles fullgraph and routes one gradient per list element.""" @@ -3186,11 +3204,16 @@ def run(fn): def fn(inp, w0, w2): args = _ToyListFwdArgs( - inp=inp, weights=[w0, None, w2], scales=scales, split_metadata=split_metadata + inp=inp, + weights=[] if empty_weights else [w0, None, w2], + scales=scales, + split_metadata=split_metadata, ) return _toy_list_op(args) def ref_fn(inp, w0, w2): + if empty_weights: + return (inp * 0).exp() return (inp * w0 * scales[0] + inp * w2 * scales[2]).exp() torch._dynamo.reset() @@ -3211,6 +3234,32 @@ def ref_fn(inp, w0, w2): torch.testing.assert_close(w2grad_e, w2grad_ref) +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("grad_fields", [("inp",), ("weights",), ()]) +@pytest.mark.parametrize("compile_op", [False, True]) +def test_custom_op_single_gradient_return(grad_fields, compile_op): + """Zero or one gradient field keeps scalar/list schema returns unambiguous.""" + torch._dynamo.reset() + op = _toy_grad_ops[grad_fields] + assert op is not None + inp = torch.randn(8, 4, device="cuda", requires_grad=True) + weight = torch.randn_like(inp, requires_grad=True) + + def fn(x, w): + return op(_ToyListFwdArgs(x, [None, w], (0.0, 0.25), [])) + + reference = (inp * weight * 0.25).exp() + expected = torch.autograd.grad(reference.sum(), (inp, weight)) + actual = (torch.compile(fn, fullgraph=True) if compile_op else fn)(inp, weight) + torch.testing.assert_close(actual, reference) + actual.sum().backward() + for name, tensor, grad in zip(("inp", "weights"), (inp, weight), expected): + if name in grad_fields: + torch.testing.assert_close(tensor.grad, grad) + else: + assert tensor.grad is None + + @pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") @pytest.mark.parametrize("changed_tensor", ["inp", "weight", "output"]) def test_custom_op_saved_refs_version_check(changed_tensor): diff --git a/transformer_engine/pytorch/dynamo/custom_op.py b/transformer_engine/pytorch/dynamo/custom_op.py index d16a0d7573c..7c6151ab18b 100644 --- a/transformer_engine/pytorch/dynamo/custom_op.py +++ b/transformer_engine/pytorch/dynamo/custom_op.py @@ -17,8 +17,9 @@ The autograd-free API preserves nested tensor results; the autograd-wired API keeps its saved-tensor and context-metadata contract. -A ``torch.library`` custom op is narrower: it only accepts flat schema slots -(tensors plus opaque objects) and returns a flat ``Tensor[]``. +A ``torch.library`` custom op accepts schema slots (tensors plus opaque objects). +Forward returns a flat ``Tensor[]``; backward preserves each gradient field as +a ``Tensor`` or ``Tensor[]`` return. Bridging the two takes three parts (below): a parsed per-op *arg plan* maps the args dataclass onto the op's input slots; a per-trace *output plan*, parsed from @@ -72,7 +73,7 @@ whole tuple; otherwise ``grad_output`` receives the first output's grad), the container's optional ``setup_saved_tensors`` hook restores the saved tensors, then the *backward op* runs the real ``bwd_impl`` and returns the - flat grads (``bwd_fake_impl`` is its data-free fake). + structured grads (``bwd_fake_impl`` is its data-free fake). Two-tier op (``base`` + ``wrapper``), so a ``QuantizedTensor`` subclass can be an op *input*. The ``_base`` op carries the real schema + autograd; a custom op @@ -926,28 +927,30 @@ def _pack_fwd_result(result: Any) -> List[torch.Tensor]: return flat -def _pack_bwd_result( - grads: Any, num_grad_inputs: Optional[int], op_qualname: str -) -> List[torch.Tensor]: - """Pack a backward-impl return tuple into the op's ``Tensor[]`` payload. - - Validate one gradient entry per input field when ``num_grad_inputs`` is given. - List gradients occupy one slot per element; TensorSpec grads are materialized. - """ - grads = list(grads) +def _pack_bwd_result(grads: Any, num_grad_inputs: Optional[int], op_qualname: str) -> Any: + """Materialize gradients while preserving one return per input field.""" if num_grad_inputs is not None and len(grads) != num_grad_inputs: raise RuntimeError( f"{op_qualname} expected bwd_impl to return {num_grad_inputs} grads " f"(one per input_tensors_for_grad entry), got {len(grads)}" ) - out: List[torch.Tensor] = [] - for grad in grads: - for g in grad if isinstance(grad, (list, tuple)) else (grad,): - if isinstance(g, TensorSpec): - out.append(_encode_none(g.create_tensor())) - else: - out.append(_encode_none(g)) - return out + + def materialize(grad): + return _encode_none(grad.create_tensor() if isinstance(grad, TensorSpec) else grad) + + out = tuple( + [materialize(g) for g in grad] if isinstance(grad, (list, tuple)) else materialize(grad) + for grad in grads + ) + return out if out else None + + +def _restore_grad_shape(grad, shape): + """Decode a gradient and restore its original input shape.""" + grad = _decode_none(grad) + if grad is not None and shape is not None and grad.shape != shape: + grad = grad.view(shape) + return grad @dataclasses.dataclass(frozen=True) @@ -1074,7 +1077,7 @@ def _register_base_op( plan: _ArgPlan, impl: Callable[[Any], Any], fake_impl: Callable[[Any], Any], - pack_result: Callable[[Any], List[torch.Tensor]], + pack_result: Callable[[Any], Any], ) -> Any: """Define the op via ``torch.library.custom_op`` with the real ``impl`` + the ``fake_impl`` (spec), returning the ``CustomOpDef``. @@ -1084,11 +1087,11 @@ def _register_base_op( ``pack_result``. """ - def _impl(*flat: Any) -> List[torch.Tensor]: + def _impl(*flat: Any) -> Any: obj = plan.unpack(dict(zip(plan.slot_names, flat))) return pack_result(impl(obj)) - def _fake(*flat: Any) -> List[torch.Tensor]: + def _fake(*flat: Any) -> Any: obj = plan.unpack(dict(zip(plan.slot_names, flat))) spec_obj = _spec_view(obj, plan.tensor_field_names()) return pack_result(fake_impl(spec_obj)) @@ -1178,7 +1181,7 @@ def _autograd_backward(ctx, *grad_outputs): bwd_obj.grad_output = user_grads[0] kwargs = bwd_plan.pack(bwd_obj) bwd_args_flat = [kwargs[name] for name in bwd_plan.slot_names] - grads = [_decode_none(g) for g in bwd_op(*bwd_args_flat)] + grads = bwd_op(*bwd_args_flat) ctx.backward_objects = None # One grad per input schema slot: default None, but a ``Tensor[]`` slot # (always recorded in ``fwd_tensor_list_lengths``) needs a @@ -1186,22 +1189,13 @@ def _autograd_backward(ctx, *grad_outputs): out: List[Any] = [None] * len(fwd_plan.slot_names) for pos, length in ctx.fwd_tensor_list_lengths.items(): out[pos] = [None] * length - cursor = 0 - for pos in grad_targets: - is_list = pos in ctx.fwd_tensor_list_lengths - length = ctx.fwd_tensor_list_lengths[pos] if is_list else 1 - if cursor + length > len(grads): - raise RuntimeError("bwd op returned too few flat grads for its input fields") + for pos, grad in zip(grad_targets, grads or ()): shapes = ctx.grad_input_shapes.get(pos) - values = list(grads[cursor : cursor + length]) - for i, g in enumerate(values): - shape = shapes[i] if is_list else shapes - if g is not None and shape is not None and g.shape != shape: - values[i] = g.view(shape) - out[pos] = values if is_list else values[0] - cursor += length - if cursor != len(grads): - raise RuntimeError(f"bwd op returned {len(grads)} flat grads, expected {cursor}") + out[pos] = ( + [_restore_grad_shape(g, s) for g, s in zip(grad, shapes, strict=True)] + if pos in tensor_list_slots + else _restore_grad_shape(grad, shapes) + ) ctx.grad_input_shapes = None return tuple(out) @@ -1254,7 +1248,7 @@ def _flatten_subclass_into_slots( def _make_slot_forwarder( base_op: Any, slot_offsets: Sequence[Tuple[int, bool]], subclasses: Sequence[type] -) -> Callable[[Sequence[Any]], List[torch.Tensor]]: +) -> Callable[[Sequence[Any]], Any]: """Return ``call(args)`` forwarding to ``base_op``, first flattening any ``subclasses`` instance sitting in the tensor-or-quantized slot groups at ``slot_offsets``. @@ -1266,7 +1260,7 @@ def _make_slot_forwarder( """ enabled = bool(slot_offsets) and bool(subclasses) - def call(args: Sequence[Any]) -> List[torch.Tensor]: + def call(args: Sequence[Any]) -> Any: if not enabled: return base_op(*args) new_args = list(args) @@ -1278,7 +1272,7 @@ def call(args: Sequence[Any]) -> List[torch.Tensor]: def _make_dispatch_rule( - forward: Callable[[Sequence[Any]], List[torch.Tensor]], + forward: Callable[[Sequence[Any]], Any], ) -> Callable[..., Any]: """Adapt a slot forwarder to the ``register_torch_dispatch`` signature.""" @@ -1302,7 +1296,7 @@ def _register_wrapper_op( """ forward = _make_slot_forwarder(base_op, slot_offsets, subclasses) - def _forward(*flat: Any) -> List[torch.Tensor]: + def _forward(*flat: Any) -> Any: return forward(flat) op_def = torch.library.custom_op( @@ -1338,7 +1332,7 @@ class _RegisteredOp: wrapper_def: Any wrapper_op: Any - def __call__(self, args: Any) -> List[torch.Tensor]: + def __call__(self, args: Any) -> Any: """Pack the args dataclass into slots and call the wrapper op.""" kwargs = self.plan.pack(args) return self.wrapper_op(*[kwargs[name] for name in self.plan.slot_names]) @@ -1350,13 +1344,14 @@ def _register_op( arg_type: type, impl: Callable[[Any], Any], fake_impl: Callable[[Any], Any], - pack_result: Callable[[Any], List[torch.Tensor]], + pack_result: Callable[[Any], Any], + return_schema: str = "Tensor[]", ) -> _RegisteredOp: """Define one two-tier custom op: the base kernel, the wrapper op that lets ``QuantizedTensor`` subclasses be inputs, and the passthrough registrations. """ plan = _parse_arg_type(arg_type) - schema = f"{plan.schema_str} -> Tensor[]" + schema = f"{plan.schema_str} -> {return_schema}" subclasses = _all_tensor_subclasses() slot_offsets = plan.tensor_or_quantized_offsets() namespace = getattr(torch.ops, _TE_OP_NAMESPACE) @@ -1560,18 +1555,26 @@ def _register_custom_op_with_autograd_impl( ) bwd_qualname = f"{_TE_OP_NAMESPACE}::{op_name}_backward_base" num_grad_inputs = len(input_tensors_for_grad) + grad_targets = fwd_op.plan.resolve_grad_targets(input_tensors_for_grad) + fields = {field.name: field for field in fwd_op.plan.fields} + grad_types = ", ".join( + "Tensor[]" if fields[name].is_list else "Tensor" for name in input_tensors_for_grad + ) + if num_grad_inputs == 1: + grad_types = f"({grad_types})" # Preserve a one-element tuple in the schema. bwd_op = _register_op( name=f"{op_name}_backward", arg_type=bwd_arg_type, impl=bwd_impl, fake_impl=bwd_fake_impl, pack_result=lambda grads: _pack_bwd_result(grads, num_grad_inputs, bwd_qualname), + return_schema=f"({grad_types})", ) autograd_common = { "fwd_plan": fwd_op.plan, "bwd_plan": bwd_op.plan, - "grad_targets": fwd_op.plan.resolve_grad_targets(input_tensors_for_grad), + "grad_targets": grad_targets, "setup_context_user": setup_context, "fwd_fake_impl": fwd_fake_impl, } From 5e994640625626263aa62250c709e35601563566 Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Wed, 7 Oct 2026 11:45:04 -0700 Subject: [PATCH 08/31] Preserve tiny sigmoid scores in biased fused routing (#3637) Gather selected sigmoid scores from the saved unbiased intermediate instead of subtracting expert bias after top-k. Adding the bias can round away tiny scores, which subtraction cannot recover. Keep expert selection, QB histograms, and other score functions unchanged. Cover tiny and mixed scores across top-k sizes, QB modes, and output maps. Signed-off-by: Philip Monk Co-authored-by: Kirthi Shankar Sivamani --- tests/pytorch/test_fused_router.py | 64 +++++++++++++++++++ .../fused_topk_with_score_function.cu | 10 ++- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/tests/pytorch/test_fused_router.py b/tests/pytorch/test_fused_router.py index bd89ecebc9e..c6c82fa3d78 100644 --- a/tests/pytorch/test_fused_router.py +++ b/tests/pytorch/test_fused_router.py @@ -23,6 +23,70 @@ torch.cuda.manual_seed(seed) +@pytest.mark.parametrize("topk", [1, 10, 32]) +@pytest.mark.parametrize("histogram_mode", [None, "two_kernel", "fused_atomic"]) +@pytest.mark.parametrize("layout", ["bytemap", "bitmap_u8", "dense"]) +@pytest.mark.parametrize("mixed_scores", [False, True]) +def test_sigmoid_bias_preserves_tiny_scores(topk, histogram_mode, layout, mixed_scores): + """Bias affects selection, but must not round away the selected raw scores.""" + num_tokens, num_experts, num_bins = 17, 512, 1000 + logits = torch.linspace(-8.0, -4.0, num_experts, device="cuda").repeat(num_tokens, 1) + logits[:, :topk] = -16.0 - torch.arange(topk, device="cuda") % 9 + if mixed_scores: + logits[:, 0] = -1.0 + logits.requires_grad_(True) + expert_bias = torch.zeros(num_experts, device="cuda") + expert_bias[:topk] = 0.75 + 0.001 * torch.arange(topk, device="cuda") + expert_bias -= expert_bias.mean() + bin_bounds = torch.tensor([-1.0, 1.0], device="cuda") + histogram = torch.zeros(num_experts, num_bins, dtype=torch.int32, device="cuda") + reference_probs, reference_map = topk_score_function_pytorch( + logits, topk, scaling_factor=3.16, score_function="sigmoid", expert_bias=expert_bias + ) + kwargs = {} + if histogram_mode is not None: + kwargs.update( + qb_histogram=histogram, + qb_bin_bounds=bin_bounds, + qb_histogram_mode=histogram_mode, + ) + if layout == "dense": + kwargs["topk_indices"] = torch.empty(num_tokens, topk, dtype=torch.int16, device="cuda") + fused_logits = logits.detach().clone().requires_grad_(True) + probs, routing = fused_topk_with_score_function( + fused_logits, + topk, + False, + None, + None, + 3.16, + "sigmoid", + expert_bias, + routing_map_format="bitmap_u8" if layout == "bitmap_u8" else "bytemap", + **kwargs, + ) + if layout == "dense": + routing = topk_indices_to_routing_map(routing, num_experts) + elif layout == "bitmap_u8": + reference_map = _bytemap_to_bitmap_u8(reference_map) + torch.testing.assert_close(routing, reference_map) + # An absolute tolerance would hide precisely the tiny values under test. + torch.testing.assert_close(probs, reference_probs, atol=0, rtol=5e-6) + assert torch.all(probs[:, :topk] > 0) + if histogram_mode is not None: + reference = qb_topk_score_function_pytorch(logits, topk, expert_bias, bin_bounds, num_bins) + torch.testing.assert_close(histogram, reference["histogram"]) + # Nonuniform upstream gradients exercise the normalization derivative. + upstream = torch.linspace(-0.7, 1.3, num_experts, device="cuda").expand_as(probs) + probs.backward(upstream) + reference_probs.backward(upstream) + torch.testing.assert_close(fused_logits.grad, logits.grad, atol=2e-7, rtol=2e-4) + if mixed_scores and topk > 1: + torch.testing.assert_close( + fused_logits.grad[:, 1:topk], logits.grad[:, 1:topk], atol=0, rtol=2e-4 + ) + + def _get_tolerances(dtype: torch.dtype, num_experts: int): """Return (atol, rtol) scaled by the number of experts. diff --git a/transformer_engine/common/fused_router/fused_topk_with_score_function.cu b/transformer_engine/common/fused_router/fused_topk_with_score_function.cu index 4ea11f05dce..32d525bb027 100644 --- a/transformer_engine/common/fused_router/fused_topk_with_score_function.cu +++ b/transformer_engine/common/fused_router/fused_topk_with_score_function.cu @@ -265,7 +265,10 @@ __global__ void fused_topk_forward_simple_kernel( // Postprocess: revert bias, softmax, normalization if (expert_bias && (score_function == 0 || score_function == 2)) { for (int i = lane_id; i < topk; i += kThreadsPerWarp) { - topk_scores[i] = topk_scores[i] - static_cast(expert_bias[topk_indices[i]]); + // Recover sigmoid scores directly: subtracting the bias can erase tiny scores. + topk_scores[i] = score_function == 0 + ? intermediate_output[pos_offset + topk_indices[i]] + : topk_scores[i] - static_cast(expert_bias[topk_indices[i]]); } __syncwarp(); } @@ -593,7 +596,10 @@ __global__ void fused_topk_with_score_function_forward_kernel( if constexpr (ScoreFunc == 0 || ScoreFunc == 2) { if (expert_bias) { for (int i = lane_id; i < topk; i += kThreadsPerWarp) { - topk_scores[i] = topk_scores[i] - static_cast(expert_bias[topk_indices[i]]); + // Sigmoid intermediates contain raw scores; sqrtsoftplus intermediates are logits. + topk_scores[i] = + ScoreFunc == 0 ? intermediate_output[pos_offset + topk_indices[i]] + : topk_scores[i] - static_cast(expert_bias[topk_indices[i]]); } __syncwarp(); } From 14be43fb924af0431d99e3fcc9043574b3afbefd Mon Sep 17 00:00:00 2001 From: Varun Nakar Date: Thu, 8 Oct 2026 11:50:15 +0300 Subject: [PATCH 09/31] Build FP8 recipe quantizer params per instance (#3605) * Build FP8 recipe quantizer params per instance Float8CurrentScaling and Float8BlockScaling created fp8_quant_* QParams at class definition, so constructor values for use_power_2_scales / use_f32_scales were ignored. Construct them in __post_init__ instead. Signed-off-by: varun-51 * Rebuild FP8 recipe quantizer params when unpickling older recipes Recipes pickled by earlier releases lack the fp8_quant_* instance attributes, and unpickling does not call __post_init__. Rebuild them in __setstate__. Signed-off-by: varun-51 --------- Signed-off-by: varun-51 --- tests/pytorch/test_recipe.py | 38 ++++++++++++++++++++ transformer_engine/common/recipe/__init__.py | 24 +++++++++---- 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/tests/pytorch/test_recipe.py b/tests/pytorch/test_recipe.py index 40983c89370..446bfda8e59 100644 --- a/tests/pytorch/test_recipe.py +++ b/tests/pytorch/test_recipe.py @@ -782,3 +782,41 @@ def test_stateful_unknown_or_malformed_pickled_extra_state_requires_opt_in(paylo monkeypatch.setenv(UNSAFE_PICKLE_EXTRA_STATE_ENV, "1") assert should_load_extra_state_pickle(payload, "test") + + +@pytest.mark.parametrize("use_power_2_scales", [False, True]) +def test_float8_current_scaling_quant_params_follow_constructor(use_power_2_scales): + recipe = Float8CurrentScaling(use_power_2_scales=use_power_2_scales) + for qparams in ( + recipe.fp8_quant_fwd_inp, + recipe.fp8_quant_fwd_weight, + recipe.fp8_quant_bwd_grad, + ): + assert qparams.power_2_scale == use_power_2_scales + + +@pytest.mark.parametrize("use_f32_scales", [False, True]) +def test_float8_block_scaling_quant_params_follow_constructor(use_f32_scales): + recipe = Float8BlockScaling(use_f32_scales=use_f32_scales) + for qparams in ( + recipe.fp8_quant_fwd_inp, + recipe.fp8_quant_fwd_weight, + recipe.fp8_quant_bwd_grad, + ): + assert qparams.power_2_scale == (not use_f32_scales) + + +@pytest.mark.parametrize( + "recipe", + [Float8CurrentScaling(use_power_2_scales=True), Float8BlockScaling(use_f32_scales=True)], + ids=["current_scaling", "block_scaling"], +) +def test_fp8_recipe_unpickle_rebuilds_missing_quant_params(recipe): + state = dict(recipe.__dict__) + for name in ("fp8_quant_fwd_inp", "fp8_quant_fwd_weight", "fp8_quant_bwd_grad"): + del state[name] + restored = recipe.__class__.__new__(recipe.__class__) + restored.__setstate__(state) + assert restored.fp8_quant_fwd_inp.power_2_scale == recipe.fp8_quant_fwd_inp.power_2_scale + assert restored.fp8_quant_fwd_weight.power_2_scale == recipe.fp8_quant_fwd_weight.power_2_scale + assert restored.fp8_quant_bwd_grad.power_2_scale == recipe.fp8_quant_bwd_grad.power_2_scale diff --git a/transformer_engine/common/recipe/__init__.py b/transformer_engine/common/recipe/__init__.py index 48fa4695ce5..f9abf9de87a 100644 --- a/transformer_engine/common/recipe/__init__.py +++ b/transformer_engine/common/recipe/__init__.py @@ -3,6 +3,7 @@ # See LICENSE for license information. """This module provides predefined FP8 recipes.""" + from __future__ import annotations import abc import os @@ -11,7 +12,6 @@ from dataclasses import field from pydantic.dataclasses import dataclass - _BACKWARD_OVERRIDES = (None, "high_precision", "dequantized") _NVFP4_4OVER6_SCOPES = ("none", "weights", "activations", "all") _NVFP4_4OVER6_ERR_MODES = ("MAE", "MSE") @@ -304,9 +304,6 @@ class Float8CurrentScaling(Recipe): use_power_2_scales: bool = os.getenv("NVTE_FP8_CURRENT_SCALING_POWER_2_SCALES", "0") == "1" fp8_format: Format = Format.HYBRID - fp8_quant_fwd_inp = QParams(power_2_scale=use_power_2_scales, amax_epsilon=0.0) - fp8_quant_fwd_weight = QParams(power_2_scale=use_power_2_scales, amax_epsilon=0.0) - fp8_quant_bwd_grad = QParams(power_2_scale=use_power_2_scales, amax_epsilon=0.0) fp8_gemm_fprop: MMParams = MMParams(use_split_accumulator=False) fp8_gemm_dgrad: MMParams = MMParams(use_split_accumulator=True) fp8_gemm_wgrad: MMParams = MMParams(use_split_accumulator=True) @@ -319,6 +316,14 @@ def __post_init__(self) -> None: assert ( self.backward_override in _BACKWARD_OVERRIDES ), "NVTE_BACKWARD_OVERRIDE must be unset or one of: 'high_precision', 'dequantized'." + self.fp8_quant_fwd_inp = QParams(power_2_scale=self.use_power_2_scales, amax_epsilon=0.0) + self.fp8_quant_fwd_weight = QParams(power_2_scale=self.use_power_2_scales, amax_epsilon=0.0) + self.fp8_quant_bwd_grad = QParams(power_2_scale=self.use_power_2_scales, amax_epsilon=0.0) + + def __setstate__(self, state) -> None: + self.__dict__.update(state) + if "fp8_quant_fwd_inp" not in state: + self.__post_init__() def _make_repr(self) -> str: return ( @@ -432,9 +437,6 @@ class Float8BlockScaling(Recipe): use_f32_scales: bool = os.getenv("NVTE_FP8_BLOCK_SCALING_FP32_SCALES", "0") == "1" fp8_format: Format = Format.E4M3 - fp8_quant_fwd_inp = QParams(power_2_scale=not use_f32_scales, amax_epsilon=0.0) - fp8_quant_fwd_weight = QParams(power_2_scale=not use_f32_scales, amax_epsilon=0.0) - fp8_quant_bwd_grad = QParams(power_2_scale=not use_f32_scales, amax_epsilon=0.0) x_block_scaling_dim: int = 1 w_block_scaling_dim: int = 2 grad_block_scaling_dim: int = 1 @@ -468,6 +470,14 @@ def __post_init__(self) -> None: assert ( self.backward_override in _BACKWARD_OVERRIDES ), "NVTE_BACKWARD_OVERRIDE must be unset or one of: 'high_precision', 'dequantized'." + self.fp8_quant_fwd_inp = QParams(power_2_scale=not self.use_f32_scales, amax_epsilon=0.0) + self.fp8_quant_fwd_weight = QParams(power_2_scale=not self.use_f32_scales, amax_epsilon=0.0) + self.fp8_quant_bwd_grad = QParams(power_2_scale=not self.use_f32_scales, amax_epsilon=0.0) + + def __setstate__(self, state) -> None: + self.__dict__.update(state) + if "fp8_quant_fwd_inp" not in state: + self.__post_init__() def _make_repr(self) -> str: return ( From c25fa275259808f6f6cc4fa0b8b285cfd453287a Mon Sep 17 00:00:00 2001 From: Zhiyu Li Date: Thu, 8 Oct 2026 10:38:08 -0700 Subject: [PATCH 10/31] [PyTorch] Allow deterministic dprob for the GLU fused grouped MLP (#3635) * [PyTorch] Allow deterministic dprob for the GLU fused grouped MLP Feature-detect the cuDNN dactivation wrapper's `deterministic` argument in the base class instead of only for dSReLU, so the GLU path (dGLU) can honor NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 / torch.use_deterministic_algorithms once the installed cuDNN front-end carries NVIDIA/cudnn-frontend#1319, rather than raising. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Zhiyu Li * [PyTorch] Report cuDNN's deterministic-dprob refusal as TE's error Which configurations cuDNN can make deterministic (e.g. dGLU: no dbias, no Rubin kernel, discrete weights only from cudnn-frontend#1414) is cuDNN's to decide. Instead of mirroring those rules in TE, keep the capability a pure feature-detect of the wrapper's `deterministic` argument and translate cuDNN's NotImplementedError into TE's determinism RuntimeError at the call. Drops the cuDNN FE >= 1.31.0 gate added for the same purpose. Tests check for TE's error by exact type: cuDNN's NotImplementedError subclasses RuntimeError. New test_glu_fc1_bias_refuses covers an FC1-bias-only GLU. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Zhiyu Li * [PyTorch] Accept either TE determinism refusal in test_glu_fc1_bias_refuses On a front-end whose dGLU wrapper lacks `deterministic`, TE refuses in its early check rather than by re-raising cuDNN's refusal. Match the prefix both errors share instead of the re-raise wording; cuDNN's own NotImplementedError still fails the test. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Zhiyu Li * [PyTorch] Drop implementation-detail determinism tests Per review: test_capability_follows_the_wrapper and test_glu_fc1_bias_refuses check how the capability and the error re-raise are implemented rather than deterministic execution itself. Keep test_dprob_is_bit_exact_across_runs (SReLU and GLU). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Zhiyu Li * Update transformer_engine/pytorch/ops/fused/grouped_mlp.py Co-authored-by: Tim Moon <4406448+timmoon10@users.noreply.github.com> Signed-off-by: Zhiyu Li * [PyTorch] Accept cuDNN's NotImplementedError in the determinism refusal test With the re-raise removed, a configuration cuDNN cannot make deterministic (e.g. GLU on Rubin) now surfaces cuDNN's NotImplementedError directly. Accept it alongside TE's own refusal; any other RuntimeError still fails the test. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Zhiyu Li --------- Signed-off-by: Zhiyu Li Signed-off-by: Zhiyu Li Co-authored-by: Claude Opus 5.5 Co-authored-by: Tim Moon <4406448+timmoon10@users.noreply.github.com> --- tests/pytorch/test_grouped_mlp.py | 61 ++++++++++++------- .../pytorch/ops/fused/grouped_mlp.py | 41 ++++++------- 2 files changed, 59 insertions(+), 43 deletions(-) diff --git a/tests/pytorch/test_grouped_mlp.py b/tests/pytorch/test_grouped_mlp.py index f1ca21a4b2f..b7309aff751 100644 --- a/tests/pytorch/test_grouped_mlp.py +++ b/tests/pytorch/test_grouped_mlp.py @@ -5,7 +5,6 @@ from __future__ import annotations from collections.abc import Iterable -import contextlib import functools import os import math @@ -3097,8 +3096,9 @@ def train_step( class TestGroupedMLPDeterminism: """Determinism coverage for the CuTe DSL fused grouped MLP. - Only the dSReLU wrapper can make ``dprob`` bit-exact, and only from cuDNN FE 1.28.0 on. - Anything else must refuse a determinism request rather than run non-deterministically. + The dSReLU and dGLU wrappers can make ``dprob`` bit-exact when the installed cuDNN FE + accepts ``deterministic``. Anything else must refuse a determinism request rather than run + non-deterministically. """ @pytest.fixture @@ -3136,13 +3136,6 @@ def test_either_knob_requests_determinism( torch.use_deterministic_algorithms(torch_flag) assert grouped_mlp_module._deterministic_algorithms_required() is expected - def test_only_the_srelu_path_can_be_deterministic(self) -> None: - """The capability belongs to the wrapper, not the environment. Needs no GPU.""" - glu = grouped_mlp_module.GroupedMLP_CuTeGEMMGLU - unary = grouped_mlp_module.GroupedMLP_CuTeGEMMUnary - assert glu.grouped_gemm_dactivation_is_deterministic() is False - assert isinstance(unary.grouped_gemm_dactivation_is_deterministic(), bool) - @pytest.mark.skipif(not mxfp8_available, reason=reason_for_no_mxfp8) @pytest.mark.parametrize( "activation,fused_cls", @@ -3159,12 +3152,7 @@ def test_determinism_either_runs_or_refuses( pytest.skip("MXFP8 fused grouped MLP is not supported on this system") monkeypatch.setenv("NVTE_ALLOW_NONDETERMINISTIC_ALGO", "0") - expectation = ( - contextlib.nullcontext() - if fused_cls.grouped_gemm_dactivation_is_deterministic() - else pytest.raises(RuntimeError, match="dprob") - ) - with expectation: + try: TestGroupedMLPFusedOp().test_grouped_mlp( bias=False, hidden_size=128, @@ -3172,6 +3160,14 @@ def test_determinism_either_runs_or_refuses( single_grouped_weight=False, activation=activation, ) + except RuntimeError as e: + # Refused loudly: TE's own check, or cuDNN declining `deterministic=True` for this + # configuration (NotImplementedError, e.g. no Rubin kernel). Anything else is a bug. + assert isinstance(e, NotImplementedError) or ( + grouped_mlp_module._DETERMINISM_REQUESTED in str(e) + ) + return + assert fused_cls.grouped_gemm_dactivation_is_deterministic() @pytest.mark.skipif(not mxfp8_available, reason=reason_for_no_mxfp8) def test_scale_bias_refuses_under_the_torch_flag( @@ -3199,17 +3195,23 @@ def test_scale_bias_refuses_under_the_torch_flag( ) @pytest.mark.skipif(not mxfp8_available, reason=reason_for_no_mxfp8) - def test_dprob_is_bit_exact_across_runs(self, monkeypatch) -> None: + @pytest.mark.parametrize( + "activation,fused_cls", + ( + ("scaled_srelu", grouped_mlp_module.GroupedMLP_CuTeGEMMUnary), + ("scaled_swiglu", grouped_mlp_module.GroupedMLP_CuTeGEMMGLU), + ), + ) + def test_dprob_is_bit_exact_across_runs(self, monkeypatch, *, activation, fused_cls) -> None: """Repeated identical runs must give a bit-identical ``dprob``. An ulp of reordering passes every tolerance in this file, so only an exact comparison across runs can see it. """ - fused_cls = grouped_mlp_module.GroupedMLP_CuTeGEMMUnary if not fused_cls.is_supported(): pytest.skip("MXFP8 fused grouped MLP is not supported on this system") if not fused_cls.grouped_gemm_dactivation_is_deterministic(): - pytest.skip("dSReLU determinism needs cuDNN frontend 1.28.0 or later") + pytest.skip("the installed cuDNN frontend cannot make this dactivation deterministic") monkeypatch.setenv("NVTE_ALLOW_NONDETERMINISTIC_ALGO", "0") @@ -3240,14 +3242,29 @@ def _rand(*shape, requires_grad=True) -> torch.Tensor: probs = _rand(num_tokens) # No bias, or probs.grad comes from the Triton dbias kernel instead of cuDNN. + # GLU's FC1 produces both gate halves. GLU uses one grouped weight (dense), which + # cudnn-frontend#1319 already supports; discrete weights need cudnn-frontend#1414. + is_glu = fused_cls is grouped_mlp_module.GroupedMLP_CuTeGEMMGLU with te.quantized_model_init(enabled=True, recipe=recipe): module = te.ops.Sequential( te.ops.GroupedLinear( - group_size, hidden_size, hidden_size, bias=False, device=device, dtype=dtype + group_size, + hidden_size, + 2 * hidden_size if is_glu else hidden_size, + bias=False, + single_grouped_weight=is_glu, + device=device, + dtype=dtype, ), - te.ops.ScaledSReLU(), + _make_scaled_activation(activation, glu_interleave_size=32 if is_glu else None), te.ops.GroupedLinear( - group_size, hidden_size, hidden_size, bias=False, device=device, dtype=dtype + group_size, + hidden_size, + hidden_size, + bias=False, + single_grouped_weight=is_glu, + device=device, + dtype=dtype, ), ) diff --git a/transformer_engine/pytorch/ops/fused/grouped_mlp.py b/transformer_engine/pytorch/ops/fused/grouped_mlp.py index e30b98eea3f..9fd925d32b6 100644 --- a/transformer_engine/pytorch/ops/fused/grouped_mlp.py +++ b/transformer_engine/pytorch/ops/fused/grouped_mlp.py @@ -249,6 +249,12 @@ def _deterministic_algorithms_required() -> bool: ) +_DETERMINISM_REQUESTED = ( + "Deterministic execution was requested (NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 or" + " torch.use_deterministic_algorithms)" +) + + def _wrap_single_quantized_as_grouped( tensor: torch.Tensor, quantized: MXFP8Tensor | NVFP4Tensor | NVFP4TensorStorage, @@ -1125,9 +1131,17 @@ def grouped_gemm_dactivation_kernel(cls) -> Callable: raise NotImplementedError @classmethod + @functools.lru_cache(maxsize=None) def grouped_gemm_dactivation_is_deterministic(cls) -> bool: - """Whether this op's dactivation kernel can produce a bit-exact ``dprob``.""" - return False + """Whether this op's dactivation wrapper accepts ``deterministic`` (feature-detected).""" + try: + kernel = cls.grouped_gemm_dactivation_kernel() + except ImportError: + return False + try: + return "deterministic" in inspect.signature(kernel).parameters + except (TypeError, ValueError): + return False @classmethod @functools.lru_cache(maxsize=None) @@ -2454,12 +2468,10 @@ def fuser_backward( ) if not dprob_is_deterministic: raise RuntimeError( - "Deterministic execution was requested" - " (NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 or" - " torch.use_deterministic_algorithms), but the scale gradient (dprob) is" - " accumulated with nondeterministic atomics on this configuration." - " A bit-exact dprob requires the scaled-SReLU activation," - " nvidia-cudnn-frontend 1.28.0 or later, and an FC2 without scale_bias." + f"{_DETERMINISM_REQUESTED}, but the scale gradient (dprob) is accumulated" + " with nondeterministic atomics on this configuration. A bit-exact dprob" + " requires a cuDNN front-end whose dactivation wrapper accepts" + " `deterministic`, and an FC2 without scale_bias." ) scales_f32 = None scales_tensor = None @@ -3188,19 +3200,6 @@ def grouped_gemm_dactivation_kernel(cls) -> Callable: return grouped_gemm_dsrelu_wrapper_sm100 - @classmethod - @functools.lru_cache(maxsize=None) - def grouped_gemm_dactivation_is_deterministic(cls) -> bool: - """Feature-detect the dSReLU wrapper's ``deterministic`` argument (cuDNN FE 1.28.0+).""" - try: - kernel = cls.grouped_gemm_dactivation_kernel() - except ImportError: - return False - try: - return "deterministic" in inspect.signature(kernel).parameters - except (TypeError, ValueError): - return False - def fuse_glu_ops( ops: list[FusibleOperation], From 62b13895c2e6edabfae7cfe6d2ae761358503c3d Mon Sep 17 00:00:00 2001 From: ravimajeti <138998411+ravimajeti@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:01:45 -0700 Subject: [PATCH 11/31] [Common] Fall back from gated TMA kernels when shared memory does not fit (#3600) * [Common] Fall back from gated TMA kernels when shared memory does not fit The delayed-scaling gated activation dispatch selected the TMA kernel (cast_fp8_gated_kernel) on every device with compute capability 10.0+, without checking its dynamic shared memory request against the device limit. On SM 12.0 (opt-in limit 99 KiB per block) FP32 configurations request up to 160 KiB, so cudaFuncSetAttribute fails with "invalid argument" in both forward and backward (issue #3299). Compute the kernel's shared memory requirement in one helper shared by dispatch and launch, compare it with the device's cached opt-in per-block limit, and use the existing non-TMA kernels when it does not fit. Configurations that fit (e.g. BF16/FP16 on SM 12.0, everything on SM 10.0) keep using the TMA kernel. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ravi M * [Common] Read gated TMA kernel static shared memory from the compiled kernel The fit check hard-coded the static shared memory of cast_fp8_gated_kernel as its mbarrier array (32 B) and missed the staging array declared in reduce_max (64 B). Query the compiled kernel instance instead via cudaFuncGetAttributes, exposed as cuda::static_shared_memory_size() and cached per kernel and device, so __shared__ arrays in called device functions are always included. Also replace the file-local get_max_dynamic_smem() in swizzle.cu with cuda::max_shared_memory_per_block_optin(). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ravi M --------- Signed-off-by: Ravi M Co-authored-by: Claude Opus 5.5 --- .../common/cast/dispatch/gated.cuh | 10 +++- .../common/cast/fp8/gated_fp8.cuh | 55 ++++++++++++++----- transformer_engine/common/swizzle/swizzle.cu | 26 +++------ .../common/util/cuda_runtime.cpp | 33 +++++++++++ transformer_engine/common/util/cuda_runtime.h | 24 ++++++++ 5 files changed, 115 insertions(+), 33 deletions(-) diff --git a/transformer_engine/common/cast/dispatch/gated.cuh b/transformer_engine/common/cast/dispatch/gated.cuh index 06e8f0e306c..0c1761264a1 100644 --- a/transformer_engine/common/cast/dispatch/gated.cuh +++ b/transformer_engine/common/cast/dispatch/gated.cuh @@ -46,7 +46,10 @@ void quantize_gated_fwd_helper(const NVTETensor nvte_input, NVTETensor nvte_outp switch (output->scaling_mode) { case NVTE_DELAYED_TENSOR_SCALING: { - const bool use_tma_kernels = (cols % 32 == 0) && is_supported_by_CC_100(); + const bool use_tma_kernels = + (cols % 32 == 0) && is_supported_by_CC_100() && + fp8::cast_gated_tma_fits_device( + input.dtype(), output->dtype()); if (use_tma_kernels) { Tensor dummy_grad_tensor; fp8::cast_gated_tma(input, dummy_grad_tensor, @@ -137,7 +140,10 @@ void quantize_gated_bwd_helper(const NVTETensor nvte_grad, const NVTETensor nvte switch (output->scaling_mode) { case NVTE_DELAYED_TENSOR_SCALING: { - const bool use_tma_kernels = (cols % 32 == 0) && is_supported_by_CC_100(); + const bool use_tma_kernels = + (cols % 32 == 0) && is_supported_by_CC_100() && + fp8::cast_gated_tma_fits_device( + gated_input.dtype(), output->dtype()); if (use_tma_kernels) { fp8::cast_gated_tma(gated_input, grad, output, p, stream); diff --git a/transformer_engine/common/cast/fp8/gated_fp8.cuh b/transformer_engine/common/cast/fp8/gated_fp8.cuh index 631143c6aef..124ec48b91b 100644 --- a/transformer_engine/common/cast/fp8/gated_fp8.cuh +++ b/transformer_engine/common/cast/fp8/gated_fp8.cuh @@ -17,6 +17,7 @@ #include #include "../../common.h" +#include "../../util/cuda_runtime.h" #include "../../util/math.h" #include "../../util/ptx.cuh" #include "../../util/vectorized_pointwise.h" @@ -279,6 +280,45 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) } } // namespace kernel +// Dynamic shared memory requested by cast_fp8_gated_kernel. Must match the +// buffer layout inside the kernel. +inline size_t cast_gated_tma_dynamic_shmem_size(const bool is_bwd, const DType itype, + const DType otype) { + using namespace kernel; + const size_t buff_elems_total = BUFFERS_NUM * SHMEM_DIM_Y * SHMEM_DIM_X; + const size_t buff_size_aligned_in = + DIVUP_TO_MULTIPLE(buff_elems_total * typeToNumBits(itype) / 8, TMA_SHMEM_ALIGNMENT); + const size_t buff_size_aligned_out = + DIVUP_TO_MULTIPLE(buff_elems_total * typeToNumBits(otype) / 8, TMA_SHMEM_ALIGNMENT); + const size_t grad_mem = (is_bwd ? buff_size_aligned_in : 0); + const size_t in_act_mem = buff_size_aligned_in; + const size_t in_gate_mem = buff_size_aligned_in; + const size_t out_act_mem = buff_size_aligned_out; + const size_t out_gate_mem = buff_size_aligned_out; + return grad_mem + (in_act_mem + in_gate_mem) + (out_act_mem + out_gate_mem) + TMA_SHMEM_ALIGNMENT; +} + +// Whether cast_fp8_gated_kernel fits in the shared memory of the current device. +// Devices with compute capability 10.0+ differ in shared memory per block +// (e.g. SM 12.0 has much less than SM 10.0), so FP32 configurations may not fit. +// The static shared memory is read from the compiled kernel so that __shared__ +// arrays in the device functions it calls (e.g. reduce_max) are included. +template +bool cast_gated_tma_fits_device(const DType itype, const DType otype) { + using namespace kernel; + size_t static_shmem_size = 0; + TRANSFORMER_ENGINE_TYPE_SWITCH_INPUT( + itype, IType, + TRANSFORMER_ENGINE_TYPE_SWITCH_OUTPUT( + otype, OType, + static_shmem_size = cuda::static_shared_memory_size(reinterpret_cast( + &cast_fp8_gated_kernel)););); + const size_t required = + cast_gated_tma_dynamic_shmem_size(IS_BWD, itype, otype) + static_shmem_size; + return required <= cuda::max_shared_memory_per_block_optin(); +} + template void cast_gated_tma(const Tensor &gated_input, const Tensor &grad, Tensor *output, ParamOP &p, @@ -329,19 +369,8 @@ void cast_gated_tma(const Tensor &gated_input, const Tensor &grad, Tensor *outpu SHMEM_DIM_X, tensor_stride_elems, cols, typeToNumBits(output->dtype())); - const size_t buff_elems_total = BUFFERS_NUM * SHMEM_DIM_Y * SHMEM_DIM_X; - const size_t buff_size_aligned_in = - DIVUP_TO_MULTIPLE(buff_elems_total * sizeof(IType), TMA_SHMEM_ALIGNMENT); - const size_t buff_size_aligned_out = - DIVUP_TO_MULTIPLE(buff_elems_total * sizeof(OType), TMA_SHMEM_ALIGNMENT); - const size_t grad_mem = (IS_BWD ? buff_size_aligned_in : 0); - const size_t in_act_mem = buff_size_aligned_in; - const size_t in_gate_mem = buff_size_aligned_in; - const size_t out_act_mem = buff_size_aligned_out; - const size_t out_gate_mem = buff_size_aligned_out; - - const size_t shmem_size = grad_mem + (in_act_mem + in_gate_mem) + - (out_act_mem + out_gate_mem) + TMA_SHMEM_ALIGNMENT; + const size_t shmem_size = + cast_gated_tma_dynamic_shmem_size(IS_BWD, gated_input.dtype(), output->dtype()); auto kernel = cast_fp8_gated_kernel; NVTE_CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, diff --git a/transformer_engine/common/swizzle/swizzle.cu b/transformer_engine/common/swizzle/swizzle.cu index f402f175cf3..6e197dda50e 100644 --- a/transformer_engine/common/swizzle/swizzle.cu +++ b/transformer_engine/common/swizzle/swizzle.cu @@ -24,20 +24,6 @@ namespace { constexpr int MXFP8_BLOCK_SIZE = 32; constexpr int NVFP4_BLOCK_SIZE = 16; -int get_max_dynamic_smem(int device_id = -1) { - static std::vector cache(cuda::num_devices(), -1); - static std::vector flags(cuda::num_devices()); - if (device_id < 0) { - device_id = cuda::current_device(); - } - auto init = [&]() { - NVTE_CHECK_CUDA(cudaDeviceGetAttribute(&cache[device_id], - cudaDevAttrMaxSharedMemoryPerBlockOptin, device_id)); - }; - std::call_once(flags[device_id], init); - return cache[device_id]; -} - constexpr __device__ __host__ int TB_DIM = 32; constexpr __device__ __host__ int NEW_SF_TILE_DIM_K = 16; constexpr __device__ __host__ int N_SF_PER_TD_PER_TILE = 4; @@ -1113,7 +1099,8 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s const int narrow_k_slm_size = TB_DIM * num_tiles_k * SF_TILE_DIM_M * SF_TILE_DIM_K * static_cast(sizeof(int8_t)); - if (num_tiles_k < TB_DIM && narrow_k_slm_size <= get_max_dynamic_smem()) { + if (num_tiles_k < TB_DIM && + static_cast(narrow_k_slm_size) <= cuda::max_shared_memory_per_block_optin()) { // Narrow-K: batch TB_DIM M-tiles per block, fully utilizing all threads. dim3 num_blocks_narrow(DIVUP(num_tiles_m, TB_DIM)); NVTE_CHECK_CUDA( @@ -1170,7 +1157,8 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s const int narrow_m_slm_size = TB_DIM * num_tiles_m * SF_TILE_DIM_M * SF_TILE_DIM_K * static_cast(sizeof(int8_t)); - if (num_tiles_m < TB_DIM && narrow_m_slm_size <= get_max_dynamic_smem()) { + if (num_tiles_m < TB_DIM && + static_cast(narrow_m_slm_size) <= cuda::max_shared_memory_per_block_optin()) { // Narrow-M: batch TB_DIM K-tiles per block, fully utilizing all threads. dim3 num_blocks_narrow(DIVUP(num_tiles_k, TB_DIM)); NVTE_CHECK_CUDA( @@ -1538,7 +1526,8 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, const int narrow_k_slm = TB_DIM * num_tiles_k * SF_TILE_DIM_M * SF_TILE_DIM_K * static_cast(sizeof(int8_t)); all_narrow_k = - all_narrow_k && (num_tiles_k < TB_DIM) && (narrow_k_slm <= get_max_dynamic_smem()); + all_narrow_k && (num_tiles_k < TB_DIM) && + (static_cast(narrow_k_slm) <= cuda::max_shared_memory_per_block_optin()); int vec_load_size_i = (num_tiles_k - 1) % 4 + 1; // We use the minimum vec_load_size across all tensors. // TODO(zhongbo): fix vec_load_size for NVFP4 @@ -1610,7 +1599,8 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, const int narrow_m_slm = TB_DIM * num_tiles_m * SF_TILE_DIM_M * SF_TILE_DIM_K * static_cast(sizeof(int8_t)); all_narrow_m = - all_narrow_m && (num_tiles_m < TB_DIM) && (narrow_m_slm <= get_max_dynamic_smem()); + all_narrow_m && (num_tiles_m < TB_DIM) && + (static_cast(narrow_m_slm) <= cuda::max_shared_memory_per_block_optin()); int vec_load_size_i = (num_tiles_k - 1) % 4 + 1; // We use the minimum vec_load_size across all tensors. vec_load_size = std::min(vec_load_size, vec_load_size_i); diff --git a/transformer_engine/common/util/cuda_runtime.cpp b/transformer_engine/common/util/cuda_runtime.cpp index d23f11ccc6d..17e5026bf09 100644 --- a/transformer_engine/common/util/cuda_runtime.cpp +++ b/transformer_engine/common/util/cuda_runtime.cpp @@ -10,7 +10,9 @@ #include #include +#include #include +#include #include "../common.h" #include "../util/cuda_driver.h" @@ -143,6 +145,37 @@ int sm_count(int device_id) { return cache[device_id]; } +size_t max_shared_memory_per_block_optin(int device_id) { + static std::vector cache(num_devices(), 0); + static std::vector flags(num_devices()); + if (device_id < 0) { + device_id = current_device(); + } + NVTE_CHECK(0 <= device_id && device_id < num_devices(), "invalid CUDA device ID"); + auto init = [&]() { + int value; + NVTE_CHECK_CUDA( + cudaDeviceGetAttribute(&value, cudaDevAttrMaxSharedMemoryPerBlockOptin, device_id)); + cache[device_id] = static_cast(value); + }; + std::call_once(flags[device_id], init); + return cache[device_id]; +} + +size_t static_shared_memory_size(const void *kernel) { + static std::map, size_t> cache; + static std::mutex mutex; + const auto key = std::make_pair(kernel, current_device()); + const std::lock_guard lock(mutex); + auto it = cache.find(key); + if (it == cache.end()) { + cudaFuncAttributes attr; + NVTE_CHECK_CUDA(cudaFuncGetAttributes(&attr, kernel)); + it = cache.emplace(key, attr.sharedSizeBytes).first; + } + return it->second; +} + void stream_priority_range(int *low_priority, int *high_priority, int device_id) { static std::vector> cache(num_devices()); static std::vector flags(num_devices()); diff --git a/transformer_engine/common/util/cuda_runtime.h b/transformer_engine/common/util/cuda_runtime.h index 0f355940010..02f95c08f1e 100644 --- a/transformer_engine/common/util/cuda_runtime.h +++ b/transformer_engine/common/util/cuda_runtime.h @@ -38,6 +38,30 @@ int sm_arch(int device_id = -1); */ int sm_count(int device_id = -1); +/* \brief Maximum shared memory per block available with opt-in + * + * This is the upper bound for the static plus dynamic shared memory + * of a single thread block, after raising the kernel's + * cudaFuncAttributeMaxDynamicSharedMemorySize attribute. + * + * \param[in] device_id CUDA device (default is current device) + * + * \return Shared memory size in bytes + */ +size_t max_shared_memory_per_block_optin(int device_id = -1); + +/* \brief Static shared memory used by a compiled kernel on the current device + * + * Size of the kernel's fixed-size __shared__ arrays, including those + * declared in device functions it calls, as reported by + * cudaFuncGetAttributes. The result is cached per kernel and device. + * + * \param[in] kernel Pointer to the __global__ function + * + * \return Shared memory size in bytes + */ +size_t static_shared_memory_size(const void *kernel); + /* \brief Minimum and maximum stream priorities supported on device * * \param[in] device_id CUDA device (default is current device) From 45223d764a28fdaae242de3f0077c34f538d00de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadzi=C5=84ski?= <62263673+pggPL@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:14:02 +0200 Subject: [PATCH 12/31] [PyTorch] Compile Linear and Bias operations with forward fusion (#3547) * [PyTorch] Compile Linear and Bias operations with forward fusion Signed-off-by: Pawel Gadzinski * [PyTorch] Preserve quantized tensors for dequantized backward Signed-off-by: Pawel Gadzinski * [PyTorch] Share Linear and Bias compute between eager and compile Signed-off-by: Pawel Gadzinski * [PyTorch] Enable compiled BasicLinear tensor parallelism and address review Signed-off-by: Pawel Gadzinski * [PyTorch] Simplify ops backward bookkeeping and clarify compile tests Signed-off-by: Pawel Gadzinski * [PyTorch] Pass custom op context directly to operation compute Signed-off-by: Pawel Gadzinski * [PyTorch] Fix ops compile regressions and clarify test scenarios Signed-off-by: Pawel Gadzinski * [PyTorch] Match saved FP8 fake tensor dtypes to sources Signed-off-by: Pawel Gadzinski * [PyTorch] Specialize BasicLinear compile ops by TP size Signed-off-by: Pawel Gadzinski * [PyTorch] Move compiled bias coverage into Linear tests Signed-off-by: Pawel Gadzinski --------- Signed-off-by: Pawel Gadzinski --- tests/pytorch/distributed/test_fusible_ops.py | 113 +++-- tests/pytorch/test_fusible_ops.py | 15 +- tests/pytorch/test_torch_compile.py | 411 ++++++++++++++++-- transformer_engine/pytorch/ops/_common.py | 7 +- .../pytorch/ops/basic/basic_linear.py | 399 ++++++++++++----- transformer_engine/pytorch/ops/basic/bias.py | 125 ++++-- .../fused/forward_linear_bias_activation.py | 143 ++---- transformer_engine/pytorch/ops/fuser.py | 12 +- transformer_engine/pytorch/ops/op.py | 14 +- 9 files changed, 918 insertions(+), 321 deletions(-) diff --git a/tests/pytorch/distributed/test_fusible_ops.py b/tests/pytorch/distributed/test_fusible_ops.py index c314bf5bb69..8057a75a84e 100644 --- a/tests/pytorch/distributed/test_fusible_ops.py +++ b/tests/pytorch/distributed/test_fusible_ops.py @@ -35,7 +35,7 @@ _current_file = pathlib.Path(__file__).resolve() # Prepend so installed packages with a top-level utils module cannot shadow the test helpers. sys.path = [str(_current_file.parent.parent)] + sys.path -from utils import dtype_tols, make_recipe, quantization_tols +from utils import assert_close, dtype_tols, make_recipe, quantization_tols # Check what quantization schemes are supported @@ -331,6 +331,7 @@ def _test_basic_linear( quantized_weight: bool = False, tensor_parallel_mode: str = "column", sequence_parallel: bool = False, + compile_model: bool = False, ) -> None: # Skip invalid configurations @@ -428,6 +429,8 @@ def _test_basic_linear( # Implementation with fusible operation recipe = make_recipe(quantization) + if compile_model and recipe is not None: + recipe.backward_override = None with te.quantized_model_init(enabled=quantized_weight, recipe=recipe): op = te_ops.BasicLinear( in_features, @@ -441,9 +444,17 @@ def _test_basic_linear( with torch.no_grad(): op.weight.copy_(w_test) del w_test - with te.autocast(enabled=quantized_compute, recipe=recipe): - y_test = op(x_test) - y_test.backward(dy_test) + model = te_ops.Sequential(op) + + def forward(x): + if not quantized_compute: + return model(x) + with te.autocast(enabled=quantized_compute, recipe=recipe): + return model(x) + + if compile_model: + torch._dynamo.reset() + forward = torch.compile(forward, fullgraph=True) # Expected numerical error tols = dtype_tols(dtype) @@ -452,13 +463,15 @@ def _test_basic_linear( if quantized_compute: tols = quantization_tols(quantization) - # Check results - y_test = y_test.to(dtype=torch.float64, device="cpu") - dx_test = x_test.grad.to(dtype=torch.float64, device="cpu") - dw_test = op.weight.grad.to(dtype=torch.float64, device="cpu") - torch.testing.assert_close(y_test, y_ref, **tols) - torch.testing.assert_close(dx_test, dx_ref, **tols) - torch.testing.assert_close(dw_test, dw_ref, **tols) + # Repeated calls also exercise the compiled backward and cached graph. + for _ in range(3 if compile_model else 1): + model.zero_grad(set_to_none=True) + x_test.grad = None + y_test = forward(x_test) + y_test.backward(dy_test) + assert_close(y_test, y_ref, **tols) + assert_close(x_test.grad, dx_ref, **tols) + assert_close(op.weight.grad, dw_ref, **tols) def _test_linear( @@ -472,6 +485,7 @@ def _test_linear( quantized_weight: bool = False, tensor_parallel_mode: str = "column", sequence_parallel: bool = False, + compile_model: bool = False, ) -> None: # Skip invalid configurations @@ -594,6 +608,8 @@ def _test_linear( # Implementation with fusible operation recipe = make_recipe(quantization) + if compile_model and recipe is not None: + recipe.backward_override = None with te.quantized_model_init(enabled=quantized_weight, recipe=recipe): model = te_ops.Sequential( te_ops.Linear( @@ -613,9 +629,16 @@ def _test_linear( model[0].bias.copy_(b_test) del w_test del b_test - with te.autocast(enabled=quantized_compute, recipe=recipe): - y_test = model(x_test) - y_test.backward(dy_test) + + def forward(x): + if not quantized_compute: + return model(x) + with te.autocast(enabled=quantized_compute, recipe=recipe): + return model(x) + + if compile_model: + torch._dynamo.reset() + forward = torch.compile(forward, fullgraph=True) # Expected numerical error tols = dtype_tols(dtype) @@ -624,16 +647,16 @@ def _test_linear( if quantized_compute: tols = quantization_tols(quantization) - # Check results - y_test = y_test.to(dtype=torch.float64, device="cpu") - dx_test = x_test.grad.to(dtype=torch.float64, device="cpu") - dw_test = model[0].weight.grad.to(dtype=torch.float64, device="cpu") - torch.testing.assert_close(y_test, y_ref, **tols) - torch.testing.assert_close(dx_test, dx_ref, **tols) - torch.testing.assert_close(dw_test, dw_ref, **tols) - if bias: - db_test = model[0].bias.grad.to(dtype=torch.float64, device="cpu") - torch.testing.assert_close(db_test, db_ref, **tols) + for _ in range(3 if compile_model else 1): + model.zero_grad(set_to_none=True) + x_test.grad = None + y_test = forward(x_test) + y_test.backward(dy_test) + assert_close(y_test, y_ref, **tols) + assert_close(x_test.grad, dx_ref, **tols) + assert_close(model[0].weight.grad, dw_ref, **tols) + if bias: + assert_close(model[0].bias.grad, db_ref, **tols) def _test_mlp( @@ -1034,6 +1057,35 @@ def run_parallel_tests() -> None: _test_fp8_scale_update() +def run_compile_parallel_tests() -> None: + """Compile the existing TP numerical test, including the Linear+Bias fusion.""" + quantizations = [None, "fp8_current_scaling"] if fp8_available else [None] + for quantization, mode, sp in itertools.product( + quantizations, ("column", "row"), (False, True) + ): + if torch.distributed.get_rank(world_group()) == 0: + print(f"Compile BasicLinear: {quantization=}, {mode=}, {sp=}", flush=True) + _test_basic_linear( + dtype=torch.bfloat16, + quantization=quantization, + tensor_parallel_mode=mode, + sequence_parallel=sp, + compile_model=True, + ) + # Row-parallel Linear uses standalone collectives without compile support. + if mode == "column": + if torch.distributed.get_rank(world_group()) == 0: + print(f"Compile Linear: {quantization=}, {mode=}, {sp=}", flush=True) + _test_linear( + bias=True, + dtype=torch.bfloat16, + quantization=quantization, + tensor_parallel_mode=mode, + sequence_parallel=sp, + compile_model=True, + ) + + # Parallel job sizes _world_sizes = [torch.cuda.device_count()] if 1 not in _world_sizes: @@ -1043,7 +1095,8 @@ def run_parallel_tests() -> None: @pytest.mark.parametrize("world_size", _world_sizes) -def test_distributed_fuser_ops(world_size: int) -> None: +@pytest.mark.parametrize("compile_model", [False, True], ids=["eager", "compile"]) +def test_distributed_fuser_ops(world_size: int, compile_model: bool) -> None: """Launch parallel job that runs parallel tests""" python_exe = pathlib.Path(sys.executable) current_file = pathlib.Path(__file__).resolve() @@ -1055,6 +1108,8 @@ def test_distributed_fuser_ops(world_size: int) -> None: current_file, "--parallel", ] + if compile_model: + command.append("--compile") with tempfile.TemporaryDirectory(prefix="te-test-fusible-ops-") as temp_dir: env = dict(os.environ) env["NVTE_TEST_RDZV_PATH"] = str(pathlib.Path(temp_dir) / "rdzv") @@ -1064,9 +1119,15 @@ def test_distributed_fuser_ops(world_size: int) -> None: def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--parallel", action="store_true", help="Run parallel tests") + parser.add_argument("--compile", action="store_true", help="Use the compiled TP test suite") args = parser.parse_args() + if args.compile and not args.parallel: + parser.error("--compile requires --parallel") if args.parallel: - run_parallel_tests() + if args.compile: + run_compile_parallel_tests() + else: + run_parallel_tests() if __name__ == "__main__": diff --git a/tests/pytorch/test_fusible_ops.py b/tests/pytorch/test_fusible_ops.py index 34049523501..bdee6f7ffe8 100644 --- a/tests/pytorch/test_fusible_ops.py +++ b/tests/pytorch/test_fusible_ops.py @@ -48,6 +48,7 @@ # Import utility functions from utils import ( + MegatronTrainingHelper, assert_close, assert_close_grads, dtype_tols, @@ -1722,6 +1723,10 @@ def _test_basic_linear( op.weight.copy_(w_test) del w_test op.weight.main_grad = torch.full_like(op.weight, 0.5, dtype=torch.float32) + if accumulate_into_main_grad: + MegatronTrainingHelper.init_main_grad_buffers( + [op.weight], fill_value=0.5, overwrite_main_grad=False, zero_out_wgrad=True + ) with te.autocast(enabled=quantized_compute, recipe=recipe): y_test = forward(x_test) y_test.backward(dy_test) @@ -1739,13 +1744,9 @@ def _test_basic_linear( torch.testing.assert_close(y_test, y_ref, **tols) torch.testing.assert_close(dx_test, x_ref.grad, **tols) if accumulate_into_main_grad: - if op.weight.grad is not None: - torch.testing.assert_close( - op.weight.grad, - torch.zeros_like(op.weight.grad), - rtol=0, - atol=0, - ) + MegatronTrainingHelper.verify_main_grad_accumulation( + [op.weight], expected_main_grads=[w_ref.grad + 0.5], **tols + ) dw_test = op.weight.main_grad.to(dtype=torch.float64, device="cpu") - 0.5 else: dw_test = op.weight.grad.to(dtype=torch.float64, device="cpu") diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index eae6f0a8a24..49aa5033cb4 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -4,12 +4,15 @@ import abc import contextlib +import copy import dataclasses import os import re +import subprocess import sys +import textwrap import warnings -from typing import Union +from typing import Literal, NamedTuple, Union import pytest import torch @@ -2412,7 +2415,7 @@ def __init__(self, weight=2.0, dtype=torch.float32): self.weight = torch.nn.Parameter(torch.tensor(weight, dtype=dtype, device="cuda")) @classmethod - def forward_compute(cls, args): + def forward_compute(cls, args, *, in_custom_op=False): output = args.input_ * args.weight * args.gain offset = args.offset if isinstance(offset, QuantizedTensorStorage): @@ -2426,7 +2429,7 @@ def forward_compute_fake(cls, args): return args.input_, [()], () @classmethod - def backward_compute(cls, args): + def backward_compute(cls, args, *, in_custom_op=False): dy = args.grad_output return dy * args.weight * args.gain, [((dy * args.input_).sum() * args.gain,)], [()] @@ -2484,7 +2487,7 @@ class _AffinePair(te.ops.FusedOperation): bwd_args_type = _AffinePairBwdArgs @classmethod - def forward_compute(cls, args): + def forward_compute(cls, args, *, in_custom_op=False): intermediate = args.input_ * args.weight0 output = intermediate * args.weight1 + args.residual return output, [(), (intermediate.square(), None)], (intermediate,) @@ -2494,7 +2497,7 @@ def forward_compute_fake(cls, args): return args.input_, [(), (args.input_, None)], (args.input_,) @classmethod - def backward_compute(cls, args): + def backward_compute(cls, args, *, in_custom_op=False): dy = args.grad_output du = dy * args.weight1 + 2 * args.intermediate * args.grad_extra_output return ( @@ -2539,7 +2542,16 @@ def backend(graph, inputs): return torch.compile(fn, fullgraph=True, backend=backend), graphs -def _assert_custom_ops(graphs, name, present=True): +_ExecutionPath = Literal["custom_op", "eager"] + + +def _assert_custom_ops( + graphs, + name, + *, + forward: bool = True, + backward: bool = True, +): targets = { str(node.target).removesuffix(".default").removesuffix("_base") for graph in graphs @@ -2548,9 +2560,7 @@ def _assert_custom_ops(graphs, name, present=True): for node in module.graph.nodes if node.op == "call_function" } - if isinstance(present, bool): - present = (present, present) - for suffix, expected in zip(("", "_backward"), present): + for suffix, expected in (("", forward), ("_backward", backward)): assert (f"transformer_engine_compile.{name}{suffix}" in targets) == expected, targets @@ -2572,32 +2582,79 @@ def _check_ops(fn, model, x, dy, kwargs=None): torch.testing.assert_close(actual, expected) +class _Scenario(NamedTuple): + variant: str + expected_warning: str | None + forward_path: _ExecutionPath + backward_path: _ExecutionPath + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) @pytest.mark.parametrize( - "case,reason,compiled_passes", + "scenario", [ - ("single", None, (True, True)), - ("single_forward", "without a custom op for backward", (True, False)), - ("single_backward", "without a custom op for forward", (False, True)), - ("single_eager", "without a custom op", (False, False)), - ("multi", "several operations", (False, False)), - ("backward_fusion", "backward fusion", (False, False)), - ("legacy", "without a custom op", (False, False)), + _Scenario( + variant="single", + expected_warning=None, + forward_path="custom_op", + backward_path="custom_op", + ), + _Scenario( + variant="single_forward", + expected_warning="without a custom op for backward", + forward_path="custom_op", + backward_path="eager", + ), + _Scenario( + variant="single_backward", + expected_warning="without a custom op for forward", + forward_path="eager", + backward_path="custom_op", + ), + _Scenario( + variant="single_eager", + expected_warning="without a custom op", + forward_path="eager", + backward_path="eager", + ), + # A pipeline can dispatch each operation through its own custom op. + _Scenario( + variant="multi", + expected_warning=None, + forward_path="custom_op", + backward_path="custom_op", + ), + # This test fusion only implements eager backward; forward still compiles. + _Scenario( + variant="backward_fusion", + expected_warning="without a custom op for backward", + forward_path="custom_op", + backward_path="eager", + ), + _Scenario( + variant="legacy", + expected_warning="without a custom op", + forward_path="eager", + backward_path="eager", + ), ], + ids=lambda scenario: scenario.variant, ) -def test_te_ops_pipeline(case, reason, compiled_passes, dtype, monkeypatch): +def test_te_ops_pipeline(scenario, dtype, monkeypatch): torch._dynamo.reset() - if case.startswith("single"): + if scenario.variant.startswith("single"): monkeypatch.setattr( _AffineOp, "compile_ops", tuple( - op if enabled else None - for op, enabled in zip(_AffineOp.compile_ops, compiled_passes) + op if path == "custom_op" else None + for op, path in zip( + _AffineOp.compile_ops, (scenario.forward_path, scenario.backward_path) + ) ), ) - if case == "backward_fusion": + if scenario.variant == "backward_fusion": def fuse(ops, **unused): if len(ops) == 2 and all(isinstance(op, _AffineOp) for op in ops): @@ -2607,23 +2664,35 @@ def fuse(ops, **unused): monkeypatch.setattr(OperationFuser, "backward_fusion_functions", [fuse]) ops = ( [te.ops.Identity()] - if case == "legacy" - else [_AffineOp(float(i + 2), dtype) for i in range(1 if case.startswith("single") else 2)] + if scenario.variant == "legacy" + else [ + _AffineOp(float(i + 2), dtype) + for i in range(1 if scenario.variant.startswith("single") else 2) + ] ) model = te.ops.Sequential(*ops) compiled, graphs = _compile_with_graphs(model) # Keep products exact in BF16 across eager and fused reductions. x = (torch.randint(-8, 9, (8, 16), device="cuda").to(dtype) / 8).requires_grad_() dy = torch.randint_like(x, -8, 9) / 8 - with pytest.warns(UserWarning, match=reason) if reason else contextlib.nullcontext(): + with ( + pytest.warns(UserWarning, match=scenario.expected_warning) + if scenario.expected_warning + else contextlib.nullcontext() + ): _check_ops(compiled, model, x, dy) _check_ops(model, model, x, dy) - _assert_custom_ops(graphs, "_affineop", present=compiled_passes) - if case == "single_forward": + _assert_custom_ops( + graphs, + "_affineop", + forward=scenario.forward_path == "custom_op", + backward=scenario.backward_path == "custom_op", + ) + if scenario.variant == "single_forward": compiled, graphs = _compile_with_graphs(model) with torch.no_grad(): torch.testing.assert_close(compiled(x), x * ops[0].weight) - _assert_custom_ops(graphs, "_affineop", present=(True, False)) + _assert_custom_ops(graphs, "_affineop", backward=False) @pytest.mark.parametrize( @@ -2636,7 +2705,12 @@ def register(**kwargs): registered.append(kwargs["op_name"]) return kwargs["impl"] + def compute(args, *, in_custom_op=False): + return in_custom_op + monkeypatch.setattr("transformer_engine.pytorch.ops.op.register_custom_op", register) + monkeypatch.setattr(_AffineOp, "forward_compute", staticmethod(compute)) + monkeypatch.setattr(_AffineOp, "backward_compute", staticmethod(compute)) monkeypatch.setattr(_AffineOp, "fwd_args_type", _AffineFwdArgs if forward else None) monkeypatch.setattr(_AffineOp, "bwd_args_type", _AffineBwdArgs if backward else None) monkeypatch.setattr(_AffineOp, "compile_ops", (None, None)) @@ -2644,10 +2718,13 @@ def register(**kwargs): assert registered == (["_affineop"] if forward else []) + ( ["_affineop_backward"] if backward else [] ) - assert _AffineOp.compile_ops == ( - _AffineOp.forward_compute if forward else None, - _AffineOp.backward_compute if backward else None, - ) + for custom_op, enabled in zip(_AffineOp.compile_ops, (forward, backward)): + if enabled: + assert custom_op(None) is True + else: + assert custom_op is None + assert _AffineOp.forward_compute(None) is False + assert _AffineOp.backward_compute(None) is False @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @@ -2694,7 +2771,7 @@ def run(input_, extra_input, grad_output, grad_extra_output): ) expected = output, [(), (extra, None)], (dx, [(dw0,), (dw1,)], [(), (dr,)]) torch.testing.assert_close(actual, expected) - _assert_custom_ops(graphs, "_affinepair", present=use_custom_ops) + _assert_custom_ops(graphs, "_affinepair", forward=use_custom_ops, backward=use_custom_ops) @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @@ -2719,4 +2796,270 @@ def test_te_ops_forward_kwargs_compile(): with pytest.warns(UserWarning, match="non-tensor keyword arguments"): for gain in (3.0, 5.0): _check_ops(compiled, model, x, dy, {"gain": gain}) - _assert_custom_ops(graphs[-1:], "_affineop", present=False) + _assert_custom_ops(graphs[-1:], "_affineop", forward=False, backward=False) + + +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +@pytest.mark.parametrize("backward", [False, True]) +def test_te_ops_bias_compile_with_fp8_input(backward): + """A quantized tensor subclass must cross the Bias custom-op boundary.""" + torch._dynamo.reset() + quantizer = Float8CurrentScalingQuantizer(fp8_dtype=tex.DType.kFloat8E4M3, device="cuda") + x = quantizer(torch.randn(32, 64, device="cuda", dtype=torch.bfloat16)) + model = te.ops.Bias(64, device="cuda", dtype=torch.bfloat16) + compiled = torch.compile(lambda inp: model(inp), fullgraph=True) + + with contextlib.nullcontext() if backward else torch.no_grad(): + expected = model(x) + actual = compiled(x) + torch.testing.assert_close(actual, expected) + if backward: + dy = torch.randn_like(expected) + expected_db = torch.autograd.grad(expected, model.bias, dy)[0] + actual_db = torch.autograd.grad(actual, model.bias, dy)[0] + torch.testing.assert_close(actual_db, expected_db) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("quantization", [None, "fp8"]) +@pytest.mark.parametrize("case", ["linear", "bias", "pair", "unfused"]) +@pytest.mark.parametrize("grads", ["all", "input", "weight", "bias", "none"]) +def test_te_ops_linear_bias_compile(dtype, quantization, case, grads, monkeypatch): + _check_linear_bias_compile(dtype, quantization, case, grads, monkeypatch) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("backward_override", ["high_precision", "dequantized"]) +def test_te_ops_linear_bias_backward_override(dtype, backward_override, monkeypatch): + _check_linear_bias_compile( + dtype, "fp8", "pair", "all", monkeypatch, backward_override=backward_override + ) + + +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +@pytest.mark.parametrize( + "input_dtype,weight_dtype", + [(torch.float32, torch.bfloat16), (torch.bfloat16, torch.float32)], +) +def test_te_ops_linear_saved_fp8_dtype_with_autocast(input_dtype, weight_dtype): + """Saved FP8 tensors retain their source dtype across the custom-op boundary.""" + torch._dynamo.reset() + model = te.ops.BasicLinear(32, 64, device="cuda", dtype=weight_dtype) + eager_model = copy.deepcopy(model) + fp8_recipe = recipe.Float8CurrentScaling() + + def run(module, inp): + with torch.autocast(device_type="cuda", dtype=torch.bfloat16): + with te.autocast(recipe=fp8_recipe): + return module(inp) + + compiled, graphs = _compile_with_graphs(lambda inp: run(model, inp)) + x = torch.randn(16, 32, device="cuda", dtype=input_dtype, requires_grad=True) + eager_x = x.detach().clone().requires_grad_() + actual = compiled(x) + expected = run(eager_model, eager_x) + torch.testing.assert_close(actual, expected) + + dy = torch.randn_like(actual) + actual_grads = torch.autograd.grad(actual, (x, model.weight), dy) + expected_grads = torch.autograd.grad(expected, (eager_x, eager_model.weight), dy) + torch.testing.assert_close(actual_grads, expected_grads) + _assert_custom_ops(graphs, "basiclinear") + + with torch.autocast(device_type="cuda", dtype=torch.bfloat16): + with te.autocast(recipe=fp8_recipe): + args = model.pack_forward_args( + [OperationContext()], + x, + prev_op_grad_output_quantizer=None, + next_op_input_quantizer=None, + basic_op_kwargs=[{}], + ) + _, _, (saved_input, saved_weight) = model.forward_compute_fake(args) + assert saved_input.dtype == x.dtype + assert saved_weight.dtype == model.weight.dtype + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_te_ops_linear_workspace_fake_mode(monkeypatch): + def unexpected_allocation(*args, **kwargs): + pytest.fail("Fake initialization must not populate the real workspace cache") + + monkeypatch.setattr( + "transformer_engine.pytorch.ops.basic.basic_linear.get_cublas_workspace", + unexpected_allocation, + ) + with FakeTensorMode(): + te.ops.BasicLinear(32, 64, device="cuda", dtype=torch.bfloat16) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +@pytest.mark.parametrize("training", [False, True]) +@pytest.mark.parametrize("initial_device", ["cuda", "cpu", "meta"]) +def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device): + """A fresh process prevents earlier GEMMs from hiding lazy workspace allocation.""" + script = textwrap.dedent( + """ + import contextlib + import sys + import torch + import transformer_engine.pytorch as te + from transformer_engine.pytorch.cpp_extensions.gemm import get_cublas_workspace + from torch._dynamo.utils import counters + + training, initial_device = sys.argv[1:] + training = training == "True" + assert get_cublas_workspace.cache_info().currsize == 0 + linear = te.ops.BasicLinear(32, 64, device=initial_device, dtype=torch.bfloat16) + if initial_device == "cpu": + assert get_cublas_workspace.cache_info().currsize == 0 + linear.cuda() + elif initial_device == "meta": + assert get_cublas_workspace.cache_info().currsize == 0 + linear.to_empty(device="cuda") + linear.reset_parameters() + assert get_cublas_workspace.cache_info().currsize > 0 + model = te.ops.Sequential(linear, te.ops.Bias(64, dtype=torch.bfloat16)) + x = torch.randn(32, 32, device="cuda", dtype=torch.bfloat16, requires_grad=training) + targets = (x, *model.parameters()) + dy = torch.randn(32, 64, device="cuda", dtype=torch.bfloat16) + + def forward(x): + return model(x) + + compiled = torch.compile(forward, fullgraph=True, mode="reduce-overhead") + counters.clear() + with contextlib.nullcontext() if training else torch.no_grad(): + for _ in range(5): + torch.compiler.cudagraph_mark_step_begin() + actual = compiled(x) + if training: + grads = torch.autograd.grad(actual, targets, dy) + # Eager reference runs only after the first compiled invocation. + expected = forward(x) + torch.testing.assert_close(actual, expected) + if training: + expected_grads = torch.autograd.grad(expected, targets, dy) + torch.testing.assert_close(grads, expected_grads) + del grads, expected_grads + del actual, expected + torch.cuda.synchronize() + assert not counters["inductor"]["cudagraph_skips"], counters["inductor"] + assert counters["inductor"]["cudagraph_recorded_non_static_inputs"] > 0, dict( + counters["inductor"] + ) + """ + ) + result = subprocess.run( + [sys.executable, "-c", script, str(training), initial_device], + capture_output=True, + text=True, + check=False, + timeout=180, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def _check_linear_bias_compile( + dtype, quantization, case, grads, monkeypatch, *, backward_override=None +): + if quantization == "fp8" and not fp8_available: + pytest.skip(reason_for_no_fp8) + if (case == "linear" and grads == "bias") or (case == "bias" and grads == "weight"): + pytest.skip("Requested parameter is absent") + torch._dynamo.reset() + if case == "unfused": + monkeypatch.setattr(OperationFuser, "forward_fusion_functions", []) + ops = [] + if case != "bias": + ops.append(te.ops.BasicLinear(32, 64, dtype=dtype)) + ops[-1].weight.requires_grad_(grads in ("all", "weight")) + if case != "linear": + ops.append(te.ops.Bias(32 if case == "bias" else 64, dtype=dtype)) + ops[-1].bias.requires_grad_(grads in ("all", "bias")) + model = te.ops.Sequential(*ops) + eager_model = copy.deepcopy(model) + quant_recipe = ( + recipe.Float8CurrentScaling(backward_override=backward_override) + if quantization == "fp8" + else None + ) + + def run(x, module=model): + if quant_recipe is None: + return module(x) + with te.autocast(recipe=quant_recipe): + return module(x) + + compiled, graphs = _compile_with_graphs(run) + x = torch.randn(2, 16, 32, device="cuda", dtype=dtype) + x.requires_grad_(grads in ("all", "input")) + targets = tuple(t for t in (x, *model.parameters()) if t.requires_grad) + eager_targets = tuple(t for t in (x, *eager_model.parameters()) if t.requires_grad) + graph_count = None + with torch.no_grad() if grads == "none" else contextlib.nullcontext(): + for iteration in range(3): + with torch.no_grad(): + for tensor in (x, *model.parameters()): + if iteration == 0: + # These binary fractions survive FP8 quantization exactly, + # allowing a strict comparison with native PyTorch. + tensor.copy_(torch.randint(-2, 3, tensor.shape, device=tensor.device) / 16) + else: + tensor.uniform_(-0.5, 0.5) + for param, eager_param in zip(model.parameters(), eager_model.parameters()): + eager_param.copy_(param) + actual = compiled(x) + expected = run(x, eager_model) + torch.testing.assert_close(actual, expected) + if iteration == 0: + reference = x.double() + for op in ops: + reference = ( + torch.nn.functional.linear(reference, op.weight.double()) + if isinstance(op, BasicLinear) + else reference + op.bias.double() + ) + torch.testing.assert_close( + actual, reference, check_dtype=False, **dtype_tols(dtype) + ) + if targets: + dy = ( + torch.randint_like(actual, -2, 3) / 16 + if iteration == 0 + else torch.randn_like(actual) + ) + actual_grads = torch.autograd.grad(actual, targets, dy) + torch.testing.assert_close( + actual_grads, torch.autograd.grad(expected, eager_targets, dy) + ) + if iteration == 0: + torch.testing.assert_close( + actual_grads, + torch.autograd.grad(reference, targets, dy.double()), + **dtype_tols(dtype), + ) + if iteration == 1: + graph_count = len(graphs) + elif iteration == 2: + assert len(graphs) == graph_count + _assert_custom_ops( + graphs, + "forwardlinearbiasactivation", + forward=case == "pair", + backward=False, + ) + _assert_custom_ops( + graphs, + "basiclinear", + forward=case in ("linear", "unfused"), + backward=case != "bias" and grads in ("all", "input", "weight"), + ) + _assert_custom_ops( + graphs, + "bias", + forward=case in ("bias", "unfused"), + backward=case != "linear" and bool(targets), + ) diff --git a/transformer_engine/pytorch/ops/_common.py b/transformer_engine/pytorch/ops/_common.py index 19a5eaf473b..50abe957e19 100644 --- a/transformer_engine/pytorch/ops/_common.py +++ b/transformer_engine/pytorch/ops/_common.py @@ -11,6 +11,7 @@ import torch from transformer_engine_torch import FP8TensorMeta +from ..dynamo.tensor_spec import TensorSpec from ..ep import ( EpBuffer, EpConfig, @@ -156,8 +157,10 @@ def validate_or_alloc_output( return buffer -def is_quantized_tensor(tensor: torch.Tensor | QuantizedTensorStorage) -> bool: - """Check if tensor is a quantized tensor""" +def is_quantized_tensor(tensor: torch.Tensor | QuantizedTensorStorage | TensorSpec) -> bool: + """Check whether a tensor or tensor spec represents quantized data.""" + if isinstance(tensor, TensorSpec): + return tensor.is_quantized return isinstance(tensor, QuantizedTensorStorage) diff --git a/transformer_engine/pytorch/ops/basic/basic_linear.py b/transformer_engine/pytorch/ops/basic/basic_linear.py index cb429055a4f..a7b09198250 100644 --- a/transformer_engine/pytorch/ops/basic/basic_linear.py +++ b/transformer_engine/pytorch/ops/basic/basic_linear.py @@ -5,14 +5,16 @@ """Fusible operation for linear layer without bias.""" from __future__ import annotations -from collections.abc import Callable, Iterable +from collections.abc import Callable import contextlib +from dataclasses import dataclass import math from typing import Any, Optional import torch -from ...cpp_extensions import general_gemm +from ...cpp_extensions import general_gemm, get_cublas_workspace +from ...dynamo import TensorOrQuantized, TensorSpec from ...cpu_offload import is_cpu_offload_enabled, mark_activation_offload from ...distributed import ( CudaRNGStatesTracker, @@ -35,7 +37,7 @@ clear_tensor_data, devices_match, ) -from ..op import BasicOperation, OperationContext +from ..op import BasicOperation from .._common import ( get_accumulate_flag_in_param, get_dummy_wgrads_for_params, @@ -45,6 +47,75 @@ ) +@dataclass(slots=True) +class BasicLinearFwdArgs: + """Tensor inputs and configuration for the linear forward custom op.""" + + input_: TensorOrQuantized + weight: TensorOrQuantized + dtype: torch.dtype + input_requires_grad: bool + weight_requires_grad: bool + with_quantized_compute: bool + backward_override: Optional[str] + input_quantizer: Optional[Quantizer] + weight_quantizer: Optional[Quantizer] + output_quantizer: Optional[Quantizer] + grad_output_quantizer: Optional[Quantizer] + grad_input_quantizer: Optional[Quantizer] + tensor_parallel_mode: Optional[str] + tensor_parallel_group: Optional[torch.distributed.ProcessGroup] + tensor_parallel_size: int + sequence_parallel: bool + bias: Optional[torch.Tensor] = None + + +@dataclass(slots=True) +class BasicLinearBwdArgs: + """Saved tensors and configuration for the linear backward custom op.""" + + grad_output: TensorOrQuantized + input_: Optional[TensorOrQuantized] + weight: Optional[TensorOrQuantized] + dtype: torch.dtype + input_requires_grad: bool + weight_requires_grad: bool + with_quantized_compute: bool + input_quantizer: Optional[Quantizer] + weight_quantizer: Optional[Quantizer] + grad_output_quantizer: Optional[Quantizer] + grad_input_quantizer: Optional[Quantizer] + tensor_parallel_mode: Optional[str] + tensor_parallel_group: Optional[torch.distributed.ProcessGroup] + tensor_parallel_size: int + sequence_parallel: bool + grad_weight: Optional[torch.Tensor] + accumulate_into_grad_weight: bool + # Original parameter for eager-only main_grad flags and dummy wgrad. + weight_param: Optional[torch.Tensor] = None + + +def _save_quantized_input(args: BasicLinearFwdArgs) -> bool: + return ( + args.with_quantized_compute + and args.backward_override != "high_precision" + and args.weight_requires_grad + # Column SP gathers/quantizes a separate tensor; the saved local input + # is still the caller's tensor and must not be returned as fresh FP8 aux. + and not (args.tensor_parallel_mode == "column" and args.sequence_parallel) + and not is_quantized_tensor(args.input_) + ) + + +def _save_quantized_weight(args: BasicLinearFwdArgs) -> bool: + return ( + args.with_quantized_compute + and args.backward_override != "high_precision" + and args.input_requires_grad + and not is_quantized_tensor(args.weight) + ) + + def _wait_async(handle: Optional[Any]) -> None: """Wait for asynchronous communication to finish, if needed""" if handle is not None: @@ -340,6 +411,23 @@ def reset_parameters(self) -> None: if not isinstance(weight, torch.nn.Parameter): weight = torch.nn.Parameter(weight) self.weight = weight + self._prealloc_cublas_workspace() + + def _apply(self, *args, **kwargs): + out = super()._apply(*args, **kwargs) + self._prealloc_cublas_workspace() + return out + + def _prealloc_cublas_workspace(self) -> None: + """Keep the cached GEMM workspace outside CUDA graph private pools.""" + # pylint: disable=import-outside-toplevel + from torch._guards import detect_fake_mode + from torch._subclasses.fake_tensor import is_fake + + weight = self.weight + if weight.device.type != "cuda" or is_fake(weight) or detect_fake_mode(): + return + get_cublas_workspace(weight.device.index, False, False) def pre_first_fuser_forward(self) -> None: super().pre_first_fuser_forward() @@ -990,127 +1078,228 @@ def _functional_backward( _wait_async(dx_async) return dx, dw - def op_forward( - self, - ctx: OperationContext, - input_: torch.Tensor, - prev_op_grad_output_quantizer: Optional[Quantizer], - next_op_input_quantizer: Optional[Quantizer], - ) -> torch.Tensor: + fwd_args_type = BasicLinearFwdArgs + bwd_args_type = BasicLinearBwdArgs - # Check which grads are required - input_requires_grad = ctx.requires_grad - weight_requires_grad = ctx.requires_grad and self.weight.requires_grad + def compile_unsupported_reason(self, mode: str) -> Optional[str]: + if self._accumulate_into_main_grad: + return "accumulate_into_main_grad" + if is_cpu_offload_enabled(): + return "CPU offloading" + return super().compile_unsupported_reason(mode) - # Quantizers - input_quantizer = self.get_quantizer("forward", 0) - weight_quantizer = self.get_quantizer("forward", 1) - output_quantizer = next_op_input_quantizer - grad_output_quantizer = self.get_quantizer("backward", 0) - grad_input_quantizer = prev_op_grad_output_quantizer + def pack_forward_args( + self, + basic_op_ctxs, + input_, + *, + prev_op_grad_output_quantizer, + next_op_input_quantizer, + basic_op_kwargs, + **unused, # pylint: disable=unused-argument + ) -> BasicLinearFwdArgs: + if basic_op_kwargs[0]: + raise ValueError("BasicLinear forward does not expect keyword arguments") with_quantized_compute = FP8GlobalStateManager.is_fp8_enabled() - if with_quantized_compute: - backward_override = FP8GlobalStateManager.get_fp8_recipe().backward_override - else: - backward_override = None - - # Get autocast dtype if needed - if torch.is_autocast_enabled(): - dtype = torch.get_autocast_dtype("cuda") - else: - dtype = self.weight.dtype - - # Linear forward - output, x_local, w = BasicLinear._functional_forward( - input=input_, + backward_override = ( + FP8GlobalStateManager.get_fp8_recipe().backward_override + if with_quantized_compute + else None + ) + return BasicLinearFwdArgs( + input_=input_, weight=self.weight, - dtype=dtype, + dtype=( + torch.get_autocast_dtype("cuda") + if torch.is_autocast_enabled() + else self.weight.dtype + ), + input_requires_grad=basic_op_ctxs[0].requires_grad, + weight_requires_grad=basic_op_ctxs[0].requires_grad and self.weight.requires_grad, + with_quantized_compute=with_quantized_compute, + backward_override=backward_override, + input_quantizer=self.get_quantizer("forward", 0), + weight_quantizer=self.get_quantizer("forward", 1), + output_quantizer=next_op_input_quantizer, + grad_output_quantizer=self.get_quantizer("backward", 0), + grad_input_quantizer=prev_op_grad_output_quantizer, tensor_parallel_mode=self.tensor_parallel_mode, tensor_parallel_group=self.tensor_parallel_group, + tensor_parallel_size=self.tensor_parallel_size, sequence_parallel=self.sequence_parallel, - with_quantized_compute=with_quantized_compute, - backward_override=backward_override, - input_quantizer=input_quantizer, - weight_quantizer=weight_quantizer, - output_quantizer=output_quantizer, - input_requires_grad=input_requires_grad, - weight_requires_grad=weight_requires_grad, ) - # Save state for backward pass - if ctx.requires_grad: - if backward_override == "high_precision": - saved_input = input_ if weight_requires_grad else None - saved_weight = self.weight if input_requires_grad else None - else: - saved_input = x_local - saved_weight = w - if is_cpu_offload_enabled(): - # No special CPU offloading logic is needed for weights. saved_weight is - # either self.weight (nn.Parameter, auto-excluded from offload) or a - # workspace freshly created each forward pass. - mark_activation_offload(saved_input) - ctx.save_for_backward(saved_input, saved_weight) - ctx.with_quantized_compute = with_quantized_compute and backward_override is None - ctx.backward_override = backward_override - ctx.input_quantizer = input_quantizer - ctx.weight_quantizer = weight_quantizer - ctx.grad_output_quantizer = grad_output_quantizer - ctx.grad_input_quantizer = grad_input_quantizer - ctx.dtype = dtype - ctx.input_requires_grad = input_requires_grad - ctx.weight_requires_grad = weight_requires_grad - - return output - - def op_backward( - self, - ctx: OperationContext, - grad_output: torch.Tensor, - ) -> tuple[torch.Tensor, Iterable[Optional[torch.Tensor]]]: - - # Saved tensors from forward pass - (x_local, w) = ctx.saved_tensors + @classmethod + def forward_compute( + cls, args: BasicLinearFwdArgs, *, in_custom_op: bool = False + ): # pylint: disable=unused-argument + """Shared GEMM and saved tensors, optionally with a fused bias epilogue.""" + output, saved_input, saved_weight = cls._functional_forward( + input=args.input_, + weight=args.weight, + bias=args.bias, + dtype=args.dtype, + input_requires_grad=args.input_requires_grad, + weight_requires_grad=args.weight_requires_grad, + with_quantized_compute=args.with_quantized_compute, + backward_override=args.backward_override, + input_quantizer=args.input_quantizer, + weight_quantizer=args.weight_quantizer, + output_quantizer=args.output_quantizer, + tensor_parallel_mode=args.tensor_parallel_mode, + tensor_parallel_group=args.tensor_parallel_group, + sequence_parallel=args.sequence_parallel, + ) + return ( + output, + [()], + ( + saved_input if _save_quantized_input(args) else None, + saved_weight if _save_quantized_weight(args) else None, + ), + ) - # Megatron-LM wgrad fusion - # Note: Get grad tensor from param so we can accumulate - # directly into it. - accumulate_into_main_grad = self._accumulate_into_main_grad + @classmethod + def forward_compute_fake(cls, args: BasicLinearFwdArgs): + shape = list(args.input_.shape) + shape[-1] = args.weight.shape[0] + if args.sequence_parallel: + if args.tensor_parallel_mode == "column": + shape[0] *= args.tensor_parallel_size + elif args.tensor_parallel_mode == "row": + shape[0] //= args.tensor_parallel_size + output = TensorSpec( + shape=tuple(shape), + dtype=args.dtype, + device=args.input_.device, + quantizer=( + args.output_quantizer + if args.with_quantized_compute and args.tensor_parallel_mode != "row" + else None + ), + ) + saved = [] + for needed, value, quantizer in ( + (_save_quantized_input(args), args.input_, args.input_quantizer), + (_save_quantized_weight(args), args.weight, args.weight_quantizer), + ): + spec = None + if needed: + quantizer = quantizer.copy() + quantizer.set_usage( + rowwise=args.backward_override is not None, + columnwise=args.backward_override is None, + ) + spec = TensorSpec( + shape=value.shape, dtype=value.dtype, device=value.device, quantizer=quantizer + ) + saved.append(spec) + return output, [()], tuple(saved) + + def forward_setup_context(self, basic_op_ctxs, args, aux) -> None: + ctx = basic_op_ctxs[0] + if not ctx.requires_grad: + return + saved_input, saved_weight = aux + if saved_input is None and args.weight_requires_grad: + saved_input = args.input_ + if saved_weight is None and args.input_requires_grad: + saved_weight = args.weight + if is_cpu_offload_enabled(): + mark_activation_offload(saved_input) + ctx.save_for_backward(saved_input, saved_weight) + ctx.with_quantized_compute = args.with_quantized_compute and args.backward_override is None + ctx.backward_override = args.backward_override + ctx.input_quantizer = args.input_quantizer + ctx.weight_quantizer = args.weight_quantizer + ctx.grad_output_quantizer = args.grad_output_quantizer + ctx.grad_input_quantizer = args.grad_input_quantizer + ctx.dtype = args.dtype + ctx.input_requires_grad = args.input_requires_grad + ctx.weight_requires_grad = args.weight_requires_grad + + def pack_backward_args( + self, basic_op_ctxs, grad_output, **unused # pylint: disable=unused-argument + ) -> BasicLinearBwdArgs: + ctx = basic_op_ctxs[0] grad_weight = None - if ctx.weight_requires_grad and accumulate_into_main_grad: - weight_param = self.weight - main_grad = get_main_grad_from_param(weight_param, op_label="BasicLinear") - accumulate_into_main_grad = get_accumulate_flag_in_param(weight_param) - grad_weight = main_grad.detach() - else: - accumulate_into_main_grad = False - - # Linear backward pass - grad_input, grad_weight = BasicLinear._functional_backward( + accumulate_into_grad_weight = False + if ctx.weight_requires_grad and self._accumulate_into_main_grad: + grad_weight = get_main_grad_from_param(self.weight, op_label="BasicLinear").detach() + accumulate_into_grad_weight = get_accumulate_flag_in_param(self.weight) + return BasicLinearBwdArgs( grad_output=grad_output, - input=x_local, - weight=w, + input_=ctx.saved_tensors[0], + weight=ctx.saved_tensors[1], + dtype=ctx.dtype, input_requires_grad=ctx.input_requires_grad, weight_requires_grad=ctx.weight_requires_grad, - dtype=ctx.dtype, - grad_weight=grad_weight, - accumulate_into_grad_weight=accumulate_into_main_grad, - tensor_parallel_mode=self.tensor_parallel_mode, - tensor_parallel_group=self.tensor_parallel_group, - sequence_parallel=self.sequence_parallel, with_quantized_compute=ctx.with_quantized_compute, input_quantizer=ctx.input_quantizer, weight_quantizer=ctx.weight_quantizer, grad_output_quantizer=ctx.grad_output_quantizer, grad_input_quantizer=ctx.grad_input_quantizer, + tensor_parallel_mode=self.tensor_parallel_mode, + tensor_parallel_group=self.tensor_parallel_group, + tensor_parallel_size=self.tensor_parallel_size, + sequence_parallel=self.sequence_parallel, + grad_weight=grad_weight, + accumulate_into_grad_weight=accumulate_into_grad_weight, + weight_param=self.weight if accumulate_into_grad_weight else None, ) - # Clear input tensor if possible - clear_tensor_data(x_local) - - # Megatron-LM wgrad fusion - # Note: Return dummy tensor for grad weight if needed. - if accumulate_into_main_grad: - grad_weight = get_dummy_wgrads_for_params([self.weight])[0] + @classmethod + def backward_compute(cls, args: BasicLinearBwdArgs, *, in_custom_op: bool = False): + dx, dw = cls._functional_backward( + grad_output=args.grad_output, + input=args.input_, + weight=args.weight, + dtype=args.dtype, + input_requires_grad=args.input_requires_grad, + weight_requires_grad=args.weight_requires_grad, + with_quantized_compute=args.with_quantized_compute, + input_quantizer=args.input_quantizer, + weight_quantizer=args.weight_quantizer, + grad_output_quantizer=args.grad_output_quantizer, + grad_input_quantizer=args.grad_input_quantizer, + tensor_parallel_mode=args.tensor_parallel_mode, + tensor_parallel_group=args.tensor_parallel_group, + sequence_parallel=args.sequence_parallel, + grad_weight=args.grad_weight, + accumulate_into_grad_weight=args.accumulate_into_grad_weight, + ) + if not in_custom_op: + clear_tensor_data(args.input_) + if args.accumulate_into_grad_weight: + dw = get_dummy_wgrads_for_params([args.weight_param])[0] + return dx, [(dw,)], [()] - return grad_input, [grad_weight] + @classmethod + def backward_compute_fake(cls, args: BasicLinearBwdArgs): + dx = None + if args.input_requires_grad: + shape = list(args.grad_output.shape) + shape[-1] = args.weight.shape[-1] + if args.sequence_parallel: + if args.tensor_parallel_mode == "column": + shape[0] //= args.tensor_parallel_size + elif args.tensor_parallel_mode == "row": + shape[0] *= args.tensor_parallel_size + dx = TensorSpec( + shape=tuple(shape), + dtype=args.dtype, + device=args.grad_output.device, + quantizer=( + args.grad_input_quantizer + if args.with_quantized_compute and args.tensor_parallel_mode != "column" + else None + ), + ) + dw = None + if args.weight_requires_grad: + dw = TensorSpec( + shape=(args.grad_output.shape[-1], args.input_.shape[-1]), + dtype=args.dtype, + device=args.grad_output.device, + ) + return dx, [(dw,)], [()] diff --git a/transformer_engine/pytorch/ops/basic/bias.py b/transformer_engine/pytorch/ops/basic/bias.py index 88f563b2c5b..70b34da1851 100644 --- a/transformer_engine/pytorch/ops/basic/bias.py +++ b/transformer_engine/pytorch/ops/basic/bias.py @@ -5,17 +5,36 @@ """Fusible operation for bias.""" from __future__ import annotations +from dataclasses import dataclass from typing import Optional import torch import transformer_engine_torch as tex +from ...dynamo import TensorOrQuantized, TensorSpec from ...quantization import FP8GlobalStateManager -from ..op import BasicOperation, OperationContext +from ..op import BasicOperation from ...utils import canonicalize_device, canonicalize_dtype from ...tensor import Quantizer +@dataclass(slots=True) +class BiasFwdArgs: + """Tensor inputs and backward quantizer for the bias forward custom op.""" + + input_: TensorOrQuantized + bias: torch.Tensor + grad_input_quantizer: Optional[Quantizer] + + +@dataclass(slots=True) +class BiasBwdArgs: + """Output gradient and optional quantizer for the bias backward custom op.""" + + grad_output: TensorOrQuantized + grad_input_quantizer: Optional[Quantizer] + + class Bias(BasicOperation): """Apply additive bias @@ -113,37 +132,75 @@ def pre_first_fuser_forward(self) -> None: if self.bias.device.type == "meta": self.reset_parameters() - def op_forward( - self, - ctx: OperationContext, - input_: torch.Tensor, - prev_op_grad_output_quantizer: Optional[Quantizer], - next_op_input_quantizer: Optional[Quantizer], - ) -> torch.Tensor: - x = input_ - b = self.bias.view([1] * (x.dim() - 1) + [self.local_size]) - - if ctx.requires_grad: - ctx.grad_input_quantizer = prev_op_grad_output_quantizer - if FP8GlobalStateManager.is_fp8_enabled(): - fp8_recipe = FP8GlobalStateManager.get_fp8_recipe() - if fp8_recipe.backward_override is not None: - ctx.grad_input_quantizer = None - - return x + b - - def op_backward( + fwd_args_type = BiasFwdArgs + bwd_args_type = BiasBwdArgs + + def pack_forward_args( self, - ctx: OperationContext, - grad_output: torch.Tensor, - ) -> tuple[torch.Tensor, tuple[()]]: - dy = grad_output - if dy.dim() > 1: - quantizer = ctx.grad_input_quantizer - if quantizer is None: - db = dy.sum(tuple(range(dy.dim() - 1))) - else: - db, dy = tex.bgrad_quantize(dy, quantizer) - else: - db = dy - return dy, (db,) + basic_op_ctxs, + input_, + *, + prev_op_grad_output_quantizer, + basic_op_kwargs, + **unused, # pylint: disable=unused-argument + ) -> BiasFwdArgs: + if basic_op_kwargs[0]: + raise ValueError("Bias forward does not expect keyword arguments") + quantizer = prev_op_grad_output_quantizer + if FP8GlobalStateManager.is_fp8_enabled(): + if FP8GlobalStateManager.get_fp8_recipe().backward_override is not None: + quantizer = None + return BiasFwdArgs(input_, self.bias, quantizer) + + @classmethod + def forward_compute( + cls, args: BiasFwdArgs, *, in_custom_op: bool = False + ): # pylint: disable=unused-argument + return args.input_ + args.bias, [()], () + + @classmethod + def forward_compute_fake(cls, args: BiasFwdArgs): + return ( + TensorSpec( + shape=args.input_.shape, + dtype=torch.promote_types(args.input_.dtype, args.bias.dtype), + device=args.input_.device, + ), + [()], + (), + ) + + def forward_setup_context(self, basic_op_ctxs, args, aux) -> None: + basic_op_ctxs[0].grad_input_quantizer = args.grad_input_quantizer + + def pack_backward_args( + self, basic_op_ctxs, grad_output, **unused # pylint: disable=unused-argument + ) -> BiasBwdArgs: + quantizer = basic_op_ctxs[0].grad_input_quantizer if len(grad_output.shape) > 1 else None + return BiasBwdArgs(grad_output, quantizer) + + @classmethod + def backward_compute( + cls, args: BiasBwdArgs, *, in_custom_op: bool = False + ): # pylint: disable=unused-argument + dy = args.grad_output + if dy.dim() == 1: + return None, [(dy.clone(),)], [()] + if args.grad_input_quantizer is None: + return None, [(dy.sum(tuple(range(dy.dim() - 1))),)], [()] + db, dx = tex.bgrad_quantize(dy, args.grad_input_quantizer) + return dx, [(db,)], [()] + + @classmethod + def backward_compute_fake(cls, args: BiasBwdArgs): + dy = args.grad_output + dx = None + if len(dy.shape) > 1 and args.grad_input_quantizer is not None: + dx = TensorSpec( + shape=dy.shape, + dtype=dy.dtype, + device=dy.device, + quantizer=args.grad_input_quantizer, + ) + db = TensorSpec(shape=(dy.shape[-1],), dtype=dy.dtype, device=dy.device) + return dx, [(db,)], [()] diff --git a/transformer_engine/pytorch/ops/fused/forward_linear_bias_activation.py b/transformer_engine/pytorch/ops/fused/forward_linear_bias_activation.py index 6fa63675b4c..e45164a04c9 100644 --- a/transformer_engine/pytorch/ops/fused/forward_linear_bias_activation.py +++ b/transformer_engine/pytorch/ops/fused/forward_linear_bias_activation.py @@ -5,16 +5,13 @@ """Fused operation for forward GEMM + bias + activation.""" from __future__ import annotations -from collections.abc import Sequence -from typing import Any, Optional +from typing import Optional import torch -from ...cpu_offload import is_cpu_offload_enabled, mark_activation_offload -from ...quantization import FP8GlobalStateManager -from ...tensor import Quantizer from ..basic import BasicLinear, Bias -from ..op import FusedOperation, FusibleOperation, OperationContext +from ..basic.basic_linear import BasicLinearFwdArgs +from ..op import FusedOperation, FusibleOperation class ForwardLinearBiasActivation(FusedOperation): @@ -50,105 +47,47 @@ def __init__( # Index of each basic operations self._op_idxs: dict[str, Optional[int]] = op_idxs - def fuser_forward( - self, - basic_op_ctxs: list[OperationContext], - input_: torch.Tensor, - *, - basic_op_extra_inputs: list[tuple[torch.Tensor, ...]], - prev_op_grad_output_quantizer: Optional[Quantizer], - next_op_input_quantizer: Optional[Quantizer], - basic_op_kwargs: list[dict[str, Any]], - ) -> tuple[torch.Tensor, Sequence[Sequence[torch.Tensor]]]: - - # Get basic operations - idx = self._op_idxs["linear"] - linear_op = self.basic_ops[idx] - linear_op_ctx = basic_op_ctxs[idx] - if self._op_idxs["bias"] is None: - bias_op = None - bias_op_ctx = None - bias = None - else: - idx = self._op_idxs["bias"] - bias_op = self.basic_ops[idx] - bias_op_ctx = basic_op_ctxs[idx] - bias = bias_op.bias - if basic_op_kwargs[idx]: - raise ValueError("Bias operation forward does not expect keyword arguments") - if self._op_idxs["activation"] is None: - activation_op = None # pylint: disable=unused-variable - else: - raise NotImplementedError("Activations are not yet supported") + fwd_args_type = BasicLinearFwdArgs - # Check which grads are required - input_requires_grad = linear_op_ctx.requires_grad - weight_requires_grad = linear_op_ctx.requires_grad and linear_op.weight.requires_grad - - # Quantizers - input_quantizer = linear_op.get_quantizer("forward", 0) - weight_quantizer = linear_op.get_quantizer("forward", 1) - output_quantizer = next_op_input_quantizer - grad_output_quantizer = linear_op.get_quantizer("backward", 0) - grad_input_quantizer = prev_op_grad_output_quantizer - with_quantized_compute = FP8GlobalStateManager.is_fp8_enabled() - if with_quantized_compute: - backward_override = FP8GlobalStateManager.get_fp8_recipe().backward_override - else: - backward_override = None - - # Get autocast dtype if needed - if torch.is_autocast_enabled(): - dtype = torch.get_autocast_dtype("cuda") - else: - dtype = linear_op.weight.dtype - - # Linear forward - output, x_local, w = BasicLinear._functional_forward( - input=input_, - weight=linear_op.weight, - bias=bias, - dtype=dtype, - tensor_parallel_mode=linear_op.tensor_parallel_mode, - tensor_parallel_group=linear_op.tensor_parallel_group, - sequence_parallel=linear_op.sequence_parallel, - with_quantized_compute=with_quantized_compute, - backward_override=backward_override, - input_quantizer=input_quantizer, - weight_quantizer=weight_quantizer, - output_quantizer=output_quantizer, - input_requires_grad=input_requires_grad, - weight_requires_grad=weight_requires_grad, - ) + def compile_unsupported_reason(self, mode: str) -> Optional[str]: + reason = super().compile_unsupported_reason(mode) + if reason is not None: + return reason + return self.basic_ops[0].compile_unsupported_reason(mode) - # Save state for backward pass - if linear_op_ctx.requires_grad: - if backward_override == "high_precision": - saved_input = input_ if weight_requires_grad else None - saved_weight = linear_op.weight if input_requires_grad else None - else: - saved_input = x_local - saved_weight = w - if is_cpu_offload_enabled(): - mark_activation_offload(saved_input) - linear_op_ctx.save_for_backward(saved_input, saved_weight) - linear_op_ctx.with_quantized_compute = ( - with_quantized_compute and backward_override is None + def pack_forward_args(self, basic_op_ctxs, input_, *, basic_op_kwargs, **kwargs): + if self._op_idxs["activation"] is not None: + raise NotImplementedError("Activations are not yet supported") + args = self.basic_ops[0].pack_forward_args( + basic_op_ctxs[:1], input_, basic_op_kwargs=basic_op_kwargs[:1], **kwargs + ) + if self._op_idxs["bias"] is not None: + idx = self._op_idxs["bias"] + if basic_op_kwargs[idx]: + raise ValueError("Bias forward does not expect keyword arguments") + args.bias = self.basic_ops[idx].bias + return args + + @classmethod + def forward_compute(cls, args: BasicLinearFwdArgs, *, in_custom_op: bool = False): + output, extras, aux = BasicLinear.forward_compute(args, in_custom_op=in_custom_op) + if args.bias is not None: + extras.append(()) + return output, extras, aux + + @classmethod + def forward_compute_fake(cls, args: BasicLinearFwdArgs): + output, extras, aux = BasicLinear.forward_compute_fake(args) + if args.bias is not None: + extras.append(()) + return output, extras, aux + + def forward_setup_context(self, basic_op_ctxs, args, aux) -> None: + self.basic_ops[0].forward_setup_context(basic_op_ctxs[:1], args, aux) + if self._op_idxs["bias"] is not None: + basic_op_ctxs[self._op_idxs["bias"]].grad_input_quantizer = ( + args.grad_output_quantizer if args.backward_override is None else None ) - linear_op_ctx.backward_override = backward_override - linear_op_ctx.input_quantizer = input_quantizer - linear_op_ctx.weight_quantizer = weight_quantizer - linear_op_ctx.grad_output_quantizer = grad_output_quantizer - linear_op_ctx.grad_input_quantizer = grad_input_quantizer - linear_op_ctx.dtype = dtype - linear_op_ctx.input_requires_grad = input_requires_grad - linear_op_ctx.weight_requires_grad = weight_requires_grad - if bias_op is not None and bias_op_ctx.requires_grad: - bias_op_ctx.grad_input_quantizer = linear_op.get_grad_output_quantizer() - if backward_override is not None: - bias_op_ctx.grad_input_quantizer = None - - return output, [() for _ in range(len(self.basic_ops))] @staticmethod def fuse_forward_ops( diff --git a/transformer_engine/pytorch/ops/fuser.py b/transformer_engine/pytorch/ops/fuser.py index 285f29ed54c..5c7371ce644 100644 --- a/transformer_engine/pytorch/ops/fuser.py +++ b/transformer_engine/pytorch/ops/fuser.py @@ -742,13 +742,6 @@ def _custom_ops_unsupported_reason( ) -> Optional[str]: """Why this pass may not run through its operations' custom ops.""" ops = self._forward_ops if mode == "forward" else self._backward_ops - if len(ops) != self._num_basic_ops or any( - op is not self._basic_ops[idx] or basic_op_idxs != [idx] - for idx, (op, basic_op_idxs) in enumerate(ops) - ): - return f"a {mode} fusion" - if self._num_basic_ops != 1: - return "a group of several operations" for op, kwargs in zip(self._basic_ops, basic_op_kwargs, strict=True): # Only tensors. The other fields of an args container are values read # off the module, constant across calls and baked into the graph; a @@ -761,6 +754,11 @@ def _custom_ops_unsupported_reason( for op in self._basic_ops: if op.num_extra_inputs or op.num_extra_outputs: return f"{type(op).__name__} with extra tensor inputs or outputs" + for op, basic_op_idxs in ops: + # A frozen prefix is not visited by backward, so its missing + # backward custom ops must not force the trainable suffix to eager. + if mode == "backward" and basic_op_idxs[-1] < self.first_op_requiring_backward: + continue reason = op.compile_unsupported_reason(mode) if reason is not None: return reason diff --git a/transformer_engine/pytorch/ops/op.py b/transformer_engine/pytorch/ops/op.py index 8d018e273a7..a4ff7503cae 100644 --- a/transformer_engine/pytorch/ops/op.py +++ b/transformer_engine/pytorch/ops/op.py @@ -8,6 +8,7 @@ import abc from collections.abc import Iterable, Sequence import dataclasses +import functools import pickle from typing import Any, Callable, Optional @@ -93,7 +94,7 @@ def _register_compile_ops(cls) -> None: register_custom_op( op_name=op_name, arg_type=arg_type, - impl=getattr(cls, f"{mode}_compute"), + impl=functools.partial(getattr(cls, f"{mode}_compute"), in_custom_op=True), fake_impl=getattr(cls, f"{mode}_compute_fake"), ) ) @@ -248,8 +249,12 @@ def fuser_backward( return grad_input, grad_params, grad_extra_inputs @classmethod - def forward_compute(cls, args: Any) -> tuple: - """Return (output, extra_outputs per basic op, fresh aux tensors).""" + def forward_compute(cls, args: Any, *, in_custom_op: bool = False) -> tuple: + """Return (output, extra_outputs per basic op, fresh aux tensors). + + The registered custom op passes in_custom_op=True; direct eager calls + use False. Inside a custom op, do not mutate tensors supplied in args. + """ raise NotImplementedError @classmethod @@ -258,10 +263,11 @@ def forward_compute_fake(cls, args: Any) -> tuple: raise NotImplementedError @classmethod - def backward_compute(cls, args: Any) -> tuple: + def backward_compute(cls, args: Any, *, in_custom_op: bool = False) -> tuple: """Return (grad_input, grad_params per basic op, grad_extra_inputs per basic op). A None grad_input passes grad_output through unchanged. + in_custom_op has the same meaning as in forward_compute. """ raise NotImplementedError From 7e7898fa82719ebc1069cfa29b9ccd3b6eec4a24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadzi=C5=84ski?= <62263673+pggPL@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:24:38 +0200 Subject: [PATCH 13/31] [PyTorch] torch.compile support for FusedAttention (#3472) * [PyTorch] Split FusedAttnFunc into single-argument forward/backward helpers Behavior-neutral restructuring of the cuDNN fused attention autograd function, mirroring what #2967 did for Linear: FusedAttnFwdArgs / FusedAttnBwdArgs dataclasses, module-level _fused_attn_forward_impl, _fused_attn_setup_ctx and _fused_attn_backward_impl, and a thin FusedAttnFunc wrapper taking the differentiable tensors plus one args object instead of 37 positional arguments. Signed-off-by: Pawel Gadzinski * [PyTorch] Match the fused attention aux pack split to the C++ conditions The bias / softmax_offset aux entries exist only when the tensor was passed, not merely when the bias / softmax type asks for one. Signed-off-by: Pawel Gadzinski * [PyTorch] torch.compile support for FusedAttention Register the cuDNN fused attention forward/backward as a torch custom op through dynamo/custom_op.py, reusing the single-argument helpers behind FusedAttnFunc, so DotProductAttention with the fused backend traces under torch.compile(fullgraph=True), including KV caching. F16/BF16 only; FP8, context parallelism, score_mod, FAv2 backward and CPU offloading keep falling back to eager via no_torch_dynamo(when=...). Signed-off-by: Pawel Gadzinski * [PyTorch] Preserve packed gradients in compiled fused attention Differentiate packed QKV and KV inputs directly and return their native packed gradient buffers without contiguous copies or repacking. Cover packed backward graphs, FP16/BF16 gradients and CUDA graph replay, and rename the custom-op-unavailable fallback test. Signed-off-by: Pawel Gadzinski * [PyTorch] Simplify compiled fused attention output handling Signed-off-by: Pawel Gadzinski * [PyTorch] Remove redundant fused attention gradient conversions DPA routes undeclared packed QKV views to eager before custom-op dispatch. Return the independently allocated gradients directly for the remaining unpacked path. Signed-off-by: Pawel Gadzinski * [PyTorch] Unify fused attention backward state setup Store packed gradient metadata in FusedAttnBwdArgs and initialize all backward state in the shared setup function. Reconstruct omitted QKV views in the shared forward and setup, removing the custom-op-specific dataclass and forwarding wrappers. Signed-off-by: Pawel Gadzinski * Keep fused attention packed inputs in saved backward state Signed-off-by: Pawel Gadzinski * Consolidate packed attention compile tests into shared configurations Signed-off-by: Pawel Gadzinski * Declare fused attention RNG effects and align fake statistics Signed-off-by: Pawel Gadzinski * Address fused attention compile review feedback Signed-off-by: Pawel Gadzinski --------- Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 85 ++++-- .../dot_product_attention/backends.py | 261 ++++++++++++++++-- .../pytorch/dynamo/custom_op.py | 19 +- 3 files changed, 312 insertions(+), 53 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 49aa5033cb4..472917d56f0 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -700,6 +700,9 @@ def _packed_layout(qkv_format: str, packed_dim: int, interleave_dim: int) -> str _DPA_COMPILE_CONFIGS = { "self_bshd_causal": _cfg(ModelConfig(2, 128, 4, 64, attn_mask_type="causal")), + "self_bshd_causal_dropout": _cfg( + ModelConfig(2, 128, 4, 64, attn_mask_type="causal", dropout_p=0.1) + ), "self_sbhd_no_mask": _cfg(ModelConfig(2, 128, 4, 64, attn_mask_type="no_mask"), "sbhd"), "self_bshd_swa": _cfg(ModelConfig(2, 128, 4, 64, attn_mask_type="causal", window_size=(16, 0))), "gqa_bshd_causal": _cfg(ModelConfig(2, 128, 8, 64, num_gqa_groups=2, attn_mask_type="causal")), @@ -720,6 +723,21 @@ def _packed_layout(qkv_format: str, packed_dim: int, interleave_dim: int) -> str ModelConfig(2, 128, 4, 64, attn_mask_type="causal"), packed="qkv", interleave_dim=-2 ), "packed_kv_bshd_bs2hd": _cfg(ModelConfig(2, 128, 4, 64, attn_mask_type="causal"), packed="kv"), + "packed_kv_bshd_bsh2d": _cfg( + ModelConfig(2, 128, 4, 64, attn_mask_type="causal"), packed="kv", interleave_dim=-2 + ), + "packed_qkv_bs3hd_singleton": _cfg( + ModelConfig(1, 1, 1, 64, attn_mask_type="causal"), packed="qkv" + ), + "packed_qkv_bsh3d_singleton": _cfg( + ModelConfig(1, 1, 1, 64, attn_mask_type="causal"), packed="qkv", interleave_dim=-2 + ), + "packed_kv_bshd_bs2hd_singleton": _cfg( + ModelConfig(1, 1, 1, 64, attn_mask_type="causal"), packed="kv" + ), + "packed_kv_bshd_bsh2d_singleton": _cfg( + ModelConfig(1, 1, 1, 64, attn_mask_type="causal"), packed="kv", interleave_dim=-2 + ), "packed_kv_thd_th2d": _cfg( ModelConfig(2, 128, 4, 64, attn_mask_type="padding_causal"), "thd", @@ -904,16 +922,8 @@ def _packed(seqlen, tokens, packed_dim): return args, kwargs, grad_tensors -def _skip_unsupported( - spec: dict, backend: str, dtype, compiled: bool = True, inference_params=None -) -> None: - """Skip what the backend under test cannot run, or -- for a test that - compiles it -- cannot be compiled.""" - if compiled and backend == "fused": - # FusedAttention's forward carries @no_torch_dynamo, so there is nothing - # to compile: it runs as an eager island. Drop this skip once it traces, - # and the tests below cover it as they do the others. - pytest.skip("FusedAttention is an eager island and does not compile") +def _skip_unsupported(spec: dict, backend: str, dtype, inference_params=None) -> None: + """Skip configurations the backend under test cannot run.""" available, _, _ = get_available_attention_backends( spec["model_config"], dtype, @@ -1021,12 +1031,17 @@ def _compare_compiled_to_eager( ) -> None: """Run the module eagerly and compiled on the same inputs, and compare both the output and every input gradient.""" + cpu_rng_state = torch.get_rng_state() + cuda_rng_state = torch.cuda.get_rng_state() eager = _run_and_capture(module, args, kwargs, grads) torch._dynamo.reset() # Force backend selection to be re-run (and traced) inside the compiled # region instead of being served from the cache the eager call populated. _force_dpa_backend(monkeypatch, backend) + # Reuse the eager dropout mask when comparing outputs and gradients. + torch.set_rng_state(cpu_rng_state) + torch.cuda.set_rng_state(cuda_rng_state) compiled = _run_and_capture(torch.compile(module, **compile_kwargs), args, kwargs, grads) _assert_dpa_backend(backend) @@ -1035,7 +1050,8 @@ def _compare_compiled_to_eager( @pytest.mark.parametrize("backend", ["flash", "fused", "unfused"]) @pytest.mark.parametrize("config", _DPA_COMPILE_CONFIGS.keys()) -def test_dpa_torch_compile(monkeypatch, backend, config): +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +def test_dpa_torch_compile(monkeypatch, backend, config, dtype): """`DotProductAttention` under `torch.compile(fullgraph=True)` must match eager in forward and backward, for every backend that supports the configuration. @@ -1044,45 +1060,59 @@ def test_dpa_torch_compile(monkeypatch, backend, config): whole module: input unpacking, qkv layout, backend selection and the backend itself. """ - dtype = torch.bfloat16 spec = _DPA_COMPILE_CONFIGS[config] module = _make_dpa(spec, dtype) args, kwargs, grads = _make_dpa_inputs(spec, dtype) _skip_unsupported(spec, backend, dtype, inference_params=kwargs.get("inference_params")) _force_dpa_backend(monkeypatch, backend) + # Inductor uses a different RNG for unfused dropout by default. Keep the + # eager RNG here so this numerical comparison uses the same dropout mask. + options = ( + {"fallback_random": True} if backend == "unfused" and spec["model_config"].dropout_p else {} + ) _compare_compiled_to_eager( - module, args, kwargs, grads, monkeypatch, backend, dtype, fullgraph=True + module, args, kwargs, grads, monkeypatch, backend, dtype, fullgraph=True, options=options ) -def test_dpa_torch_compile_around_fused(monkeypatch): - """FusedAttention itself is an eager island, but everything around it is - compiled: DotProductAttention traces up to the backend call, breaks the - graph there and resumes afterwards. What crosses that break has to survive - it -- the sub-backend enum did not, and reached cuDNN as the function that - produced it.""" +def test_dpa_torch_compile_fused_op_unavailable(monkeypatch): + """Without the fused attention custom op, FusedAttention is an eager island + and everything around it is compiled: DotProductAttention traces up to the + backend call, breaks the graph there and resumes afterwards. What crosses + that break has to survive it -- the sub-backend enum did not, and reached + cuDNN as the function that produced it.""" + from transformer_engine.pytorch.attention.dot_product_attention import backends + dtype = torch.bfloat16 spec = _DPA_COMPILE_CONFIGS["self_bshd_causal"] - _skip_unsupported(spec, "fused", dtype, compiled=False) + _skip_unsupported(spec, "fused", dtype) _force_dpa_backend(monkeypatch, "fused") + monkeypatch.setattr(backends, "_fused_attn_op", None) module = _make_dpa(spec, dtype) args, kwargs, grads = _make_dpa_inputs(spec, dtype) # No fullgraph: the graph break at the eager island is the point here. - _compare_compiled_to_eager(module, args, kwargs, grads, monkeypatch, "fused", dtype) + with pytest.warns(UserWarning, match="Falling back to eager execution"): + _compare_compiled_to_eager(module, args, kwargs, grads, monkeypatch, "fused", dtype) -@pytest.mark.parametrize("backend", ["flash", "unfused"]) -@pytest.mark.parametrize("config", ["self_bshd_causal", "kv_cache_bshd"]) +@pytest.mark.parametrize("backend", ["flash", "fused", "unfused"]) +@pytest.mark.parametrize( + "config", + ["self_bshd_causal", "kv_cache_bshd", "packed_qkv_bsh3d", "packed_kv_bshd_bsh2d"], +) def test_dpa_torch_compile_cudagraphs(monkeypatch, backend, config): """`mode="reduce-overhead"`: forward and backward of DotProductAttention are captured into CUDA graphs and replayed on subsequent iterations.""" dtype = torch.bfloat16 spec = _DPA_COMPILE_CONFIGS[config] - _force_dpa_backend(monkeypatch, backend) - module = _make_dpa(spec, dtype) + _, kwargs, _ = _make_dpa_inputs(spec, dtype) + # Before forcing the backend: probing the available backends re-runs the + # selection and would otherwise be cached over the forced one. + _skip_unsupported(spec, backend, dtype, inference_params=kwargs.get("inference_params")) + _force_dpa_backend(monkeypatch, backend) torch._dynamo.reset() counters.clear() @@ -1102,7 +1132,7 @@ def test_dpa_torch_compile_cudagraphs(monkeypatch, backend, config): assert not counters["inductor"]["cudagraph_skips"], "inductor skipped CUDA graphs" -@pytest.mark.parametrize("backend", ["flash", "unfused"]) +@pytest.mark.parametrize("backend", ["flash", "fused", "unfused"]) @pytest.mark.parametrize("paged", [False, True], ids=["non_paged", "paged"]) @pytest.mark.parametrize("cuda_graphs", [False, True], ids=["default", "cudagraphs"]) def test_dpa_torch_compile_kv_cache_decoding(monkeypatch, backend, paged, cuda_graphs): @@ -1241,7 +1271,7 @@ def _thd_without_max_seqlen_inputs(spec, dtype): } -@pytest.mark.parametrize("backend", ["flash", "unfused"]) +@pytest.mark.parametrize("backend", ["flash", "fused", "unfused"]) @pytest.mark.parametrize("case", _EAGER_FALLBACK_CASES.keys()) def test_dpa_torch_compile_eager_fallback(monkeypatch, backend, case): """Calls that cannot be traced run as an eager island instead, with a @@ -1252,6 +1282,7 @@ def test_dpa_torch_compile_eager_fallback(monkeypatch, backend, case): dtype = torch.bfloat16 config_name, make_inputs = _EAGER_FALLBACK_CASES[case] spec = _DPA_COMPILE_CONFIGS[config_name] + _skip_unsupported(spec, backend, dtype) _force_dpa_backend(monkeypatch, backend) module = _make_dpa(spec, dtype) diff --git a/transformer_engine/pytorch/attention/dot_product_attention/backends.py b/transformer_engine/pytorch/attention/dot_product_attention/backends.py index f90fc46c032..6c4ac477781 100644 --- a/transformer_engine/pytorch/attention/dot_product_attention/backends.py +++ b/transformer_engine/pytorch/attention/dot_product_attention/backends.py @@ -59,7 +59,11 @@ fa_prepare_fwd, ) from transformer_engine.pytorch.jit import no_torch_dynamo -from transformer_engine.pytorch.dynamo.custom_op import TensorOrQuantized +from transformer_engine.pytorch.dynamo.custom_op import ( + TensorOrQuantized, + register_custom_op_with_autograd, +) +from transformer_engine.pytorch.dynamo.tensor_spec import TensorSpec from transformer_engine.pytorch.attention.dot_product_attention.context_parallel import ( attn_forward_func_with_cp, ) @@ -1523,12 +1527,14 @@ def convert_to_torch_float8(tensor, dtype): class FusedAttnFwdArgs: """Single-argument bag for the forward path of :class:`FusedAttnFunc`.""" - # --- Differentiable tensors (also passed positionally to autograd) --- - q: TensorOrQuantized - k: TensorOrQuantized - v: TensorOrQuantized + # --- Differentiable tensors --- + q: Optional[TensorOrQuantized] + k: Optional[TensorOrQuantized] + v: Optional[TensorOrQuantized] attn_bias: Optional[torch.Tensor] softmax_offset: Optional[torch.Tensor] + packed_qkv: Optional[torch.Tensor] + packed_kv: Optional[torch.Tensor] # --- Non-differentiable tensors --- cu_seqlens_q: torch.Tensor @@ -1537,8 +1543,6 @@ class FusedAttnFwdArgs: cu_seqlens_kv_padded: Optional[torch.Tensor] page_table_k: Optional[torch.Tensor] page_table_v: Optional[torch.Tensor] - packed_qkv: Optional[torch.Tensor] - packed_kv: Optional[torch.Tensor] # --- Attention config --- is_training: bool @@ -1581,6 +1585,8 @@ class FusedAttnBwdArgs: q: Optional[torch.Tensor] = None k: Optional[torch.Tensor] = None v: Optional[torch.Tensor] = None + packed_qkv: Optional[torch.Tensor] = None + packed_kv: Optional[torch.Tensor] = None out: Optional[torch.Tensor] = None cu_seqlens_q: Optional[torch.Tensor] = None cu_seqlens_kv: Optional[torch.Tensor] = None @@ -1644,6 +1650,8 @@ def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: self.rng_state, self.aux_bias, self.aux_softmax_offset, + self.packed_qkv, + self.packed_kv, ) = restore_from_func_ctx( ctx ) # pylint: disable=unbalanced-tuple-unpacking @@ -1658,6 +1666,16 @@ def aux_ctx_tensors(self) -> List[torch.Tensor]: return aux +def _fused_attn_qkv(args): + if args.q is None and args.packed_qkv is not None: + dim = -2 if "h3d" in args.qkv_layout else -3 + return args.packed_qkv.unbind(dim) + if args.k is None and args.packed_kv is not None: + dim = -2 if "h2d" in args.qkv_layout else -3 + return args.q, *args.packed_kv.unbind(dim) + return args.q, args.k, args.v + + def _fused_attn_forward_impl( args: FusedAttnFwdArgs, ) -> Tuple[Any, Optional[torch.Tensor], Tuple[Any, ...], Dict[str, Any]]: @@ -1670,7 +1688,7 @@ def _fused_attn_forward_impl( input / the output is replaced by ``None`` and named in ``ctx_attrs["saved_from"]``, so backward takes it from there instead. """ - q, k, v = args.q, args.k, args.v + q, k, v = _fused_attn_qkv(args) is_training = args.is_training fp8 = args.fp8 fp8_meta = args.fp8_meta @@ -1841,17 +1859,17 @@ def _fused_attn_forward_impl( # save q, k, v, o tensors fp8_tensors = (None, None, None, None) - f16_tensors = (None, None, None, None) + f16_tensors = (None, None, None, None, None, None) if is_bwd_fp8: if (qkv_type == "current" and _dpa_fp8_cs_o_in_f16) or qkv_type == "mxfp8": fp8_tensors = (q_fp8, k_fp8, v_fp8, None) - f16_tensors = (None, None, None, out_f16) + f16_tensors = (None, None, None, out_f16, None, None) elif qkv_type == "delayed" or (qkv_type == "current" and not _dpa_fp8_cs_o_in_f16): fp8_tensors = (q_fp8, k_fp8, v_fp8, out_fp8) else: if is_input_fp8: q, k, v = combine_and_dequantize(qkv_layout, q_fp8, k_fp8, v_fp8) - f16_tensors = (q, k, v, out_f16) + f16_tensors = (q, k, v, out_f16, None, None) else: # q, k, v, out_: torch.Tensor; dtype = torch.float16 or torch.bfloat16 out_, aux_ctx_tensors, *max_logit = fused_attn_fwd( @@ -1892,7 +1910,14 @@ def _fused_attn_forward_impl( out_f16 = out_ out_ret = out_ fp8_tensors = (None, None, None, None) - f16_tensors = (q, k, v, out_f16) + f16_tensors = ( + args.q, + args.k, + args.v, + out_f16, + args.packed_qkv if args.q is None else None, + args.packed_kv if args.k is None else None, + ) nvtx_range_pop(f"{nvtx_label}") @@ -1919,6 +1944,8 @@ def _fused_attn_forward_impl( "k" if f16_tensors[1] is args.k else None, "v" if f16_tensors[2] is args.v else None, "out" if f16_tensors[3] is out_ret else None, + "packed_qkv" if f16_tensors[4] is not None else None, + "packed_kv" if f16_tensors[5] is not None else None, None, None, "attn_bias" if has_bias else None, @@ -2046,6 +2073,8 @@ def _fused_attn_setup_ctx( "q": fwd_args.q, "k": fwd_args.k, "v": fwd_args.v, + "packed_qkv": fwd_args.packed_qkv, + "packed_kv": fwd_args.packed_kv, "attn_bias": fwd_args.attn_bias, "softmax_offset": fwd_args.softmax_offset, "out": out, @@ -2059,6 +2088,8 @@ def _fused_attn_setup_ctx( k, v, out_f16, + packed_qkv, + packed_kv, softmax_stats, rng_state, aux_bias, @@ -2084,6 +2115,8 @@ def _fused_attn_setup_ctx( rng_state, aux_bias, aux_softmax_offset, + packed_qkv, + packed_kv, ) @@ -2111,7 +2144,8 @@ def _fused_attn_backward_impl( else: d_out_fp8 = args.dO_quantizer(d_out) q_fp8, k_fp8, v_fp8, out_fp8 = args.q_fp8, args.k_fp8, args.v_fp8, args.out_fp8 - q, k, v, out = args.q, args.k, args.v, args.out + q, k, v = _fused_attn_qkv(args) + out = args.out cu_seqlens_q, cu_seqlens_kv = args.cu_seqlens_q, args.cu_seqlens_kv cu_seqlens_q_padded, cu_seqlens_kv_padded = args.cu_seqlens_q_padded, args.cu_seqlens_kv_padded aux_ctx_tensors = args.aux_ctx_tensors() @@ -2369,6 +2403,176 @@ def backward(ctx, d_out, *_args): return (*_fused_attn_backward_impl(bwd_args), None) +def _fused_attn_q_shape(args) -> Tuple[int, ...]: + if args.q is not None: + return tuple(args.q.shape) + shape = list(args.packed_qkv.shape) + del shape[-2 if "h3d" in args.qkv_layout else -3] + return tuple(shape) + + +def _fused_attn_stats_shape( + args: FusedAttnFwdArgs, q_format: str, q_shape: Tuple[int, ...] +) -> Tuple[int, ...]: + """Shape of the softmax stats auxiliary tensor cuDNN returns.""" + if q_format == "thd": + num_heads = q_shape[1] + sm = get_device_compute_capability() + if sm >= (9, 0) and sm != (12, 0): + return (q_shape[0], num_heads, 1) + batch_size = args.cu_seqlens_q.shape[0] - 1 + return (batch_size, num_heads, args.max_seqlen_q, 1) + if q_format == "bshd": + batch_size, num_heads = q_shape[0], q_shape[2] + else: + batch_size, num_heads = q_shape[1], q_shape[2] + return (batch_size, num_heads, args.max_seqlen_q, 1) + + +def _fused_attn_forward_fake( + args: FusedAttnFwdArgs, +) -> Tuple[TensorSpec, Optional[TensorSpec], Tuple[Any, ...], Dict[str, Any]]: + """Data-free twin of :func:`_fused_attn_forward_impl` (non-FP8 only).""" + q = args.q if args.q is not None else args.packed_qkv + v = args.v + if v is None: + v = args.packed_qkv if args.packed_qkv is not None else args.packed_kv + q_shape = _fused_attn_q_shape(args) + _, o_format, _ = dpa_utils.get_qkv_format(args.qkv_layout) + out = TensorSpec(shape=(*q_shape[:-1], v.shape[-1]), dtype=q.dtype, device=q.device) + max_logit = None + if args.return_max_logit: + max_logit = TensorSpec(shape=(q_shape[-2],), dtype=q.dtype, device=q.device) + softmax_stats = TensorSpec( + shape=_fused_attn_stats_shape(args, o_format, q_shape), dtype=torch.float32, device=q.device + ) + rng_state = TensorSpec(shape=(2,), dtype=torch.int64, device=q.device) + has_bias = args.attn_bias_type not in ["no_bias", "alibi"] and args.attn_bias is not None + has_softmax_offset = args.softmax_type != "vanilla" and args.softmax_offset is not None + tensors_to_save = (*(None,) * 10, softmax_stats, rng_state, None, None) + saved_from = ( + *(None,) * 4, + "q", + "k", + "v", + "out", + "packed_qkv" if args.q is None and args.packed_qkv is not None else None, + "packed_kv" if args.k is None and args.packed_kv is not None else None, + None, + None, + "attn_bias" if has_bias else None, + "softmax_offset" if has_softmax_offset else None, + ) + ctx_attrs = { + "fp8": False, + "is_input_fp8": False, + "saved_from": saved_from, + "fused_attention_backend": args.fused_attention_backend, + "nominal_dtype": q.dtype, + "qkv_type": None, + "qkv_layout": args.qkv_layout, + "original_qkv_layout": args.qkv_layout, + "o_format": o_format, + "qkv_scale_inv_format": None, + "QKV_quantizer": None, + "O_quantizer": None, + "S_quantizer": None, + "dQKV_quantizer": None, + "dO_quantizer": None, + "dP_quantizer": None, + } + return out, max_logit, tensors_to_save, ctx_attrs + + +def _fused_attn_pack_grad(grad, dim, count): + """Recover the packed allocation produced by native attention backward.""" + shape, strides = list(grad.shape), list(grad.stride()) + packed_stride = shape[dim + 1] * strides[dim + 1] + shape.insert(dim + 1, count) + strides.insert(dim + 1, packed_stride) + return grad.as_strided(shape, strides) + + +def _fused_attn_backward_op_impl( + args: FusedAttnBwdArgs, +) -> Tuple[Optional[torch.Tensor], ...]: + dq, dk, dv, d_bias, d_softmax_offset = _fused_attn_backward_impl(args) + dqkv, dkv = None, None + if args.packed_qkv is not None: + dim = -2 if "h3d" in args.dqkv_layout else -3 + dqkv = _fused_attn_pack_grad(dq, dim, 3) + dq, dk, dv = None, None, None + elif args.packed_kv is not None: + dim = -2 if "h2d" in args.dqkv_layout else -3 + dkv = _fused_attn_pack_grad(dk, dim, 2) + dk, dv = None, None + return dq, dk, dv, d_bias, d_softmax_offset, dqkv, dkv + + +def _fused_attn_backward_fake( + args: FusedAttnBwdArgs, +) -> Tuple[Optional[TensorSpec], ...]: + """Data-free twin of :func:`_fused_attn_backward_op_impl`.""" + dq, dk, dv, dqkv, dkv = ( + TensorSpec(shape=tuple(t.shape), dtype=t.dtype, device=t.device) if t is not None else None + for t in (args.q, args.k, args.v, args.packed_qkv, args.packed_kv) + ) + q = args.q if args.q is not None else args.packed_qkv + d_bias = None + if args.aux_bias is not None: + d_bias = TensorSpec(shape=tuple(args.aux_bias.shape), dtype=q.dtype, device=q.device) + d_softmax_offset = None + if args.aux_softmax_offset is not None: + d_softmax_offset = TensorSpec( + shape=(1, _fused_attn_q_shape(args)[-2], 1, 1), dtype=torch.float32, device=q.device + ) + return dq, dk, dv, d_bias, d_softmax_offset, dqkv, dkv + + +# Custom op used under ``torch.compile``. +_fused_attn_op = register_custom_op_with_autograd( + op_name="fused_attn", + input_tensors_for_grad=[ + "q", + "k", + "v", + "attn_bias", + "softmax_offset", + "packed_qkv", + "packed_kv", + ], + fwd_arg_type=FusedAttnFwdArgs, + fwd_impl=_fused_attn_forward_impl, + fwd_fake_impl=_fused_attn_forward_fake, + setup_context=_fused_attn_setup_ctx, + bwd_arg_type=FusedAttnBwdArgs, + bwd_impl=_fused_attn_backward_op_impl, + bwd_fake_impl=_fused_attn_backward_fake, + fwd_tags=(torch.Tag.nondeterministic_seeded,), +) + + +def _needs_eager_fused_attention(call: Dict[str, Any]) -> Optional[str]: + """Why this FusedAttention call has to run outside the graph, or None. + + `call` maps `FusedAttention.forward`'s parameter names to the arguments + this call passed, including `self`. + """ + if _fused_attn_op is None: + return "fused attention without custom-op support" + if call.get("fp8", False): + return "FP8 attention" + if call.get("cp_group") is not None: + return "context parallelism" + if call.get("score_mod") is not None: + return "score_mod" + if call["self"].use_FAv2_bwd: + return "NVTE_FUSED_ATTN_USE_FAv2_BWD" + if is_cpu_offload_enabled(): + return "CPU activation offloading" + return None + + class FusedAttention(torch.nn.Module): """Dot product attention using `cuDNN attention `_: @@ -2441,7 +2645,7 @@ def remove_extra_states_check(self, incompatible_keys): # pylint: disable=unuse self.register_load_state_dict_post_hook(remove_extra_states_check) - @no_torch_dynamo() + @no_torch_dynamo(when=_needs_eager_fused_attention) def forward( self, query_layer: torch.Tensor, @@ -2697,21 +2901,30 @@ def forward( return_max_logit=self.return_max_logit, rng_gen=None, fp8=fp8, - fp8_meta=fp8_meta, - quantizers=quantizers, + fp8_meta=fp8_meta if fp8 else None, + quantizers=quantizers if fp8 else None, fp8_output=fp8_output, bf16_backward=bf16_backward, layer_number=self.layer_number, ) with self.attention_dropout_ctx(): - output = FusedAttnFunc.apply( - query_layer, - key_layer, - value_layer, - core_attention_bias, - softmax_offset, - fwd_args, - ) + if torch.compiler.is_compiling() and _fused_attn_op is not None: + if packed_qkv is not None: + fwd_args.q, fwd_args.k, fwd_args.v = None, None, None + elif packed_kv is not None: + fwd_args.k, fwd_args.v = None, None + output = _fused_attn_op(fwd_args) + if not self.return_max_logit: + output = output[0] + else: + output = FusedAttnFunc.apply( + query_layer, + key_layer, + value_layer, + core_attention_bias, + softmax_offset, + fwd_args, + ) if self.return_max_logit: # ...hd -> ...(hd) diff --git a/transformer_engine/pytorch/dynamo/custom_op.py b/transformer_engine/pytorch/dynamo/custom_op.py index a87374095a7..39d59dc2907 100644 --- a/transformer_engine/pytorch/dynamo/custom_op.py +++ b/transformer_engine/pytorch/dynamo/custom_op.py @@ -975,6 +975,7 @@ def _register_base_op( impl: Callable[[Any], Any], fake_impl: Callable[[Any], Any], pack_result: Callable[[Any], List[torch.Tensor]], + tags: Sequence[torch.Tag] = (), ) -> Any: """Define the op via ``torch.library.custom_op`` with the real ``impl`` + the ``fake_impl`` (spec), returning the ``CustomOpDef``. @@ -994,7 +995,7 @@ def _fake(*flat: Any) -> List[torch.Tensor]: return pack_result(fake_impl(spec_obj)) op = torch.library.custom_op( - f"{_TE_OP_NAMESPACE}::{op_name}", _impl, mutates_args=(), schema=schema_str + f"{_TE_OP_NAMESPACE}::{op_name}", _impl, mutates_args=(), schema=schema_str, tags=tags ) op.register_fake(_fake) _mark_effectful(op) @@ -1144,6 +1145,7 @@ def _register_wrapper_op( base_op: Any, slot_offsets: Sequence[int] = (), subclasses: Sequence[type] = (), + tags: Sequence[torch.Tag] = (), ) -> Any: """Define the wrapper op via ``torch.library.custom_op``: forward to the base op through :func:`_make_slot_forwarder`. Returns the ``CustomOpDef``. @@ -1154,7 +1156,11 @@ def _forward(*flat: Any) -> List[torch.Tensor]: return forward(flat) op_def = torch.library.custom_op( - f"{_TE_OP_NAMESPACE}::{wrapper_op_name}", _forward, mutates_args=(), schema=schema_str + f"{_TE_OP_NAMESPACE}::{wrapper_op_name}", + _forward, + mutates_args=(), + schema=schema_str, + tags=tags, ) op_def.register_fake(_forward) _mark_effectful(op_def) @@ -1198,6 +1204,7 @@ def _register_op( impl: Callable[[Any], Any], fake_impl: Callable[[Any], Any], pack_result: Callable[[Any], List[torch.Tensor]], + tags: Sequence[torch.Tag] = (), ) -> _RegisteredOp: """Define one two-tier custom op: the base kernel, the wrapper op that lets ``QuantizedTensor`` subclasses be inputs, and the passthrough registrations. @@ -1219,6 +1226,7 @@ def _register_op( impl=impl, fake_impl=fake_impl, pack_result=pack_result, + tags=tags, ) base_op = getattr(namespace, f"{name}_base") wrapper_def = _register_wrapper_op( @@ -1227,6 +1235,7 @@ def _register_op( base_op=base_op, slot_offsets=slot_offsets, subclasses=subclasses, + tags=tags, ) wrapper_op = getattr(namespace, name) @@ -1303,6 +1312,7 @@ def register_custom_op_with_autograd( bwd_impl: Callable[[Any], Any], fwd_fake_impl: Callable[[Any], Tuple[Any, ...]], bwd_fake_impl: Callable[[Any], Tuple[Any, ...]], + fwd_tags: Sequence[torch.Tag] = (), ) -> Optional[Callable[..., Any]]: """Register a TE module's forward + backward as torch custom ops. @@ -1310,6 +1320,8 @@ def register_custom_op_with_autograd( ``Function.apply`` under ``torch.compiler.is_compiling()`` that dispatches through the op and returns the user-facing outputs. + ``fwd_tags`` apply to both the base and wrapper forward operators. + ``fwd_arg_type`` and ``bwd_arg_type`` are ``@dataclass``es whose *field annotations* define the op schema (see the module docstring for the field <-> slot mapping). The caller builds a ``fwd_arg_type`` instance and @@ -1367,6 +1379,7 @@ def register_custom_op_with_autograd( bwd_impl=bwd_impl, fwd_fake_impl=fwd_fake_impl, bwd_fake_impl=bwd_fake_impl, + fwd_tags=fwd_tags, ) except (ImportError, AttributeError, RuntimeError, TypeError) as e: record_compile_disabled( @@ -1386,6 +1399,7 @@ def _register_custom_op_with_autograd_impl( bwd_impl: Callable[[Any], Any], fwd_fake_impl: Callable[[Any], Tuple[Any, ...]], bwd_fake_impl: Callable[[Any], Tuple[Any, ...]], + fwd_tags: Sequence[torch.Tag], ) -> Callable[..., Any]: """Body of :func:`register_custom_op_with_autograd`; see it for semantics.""" # Existence check at the API boundary: every ``input_tensors_for_grad`` name @@ -1403,6 +1417,7 @@ def _register_custom_op_with_autograd_impl( impl=fwd_impl, fake_impl=fwd_fake_impl, pack_result=_pack_fwd_result, + tags=fwd_tags, ) bwd_qualname = f"{_TE_OP_NAMESPACE}::{op_name}_backward_base" num_grad_inputs = len(input_tensors_for_grad) From 39c30c577f5dd4f9fba921ee011b5cd797ae667e Mon Sep 17 00:00:00 2001 From: Oleg Goncharov <64355998+Oleg-Goncharov@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:38:07 +0200 Subject: [PATCH 14/31] [Common] Fix TMA synchronization in quantization kernels (#3417) * Fix TMA shared-memory reuse synchronization Signed-off-by: Oleg Goncharov * Fix remaining TMA synchronization gaps Signed-off-by: Oleg Goncharov * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Oleg Goncharov Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Przemyslaw Tredak --- .../common/cast/core/grouped_tma.cuh | 3 +- .../common/cast/fp8/gated_fp8.cuh | 9 ++++- .../common/cast/fp8/quantize_fp8.cuh | 22 +++++++++--- .../group_quantize_fp8_blockwise.cuh | 6 ++-- .../common/cast/fused_group_requantize.cu | 2 +- .../common/cast/mxfp8/dequantize_mxfp8.cuh | 11 ++++-- .../common/cast/mxfp8/gated_mxfp8.cuh | 13 ++++--- .../cast/mxfp8/group_dequantize_mxfp8.cuh | 11 ++++-- .../cast/mxfp8/group_scaled_swiglu_mxfp8.cuh | 16 ++++++--- .../common/cast/mxfp8/quantize_mxfp8.cuh | 20 ++++++++--- .../cast/mxfp8/specialized/quantize_mxfp8.cuh | 22 ++++++++---- .../nvfp4/group_quantize_transpose_nvfp4.cuh | 18 +++++++--- .../cast/nvfp4/quantize_transpose_nvfp4.cuh | 36 ++++++++++++++----- .../quantize_transpose_nvfp4_tuned_1D.cuh | 18 ++++++++-- ...cast_col_hadamard_transform_cast_fusion.cu | 2 ++ .../group_hadamard_transform_cast_fusion.cu | 2 ++ ...cast_col_hadamard_transform_cast_fusion.cu | 2 ++ .../hadamard_transform_cast_fusion.cu | 2 ++ ...cast_col_hadamard_transform_cast_fusion.cu | 2 ++ .../quantize_transpose_square_blockwise.cu | 4 +-- transformer_engine/common/util/ptx.cuh | 22 ++++++------ 21 files changed, 184 insertions(+), 59 deletions(-) diff --git a/transformer_engine/common/cast/core/grouped_tma.cuh b/transformer_engine/common/cast/core/grouped_tma.cuh index a919b0a5eec..6b3fd43f001 100644 --- a/transformer_engine/common/cast/core/grouped_tma.cuh +++ b/transformer_engine/common/cast/core/grouped_tma.cuh @@ -173,7 +173,8 @@ __global__ void __launch_bounds__(THREADS_PER_WARP) __device__ __forceinline__ void fence_acquire_tensormap(const CUtensorMap *tensor_map) { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) // The descriptor updater and consumer execute in different CTAs, so CTA scope is insufficient. - asm volatile("fence.proxy.tensormap::generic.acquire.gpu [%0], 128;" ::"l"(tensor_map)); + asm volatile("fence.proxy.tensormap::generic.acquire.gpu [%0], 128;" ::"l"(tensor_map) + : "memory"); #else NVTE_DEVICE_ERROR("fence_acquire_tensormap is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) diff --git a/transformer_engine/common/cast/fp8/gated_fp8.cuh b/transformer_engine/common/cast/fp8/gated_fp8.cuh index 124ec48b91b..08583f6298d 100644 --- a/transformer_engine/common/cast/fp8/gated_fp8.cuh +++ b/transformer_engine/common/cast/fp8/gated_fp8.cuh @@ -248,8 +248,15 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) // Wait for TMA transfer to have finished reading shared memory. ptx::cp_async_bulk_wait_group_read(); } + // The bulk async-group is owned by the issuing thread. Hand its completion + // off to all cooperative writers before the output ring can be reused. + if (next_it < ITERATIONS && next_it >= BUFFERS_NUM) { + __syncthreads(); + } + } + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); } - ptx::cp_async_bulk_wait_group_read<0>(); __syncthreads(); if (amax_ptr != nullptr) { diff --git a/transformer_engine/common/cast/fp8/quantize_fp8.cuh b/transformer_engine/common/cast/fp8/quantize_fp8.cuh index 9b1ebf5c040..1f6e33e441a 100644 --- a/transformer_engine/common/cast/fp8/quantize_fp8.cuh +++ b/transformer_engine/common/cast/fp8/quantize_fp8.cuh @@ -198,10 +198,17 @@ __global__ void __launch_bounds__(FP8_THREADS_PER_CHUNK) ptx::cp_async_bulk_commit_group(); // Wait for TMA transfer to have finished reading shared memory. - ptx::cp_async_bulk_wait_group_read(); + ptx::cp_async_bulk_wait_group_read(); } + // The bulk async-group is owned by the issuing thread. Hand its completion + // off to all cooperative writers before the output ring can be reused. + if (iter + 1 < FP8_ITERATIONS && iter + 1 >= FP8_BUFFERS_NUM) { + __syncthreads(); + } + } + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); } - ptx::cp_async_bulk_wait_group_read<0>(); __syncthreads(); parity ^= 1; @@ -331,10 +338,17 @@ __global__ void __launch_bounds__(THREADS_PER_BLOCK) ptx::cp_async_bulk_commit_group(); // Wait for TMA transfer to have finished reading shared memory. - ptx::cp_async_bulk_wait_group_read<1>(); + ptx::cp_async_bulk_wait_group_read(); } + // The bulk async-group is owned by the issuing thread. Hand its completion + // off to all cooperative writers before the output ring can be reused. + if (next_iter < ITERATIONS && next_iter >= SHMEM_BUFFERS) { + __syncthreads(); + } + } + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); } - ptx::cp_async_bulk_wait_group_read<0>(); __syncthreads(); if (amax_ptr != nullptr) { diff --git a/transformer_engine/common/cast/fp8_blockwise/group_quantize_fp8_blockwise.cuh b/transformer_engine/common/cast/fp8_blockwise/group_quantize_fp8_blockwise.cuh index 31feaf833dc..bf73e935014 100644 --- a/transformer_engine/common/cast/fp8_blockwise/group_quantize_fp8_blockwise.cuh +++ b/transformer_engine/common/cast/fp8_blockwise/group_quantize_fp8_blockwise.cuh @@ -341,8 +341,9 @@ __global__ void __launch_bounds__(kThreadsPerBlock, 4) group_block_scaled_2d_tma static_cast(global_col_base), static_cast(global_row_base), &tma_mbar); } ptx::mbarrier_wait_parity(&tma_mbar, 0); - if (leading_thread) ptx::mbarrier_invalid(&tma_mbar); + // Every thread must finish waiting before the leader invalidates the barrier. __syncthreads(); + if (leading_thread) ptx::mbarrier_invalid(&tma_mbar); // ---- Optional dbias: per-tile column sum of the high-precision input (smem-resident) ---- if (dbias_workspace != nullptr) { @@ -628,8 +629,9 @@ __global__ void __launch_bounds__(kThreadsPerBlock) group_block_scaled_1d_tma_ke static_cast(global_col_base), static_cast(global_row_base), &tma_mbar); } ptx::mbarrier_wait_parity(&tma_mbar, 0); - if (leading_thread) ptx::mbarrier_invalid(&tma_mbar); + // Every thread must finish waiting before the leader invalidates the barrier. __syncthreads(); + if (leading_thread) ptx::mbarrier_invalid(&tma_mbar); // ---- Optional dbias: per-tile column sum of the high-precision input (smem-resident) ---- if (dbias_workspace != nullptr) { diff --git a/transformer_engine/common/cast/fused_group_requantize.cu b/transformer_engine/common/cast/fused_group_requantize.cu index a2156f369a9..7efe629d112 100644 --- a/transformer_engine/common/cast/fused_group_requantize.cu +++ b/transformer_engine/common/cast/fused_group_requantize.cu @@ -380,7 +380,7 @@ __global__ void __launch_bounds__(kThreads) reinterpret_cast(&output_tensor_map), col_base, row_base, reinterpret_cast(quantized)); ptx::cp_async_bulk_commit_group(); - ptx::cp_async_bulk_wait_group_read<0>(); + ptx::cp_async_bulk_wait_group(); } __syncthreads(); #else diff --git a/transformer_engine/common/cast/mxfp8/dequantize_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/dequantize_mxfp8.cuh index 1face261bd6..91af8b139d9 100644 --- a/transformer_engine/common/cast/mxfp8/dequantize_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/dequantize_mxfp8.cuh @@ -213,10 +213,17 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) ptx::cp_async_bulk_commit_group(); // Wait for TMA transfer to have finished reading shared memory. - ptx::cp_async_bulk_wait_group_read<1>(); + ptx::cp_async_bulk_wait_group_read(); } + // The bulk async-group is owned by the issuing thread. Hand its completion + // off to all cooperative writers before the output ring can be reused. + if (next_iter < ITERATIONS && next_iter >= BUFFERS_NUM) { + __syncthreads(); + } + } + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); } - ptx::cp_async_bulk_wait_group_read<0>(); __syncthreads(); parity ^= 1; diff --git a/transformer_engine/common/cast/mxfp8/gated_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/gated_mxfp8.cuh index 44dea1a7c76..31e445d4fb4 100644 --- a/transformer_engine/common/cast/mxfp8/gated_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/gated_mxfp8.cuh @@ -74,6 +74,8 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) constexpr size_t STAGES = CHUNK_DIM_Y / BUFF_DIM_Y; static_assert(STAGES >= 1); + static_assert(STAGES <= BUFFS_NUM, + "Output-buffer reuse requires a TMA completion handoff to the CTA"); constexpr bool IS_CACHED_ACT_OP = ROWWISE_SCALING && COLWISE_SCALING; constexpr bool ONLY_COLWISE_SCALING = COLWISE_SCALING && (!ROWWISE_SCALING); @@ -195,10 +197,6 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) const size_t stage_offset_Y = stage * BUFF_DIM_Y; if (next_stage < STAGES) { - // Wait for TMA transfer to have finished reading shared memory. - // I.e. the buffer is ready to be written to - ptx::cp_async_bulk_wait_group_read<1>(); - const size_t next_buff = next_stage % BUFFS_NUM; const size_t next_stage_offset_Y = next_stage * BUFF_DIM_Y; const size_t global_offset_Y = block_offset_Y + next_stage_offset_Y; @@ -704,6 +702,13 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) } parity ^= 1; + + // Ensure all S2G operations issued by the master have completed before the CTA exits. + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); + } + __syncthreads(); + destroy_barriers(mbar, is_master_thread); #endif // #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) } // NOLINT(readability/fn_size) diff --git a/transformer_engine/common/cast/mxfp8/group_dequantize_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/group_dequantize_mxfp8.cuh index 141efabbaf5..fcf8819e5f8 100644 --- a/transformer_engine/common/cast/mxfp8/group_dequantize_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/group_dequantize_mxfp8.cuh @@ -359,10 +359,17 @@ __global__ void __launch_bounds__(128) chunk_it_offset_y, reinterpret_cast(&out_sh[buff])); ptx::cp_async_bulk_commit_group(); - ptx::cp_async_bulk_wait_group_read<1>(); + ptx::cp_async_bulk_wait_group_read(); } + // The bulk async-group is owned by the issuing thread. Hand its completion + // off to all cooperative writers before the output ring can be reused. + if (next_iter < ITERATIONS && next_iter >= BUFFERS_NUM) { + __syncthreads(); + } + } + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); } - ptx::cp_async_bulk_wait_group_read<0>(); __syncthreads(); destroy_barriers(mbar, is_master_thread); diff --git a/transformer_engine/common/cast/mxfp8/group_scaled_swiglu_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/group_scaled_swiglu_mxfp8.cuh index 878fc93107c..0dab09f163e 100644 --- a/transformer_engine/common/cast/mxfp8/group_scaled_swiglu_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/group_scaled_swiglu_mxfp8.cuh @@ -344,11 +344,11 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) group_scaled_swiglu_mxfp8_k ptx::mbarrier_wait_parity_acquire_cta_shared_cta(&IN_buff_readable_mbar[buff_in], IN_buff_readable_parity[buff_in]); IN_buff_readable_parity[buff_in] ^= 1; - // Wait until the store groups still holding an output slice have drained. Only - // the leading thread commits those groups, so the wait is a no-op on the other - // threads and the barrier is what stops them from overwriting a slice the TMA - // unit has not finished reading. - ptx::cp_async_bulk_wait_group_read(); + // Only the issuing thread can wait for the TMA read of the output slice. + // Hand its completion off to all cooperative writers before reusing the slice. + if (leading_thread) { + ptx::cp_async_bulk_wait_group_read(); + } __syncthreads(); const size_t buff = buff_in; @@ -379,6 +379,12 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) group_scaled_swiglu_mxfp8_k total_work_blocks, work_blocks_X); } + // The last stage has no following iteration to wait for its TMA store. + if (leading_thread) { + ptx::cp_async_bulk_wait_group(); + } + __syncthreads(); + destroy_barriers(IN_buff_readable_mbar, leading_thread); } // if constexpr (is_single_tensor) #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) diff --git a/transformer_engine/common/cast/mxfp8/quantize_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/quantize_mxfp8.cuh index f7749f65dd7..e04a72e5d9d 100644 --- a/transformer_engine/common/cast/mxfp8/quantize_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/quantize_mxfp8.cuh @@ -206,11 +206,17 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) const size_t next_stage = stage + 1; const size_t stage_offset_Y = stage * BUFF_DIM_Y; - if (next_stage < STAGES) { - // Wait for TMA transfer to have finished reading shared memory. - // I.e. the buffer is ready to be written to - ptx::cp_async_bulk_wait_group_read<1>(); + // S2G bulk async-groups are thread-local, so only the issuing thread can wait for the + // previous TMA read of this output buffer to complete. Hand that completion off to the + // rest of the CTA before any thread overwrites the reused buffer. + if (stage >= BUFFS_NUM) { + if (is_master_thread) { + ptx::cp_async_bulk_wait_group_read(); + } + __syncthreads(); + } + if (next_stage < STAGES) { const size_t next_buff = next_stage % BUFFS_NUM; const size_t next_stage_offset_Y = next_stage * BUFF_DIM_Y; const size_t global_offset_Y = block_offset_Y + next_stage_offset_Y; @@ -633,6 +639,12 @@ __global__ void __launch_bounds__(THREADS_PER_CHUNK) atomicMaxFloat(amax_ptr, block_amax); } + // Ensure all S2G operations issued by the master have completed before the CTA exits. + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); + } + __syncthreads(); + destroy_barriers(mbar, is_master_thread); #endif // #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) } diff --git a/transformer_engine/common/cast/mxfp8/specialized/quantize_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/specialized/quantize_mxfp8.cuh index 2aad1aae165..baf9e5a6cec 100644 --- a/transformer_engine/common/cast/mxfp8/specialized/quantize_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/specialized/quantize_mxfp8.cuh @@ -1027,8 +1027,8 @@ __global__ void quantize_mxfp8_kernel_cast_only( ptx::cp_async_bulk_wait_group_read(); ptx::mbarrier_arrive_expect_tx(&stg_consumer[read_state.index()], 0u); } + ptx::cp_async_bulk_wait_group(); } - ptx::cp_async_bulk_wait_group_read<0>(); } else { PipeState read_state; @@ -1510,6 +1510,17 @@ __global__ void quantize_mxfp8_kernel_cast_only( } #pragma unroll 1 for (int32_t iter = 0; iter < CastTraits::iterLayout::num; iter++) { + // S2G bulk async-groups are thread-local. Once the output ring wraps, the issuing + // thread must observe completion and hand it off to the CTA before cooperative STS. + if constexpr (CastTraits::_need_wait_group) { + if (iter >= CastTraits::numStages) { + if (warpId == 0 && leader) { + ptx::cp_async_bulk_wait_group_read(); + } + __syncthreads(); + } + } + { int32_t next = iter + (CastTraits::numStages - 1); int32_t next_stage = next % CastTraits::numStages; @@ -1520,10 +1531,6 @@ __global__ void quantize_mxfp8_kernel_cast_only( coords.x = block_coords.x + iter_n * CastTraits::blockIterDim::N; if (next < CastTraits::iterLayout::num && coords.x < cols && coords.y < rows) { if (warpId == 0 && leader) { - if constexpr (CastTraits::_need_wait_group) { - ptx::cp_async_bulk_wait_group_read(); - } - ptx::cp_async_bulk_tensor_2d_global_to_shared( reinterpret_cast(sInput + next_stage * CastTraits::blockIterDim::num), reinterpret_cast(&tensor_map_input), @@ -1984,7 +1991,10 @@ __global__ void quantize_mxfp8_kernel_cast_only( } } - ptx::cp_async_bulk_wait_group_read<0>(); + if (warpId == 0 && leader) { + ptx::cp_async_bulk_wait_group(); + } + __syncthreads(); #endif // #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) } // NOLINT(readability/fn_size) diff --git a/transformer_engine/common/cast/nvfp4/group_quantize_transpose_nvfp4.cuh b/transformer_engine/common/cast/nvfp4/group_quantize_transpose_nvfp4.cuh index f506b764947..2a481074d27 100644 --- a/transformer_engine/common/cast/nvfp4/group_quantize_transpose_nvfp4.cuh +++ b/transformer_engine/common/cast/nvfp4/group_quantize_transpose_nvfp4.cuh @@ -655,6 +655,15 @@ __global__ void __launch_bounds__(THREADS_NUM) const size_t buff_offset_out = buff * BUFF_OUT_SIZE; const size_t buff_offset_out_t = buff * BUFF_OUT_T_SIZE; + if (stage >= BUFFS_NUM) { + if (is_master_thread) { + ptx::cp_async_bulk_wait_group_read(); + } + // Bulk async-groups are thread-local. Publish the master's completion before any thread + // overwrites this reused output buffer. + __syncthreads(); + } + // for stages from 1 to STAGES - 1, we need to update the tensor id // skip updating tensor id if it's the last CTA, and some stages will be out of bounds if (need_update_tensor_id && stage > 0 && (block_offset_Y + stage_offset_Y < rows)) { @@ -673,10 +682,6 @@ __global__ void __launch_bounds__(THREADS_NUM) } if (next_stage < STAGES) { - // Wait for TMA transfer to have finished reading shared memory. - // I.e. the buffer is ready to be written to - ptx::cp_async_bulk_wait_group_read<1>(); - const size_t next_buff = next_stage % BUFFS_NUM; const size_t next_stage_offset_Y = next_stage * BUFF_DIM_Y; const size_t global_offset_Y = block_offset_Y + next_stage_offset_Y; @@ -1050,6 +1055,11 @@ __global__ void __launch_bounds__(THREADS_NUM) // } // } + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); + } + __syncthreads(); + destroy_barriers(mbar, is_master_thread); #else NVTE_DEVICE_ERROR("sm_100 or higher is required."); diff --git a/transformer_engine/common/cast/nvfp4/quantize_transpose_nvfp4.cuh b/transformer_engine/common/cast/nvfp4/quantize_transpose_nvfp4.cuh index 33371de1349..6a76838695c 100644 --- a/transformer_engine/common/cast/nvfp4/quantize_transpose_nvfp4.cuh +++ b/transformer_engine/common/cast/nvfp4/quantize_transpose_nvfp4.cuh @@ -827,11 +827,16 @@ __global__ void __launch_bounds__(THREADS_NUM) const size_t buff_offset_out = buff * BUFF_OUT_SIZE; const size_t buff_offset_out_t = buff * BUFF_OUT_T_SIZE; - if (next_stage < STAGES) { - // Wait for TMA transfer to have finished reading shared memory. - // I.e. the buffer is ready to be written to - ptx::cp_async_bulk_wait_group_read<1>(); + if (stage >= BUFFS_NUM) { + if (is_master_thread) { + ptx::cp_async_bulk_wait_group_read(); + } + // Bulk async-groups are thread-local. Publish the master's completion before any thread + // overwrites this reused output buffer. + __syncthreads(); + } + if (next_stage < STAGES) { const size_t next_buff = next_stage % BUFFS_NUM; const size_t next_stage_offset_Y = next_stage * BUFF_DIM_Y; const size_t global_offset_Y = block_offset_Y + next_stage_offset_Y; @@ -1242,6 +1247,11 @@ __global__ void __launch_bounds__(THREADS_NUM) } } + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); + } + __syncthreads(); + destroy_barriers(mbar, is_master_thread); if constexpr (APPLY_COLUMNWISE_RHT) { destroy_barriers<1>(mbar_rht, is_master_thread); @@ -1413,11 +1423,16 @@ __global__ void __launch_bounds__(THREADS_NUM) const size_t buff_offset_out = buff * BUFF_OUT_SIZE; const size_t buff_offset_out_t = buff * BUFF_OUT_T_SIZE; - if (next_stage < STAGES) { - // Wait for TMA transfer to have finished reading shared memory. - // I.e. the buffer is ready to be written to - ptx::cp_async_bulk_wait_group_read<1>(); + if (stage >= BUFFS_NUM) { + if (is_master_thread) { + ptx::cp_async_bulk_wait_group_read(); + } + // Bulk async-groups are thread-local. Publish the master's completion before any thread + // overwrites this reused output buffer. + __syncthreads(); + } + if (next_stage < STAGES) { const size_t next_buff = next_stage % BUFFS_NUM; const size_t next_stage_offset_Y = next_stage * BUFF_DIM_Y; const size_t global_offset_Y = block_offset_Y + next_stage_offset_Y; @@ -1804,6 +1819,11 @@ __global__ void __launch_bounds__(THREADS_NUM) } } + if (is_master_thread) { + ptx::cp_async_bulk_wait_group(); + } + __syncthreads(); + destroy_barriers(mbar, is_master_thread); #endif // #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) } diff --git a/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh b/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh index 4aa7aa36ee5..3c24f9a4111 100644 --- a/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh +++ b/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh @@ -81,7 +81,11 @@ constexpr int STAGES = STAGES_Y * STAGES_X; constexpr int BUFFS_NUM = TunableConfig::PREFETCH_STAGES + 1; constexpr int BUFFS_NUM_IN = BUFFS_NUM; constexpr int BUFFS_NUM_OUT = BUFFS_NUM; -constexpr int BUFFS_NUM_OUT_TR = 2; +constexpr int BUFFS_NUM_OUT_TR = TunableConfig::PREFETCH_STAGES + 1; +static_assert(BUFFS_NUM_OUT >= TunableConfig::PREFETCH_STAGES + 1, + "The output buffer ring must be larger than the TMA store wait depth"); +static_assert(BUFFS_NUM_OUT_TR >= TunableConfig::PREFETCH_STAGES + 1, + "The transposed output buffer ring must be larger than the TMA store wait depth"); constexpr int BUFF_DIM_Y = TILE_DIM_Y; constexpr int BUFF_DIM_X = TILE_DIM_X; constexpr int BUFF_SIZE = BUFF_DIM_Y * BUFF_DIM_X; @@ -586,7 +590,12 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D // Wait for TMA transfer to have finished reading shared memory // I.e. the OUT buffer is ready to be written to - ptx::cp_async_bulk_wait_group_read(); + if (leading_thread) { + ptx::cp_async_bulk_wait_group_read(); + } + // Bulk async-groups are thread-local. Publish the leading thread's completion to all + // threads before they cooperatively overwrite a reused output buffer. + __syncthreads(); // NVFP4 Quantization rowwise_scaling( @@ -674,6 +683,11 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D } } + if (leading_thread) { + ptx::cp_async_bulk_wait_group(); + } + __syncthreads(); + if (leading_thread) { #pragma unroll for (int buff = 0; buff < BUFFS_NUM; ++buff) { diff --git a/transformer_engine/common/hadamard_transform/graph_safe_group_row_cast_col_hadamard_transform_cast_fusion.cu b/transformer_engine/common/hadamard_transform/graph_safe_group_row_cast_col_hadamard_transform_cast_fusion.cu index f2c0c6b00fe..df19e907fd0 100644 --- a/transformer_engine/common/hadamard_transform/graph_safe_group_row_cast_col_hadamard_transform_cast_fusion.cu +++ b/transformer_engine/common/hadamard_transform/graph_safe_group_row_cast_col_hadamard_transform_cast_fusion.cu @@ -500,6 +500,8 @@ __launch_bounds__(512, 1) __global__ static void group_row_col_rht_gemm_device_g if (warp_idx == 2 && elect_one_sync()) { cute::initialize_barrier(shared_storage.tma_barrier[0], /* num_threads */ 1); + // Publish this separately initialized barrier before TMA can complete against it. + cutlass::arch::fence_barrier_init(); } __syncthreads(); diff --git a/transformer_engine/common/hadamard_transform/group_hadamard_transform_cast_fusion.cu b/transformer_engine/common/hadamard_transform/group_hadamard_transform_cast_fusion.cu index 11d4620c594..e16d73862c7 100644 --- a/transformer_engine/common/hadamard_transform/group_hadamard_transform_cast_fusion.cu +++ b/transformer_engine/common/hadamard_transform/group_hadamard_transform_cast_fusion.cu @@ -351,6 +351,8 @@ __global__ static void group_rht_gemm_device( if (warp_idx == 2 && elect_one_sync()) { cute::initialize_barrier(shared_storage.tma_barrier[0], /* num_threads */ 1); + // Publish this separately initialized barrier before TMA can complete against it. + cutlass::arch::fence_barrier_init(); } __syncthreads(); using TMEM_LOAD_NEW = cute::SM100::TMEM::LOAD::SM100_TMEM_LOAD_32dp32b64x; diff --git a/transformer_engine/common/hadamard_transform/group_row_cast_col_hadamard_transform_cast_fusion.cu b/transformer_engine/common/hadamard_transform/group_row_cast_col_hadamard_transform_cast_fusion.cu index e68581f4895..6faaa10ed44 100644 --- a/transformer_engine/common/hadamard_transform/group_row_cast_col_hadamard_transform_cast_fusion.cu +++ b/transformer_engine/common/hadamard_transform/group_row_cast_col_hadamard_transform_cast_fusion.cu @@ -488,6 +488,8 @@ __launch_bounds__(512, 1) __global__ static void group_row_col_rht_gemm_device( if (warp_idx == 2 && elect_one_sync()) { cute::initialize_barrier(shared_storage.tma_barrier[0], /* num_threads */ 1); + // Publish this separately initialized barrier before TMA can complete against it. + cutlass::arch::fence_barrier_init(); } __syncthreads(); diff --git a/transformer_engine/common/hadamard_transform/hadamard_transform_cast_fusion.cu b/transformer_engine/common/hadamard_transform/hadamard_transform_cast_fusion.cu index 856046f1f70..df73afeb671 100644 --- a/transformer_engine/common/hadamard_transform/hadamard_transform_cast_fusion.cu +++ b/transformer_engine/common/hadamard_transform/hadamard_transform_cast_fusion.cu @@ -316,6 +316,8 @@ rht_gemm_device(MShape M, NShape N, KShape K, ClusterTileShape cluster_tile, if (warp_idx == 2 && elect_one_sync()) { cute::initialize_barrier(shared_storage.tma_barrier[0], /* num_threads */ 1); + // Publish this separately initialized barrier before TMA can complete against it. + cutlass::arch::fence_barrier_init(); } __syncthreads(); using TMEM_LOAD_NEW = cute::SM100::TMEM::LOAD::SM100_TMEM_LOAD_32dp32b64x; diff --git a/transformer_engine/common/hadamard_transform/row_cast_col_hadamard_transform_cast_fusion.cu b/transformer_engine/common/hadamard_transform/row_cast_col_hadamard_transform_cast_fusion.cu index 4a81a5e966d..94d395526be 100644 --- a/transformer_engine/common/hadamard_transform/row_cast_col_hadamard_transform_cast_fusion.cu +++ b/transformer_engine/common/hadamard_transform/row_cast_col_hadamard_transform_cast_fusion.cu @@ -494,6 +494,8 @@ __global__ static void row_col_rht_gemm_device( if (warp_idx == 2 && elect_one_sync()) { cute::initialize_barrier(shared_storage.tma_barrier[0], /* num_threads */ 1); + // Publish this separately initialized barrier before TMA can complete against it. + cutlass::arch::fence_barrier_init(); } __syncthreads(); diff --git a/transformer_engine/common/transpose/quantize_transpose_square_blockwise.cu b/transformer_engine/common/transpose/quantize_transpose_square_blockwise.cu index 02d64bcfffa..2d1a62e0fae 100644 --- a/transformer_engine/common/transpose/quantize_transpose_square_blockwise.cu +++ b/transformer_engine/common/transpose/quantize_transpose_square_blockwise.cu @@ -230,8 +230,8 @@ __global__ void __launch_bounds__(THREADS_PER_BLOCK) // Wait for TMA transfer to have finished reading shared memory. // Create a "bulk async-group" out of the previous bulk copy operation. ptx::cp_async_bulk_commit_group(); - // Wait for the group to have completed reading from shared memory. - ptx::cp_async_bulk_wait_group_read<0>(); + // Wait for the group to have completed the shared-to-global transfer. + ptx::cp_async_bulk_wait_group(); } #else // Step 4 Alternative (when TMA is not available, skip writing to shared memory) diff --git a/transformer_engine/common/util/ptx.cuh b/transformer_engine/common/util/ptx.cuh index 49686437c63..74fd2261f58 100644 --- a/transformer_engine/common/util/ptx.cuh +++ b/transformer_engine/common/util/ptx.cuh @@ -198,7 +198,7 @@ __device__ __forceinline__ void mbarrier_arrive_expect_tx_cta_relaxed_shared_cta __device__ __forceinline__ void fence_mbarrier_init_release_cluster() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) - asm volatile("fence.mbarrier_init.release.cluster;"); + asm volatile("fence.mbarrier_init.release.cluster;" ::: "memory"); #else NVTE_DEVICE_ERROR("fence_mbarrier_init_release_cluster is only supported on SM 10.0+."); #endif // #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) @@ -391,7 +391,7 @@ __device__ __forceinline__ void cp_async_bulk_tensor_2d_shared_to_global( // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-wait-group __device__ __forceinline__ void cp_async_bulk_wait_group() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("cp.async.bulk.wait_group 0;"); + asm volatile("cp.async.bulk.wait_group 0;" ::: "memory"); #else NVTE_DEVICE_ERROR("cp_async_bulk_wait_group is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -401,7 +401,7 @@ __device__ __forceinline__ void cp_async_bulk_wait_group() { template __device__ __forceinline__ void cp_async_bulk_wait_group_read() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("cp.async.bulk.wait_group.read 0;"); + asm volatile("cp.async.bulk.wait_group.read 0;" ::: "memory"); #else NVTE_DEVICE_ERROR("cp_async_bulk_wait_group_read is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -410,7 +410,7 @@ __device__ __forceinline__ void cp_async_bulk_wait_group_read() { template <> __device__ __forceinline__ void cp_async_bulk_wait_group_read<0>() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("cp.async.bulk.wait_group.read 0;"); + asm volatile("cp.async.bulk.wait_group.read 0;" ::: "memory"); #else NVTE_DEVICE_ERROR("cp_async_bulk_wait_group_read is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -418,7 +418,7 @@ __device__ __forceinline__ void cp_async_bulk_wait_group_read<0>() { template <> __device__ __forceinline__ void cp_async_bulk_wait_group_read<1>() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("cp.async.bulk.wait_group.read 1;"); + asm volatile("cp.async.bulk.wait_group.read 1;" ::: "memory"); #else NVTE_DEVICE_ERROR("cp_async_bulk_wait_group_read is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -426,7 +426,7 @@ __device__ __forceinline__ void cp_async_bulk_wait_group_read<1>() { template <> __device__ __forceinline__ void cp_async_bulk_wait_group_read<2>() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("cp.async.bulk.wait_group.read 2;"); + asm volatile("cp.async.bulk.wait_group.read 2;" ::: "memory"); #else NVTE_DEVICE_ERROR("cp_async_bulk_wait_group_read is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -434,7 +434,7 @@ __device__ __forceinline__ void cp_async_bulk_wait_group_read<2>() { template <> __device__ __forceinline__ void cp_async_bulk_wait_group_read<4>() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("cp.async.bulk.wait_group.read 4;"); + asm volatile("cp.async.bulk.wait_group.read 4;" ::: "memory"); #else NVTE_DEVICE_ERROR("cp_async_bulk_wait_group_read is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -443,7 +443,7 @@ __device__ __forceinline__ void cp_async_bulk_wait_group_read<4>() { // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-commit-group __device__ __forceinline__ void cp_async_bulk_commit_group() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("cp.async.bulk.commit_group;"); + asm volatile("cp.async.bulk.commit_group;" ::: "memory"); #else NVTE_DEVICE_ERROR("cp_async_bulk_commit_group is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -452,7 +452,7 @@ __device__ __forceinline__ void cp_async_bulk_commit_group() { // Proxy fence (bi-directional): __device__ __forceinline__ void fence_proxy_async() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("fence.proxy.async;"); + asm volatile("fence.proxy.async;" ::: "memory"); #else NVTE_DEVICE_ERROR("fence_proxy_async is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -460,7 +460,7 @@ __device__ __forceinline__ void fence_proxy_async() { __device__ __forceinline__ void fence_proxy_async_shared_cta() { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - asm volatile("fence.proxy.async.shared::cta;"); + asm volatile("fence.proxy.async.shared::cta;" ::: "memory"); #else NVTE_DEVICE_ERROR("fence_proxy_async_shared_cta is only supported on SM 9.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -752,7 +752,7 @@ __device__ __forceinline__ int32_t elect_one_sync(uint32_t mask = 0xFFFFFFFFu) { __device__ __forceinline__ void numbered_barrier_sync(uint32_t num_threads, uint32_t barrier_id = 1u) { - asm volatile("bar.sync %0, %1;\n" ::"r"(barrier_id), "r"(num_threads)); + asm volatile("bar.sync %0, %1;\n" ::"r"(barrier_id), "r"(num_threads) : "memory"); } __device__ __forceinline__ void fma_f32_f16(float &out, uint16_t const &a, uint16_t const &b, From d9beab1f5961acfd46179db1acdfdf0e01ff72f9 Mon Sep 17 00:00:00 2001 From: Siddhartha Raman Sundara Raman Date: Fri, 9 Oct 2026 11:29:12 -0500 Subject: [PATCH 15/31] [Common] Fuse RMSNorm forward with MXFP8 quantization (#3657) * [Common] Fuse RMSNorm forward with MXFP8 quantization Signed-off-by: Siddhartha Raman Sundara Raman Signed-off-by: Tim Moon <4406448+timmoon10@users.noreply.github.com> Co-authored-by: Tim Moon <4406448+timmoon10@users.noreply.github.com> Co-authored-by: vthumbe1503 --- .../cpp/operator/test_normalization_mxfp8.cu | 150 +++++- tests/pytorch/mxfp8/test_mxfp8_rmsnorm_fwd.py | 132 +++++ transformer_engine/common/CMakeLists.txt | 1 + .../common/normalization/common.h | 7 + .../normalization/rmsnorm/rmsnorm_api.cpp | 25 +- .../rmsnorm/rmsnorm_fwd_mxfp8.cu | 456 ++++++++++++++++++ .../pytorch/csrc/extensions/normalization.cpp | 19 +- 7 files changed, 773 insertions(+), 17 deletions(-) create mode 100644 tests/pytorch/mxfp8/test_mxfp8_rmsnorm_fwd.py create mode 100644 transformer_engine/common/normalization/rmsnorm/rmsnorm_fwd_mxfp8.cu diff --git a/tests/cpp/operator/test_normalization_mxfp8.cu b/tests/cpp/operator/test_normalization_mxfp8.cu index 10b33f8e2a1..84c244ff1ff 100644 --- a/tests/cpp/operator/test_normalization_mxfp8.cu +++ b/tests/cpp/operator/test_normalization_mxfp8.cu @@ -102,7 +102,8 @@ void dequantize_2x(Tensor& input, Tensor& output, bool is_training) } template -void performTest(const size_t N, const size_t H, const bool zero_centered_gamma, NormType norm_type, bool is_training, const bool zero_centered_gamma_in_weight_dtype) { +void performTest(const size_t N, const size_t H, const bool zero_centered_gamma, NormType norm_type, bool is_training, const bool zero_centered_gamma_in_weight_dtype, + const bool use_cudnn) { cudaDeviceProp prop; cudaGetDeviceProperties(&prop, 0); @@ -134,6 +135,9 @@ void performTest(const size_t N, const size_t H, const bool zero_centered_gamma, } else { nvte_enable_zero_centered_gamma_in_weight_dtype(false); } + // RMSNorm shapes that are multiples of 128 use Transformer Engine's fused MXFP8 kernel + // unless cuDNN is requested. + nvte_enable_cudnn_norm_fwd(use_cudnn); // Forward kernel float epsilon = 1e-5; @@ -163,6 +167,7 @@ void performTest(const size_t N, const size_t H, const bool zero_centered_gamma, if (zero_centered_gamma_in_weight_dtype) { nvte_enable_zero_centered_gamma_in_weight_dtype(false); } + nvte_enable_cudnn_norm_fwd(false); Tensor dequantized_output("dequantized_output", std::vector{ N, H }, DType::kFloat32, true, true); @@ -197,7 +202,7 @@ void performTest(const size_t N, const size_t H, const bool zero_centered_gamma, nullptr, // amax 1.f, // scale zero_centered_gamma, - true, // CuDNN is the only MXFP8 backend currently + true, // The MXFP8 backends add 1 to a zero-centered gamma as cuDNN does zero_centered_gamma_in_weight_dtype); cudaDeviceSynchronize(); @@ -235,6 +240,95 @@ std::vector> test_cases = { {2048, 12288}, }; +// Index of compact scaling factor (i, j) in the GEMM-swizzled layout: 128x4 tiles of i x j, +// num_tiles_j tiles along j. +size_t swizzled_scale_idx(size_t i, size_t j, size_t num_tiles_j) { + return ((i / 128) * num_tiles_j + j / 4) * 512 + (i % 32) * 16 + ((i % 128) / 32) * 4 + j % 4; +} + +// RMSNorm with MXFP8 output written with GEMM-swizzled scaling factors must give the same +// data as with compact scaling factors, and the compact scaling factors in swizzled order. +// The swizzled output is computed on a single SM, as with an SM margin, which launches the +// kernels over chunks of rows. +template +void performSwizzledScalesTest(const size_t N, const size_t H, const bool is_training) { + if (getDeviceComputeCapability() < blackwellComputeCapability) { + GTEST_SKIP(); + } + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, 0); + + const DType itype = TypeInfo::dtype; + const DType otype = TypeInfo::dtype; + Tensor input("input", std::vector{ N, H }, itype); + Tensor gamma("gamma", std::vector{ H }, itype); + Tensor rsigma("rsigma", std::vector{ N }, DType::kFloat32); + Tensor rsigma_swizzled("rsigma_swizzled", std::vector{ N }, DType::kFloat32); + Tensor z("z", std::vector{ N, H }, otype, true, is_training, NVTE_MXFP8_1D_SCALING); + Tensor z_swizzled("z_swizzled", std::vector{ N, H }, otype, true, is_training, + NVTE_MXFP8_1D_SCALING); + z_swizzled.set_with_gemm_swizzled_scales(true); + fillUniform(&input); + fillUniform(&gamma); + + const auto run = [&](Tensor &out, Tensor &rs, const int sm_count) { + Tensor workspace; + nvte_rmsnorm_fwd(input.data(), gamma.data(), 1e-5f, out.data(), rs.data(), + workspace.data(), sm_count, false, 0); + workspace = Tensor("workspace", workspace.rowwise_shape(), workspace.dtype()); + nvte_rmsnorm_fwd(input.data(), gamma.data(), 1e-5f, out.data(), rs.data(), + workspace.data(), sm_count, false, 0); + }; + run(z, rsigma, prop.multiProcessorCount); + run(z_swizzled, rsigma_swizzled, 1); + cudaDeviceSynchronize(); + auto err = cudaGetLastError(); + ASSERT_EQ(err, cudaSuccess) << cudaGetErrorString(err); + z.to_cpu(); + z_swizzled.to_cpu(); + rsigma.to_cpu(); + compareResults("rsigma", rsigma_swizzled, rsigma.rowwise_cpu_dptr(), true, 0, 0); + + const auto bytes = [](const OutputType *p) { return reinterpret_cast(p); }; + compareResults("rowwise_data", bytes(z_swizzled.rowwise_cpu_dptr()), + bytes(z.rowwise_cpu_dptr()), N * H); + const NVTEShape row_shape = z.rowwise_scale_inv_shape(); + const size_t row_dim0 = row_shape.data[0], row_dim1 = row_shape.data[1]; + std::vector ref(row_dim0 * row_dim1); + const uint8_t *compact = z.rowwise_cpu_scale_inv_ptr(); + for (size_t i = 0; i < row_dim0; ++i) { + for (size_t j = 0; j < row_dim1; ++j) { + ref[swizzled_scale_idx(i, j, row_dim1 / 4)] = compact[i * row_dim1 + j]; + } + } + compareResults("rowwise_scales", z_swizzled.rowwise_cpu_scale_inv_ptr(), ref.data(), + ref.size()); + + if (is_training) { + compareResults("colwise_data", bytes(z_swizzled.columnwise_cpu_dptr()), + bytes(z.columnwise_cpu_dptr()), N * H); + // Compact column-wise scaling factors are [rows / 32, cols]; the swizzled layout tiles + // columns by 128 and scale rows by 4. + const NVTEShape col_shape = z.columnwise_scale_inv_shape(); + const size_t col_dim0 = col_shape.data[0], col_dim1 = col_shape.data[1]; + ref.assign(col_dim0 * col_dim1, 0); + compact = z.columnwise_cpu_scale_inv_ptr(); + for (size_t i = 0; i < col_dim0; ++i) { + for (size_t j = 0; j < col_dim1; ++j) { + ref[swizzled_scale_idx(j, i, col_dim0 / 4)] = compact[i * col_dim1 + j]; + } + } + compareResults("colwise_scales", z_swizzled.columnwise_cpu_scale_inv_ptr(), + ref.data(), ref.size()); + } +} + +std::vector> swizzled_scales_test_cases = { + {128, 128}, + {768, 2304}, + {256, 7168}, +}; + std::vector norms = { NormType::LayerNorm, NormType::RMSNorm @@ -246,7 +340,7 @@ class MxNormTestSuite : public ::testing::TestWithParam< std::tuple, - bool, bool, bool>> {}; + bool, bool, bool, bool>> {}; TEST_P(MxNormTestSuite, TestMxNorm) { using namespace transformer_engine; @@ -259,10 +353,15 @@ TEST_P(MxNormTestSuite, TestMxNorm) { const bool zero_centered_gamma = std::get<4>(GetParam()); const bool is_training = std::get<5>(GetParam()); const bool zero_centered_gamma_in_weight_dtype = std::get<6>(GetParam()); + const bool use_cudnn = std::get<7>(GetParam()); + if (norm_type == NormType::LayerNorm && use_cudnn) { + GTEST_SKIP() << "LayerNorm with MXFP8 output always uses cuDNN"; + } TRANSFORMER_ENGINE_TYPE_SWITCH_FP16_FP32_ONLY(input_type, InputType, TRANSFORMER_ENGINE_TYPE_SWITCH_FP8_ONLY(output_type, OutputType, - performTest(size.first, size.second, zero_centered_gamma, norm_type, is_training, zero_centered_gamma_in_weight_dtype); + performTest(size.first, size.second, zero_centered_gamma, norm_type, is_training, zero_centered_gamma_in_weight_dtype, + use_cudnn); ); ); } @@ -277,7 +376,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(test_cases), ::testing::Values(true, false), ::testing::Values(true, false), - ::testing::Values(true, false)), + ::testing::Values(true, false), + ::testing::Values(false, true)), [](const testing::TestParamInfo& info) { std::string name = normToString.at(std::get<0>(info.param)) + "_" + test::typeName(std::get<1>(info.param)) + "X" + @@ -286,6 +386,44 @@ INSTANTIATE_TEST_SUITE_P( std::to_string(std::get<3>(info.param).second) + "X" + std::to_string(std::get<4>(info.param)) + "out" + std::to_string(int(std::get<5>(info.param)) + 1) + "x" + - std::to_string(std::get<6>(info.param)); + std::to_string(std::get<6>(info.param)) + + (std::get<7>(info.param) ? "Xcudnn" : ""); return name; }); + +class MxNormSwizzledScalesTestSuite + : public ::testing::TestWithParam, bool>> {}; + +TEST_P(MxNormSwizzledScalesTestSuite, TestMxRMSNormSwizzledScales) { + using namespace transformer_engine; + using namespace test; + + const DType input_type = std::get<0>(GetParam()); + const DType output_type = std::get<1>(GetParam()); + const auto size = std::get<2>(GetParam()); + const bool is_training = std::get<3>(GetParam()); + + TRANSFORMER_ENGINE_TYPE_SWITCH_FP16_FP32_ONLY(input_type, InputType, + TRANSFORMER_ENGINE_TYPE_SWITCH_FP8_ONLY(output_type, OutputType, + performSwizzledScalesTest(size.first, size.second, is_training); + ); + ); +} + +INSTANTIATE_TEST_SUITE_P( + OperatorTest, + MxNormSwizzledScalesTestSuite, + ::testing::Combine( + ::testing::Values(DType::kFloat32, DType::kBFloat16, DType::kFloat16), + ::testing::Values(DType::kFloat8E5M2, DType::kFloat8E4M3), + ::testing::ValuesIn(swizzled_scales_test_cases), + ::testing::Values(true, false)), + [](const testing::TestParamInfo& info) { + return test::typeName(std::get<0>(info.param)) + "X" + + test::typeName(std::get<1>(info.param)) + "X" + + std::to_string(std::get<2>(info.param).first) + "X" + + std::to_string(std::get<2>(info.param).second) + "X" + + std::to_string(int(std::get<3>(info.param)) + 1) + "x"; + }); diff --git a/tests/pytorch/mxfp8/test_mxfp8_rmsnorm_fwd.py b/tests/pytorch/mxfp8/test_mxfp8_rmsnorm_fwd.py new file mode 100644 index 00000000000..5200ec1074b --- /dev/null +++ b/tests/pytorch/mxfp8/test_mxfp8_rmsnorm_fwd.py @@ -0,0 +1,132 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""RMSNorm forward with MXFP8 output from Transformer Engine's fused kernel.""" + +import os + +import pytest +import torch + +import transformer_engine.pytorch as te +import transformer_engine_torch as tex +from transformer_engine.pytorch import MXFP8Quantizer +from transformer_engine.pytorch.constants import TE_DType + +from mxfp8_utils import swizzle_mxfp8_scale + +recipe_available, reason_for_no_recipe = te.is_mxfp8_available(return_reason=True) + + +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +@pytest.mark.skipif( + os.getenv("NVTE_NORM_FWD_USE_CUDNN", "0") == "1", + reason="Tests Transformer Engine's fused kernel, not cuDNN", +) +@pytest.mark.parametrize("shape", [(128, 128), (256, 1024), (384, 7168)]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("fp8_dtype", [tex.DType.kFloat8E4M3, tex.DType.kFloat8E5M2]) +@pytest.mark.parametrize("zero_centered_gamma", [False, True]) +@pytest.mark.parametrize("optimize_for_gemm", [False, True]) +def test_rmsnorm_fwd_mxfp8(shape, dtype, fp8_dtype, zero_centered_gamma, optimize_for_gemm): + """The fused output is the MXFP8 quantization of the FP32 normalized values.""" + torch.manual_seed(1234) + rows, cols = shape + eps = 1e-5 + x = torch.randn(rows, cols, dtype=dtype, device="cuda") + weight = torch.randn(cols, dtype=dtype, device="cuda") + + quantizer = MXFP8Quantizer(fp8_dtype) + quantizer.optimize_for_gemm = optimize_for_gemm + out, _, rsigma = tex.rmsnorm_fwd( + x, weight, eps, None, quantizer, TE_DType[dtype], 0, zero_centered_gamma + ) + assert out._with_gemm_swizzled_scales == optimize_for_gemm + + torch.testing.assert_close( + rsigma, torch.rsqrt(x.float().square().mean(dim=1) + eps), atol=0, rtol=1e-5 + ) + + # Reference: the MXFP8 quantizer on the FP32 normalized values, with the kernel's rsigma. + gamma = weight.float() + 1 if zero_centered_gamma else weight.float() + y = (x.float() * rsigma.unsqueeze(1)) * gamma + ref_quantizer = MXFP8Quantizer(fp8_dtype) + ref_quantizer.optimize_for_gemm = False + ref = ref_quantizer(y) + + for attr in ("_rowwise_data", "_columnwise_data"): + torch.testing.assert_close( + getattr(out, attr).view(torch.uint8), + getattr(ref, attr).view(torch.uint8), + atol=0, + rtol=0, + msg=attr, + ) + for attr, columnwise in (("_rowwise_scale_inv", False), ("_columnwise_scale_inv", True)): + expected = getattr(ref, attr) + if optimize_for_gemm: + expected = swizzle_mxfp8_scale(rows, cols, expected, columnwise=columnwise) + torch.testing.assert_close(getattr(out, attr), expected, atol=0, rtol=0, msg=attr) + + +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +@pytest.mark.skipif( + os.getenv("NVTE_NORM_FWD_USE_CUDNN", "0") == "1", + reason="Tests Transformer Engine's fused kernel, not cuDNN", +) +@pytest.mark.parametrize("optimize_for_gemm", [False, True]) +def test_rmsnorm_fwd_mxfp8_sm_margin(optimize_for_gemm): + """With all SMs but one reserved, the kernels run over chunks of rows with the same output.""" + torch.manual_seed(1234) + x = torch.randn(1024, 7168, dtype=torch.bfloat16, device="cuda") + weight = torch.randn(7168, dtype=torch.bfloat16, device="cuda") + sm_count = torch.cuda.get_device_properties(x.device).multi_processor_count + + outputs = [] + for sm_margin in (0, sm_count - 1): + quantizer = MXFP8Quantizer(tex.DType.kFloat8E4M3) + quantizer.optimize_for_gemm = optimize_for_gemm + outputs.append( + tex.rmsnorm_fwd(x, weight, 1e-5, None, quantizer, TE_DType[x.dtype], sm_margin, False) + ) + (out, _, rsigma), (out_margin, _, rsigma_margin) = outputs + torch.testing.assert_close(rsigma_margin, rsigma, atol=0, rtol=0) + for attr in ( + "_rowwise_data", + "_columnwise_data", + "_rowwise_scale_inv", + "_columnwise_scale_inv", + ): + torch.testing.assert_close( + getattr(out_margin, attr), getattr(out, attr), atol=0, rtol=0, msg=attr + ) + + +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +def test_rmsnorm_fwd_mxfp8_2d_quantization(dtype): + """The fused kernels quantize 1D blocks, so with 2D quantization the forward normalizes and + then quantizes 32x32 blocks.""" + torch.manual_seed(1234) + x = torch.randn(256, 1024, dtype=dtype, device="cuda") + weight = torch.randn(1024, dtype=dtype, device="cuda") + + quantizer = MXFP8Quantizer(tex.DType.kFloat8E4M3, with_2d_quantization=True) + out, _, _ = tex.rmsnorm_fwd(x, weight, 1e-5, None, quantizer, TE_DType[dtype], 0, False) + + y, _, _ = tex.rmsnorm_fwd(x, weight, 1e-5, None, None, TE_DType[dtype], 0, False) + ref = MXFP8Quantizer(tex.DType.kFloat8E4M3, with_2d_quantization=True)(y) + for attr in ( + "_rowwise_data", + "_columnwise_data", + "_rowwise_scale_inv", + "_columnwise_scale_inv", + ): + torch.testing.assert_close( + getattr(out, attr).view(torch.uint8), + getattr(ref, attr).view(torch.uint8), + atol=0, + rtol=0, + msg=attr, + ) diff --git a/transformer_engine/common/CMakeLists.txt b/transformer_engine/common/CMakeLists.txt index 843ec412c3d..ded6b6cb801 100644 --- a/transformer_engine/common/CMakeLists.txt +++ b/transformer_engine/common/CMakeLists.txt @@ -373,6 +373,7 @@ list(APPEND transformer_engine_cuda_arch_specific_sources hadamard_transform/group_row_cast_col_hadamard_transform_cast_fusion.cu hadamard_transform/graph_safe_group_row_cast_col_hadamard_transform_cast_fusion.cu multi_tensor/compute_scale.cu + normalization/rmsnorm/rmsnorm_fwd_mxfp8.cu recipe/mxfp8_scaling.cu recipe/nvfp4.cu transpose/quantize_transpose_square_blockwise.cu diff --git a/transformer_engine/common/normalization/common.h b/transformer_engine/common/normalization/common.h index dd2f3d14591..26fbaf04351 100644 --- a/transformer_engine/common/normalization/common.h +++ b/transformer_engine/common/normalization/common.h @@ -310,6 +310,13 @@ bool use_cudnn_norm_bwd(); bool& use_zero_centered_gamma_in_weight_dtype(); bool use_cudnn_mxfp8_norm_output_in_input_dtype(); +// RMSNorm forward with MXFP8 output by Transformer Engine's fused kernel +// (rmsnorm/rmsnorm_fwd_mxfp8.cu), which also writes GEMM-swizzled scaling factors. +bool is_supported_by_te_rmsnorm_fwd_mxfp8(const Tensor& x, const Tensor& gamma, const Tensor& z); +void rmsnorm_fwd_mxfp8(const Tensor& x, const Tensor& gamma, float epsilon, Tensor* z, + Tensor* rsigma, int multiprocessorCount, bool zero_centered_gamma, + cudaStream_t stream); + } // namespace normalization } // namespace transformer_engine diff --git a/transformer_engine/common/normalization/rmsnorm/rmsnorm_api.cpp b/transformer_engine/common/normalization/rmsnorm/rmsnorm_api.cpp index f56e2ef1fef..c18e6204f80 100644 --- a/transformer_engine/common/normalization/rmsnorm/rmsnorm_api.cpp +++ b/transformer_engine/common/normalization/rmsnorm/rmsnorm_api.cpp @@ -27,11 +27,6 @@ void rmsnorm_fwd(const Tensor &x, const Tensor &gamma, const float epsilon, Tens !is_mxfp8_scaling(z->scaling_mode)) { NVTE_ERROR("Not implemented scaling mode: " + to_string(z->scaling_mode) + "."); } - if (is_mxfp8_scaling(z->scaling_mode)) { - NVTE_CHECK(!z->with_gemm_swizzled_scales, - "MXFP8 output must have scales in compact format, not swizzled for GEMM."); - } - NVTE_CHECK(!x.data.shape.empty(), "x must have at least one dimension."); const auto [rows, cols] = x.flat_2d_dims(); @@ -53,6 +48,26 @@ void rmsnorm_fwd(const Tensor &x, const Tensor &gamma, const float epsilon, Tens CheckOutputTensor(*rsigma, "rsigma"); } + // MXFP8 output: Transformer Engine's fused kernel, which alone writes GEMM-swizzled scales, + // unless cuDNN is requested or the kernel does not support the tensors. + if (is_mxfp8_scaling(z->scaling_mode)) { + const bool te_supported = is_supported_by_te_rmsnorm_fwd_mxfp8(x, gamma, *z); + if (z->with_gemm_swizzled_scales || (!use_cudnn_norm_fwd() && te_supported)) { + NVTE_CHECK(te_supported, + "RMSNorm forward with GEMM-swizzled MXFP8 scales requires FP32, BF16 or FP16 " + "input and weight, and rows and columns that are multiples of 128."); + if (workspace->data.numel() == 0) { + // The fused kernel needs no workspace, but a zero size denotes a workspace query. + workspace->data.shape = {1}; + workspace->data.dtype = DType::kByte; + return; + } + rmsnorm_fwd_mxfp8(x, gamma, epsilon, z, rsigma, multiprocessorCount, zero_centered_gamma, + stream); + return; + } + } + NVTE_Norm_Backend norm_backend; bool is_aligned = true; bool cudnn_backend = use_cudnn_norm_fwd() || is_mxfp8_scaling(z->scaling_mode); diff --git a/transformer_engine/common/normalization/rmsnorm/rmsnorm_fwd_mxfp8.cu b/transformer_engine/common/normalization/rmsnorm/rmsnorm_fwd_mxfp8.cu new file mode 100644 index 00000000000..1ade354ebc2 --- /dev/null +++ b/transformer_engine/common/normalization/rmsnorm/rmsnorm_fwd_mxfp8.cu @@ -0,0 +1,456 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +/*! \file rmsnorm_fwd_mxfp8.cu + * \brief RMSNorm forward fused with MXFP8 quantization. + * + * Normalizes in FP32 and quantizes the FP32 result, row-wise and/or column-wise, with + * the scaling factors written in compact or GEMM-swizzled layout. The output is that of + * the MXFP8 quantizer applied to the FP32 normalized values, which is also what cuDNN's + * fused kernel produces; the two can differ only through the rounding of rsigma. + * + * Two kernels: a pre-pass computes rsigma per row (one warp per row), then one CTA per + * 32x128 tile normalizes the tile, quantizes it and writes its scaling factors. A tile + * only sees 128 columns of a row, so it cannot reduce the row itself; doing so with whole + * rows per CTA would limit the grid to rows / 32 CTAs. When the caller reserves SMs for other + * work, both kernels are launched over chunks of rows that fit on the remaining SMs. + */ + +#include + +#include +#include +#include + +#include "../../cast/mxfp8/swizzle.cuh" +#include "../../common.h" +#include "../../util/cuda_runtime.h" +#include "../../util/ptx_arch_spec.cuh" +#include "../../utils.cuh" +#include "../common.h" + +namespace transformer_engine { +namespace normalization { +namespace { + +constexpr int kScaleBlock = 32; // MXFP8 scaling block size. +constexpr int kTileRows = 32; // One column-wise scaling block. +constexpr int kTileCols = 128; // Four row-wise scaling blocks. +constexpr int kThreads = 128; +constexpr int kWarps = kThreads / THREADS_PER_WARP; +constexpr int kRowsPerWarp = kTileRows / kWarps; +constexpr int kColsPerLane = kTileCols / THREADS_PER_WARP; +constexpr int kLanesPerBlock = kScaleBlock / kColsPerLane; +constexpr int kRsigmaRowsPerCTA = 4; + +static_assert(kRowsPerWarp == 8); +static_assert(kColsPerLane == 4); +static_assert(kLanesPerBlock == 8); + +template +__device__ __forceinline__ void load_elements(T (&dst)[N], const T *src) { + if constexpr (sizeof(T) * N == 16) { + *reinterpret_cast(dst) = *reinterpret_cast(src); + } else if constexpr (sizeof(T) * N == 8) { + *reinterpret_cast(dst) = *reinterpret_cast(src); + } else { +#pragma unroll + for (int i = 0; i < N; ++i) { + dst[i] = src[i]; + } + } +} + +// rsigma = 1 / sqrt(mean(x^2) + epsilon), one warp per row. +template +__global__ void __launch_bounds__(kRsigmaRowsPerCTA *THREADS_PER_WARP) + rmsnorm_mxfp8_rsigma_kernel(const IType *const x, float *const rsigma, const int rows, + const int cols, const float epsilon) { + const int row = blockIdx.x * kRsigmaRowsPerCTA + threadIdx.x / THREADS_PER_WARP; + const int lane = threadIdx.x % THREADS_PER_WARP; + if (row >= rows) { + return; + } + const IType *const x_row = x + static_cast(row) * cols; + float sum = 0.0f; + if constexpr (kAligned) { + constexpr int kVec = 16 / sizeof(IType); + for (int c = lane * kVec; c < cols; c += THREADS_PER_WARP * kVec) { + IType values[kVec]; + load_elements(values, x_row + c); +#pragma unroll + for (int i = 0; i < kVec; ++i) { + const float value = static_cast(values[i]); + sum += value * value; + } + } + } else { + for (int c = lane; c < cols; c += THREADS_PER_WARP) { + const float value = static_cast(x_row[c]); + sum += value * value; + } + } +#pragma unroll + for (int offset = THREADS_PER_WARP / 2; offset > 0; offset /= 2) { + sum += __shfl_xor_sync(0xffffffff, sum, offset); + } + if (lane == 0) { + rsigma[row] = 1.0f / sqrtf(sum / static_cast(cols) + epsilon); + } +} + +// Four FP32 values times per-value multipliers, converted to four FP8 values. +template +__device__ __forceinline__ uint32_t quantize_4x(const float (&values)[kColsPerLane], + const float (&multipliers)[kColsPerLane]) { + using OTypex2 = + std::conditional_t, ptx::fp8e4m3x2, ptx::fp8e5m2x2>; + OTypex2 out[2]; + ptx::mul_cvt_2x(out[0], ptx::floatx2{values[0], values[1]}, + ptx::floatx2{multipliers[0], multipliers[1]}); + ptx::mul_cvt_2x(out[1], ptx::floatx2{values[2], values[3]}, + ptx::floatx2{multipliers[2], multipliers[3]}); + return static_cast(reinterpret_cast(out[0])) | + (static_cast(reinterpret_cast(out[1])) << 16); +} + +struct QuantizeArgs { + void *rowwise_data; + e8m0_t *rowwise_scale_inv; + void *colwise_data; + e8m0_t *colwise_scale_inv; + // Compact layouts: row strides of the scaling-factor tensors. Swizzled layouts: number of + // 4-wide scaling-factor tiles along the blocked dimension. + size_t rowwise_scale_stride; + size_t colwise_scale_stride; +}; + +// One CTA per 32x128 tile: warp w normalizes rows 8w..8w+7, lane l columns 4l..4l+3. +template +__global__ void __launch_bounds__(kThreads) + rmsnorm_mxfp8_quantize_kernel(const IType *const x, const WType *const gamma, + const float *const rsigma, const QuantizeArgs args, + const int cols, const bool zero_centered_gamma, + const bool gamma_in_weight_dtype) { +#if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + using dispatch::mxfp8::swizzle::gemm_swizzled_scale_idx; + constexpr float kMaxNormRcp = Quantized_Limits::max_norm_rcp; + __shared__ float colwise_amax[kWarps][kTileCols]; + + const int warp = threadIdx.x / THREADS_PER_WARP; + const int lane = threadIdx.x % THREADS_PER_WARP; + const int row_base = blockIdx.x * kTileRows; + const int row0 = row_base + warp * kRowsPerWarp; + const int col0 = blockIdx.y * kTileCols + lane * kColsPerLane; + + // Gamma as applied by the other normalization backends: with a zero-centered gamma, 1 is + // added in FP32, or in the weight type if requested. + float g[kColsPerLane]; + { + WType w[kColsPerLane]; + if constexpr (kAligned) { + load_elements(w, gamma + col0); + } else { +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + w[i] = gamma[col0 + i]; + } + } +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + float value = static_cast(w[i]); + if (zero_centered_gamma) { + value = gamma_in_weight_dtype ? static_cast(static_cast(value + 1.0f)) + : value + 1.0f; + } + g[i] = value; + } + } + + float y[kRowsPerWarp][kColsPerLane]; +#pragma unroll + for (int r = 0; r < kRowsPerWarp; ++r) { + const int row = row0 + r; + const float rs = rsigma[row]; + IType values[kColsPerLane]; + const IType *const src = x + static_cast(row) * cols + col0; + if constexpr (kAligned) { + load_elements(values, src); + } else { +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + values[i] = src[i]; + } + } +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + y[r][i] = (static_cast(values[i]) * rs) * g[i]; + } + } + + if constexpr (kRowwise) { + OType *const data = reinterpret_cast(args.rowwise_data); +#pragma unroll + for (int r = 0; r < kRowsPerWarp; ++r) { + const int row = row0 + r; + // Each 32-column scaling block spans 8 consecutive lanes. + float amax = 0.0f; +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + amax = fmaxf(amax, fabsf(y[r][i])); + } +#pragma unroll + for (int offset = 1; offset < kLanesPerBlock; offset *= 2) { + amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, offset)); + } + const e8m0_t biased_exponent = ptx::float_to_e8m0(amax * kMaxNormRcp); + const float multiplier = ptx::exp2f_rcp(biased_exponent); + const float multipliers[kColsPerLane] = {multiplier, multiplier, multiplier, multiplier}; + const uint32_t quantized = quantize_4x(y[r], multipliers); + OType *const dst = data + static_cast(row) * cols + col0; + if constexpr (kAligned) { + *reinterpret_cast(dst) = quantized; + } else { +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + reinterpret_cast(dst)[i] = static_cast(quantized >> (8 * i)); + } + } + if (lane % kLanesPerBlock == 0) { + const size_t scale_col = col0 / kScaleBlock; + const size_t idx = kSwizzled + ? gemm_swizzled_scale_idx(row, scale_col, args.rowwise_scale_stride) + : static_cast(row) * args.rowwise_scale_stride + scale_col; + args.rowwise_scale_inv[idx] = biased_exponent; + } + } + } + + if constexpr (kColwise) { + // Each column's scaling block is the tile's 32 rows: reduce over this warp's 8 rows, then + // over the 4 warps through shared memory. +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + float amax = 0.0f; +#pragma unroll + for (int r = 0; r < kRowsPerWarp; ++r) { + amax = fmaxf(amax, fabsf(y[r][i])); + } + colwise_amax[warp][lane * kColsPerLane + i] = amax; + } + __syncthreads(); + e8m0_t biased_exponents[kColsPerLane]; + float multipliers[kColsPerLane]; +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + float amax = colwise_amax[0][lane * kColsPerLane + i]; +#pragma unroll + for (int w = 1; w < kWarps; ++w) { + amax = fmaxf(amax, colwise_amax[w][lane * kColsPerLane + i]); + } + biased_exponents[i] = ptx::float_to_e8m0(amax * kMaxNormRcp); + multipliers[i] = ptx::exp2f_rcp(biased_exponents[i]); + } + OType *const data = reinterpret_cast(args.colwise_data); +#pragma unroll + for (int r = 0; r < kRowsPerWarp; ++r) { + const uint32_t quantized = quantize_4x(y[r], multipliers); + OType *const dst = data + static_cast(row0 + r) * cols + col0; + if constexpr (kAligned) { + *reinterpret_cast(dst) = quantized; + } else { +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + reinterpret_cast(dst)[i] = static_cast(quantized >> (8 * i)); + } + } + } + if (warp == 0) { + const size_t scale_row = row_base / kScaleBlock; +#pragma unroll + for (int i = 0; i < kColsPerLane; ++i) { + const size_t col = col0 + i; + const size_t idx = kSwizzled + ? gemm_swizzled_scale_idx(col, scale_row, args.colwise_scale_stride) + : scale_row * args.colwise_scale_stride + col; + args.colwise_scale_inv[idx] = biased_exponents[i]; + } + } + } +#else + NVTE_DEVICE_THREAD0_ERROR("RMSNorm forward with MXFP8 output requires SM 10.0+."); +#endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) +} + +bool is_aligned_to(const void *ptr, size_t alignment) { + return reinterpret_cast(ptr) % alignment == 0; +} + +// Rows per launch of a kernel with grid rows of grid_cols CTAs, each grid row covering cta_rows +// rows: all rows, unless the caller reserves SMs for other work. Then a launch runs at most as +// many CTAs as fit on the remaining num_sms SMs, in multiples of 128 rows (at least 128). +template +int rows_per_launch(Kernel kernel, const int threads, const int num_sms, const int rows, + const int cta_rows, const int grid_cols) { + if (num_sms <= 0 || num_sms >= cuda::sm_count()) { + return rows; + } + int ctas_per_sm = 0; + NVTE_CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&ctas_per_sm, kernel, threads, 0)); + const int grid_rows = std::max(ctas_per_sm, 1) * num_sms / grid_cols; + return std::min(rows, std::max(grid_rows * cta_rows / 128 * 128, 128)); +} + +template +void launch_rmsnorm_fwd_mxfp8(const Tensor &x, const Tensor &gamma, const float epsilon, Tensor *z, + Tensor *rsigma, const int num_sms, const bool zero_centered_gamma, + const bool gamma_in_weight_dtype, cudaStream_t stream) { + const auto [rows_size_t, cols_size_t] = x.flat_2d_dims(); + const int rows = static_cast(rows_size_t); + const int cols = static_cast(cols_size_t); + const bool rowwise = z->has_data(); + const bool colwise = z->has_columnwise_data(); + const bool swizzled = z->with_gemm_swizzled_scales; + + QuantizeArgs args{}; + if (rowwise) { + args.rowwise_data = z->data.dptr; + args.rowwise_scale_inv = reinterpret_cast(z->scale_inv.dptr); + const size_t scale_cols = z->scale_inv.shape.back(); + args.rowwise_scale_stride = swizzled ? scale_cols / 4 : scale_cols; + } + if (colwise) { + args.colwise_data = z->columnwise_data.dptr; + args.colwise_scale_inv = reinterpret_cast(z->columnwise_scale_inv.dptr); + args.colwise_scale_stride = + swizzled ? z->columnwise_scale_inv.shape.front() / 4 : z->columnwise_scale_inv.shape.back(); + } + + const IType *const x_ptr = reinterpret_cast(x.data.dptr); + const WType *const gamma_ptr = reinterpret_cast(gamma.data.dptr); + float *const rsigma_ptr = reinterpret_cast(rsigma->data.dptr); + + // Vector accesses: 16 bytes of a row of x in the pre-pass, 4 elements of x and gamma and 4 + // bytes of each output in the main kernel. Every row starts aligned since cols % 128 == 0. + const bool aligned = is_aligned_to(x_ptr, 16) && is_aligned_to(gamma_ptr, 4 * sizeof(WType)) && + (!rowwise || is_aligned_to(args.rowwise_data, 4)) && + (!colwise || is_aligned_to(args.colwise_data, 4)); + + TRANSFORMER_ENGINE_SWITCH_CONDITION(aligned, kAligned, { + const auto kernel = rmsnorm_mxfp8_rsigma_kernel; + constexpr int threads = kRsigmaRowsPerCTA * THREADS_PER_WARP; + const int launch_rows = rows_per_launch(kernel, threads, num_sms, rows, kRsigmaRowsPerCTA, 1); + for (int row = 0; row < rows; row += launch_rows) { + const int n = std::min(launch_rows, rows - row); + kernel<<>>( + x_ptr + static_cast(row) * cols, rsigma_ptr + row, n, cols, epsilon); + } + }); // NOLINT(*) + NVTE_CHECK_CUDA(cudaGetLastError()); + + // Arguments for the rows from row on, a multiple of 128. The scaling factors of these rows + // start at a fixed offset in each layout: row-wise, after row rows of scales (swizzled: row / + // 128 rows of 128x4 tiles, the same bytes); column-wise, after row / 32 rows of scales + // (swizzled: row / 128 tiles into each column of tiles). + const auto args_from = [&](const int row) { + QuantizeArgs a = args; + const size_t data_offset = static_cast(row) * cols; + if (rowwise) { + a.rowwise_data = static_cast(args.rowwise_data) + data_offset; + a.rowwise_scale_inv += static_cast(row) * z->scale_inv.shape.back(); + } + if (colwise) { + a.colwise_data = static_cast(args.colwise_data) + data_offset; + a.colwise_scale_inv += + swizzled ? static_cast(row / 128) * 512 + : static_cast(row / kScaleBlock) * z->columnwise_scale_inv.shape.back(); + } + return a; + }; + const auto launch = [&](auto kernel) { + const int launch_rows = + rows_per_launch(kernel, kThreads, num_sms, rows, kTileRows, cols / kTileCols); + for (int row = 0; row < rows; row += launch_rows) { + const int n = std::min(launch_rows, rows - row); + kernel<<>>( + x_ptr + static_cast(row) * cols, gamma_ptr, rsigma_ptr + row, args_from(row), + cols, zero_centered_gamma, gamma_in_weight_dtype); + } + }; + TRANSFORMER_ENGINE_SWITCH_CONDITION( + aligned, kAligned, + TRANSFORMER_ENGINE_SWITCH_CONDITION( + swizzled, kSwizzled, + if (rowwise && colwise) { + launch(rmsnorm_mxfp8_quantize_kernel); + } else if (rowwise) { + launch(rmsnorm_mxfp8_quantize_kernel); + } else { + launch(rmsnorm_mxfp8_quantize_kernel); + });); // NOLINT(*) + NVTE_CHECK_CUDA(cudaGetLastError()); +} + +} // namespace + +bool is_supported_by_te_rmsnorm_fwd_mxfp8(const Tensor &x, const Tensor &gamma, const Tensor &z) { + if (!is_mxfp8_scaling(z.scaling_mode) || !is_supported_by_CC_100()) { + return false; + } + const bool rowwise = z.has_data(); + const bool colwise = z.has_columnwise_data(); + if (!rowwise && !colwise) { + return false; + } + if ((rowwise && z.scale_inv.shape.size() != 2) || + (colwise && z.columnwise_scale_inv.shape.size() != 2)) { + return false; + } + const auto [rows, cols] = x.flat_2d_dims(); + // Full 128x128 tiles, so the scaling factors carry no padding in either layout. + if (rows == 0 || cols == 0 || rows % 128 != 0 || cols % 128 != 0 || + rows > static_cast(std::numeric_limits::max()) || cols / kTileCols > 65535) { + return false; + } + const auto is_supported_input = [](DType t) { + return t == DType::kFloat32 || t == DType::kBFloat16 || t == DType::kFloat16; + }; + if (!is_supported_input(x.data.dtype) || !is_supported_input(gamma.data.dtype)) { + return false; + } + const auto is_supported_output = [](DType t) { + return t == DType::kFloat8E4M3 || t == DType::kFloat8E5M2; + }; + if ((rowwise && !is_supported_output(z.data.dtype)) || + (colwise && !is_supported_output(z.columnwise_data.dtype)) || + (rowwise && colwise && z.data.dtype != z.columnwise_data.dtype)) { + return false; + } + return true; +} + +void rmsnorm_fwd_mxfp8(const Tensor &x, const Tensor &gamma, const float epsilon, Tensor *z, + Tensor *rsigma, const int multiprocessorCount, + const bool zero_centered_gamma, cudaStream_t stream) { + const DType otype = z->has_data() ? z->data.dtype : z->columnwise_data.dtype; + const bool gamma_in_weight_dtype = use_zero_centered_gamma_in_weight_dtype(); + TRANSFORMER_ENGINE_TYPE_SWITCH_FLOAT( + x.data.dtype, IType, + TRANSFORMER_ENGINE_TYPE_SWITCH_FLOAT( + gamma.data.dtype, WType, + TRANSFORMER_ENGINE_TYPE_SWITCH_FP8ONLY( + otype, OType, + launch_rmsnorm_fwd_mxfp8( + x, gamma, epsilon, z, rsigma, multiprocessorCount, zero_centered_gamma, + gamma_in_weight_dtype, stream);););); // NOLINT(*) +} + +} // namespace normalization +} // namespace transformer_engine diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index 43f1d32b8a3..e93c01026a9 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -337,6 +337,8 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w UNFUSED, // Compute norm directly FULLY_FUSED, + // Compute norm directly, with quantization scales in compact format + FUSED_NORM_QUANT_UNSWIZZLED, // Compute norm and amax in high precision, then quantize to FP8 FUSED_NORM_AMAX_FP8, // Compute norm and amax in high precision, then quantize to NVFP4 @@ -346,10 +348,17 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w if (quantizer.is_none() || IsFloat8Quantizers(quantizer.ptr())) { impl = Impl::FULLY_FUSED; } else if (IsMXFP8Quantizers(quantizer.ptr())) { - if (transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN") && outer_size % 128 == 0 && + auto mxfp8_quantizer_cpp = dynamic_cast(quantizer_cpp.get()); + NVTE_CHECK(mxfp8_quantizer_cpp != nullptr, "Could not cast to MXFP8 quantizer"); + // The fused MXFP8 kernels quantize 1D blocks and require full 128x128 tiles + if (!mxfp8_quantizer_cpp->with_2d_quantization && outer_size % 128 == 0 && inner_size % 128 == 0) { - // cuDNN MXFP8 kernel requires full 128x128 tiles - impl = Impl::FULLY_FUSED; + if (transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")) { + impl = Impl::FUSED_NORM_QUANT_UNSWIZZLED; + } else { + // Transformer Engine's fused RMSNorm + MXFP8 kernel + impl = Impl::FULLY_FUSED; + } } } else if (detail::IsFloat8CurrentScalingQuantizers(quantizer.ptr()) && !transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")) { @@ -375,9 +384,7 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w // Output tensor TensorWrapper out_nvte; if (out.is_none()) { - if (impl == Impl::FULLY_FUSED) { - // FP8 has no special logic to optimize for GEMM, MXFP8 cuDNN - // kernel does not support GEMM swizzled scales + if (impl == Impl::FUSED_NORM_QUANT_UNSWIZZLED) { quantizer_cpp->optimize_for_gemm = false; } std::tie(out_nvte, out) = quantizer_cpp->create_tensor(shape, out_dtype); From 074eb5d7b2a66e81bb0765ab4466d82a80b0438e Mon Sep 17 00:00:00 2001 From: Kaining Zhong <44538064+kainzhong@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:37:26 -0700 Subject: [PATCH 16/31] [Common] Link python to C++ dist tests with CuTeDSL enabled (#3648) Signed-off-by: Kaining Zhong --- tests/cpp_distributed/CMakeLists.txt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/cpp_distributed/CMakeLists.txt b/tests/cpp_distributed/CMakeLists.txt index 9ca071edfaf..2dd7b301c14 100644 --- a/tests/cpp_distributed/CMakeLists.txt +++ b/tests/cpp_distributed/CMakeLists.txt @@ -56,6 +56,10 @@ find_package(CUDAToolkit REQUIRED) find_package(OpenMP REQUIRED) find_package(MPI REQUIRED) +option(NVTE_WITH_CUTEDSL + "Link the tests against libpython, required when libtransformer_engine.so was built with NVTE_WITH_CUTEDSL=ON" + "$ENV{NVTE_WITH_CUTEDSL}") + # -- NCCL core ---------------------------------------------------------------- # Anchor on libnccl and derive nccl.h from the same install prefix so the # header and library can't drift across installs. @@ -127,6 +131,19 @@ add_executable(test_ep test_ep.cu ../cpp/test_common.cu) target_include_directories(test_ep PRIVATE ${EP_TEST_COMMON_INCLUDES}) target_link_libraries(test_ep PUBLIC ${EP_TEST_COMMON_LIBS}) +if(NVTE_WITH_CUTEDSL) + # Executables that link libtransformerengine.so must also link to the Python::Python to provide + # the definitions of these CPython symbols at runtime via dynamic linking. + # Find the interpreter too so older CMake versions use its version to locate libpython. + find_package(Python COMPONENTS Interpreter Development.Embed REQUIRED) + foreach(test_target test_comm_gemm test_ep) + target_link_libraries(${test_target} PRIVATE + "-Wl,--no-as-needed" + Python::Python + "-Wl,--as-needed") + endforeach() +endif() + # Do NOT use gtest_discover_tests - these binaries require multi-process # launch via run_test_ep.sh, not direct single-process execution. message(STATUS "EP distributed tests enabled (NCCL EP loaded from libnccl_ep.so at runtime)") From f57b8b8e6194e9a8db503c6bc8a69b0c52321114 Mon Sep 17 00:00:00 2001 From: Kaining Zhong <44538064+kainzhong@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:16:19 -0700 Subject: [PATCH 17/31] [Common] Enable all activations for MXFP8 C++ Unittest (#3616) * [Common] Enable all activations for MXFP8 C++ Unittest Signed-off-by: Kaining Zhong * nit Signed-off-by: Kaining Zhong * also enable other activations for grouped MXFP8 Signed-off-by: Kaining Zhong --------- Signed-off-by: Kaining Zhong --- tests/cpp/operator/test_cast_mxfp8.cu | 9 ++++----- tests/cpp/operator/test_cast_mxfp8_grouped.cu | 10 +++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/cpp/operator/test_cast_mxfp8.cu b/tests/cpp/operator/test_cast_mxfp8.cu index 9368d4030b5..16fb977b788 100644 --- a/tests/cpp/operator/test_cast_mxfp8.cu +++ b/tests/cpp/operator/test_cast_mxfp8.cu @@ -746,14 +746,13 @@ std::vector scaling_directions_2d_quantize = { MXFP82DScalingDirection::Bidirectional, }; -// Only GeLU activation tests are supported std::vector Activation_types = { ActivationType::Identity, ActivationType::GeLU, - // ActivationType::SiLU, - // ActivationType::ReLU, - // ActivationType::QGeLU, - // ActivationType::SReLU, + ActivationType::SiLU, + ActivationType::ReLU, + ActivationType::QGeLU, + ActivationType::SReLU, }; class FusedCastMXFP8TestSuite : public ::testing::TestWithParam diff --git a/tests/cpp/operator/test_cast_mxfp8_grouped.cu b/tests/cpp/operator/test_cast_mxfp8_grouped.cu index c2309f7aa4b..03ec1acd001 100644 --- a/tests/cpp/operator/test_cast_mxfp8_grouped.cu +++ b/tests/cpp/operator/test_cast_mxfp8_grouped.cu @@ -657,10 +657,10 @@ std::vector processing_methods = { std::vector activation_kinds = { ActivationKind::Identity, ActivationKind::GeLU, - // ActivationKind::SiLU, - // ActivationKind::ReLU, - // ActivationKind::QGeLU, - // ActivationKind::SReLU, + ActivationKind::SiLU, + ActivationKind::ReLU, + ActivationKind::QGeLU, + ActivationKind::SReLU, }; enum ScalingDirection { @@ -1012,7 +1012,7 @@ INSTANTIATE_TEST_SUITE_P( GroupedFusedCastMXFP8TestSuite, ::testing::Combine( ::testing::Values(ProcessingMethod::CAST_DBIAS_DACT), - ::testing::Values(ActivationKind::GeLU), + ::testing::ValuesIn(activation_kinds), ::testing::ValuesIn(scaling_directions), ::testing::ValuesIn(input_config_same_both_multichunk), ::testing::Values(DType::kBFloat16), From 16580f9502dcfbf3245f1dbb4670c9c7aad50162 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Fri, 9 Oct 2026 14:41:36 -0700 Subject: [PATCH 18/31] [PyTorch][CP][CI] Reset all CP process pools before retrying a failed test (#3608) Reset context-parallel pools before retry Persistent workers for different world sizes may share GPUs. Restarting only the failed worker can leave CUDA and NCCL state from another pool resident, so the retry starts under the same resource pressure. Shut down every active pool before retrying. Pool objects remain registered and restart lazily when requested, preserving reuse on the normal path. Signed-off-by: Sudhakar Singh --- tests/pytorch/attention/test_attention_with_cp.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/pytorch/attention/test_attention_with_cp.py b/tests/pytorch/attention/test_attention_with_cp.py index e4b5ad86edb..778c73fd40f 100644 --- a/tests/pytorch/attention/test_attention_with_cp.py +++ b/tests/pytorch/attention/test_attention_with_cp.py @@ -12,6 +12,7 @@ import pathlib import logging import copy +from collections.abc import Callable from collections import deque import pytest import torch @@ -104,8 +105,9 @@ class PoolWorker: _STDERR_BUFFER_LINES = 200 # ring cap (~40 KB ceiling) _STDERR_TAIL_CHARS = 4000 # how much to attach to the AssertionError - def __init__(self, world_size: int): + def __init__(self, world_size: int, reset_all_pools: Callable[[], None]): self.world_size = world_size + self._reset_all_pools = reset_all_pools self.proc: subprocess.Popen | None = None self._stderr_buf: deque[str] = deque(maxlen=self._STDERR_BUFFER_LINES) @@ -221,10 +223,13 @@ def submit(self, kwargs: dict, timeout: float = POOL_SUBMIT_TIMEOUT_SEC) -> None sys.stderr.flush() raise first_err = e + # Pools with different world sizes can occupy the same GPUs. Reset + # all pools so the retry starts without leftover CUDA/NCCL state. + self._reset_all_pools() sys.stderr.write( f"[POOL-RETRY] status=retrying test={test_id!r} " f"world_size={self.world_size} attempt={attempt + 1} " - f"error={msg_head!r}; respawning pool and retrying\n" + f"error={msg_head!r}; reset all pools and retrying\n" ) sys.stderr.flush() raise first_err # unreachable; loop either returns or raises @@ -297,11 +302,15 @@ def cp_pool(): """Returns a callable: cp_pool(world_size) -> PoolWorker.""" pools: dict[int, PoolWorker] = {} + def _reset_all_pools() -> None: + for pool in pools.values(): + pool.shutdown() + def _get(world_size: int) -> PoolWorker: if world_size > torch.cuda.device_count(): pytest.skip(f"Test requires {world_size} GPUs, but found {torch.cuda.device_count()}") if world_size not in pools: - pools[world_size] = PoolWorker(world_size) + pools[world_size] = PoolWorker(world_size, _reset_all_pools) return pools[world_size] yield _get From 15dcc848ecff8de41768053879843c0eabbe8db3 Mon Sep 17 00:00:00 2001 From: Siddhartha Raman Sundara Raman Date: Fri, 9 Oct 2026 16:48:47 -0500 Subject: [PATCH 19/31] [Common] Requantize grouped MXFP8 tensors with 128x128 tiles (#3656) Requantize grouped MXFP8 tensors with 128x128 tiles Signed-off-by: Siddhartha Raman S Co-authored-by: Phuong Nguyen --- .../common/cast/fused_group_requantize.cu | 191 +++++++++++++++++- 1 file changed, 189 insertions(+), 2 deletions(-) diff --git a/transformer_engine/common/cast/fused_group_requantize.cu b/transformer_engine/common/cast/fused_group_requantize.cu index 7efe629d112..aad83cb67c0 100644 --- a/transformer_engine/common/cast/fused_group_requantize.cu +++ b/transformer_engine/common/cast/fused_group_requantize.cu @@ -13,10 +13,12 @@ #include #include +#include #include #include #include "../common.h" +#include "../util/cuda_runtime.h" #include "../util/ptx_arch_spec.cuh" #include "../utils.cuh" #include "mxfp8/swizzle.cuh" @@ -413,6 +415,172 @@ void launch_fused_group_requantize(const Tensor &input, Tensor *output, input_scale_stride); } +// --------------------------------------------------------------------------- +// Wide-tile variant for the fast-math case without a dequantized output. +// +// Same per-element arithmetic as fused_group_requantize_kernel, so the +// outputs are bit-identical; only the work decomposition differs. The kernel above runs one 32x128 +// tile per 128-thread CTA, thread 0 binary-searches the group offsets before the tile can finish, +// and both scale outputs are written one byte at a time. Here a CTA owns a 128x128 tile: warp w +// takes rows 32w..32w+31 and lane l columns 4l..4l+3, so the data moves as coalesced 128-byte row +// segments held in registers, and each of the two swizzled scale outputs is one contiguous +// 512-byte block per tile. The grid is persistent and walks the tiles in row-block order, so a +// CTA stops at the first tile past tensor_offsets[num_groups]; a capacity-sized buffer with a +// short live prefix then costs little more than the live rows. + +constexpr int kWideTile = 128; +constexpr int kWideThreads = 128; +constexpr int kWideMaxGroups = 128; +constexpr int kWideCtasPerSM = 16; + +template +__global__ void __launch_bounds__(kWideThreads) + fused_group_requantize_wide_kernel(const IType *const input, + const e8m0_t *const input_scale_inv, + const int input_scale_stride, fp8e4m3 *const colwise_out, + e8m0_t *const rowwise_scale_inv_swizzled, + e8m0_t *const colwise_scale_inv, + const int64_t *const tensor_offsets, const int num_groups, + const int num_cols, const int num_tiles) { +#if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + static_assert(sizeof(IType) == 1); + __shared__ int64_t offsets[kWideMaxGroups + 1]; + __shared__ alignas(16) e8m0_t colwise_scale_block[kWideTile * 4]; + + const int tid = threadIdx.x; + const int warp = tid / THREADS_PER_WARP; + const int lane = tid % THREADS_PER_WARP; + const int tiles_x = num_cols / kWideTile; + const int words_per_row = num_cols / 4; + // This lane's 4 columns lie in the 32-column scaling block lane / 8 of the tile. + const int scale_shift = 8 * (lane / 8); + for (int i = tid; i <= num_groups; i += kWideThreads) { + offsets[i] = tensor_offsets[i]; + } + __syncthreads(); + const int64_t live_end = offsets[num_groups]; + + const uint32_t *const input_words = reinterpret_cast(input); + uint32_t *const output_words = reinterpret_cast(colwise_out); + for (int tile = blockIdx.x; tile < num_tiles; tile += gridDim.x) { + const int tile_x = tile % tiles_x; + const int tile_y = tile / tiles_x; + const int row_base = tile_y * kWideTile; + const int col_base = tile_x * kWideTile; + const int64_t row_element_base = static_cast(row_base) * num_cols; + // Rows at or past the live end are left untouched, as by the kernel above. Later tiles of + // this CTA lie further down, so the walk ends here. + if (row_element_base >= live_end) { + break; + } + + // 32 rows x 4 columns per thread, one 4-byte word per row. + const int row0 = row_base + 32 * warp; + const uint32_t *const src = + input_words + static_cast(row0) * words_per_row + col_base / 4 + lane; + uint32_t data[32]; +#pragma unroll + for (int r = 0; r < 32; ++r) { + data[r] = __ldg(src + static_cast(r) * words_per_row); + } + // Lane l holds the 4 rowwise scale codes (the tile's four 32-column blocks) of row row0 + l. + const uint32_t scale_word = __ldg(reinterpret_cast( + input_scale_inv + static_cast(row0 + lane) * input_scale_stride + + col_base / MXFP8_SCALE_DIM)); + + // The owning group, as found by the upper_bound lookup of the kernel above. 128-aligned group + // sizes mean a tile never straddles a group boundary. + int group = 0; + for (int i = 0; i + 1 < num_groups; ++i) { + group += (offsets[i + 1] <= row_element_base) ? 1 : 0; + } + const int group_start = static_cast(offsets[group] / num_cols); + const int group_rows = static_cast((offsets[group + 1] - offsets[group]) / num_cols); + + // GEMM-swizzled copy of the rowwise scales: the tile's 128 rows x 4 codes are one 512-byte + // block, in which row r of the tile starts at (r % 32) * 16 + (r / 32) * 4. + *reinterpret_cast(rowwise_scale_inv_swizzled + + (static_cast(tile_y) * tiles_x + tile_x) * 512 + + lane * 16 + warp * 4) = scale_word; + + // Dequantize to BF16 and take the amax of each column over the 32-row block. + ptx::bf16x2 dequantized01[32]; + ptx::bf16x2 dequantized23[32]; + ptx::bf16x2 amax01 = {static_cast(0.0f), static_cast(0.0f)}; + ptx::bf16x2 amax23 = {static_cast(0.0f), static_cast(0.0f)}; +#pragma unroll + for (int r = 0; r < 32; ++r) { + const e8m0_t scale_code = + static_cast((__shfl_sync(0xffffffff, scale_word, r) >> scale_shift) & 0xFF); + const uint16_t values01 = static_cast(data[r]); + const uint16_t values23 = static_cast(data[r] >> 16); + dequantized01[r] = + dequantize_mxfp8_2x(reinterpret_cast &>(values01), scale_code); + dequantized23[r] = + dequantize_mxfp8_2x(reinterpret_cast &>(values23), scale_code); + ptx::abs_max_2x(amax01, amax01, dequantized01[r]); + ptx::abs_max_2x(amax23, amax23, dequantized23[r]); + } + constexpr float kMaxNormRcp = Quantized_Limits::max_norm_rcp; + const e8m0_t exponent0 = ptx::float_to_e8m0(static_cast(amax01.x) * kMaxNormRcp); + const e8m0_t exponent1 = ptx::float_to_e8m0(static_cast(amax01.y) * kMaxNormRcp); + const e8m0_t exponent2 = ptx::float_to_e8m0(static_cast(amax23.x) * kMaxNormRcp); + const e8m0_t exponent3 = ptx::float_to_e8m0(static_cast(amax23.y) * kMaxNormRcp); + const ptx::bf16x2 multiplier01 = {ptx::exp2f_rcp(exponent0), + ptx::exp2f_rcp(exponent1)}; + const ptx::bf16x2 multiplier23 = {ptx::exp2f_rcp(exponent2), + ptx::exp2f_rcp(exponent3)}; + + uint32_t *const dst = + output_words + static_cast(row0) * words_per_row + col_base / 4 + lane; +#pragma unroll + for (int r = 0; r < 32; ++r) { + ptx::fp8e4m3x4 quantized; + ptx::mul_cvt_4x(quantized, dequantized01[r], multiplier01, dequantized23[r], multiplier23); + dst[static_cast(r) * words_per_row] = reinterpret_cast(quantized); + } + + // Columnwise scales: column c of the tile and 32-row block w go to (c % 32) * 16 + (c / 32) * 4 + // + w of the tile's 512-byte block in the group's swizzled layout. Stage the block in shared + // memory, then store it with one 4-byte word per thread. + const e8m0_t exponents[4] = {exponent0, exponent1, exponent2, exponent3}; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int c = 4 * lane + j; + colwise_scale_block[(c % 32) * 16 + (c / 32) * 4 + warp] = exponents[j]; + } + __syncthreads(); + const size_t colwise_scale_base = static_cast(group_start) / MXFP8_SCALE_DIM * num_cols; + const int tile_in_group = (row_base - group_start) / kWideTile; + const int colwise_scale_tiles_x = group_rows / kWideTile; + e8m0_t *const colwise_scale_dst = + colwise_scale_inv + colwise_scale_base + + (static_cast(tile_x) * colwise_scale_tiles_x + tile_in_group) * 512; + reinterpret_cast(colwise_scale_dst)[tid] = + reinterpret_cast(colwise_scale_block)[tid]; + __syncthreads(); // The next tile rewrites colwise_scale_block. + } +#else + NVTE_DEVICE_THREAD0_ERROR("Fused grouped requantization requires Blackwell (SM100+) hardware."); +#endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) +} + +template +void launch_fused_group_requantize_wide(const Tensor &input, Tensor *output, + const Tensor &tensor_offsets, const int num_groups, + const int num_tiles, const int num_cols, + const int input_scale_stride, cudaStream_t stream) { + const int grid = static_cast( + std::min(num_tiles, static_cast(cuda::sm_count()) * kWideCtasPerSM)); + fused_group_requantize_wide_kernel<<>>( + reinterpret_cast(input.data.dptr), + reinterpret_cast(input.scale_inv.dptr), input_scale_stride, + reinterpret_cast(output->columnwise_data.dptr), + reinterpret_cast(output->scale_inv.dptr), + reinterpret_cast(output->columnwise_scale_inv.dptr), + reinterpret_cast(tensor_offsets.data.dptr), num_groups, num_cols, num_tiles); +} + void fused_group_requantize(const Tensor &input, Tensor *output, const Tensor &tensor_offsets, Tensor *dequantized, const QuantizationConfig *quant_config, cudaStream_t stream) { @@ -515,11 +683,30 @@ void fused_group_requantize(const Tensor &input, Tensor *output, const Tensor &t return; } + const bool use_fast_math = quant_config != nullptr && quant_config->use_fast_math; + + // The wide-tile kernel covers the fast-math case without a dequantized output. It loads the + // rowwise scales and stores both scale outputs as 4-byte words. + const int64_t num_wide_tiles = + static_cast(num_rows / kWideTile) * static_cast(num_cols / kWideTile); + const bool use_wide_kernel = + use_fast_math && !return_dequantized && num_groups <= kWideMaxGroups && + num_wide_tiles <= static_cast(kMaxInt) && input_scale_stride % 4 == 0 && + is_aligned_ptr(input.scale_inv.dptr, 4) && is_aligned_ptr(output->scale_inv.dptr, 4) && + is_aligned_ptr(output->columnwise_scale_inv.dptr, 4); + if (use_wide_kernel) { + TRANSFORMER_ENGINE_TYPE_SWITCH_FP8ONLY( + input.data.dtype, IType, + launch_fused_group_requantize_wide(input, output, tensor_offsets, num_groups, + static_cast(num_wide_tiles), num_cols, + input_scale_stride, stream);); // NOLINT(*) + NVTE_CHECK_CUDA(cudaGetLastError()); + return; + } + alignas(64) CUtensorMap output_tensor_map{}; create_2D_tensor_map(output_tensor_map, output->columnwise_data, num_rows, num_cols, kTileRows, kTileCols, num_cols, 0, typeToNumBits(output->columnwise_data.dtype)); - - const bool use_fast_math = quant_config != nullptr && quant_config->use_fast_math; TRANSFORMER_ENGINE_SWITCH_CONDITION( use_fast_math, USE_FAST_MATH, TRANSFORMER_ENGINE_TYPE_SWITCH_FP8ONLY( From 76ae4b0981849d4b85a528d26f39981974b409f8 Mon Sep 17 00:00:00 2001 From: Kaining Zhong <44538064+kainzhong@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:27:34 -0700 Subject: [PATCH 20/31] [Common] Fix grouped MXFP8 quantization with varying last dim & swizzled scale (#3599) * fix grouped MXFP8 quantization with varying last dim & swizzled scale Signed-off-by: Kaining Zhong * nit Signed-off-by: Kaining Zhong --------- Signed-off-by: Kaining Zhong Co-authored-by: Oleg Goncharov <64355998+Oleg-Goncharov@users.noreply.github.com> --- tests/pytorch/test_grouped_tensor.py | 43 ++++++++++++------- .../cast/mxfp8/group_quantize_mxfp8.cuh | 8 +++- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/tests/pytorch/test_grouped_tensor.py b/tests/pytorch/test_grouped_tensor.py index 388b3bb7041..ae4a60eb270 100644 --- a/tests/pytorch/test_grouped_tensor.py +++ b/tests/pytorch/test_grouped_tensor.py @@ -629,11 +629,15 @@ def test_quantize_varying_shapes(self, quantization: str) -> None: "shape_case", ["varying_first", "varying_last", "varying_both"], ) + @pytest.mark.parametrize("usage", ["rowwise", "columnwise", "both"]) + @pytest.mark.parametrize("optimize_for_gemm", [False, True]) @pytest.mark.parametrize("output_dbias", [False, True]) @pytest.mark.skipif(not mxfp8_available, reason=reason_for_no_mxfp8) - def test_quantize_grouped_mxfp8(self, shape_case: str, output_dbias: bool) -> None: + def test_quantize_grouped_mxfp8( + self, shape_case: str, usage: str, optimize_for_gemm: bool, output_dbias: bool + ) -> None: """Test grouped MXFP8 quantization against per-tensor quantization for - varying first/last/both dimensions.""" + varying first/last/both dimensions and compact or GEMM-swizzled scales.""" if output_dbias and shape_case != "varying_first": pytest.skip("bgrad_group_quantize requires constant last dimension") @@ -688,7 +692,10 @@ def test_quantize_grouped_mxfp8(self, shape_case: str, output_dbias: bool) -> No ) quantizer = MXFP8Quantizer(fp8_dtype=tex.DType.kFloat8E4M3) - quantizer.set_usage(rowwise=True, columnwise=False) + rowwise = usage in ("rowwise", "both") + columnwise = usage in ("columnwise", "both") + quantizer.set_usage(rowwise=rowwise, columnwise=columnwise) + quantizer.optimize_for_gemm = optimize_for_gemm if output_dbias: grouped_output, dbias = tex.bgrad_group_quantize( @@ -700,19 +707,23 @@ def test_quantize_grouped_mxfp8(self, shape_case: str, output_dbias: bool) -> No ) # Reference: quantize each tensor independently and concatenate the - # rowwise data / scale_inv buffers in tensor order. - expected_data = [] - expected_scale_inv = [] - for tensor in input_tensors: - qtensor = quantizer(tensor) - expected_data.append(qtensor._rowwise_data.reshape(-1)) - expected_scale_inv.append(qtensor._rowwise_scale_inv.reshape(-1)) - - expected_data = torch.cat(expected_data) - expected_scale_inv = torch.cat(expected_scale_inv) - - assert torch.equal(grouped_output.rowwise_data, expected_data) - assert torch.equal(grouped_output.scale_inv, expected_scale_inv) + # data / scale_inv buffers in tensor order. Both dimensions of every member + # are multiples of 128, so swizzled scales need no padding between members. + expected_tensors = [quantizer(tensor) for tensor in input_tensors] + if rowwise: + expected_data = torch.cat([t._rowwise_data.reshape(-1) for t in expected_tensors]) + expected_scales = torch.cat( + [t._rowwise_scale_inv.reshape(-1) for t in expected_tensors] + ) + assert torch.equal(grouped_output.rowwise_data, expected_data) + assert torch.equal(grouped_output.scale_inv, expected_scales) + if columnwise: + expected_data = torch.cat([t._columnwise_data.reshape(-1) for t in expected_tensors]) + expected_scales = torch.cat( + [t._columnwise_scale_inv.reshape(-1) for t in expected_tensors] + ) + assert torch.equal(grouped_output.columnwise_data, expected_data) + assert torch.equal(grouped_output.columnwise_scale_inv, expected_scales) if output_dbias: expected_dbias = torch.stack([t.sum(dim=0) for t in input_tensors]) diff --git a/transformer_engine/common/cast/mxfp8/group_quantize_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/group_quantize_mxfp8.cuh index bd8c3052f60..08605a27867 100644 --- a/transformer_engine/common/cast/mxfp8/group_quantize_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/group_quantize_mxfp8.cuh @@ -882,7 +882,13 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ DIVUP_TO_MULTIPLE(DIVUP(cols, static_cast(SCALE_DIM_X)), scale_alignment_X_rowwise); const size_t scale_stride_colwise = DIVUP_TO_MULTIPLE(cols, scale_alignment_X_colwise); - const size_t tensor_base_for_scales = is_single_tensor ? tensor_start_offset : tensor_base; + // When is single tensor: + // - tensor_base_for_scales: offset of this tensor in the whole grouped tensor + // - scales_rowwise & scales_colwise: points to the scale of the whole grouped tensor + // When not single tensor: + // - tensor_base_for_scales: 0 + // - scales_rowwise & scales_colwise: points to the scale of this tensor + const size_t tensor_base_for_scales = is_single_tensor ? tensor_start_offset : 0; e8m0_t *const scales_rowwise = scales_rowwise_ptr + (is_single_tensor ? 0 : tensor_base / SCALE_DIM_X); e8m0_t *const scales_colwise = From 33e7ff316000c34dd976cf1d38f12e11882e987f Mon Sep 17 00:00:00 2001 From: Siddhartha Raman Sundara Raman Date: Sat, 10 Oct 2026 19:53:56 -0500 Subject: [PATCH 21/31] [Common] Swizzle MXFP8 scaling factors with one CTA per scale tile (#3658) * [Common] Swizzle MXFP8 scaling factors with one CTA per scale tile The MXFP8 scale swizzle is latency bound. The narrow-K and narrow-M kernels put up to 32 128x4 scale tiles in one 1024-thread CTA, so the scaling factors of a 4096-row activation are swizzled by a single CTA. Give every 128-row M-tile (row-wise) or 128-column x 32-row block (column-wise) its own CTA, which moves a contiguous chunk with coalesced 16-byte loads and stores through shared memory. Multi-tensor launches group tiles per CTA only when the grid would otherwise be very large. The existing kernels remain for NVFP4, unaligned buffers, and row-wise scales too wide for 48 KB of shared memory. Signed-off-by: Siddhartha Raman S Co-Authored-By: Claude Opus 5.5 * Test full and clipped 128-row column-wise swizzle chunks The column-wise multi-tensor case that selects 128-row chunks had only four scale rows. Add one with 132 scale rows per tensor: a full 128-row chunk and a clipped last chunk. Signed-off-by: Siddhartha Raman S Co-Authored-By: Claude Opus 5.5 * [Common] Make the per-tile MXFP8 swizzle kernels opt-in Select the per-tile MXFP8 scale-swizzle kernels with NVTE_MXFP8_PER_TILE_SWIZZLE=1; by default, the existing kernels run as before. Give the per-tile code descriptive names (for example TileSwizzleDesc -> PerTileSwizzleTensor, tpc -> tiles_per_block, k4_magic -> words_per_row_magic), document the variable, and run the single- and multi-tensor swizzle tests with it unset and set. Signed-off-by: Siddhartha Raman S Co-Authored-By: Claude Opus 5.5 --------- Signed-off-by: Siddhartha Raman S Co-authored-by: Claude Opus 5.5 --- docs/envvars.rst | 6 + tests/cpp/operator/test_multi_swizzle.cu | 38 +- tests/cpp/operator/test_swizzle.cu | 20 +- transformer_engine/common/swizzle/swizzle.cu | 389 ++++++++++++++++++- 4 files changed, 436 insertions(+), 17 deletions(-) diff --git a/docs/envvars.rst b/docs/envvars.rst index 46b70bbe46a..62b8ac5f4f0 100644 --- a/docs/envvars.rst +++ b/docs/envvars.rst @@ -342,6 +342,12 @@ Kernel Configuration :Default: ``0`` :Description: Disable NVRTC (CUDA Runtime Compilation) support. When set to ``1``, runtime kernel compilation is disabled. Existing transpose operations select their static fallback automatically. NVFP4 4over6 quantization, fused softmax, and normalization paths require their corresponding ``NVTE_BUILD_LEGACY_STATIC_NVFP4``, ``NVTE_BUILD_LEGACY_STATIC_FUSED_SOFTMAX``, or ``NVTE_BUILD_LEGACY_STATIC_NORM`` CMake option to have been enabled when the library was built; otherwise no static fallback is available. +.. envvar:: NVTE_MXFP8_PER_TILE_SWIZZLE + + :Type: ``int`` (0 or 1) + :Default: ``0`` + :Description: Swizzle MXFP8 scaling factors for GEMM with kernels that give each 128-row tile of scaling factors (row-wise) or each 128-column by 32-row block (column-wise) its own thread block, instead of batching 32 tiles in one block. This lowers the latency of swizzling the scaling factors of tensors with few tiles, such as activations. + .. envvar:: NVTE_USE_CUTLASS_GROUPED_GEMM :Type: ``int`` (0 or 1) diff --git a/tests/cpp/operator/test_multi_swizzle.cu b/tests/cpp/operator/test_multi_swizzle.cu index d8b8211e5b7..4bd1800cea2 100644 --- a/tests/cpp/operator/test_multi_swizzle.cu +++ b/tests/cpp/operator/test_multi_swizzle.cu @@ -300,14 +300,22 @@ void performTestMultiTensorRoundtrip(const int num_tensors, const size_t M, cons // Test suites // =================================================================== +// Sets NVTE_MXFP8_PER_TILE_SWIZZLE while it lives, so that MXFP8 swizzles use the per-tile +// kernels (enable) or the default ones. +class PerTileSwizzle { + public: + explicit PerTileSwizzle(bool enable) { + setenv("NVTE_MXFP8_PER_TILE_SWIZZLE", enable ? "1" : "0", 1); + } + ~PerTileSwizzle() { unsetenv("NVTE_MXFP8_PER_TILE_SWIZZLE"); } +}; + class MultiTensorSwizzleTestSuite - : public ::testing::TestWithParam> {}; + : public ::testing::TestWithParam, bool>> {}; TEST_P(MultiTensorSwizzleTestSuite, TestMultiTensorSwizzle) { - const auto num_tensors = std::get<0>(GetParam()); - const auto M = std::get<1>(GetParam()); - const auto K = std::get<2>(GetParam()); - const auto rowwise = std::get<3>(GetParam()); + const auto& [num_tensors, M, K, rowwise] = std::get<0>(GetParam()); + const PerTileSwizzle per_tile_swizzle(std::get<1>(GetParam())); performTestMultiTensorSwizzle(num_tensors, M, K, rowwise); } @@ -381,6 +389,14 @@ std::vector> multi_tensor_test_cases = { {2, 128, 4352, true}, // Colwise num_tiles_k = 512 / 32 / 4 = 4 selects vec_load_size = 4 (int4 kernel) {2, 512, 4096, false}, + // MXFP8 swizzle: more tensors than one launch takes, several row-wise tiles per CTA, + // and 128-row column-wise chunks + {80, 128, 128, true}, + {80, 128, 128, false}, + {20, 16384, 128, true}, + {20, 128, 8192, false}, + // A full 128-row column-wise chunk and a clipped last one (132 scale rows) + {20, 4224, 2048, false}, }; } // namespace @@ -388,12 +404,14 @@ std::vector> multi_tensor_test_cases = { INSTANTIATE_TEST_SUITE_P( OperatorTest, MultiTensorSwizzleTestSuite, - ::testing::ValuesIn(multi_tensor_test_cases), + ::testing::Combine(::testing::ValuesIn(multi_tensor_test_cases), ::testing::Values(false, true)), [](const testing::TestParamInfo& info) { - return "n" + std::to_string(std::get<0>(info.param)) + - "_M" + std::to_string(std::get<1>(info.param)) + - "_K" + std::to_string(std::get<2>(info.param)) + - (std::get<3>(info.param) ? "_row" : "_col"); + const auto& test_case = std::get<0>(info.param); + return "n" + std::to_string(std::get<0>(test_case)) + + "_M" + std::to_string(std::get<1>(test_case)) + + "_K" + std::to_string(std::get<2>(test_case)) + + (std::get<3>(test_case) ? "_row" : "_col") + + (std::get<1>(info.param) ? "_per_tile" : ""); }); INSTANTIATE_TEST_SUITE_P( diff --git a/tests/cpp/operator/test_swizzle.cu b/tests/cpp/operator/test_swizzle.cu index 8990ce8db13..a442215ae15 100644 --- a/tests/cpp/operator/test_swizzle.cu +++ b/tests/cpp/operator/test_swizzle.cu @@ -308,7 +308,17 @@ void performTestGroupedSwizzleMXFP8(const int num_tensors, const size_t M, const num_tensors * col_numel); } -class SwizzleTestSuite : public ::testing::TestWithParam, std::pair, bool>> {}; +// Sets NVTE_MXFP8_PER_TILE_SWIZZLE while it lives, so that MXFP8 swizzles use the per-tile +// kernels (enable) or the default ones. +class PerTileSwizzle { + public: + explicit PerTileSwizzle(bool enable) { + setenv("NVTE_MXFP8_PER_TILE_SWIZZLE", enable ? "1" : "0", 1); + } + ~PerTileSwizzle() { unsetenv("NVTE_MXFP8_PER_TILE_SWIZZLE"); } +}; + +class SwizzleTestSuite : public ::testing::TestWithParam, std::pair, bool, bool>> {}; TEST_P(SwizzleTestSuite, TestSwizzle) { @@ -318,6 +328,7 @@ TEST_P(SwizzleTestSuite, TestSwizzle) { const auto num_tiles = std::get<0>(GetParam()); const auto scaling_mode = std::get<1>(GetParam()); const auto transa = std::get<2>(GetParam()); + const PerTileSwizzle per_tile_swizzle(std::get<3>(GetParam())); performTestSwizzle1D(num_tiles.first, num_tiles.second, scaling_mode.first, scaling_mode.second, @@ -1004,6 +1015,7 @@ std::vector> num_tiles = { {8, 1}, // narrow_m with 8 M-tiles {31, 1}, // narrow_m at boundary (31 < TB_DIM=32) {1, 31}, // narrow_k at boundary (31 < TB_DIM=32) + {1600, 1}, // MXFP8: more row-wise tiles than one CTA per tile allows }; // Raw {M, K} data shapes for unswizzle tests. Includes aligned cases (scale dims @@ -1039,7 +1051,8 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Combine( ::testing::ValuesIn(num_tiles), ::testing::ValuesIn(scaling_mode), - ::testing::ValuesIn(transa) + ::testing::ValuesIn(transa), + ::testing::Values(false, true) ), [](const testing::TestParamInfo& info) { std::string name = "ntiles" + @@ -1047,7 +1060,8 @@ INSTANTIATE_TEST_SUITE_P( std::to_string(std::get<0>(info.param).second) + "smode" + std::to_string(std::get<1>(info.param).first) + "X"+ std::to_string(std::get<1>(info.param).second) + "trans" + - std::to_string(std::get<2>(info.param)); + std::to_string(std::get<2>(info.param)) + + (std::get<3>(info.param) ? "PerTile" : ""); return name; }); diff --git a/transformer_engine/common/swizzle/swizzle.cu b/transformer_engine/common/swizzle/swizzle.cu index 6e197dda50e..19fb8e848ad 100644 --- a/transformer_engine/common/swizzle/swizzle.cu +++ b/transformer_engine/common/swizzle/swizzle.cu @@ -16,6 +16,7 @@ #include "../common.h" #include "../util/cuda_runtime.h" #include "../util/logging.h" +#include "../util/system.h" #include "transformer_engine/transformer_engine.h" namespace transformer_engine { @@ -944,6 +945,328 @@ __global__ void __launch_bounds__(TB_DIM* TB_DIM) input, output, M, K, original_M, original_K, flat_bid, grid_dim); } +// MXFP8 scale swizzle with one thread block per scale tile, used when +// NVTE_MXFP8_PER_TILE_SWIZZLE=1. The narrow-K / narrow-M kernels above put 32 tiles in one +// 1024-thread block, so e.g. the scales of a 4096-row activation are swizzled by a single block. +// These kernels give every 128-row tile (row-wise) or 128-column x 32-row block (column-wise) its +// own block, which moves a contiguous chunk with coalesced 16-byte loads and stores through shared +// memory. Multi-tensor launches put several tiles in one block only when the grid would otherwise +// be very large. +constexpr int kRowwiseThreadsPerBlock = 128; +constexpr int kColumnwiseThreadsPerBlock = 256; +constexpr int kMaxTensorsPerLaunch = 64; +// Row-wise: the largest grid before several tiles share a block. +constexpr int kRowwiseMaxBlocks = 1536; +// Column-wise: the largest grid with 32-row chunks before switching to 128-row chunks. +constexpr int kColumnwiseMaxBlocksWith32RowChunks = 1024; + +// Whether MXFP8 swizzles use the per-tile kernels (NVTE_MXFP8_PER_TILE_SWIZZLE=1). +bool use_per_tile_swizzle() { return getenv("NVTE_MXFP8_PER_TILE_SWIZZLE"); } + +// The 4 scales packed in `word` (the first one in column first_column), with those in columns +// >= num_columns set to zero. +__device__ __forceinline__ uint32_t zero_padding_columns(uint32_t word, int first_column, + int num_columns) { + const int valid_columns = num_columns - first_column; + if (valid_columns >= 4) return word; + if (valid_columns <= 0) return 0u; + return word & ((1u << (8 * valid_columns)) - 1u); +} + +// numerator / divisor for numerator, divisor < 2^16, with magic = ceil(2^32 / divisor) (exact in +// that range); a divisor of 1 passes magic 0. +__device__ __forceinline__ uint32_t divide_by_magic(uint32_t numerator, uint32_t magic) { + return magic == 0u ? numerator : __umulhi(numerator, magic); +} + +inline uint32_t division_magic(int divisor) { + return divisor <= 1 ? 0u : static_cast(((1ull << 32) + divisor - 1) / divisor); +} + +// Row-wise: tiles [first_tile, first_tile + num_tiles) of one tensor, a contiguous chunk of the +// input and of the output. Shared-memory rows are padded to an odd number of words, so the column +// gathers are free of bank conflicts. Requires num_tiles * 128 * padded_k / 4 < 2^16 (guaranteed +// by the 48 KB shared-memory limit). +__device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict__ input, + uint8_t* __restrict__ output, int padded_k, + uint32_t words_per_row_magic, int original_m, + int original_k, int first_tile, int num_tiles, + uint32_t* smem) { + const int words_per_row = padded_k >> 2; + const int smem_row_words = words_per_row | 1; + const int num_vectors = + num_tiles * 8 * padded_k; // 16-byte vectors: num_tiles * 128 * padded_k / 16 + const size_t chunk_offset = static_cast(first_tile) * 128 * padded_k; + const int4* input_vectors = reinterpret_cast(input + chunk_offset); + for (int i = threadIdx.x; i < num_vectors; i += kRowwiseThreadsPerBlock) { + const int4 vector = __ldg(input_vectors + i); + const uint32_t words[4] = {static_cast(vector.x), static_cast(vector.y), + static_cast(vector.z), static_cast(vector.w)}; +#pragma unroll + for (int w = 0; w < 4; w++) { + const uint32_t word_index = 4 * i + w; + const uint32_t row = divide_by_magic(word_index, words_per_row_magic); + smem[row * smem_row_words + (word_index - row * words_per_row)] = words[w]; + } + } + __syncthreads(); + + // An output tile holds padded_k / 4 blocks of 512 bytes, i.e. 32 vectors each; vector r of a + // block packs rows r, r + 32, r + 64 and r + 96 of its 4 scale columns. + const int vectors_per_tile = 8 * padded_k; + for (int tile = 0; tile < num_tiles; tile++) { + int4* output_vectors = + reinterpret_cast(output + chunk_offset + static_cast(tile) * 128 * padded_k); + const int tile_first_row = tile * 128; + const int global_first_row = first_tile * 128 + tile_first_row; + for (int v = threadIdx.x; v < vectors_per_tile; v += kRowwiseThreadsPerBlock) { + const int k_block = v >> 5; + const int row_in_group = v & 31; + uint32_t words[4]; +#pragma unroll + for (int group = 0; group < 4; group++) { + const int row = row_in_group + 32 * group; + uint32_t word = smem[(tile_first_row + row) * smem_row_words + k_block]; + if (global_first_row + row >= original_m) word = 0u; + words[group] = zero_padding_columns(word, k_block * 4, original_k); + } + output_vectors[v] = make_int4(static_cast(words[0]), static_cast(words[1]), + static_cast(words[2]), static_cast(words[3])); + } + } +} + +// Column-wise: tile m_tile (128 columns), scale rows [first_k, first_k + kChunkRows) clipped to +// padded_k. Each input row segment is 128 contiguous bytes; each output block of 4 scale rows is +// 512 contiguous bytes. +template +__device__ __forceinline__ void swizzle_columnwise_chunk(const uint8_t* __restrict__ input, + uint8_t* __restrict__ output, int padded_m, + int padded_k, int original_m, + int original_k, int m_tile, int first_k, + int4* smem) { + const int num_rows = min(kChunkRows, padded_k - first_k); // multiple of 4 + const int num_vectors = num_rows * 8; + for (int i = threadIdx.x; i < num_vectors; i += kColumnwiseThreadsPerBlock) { + const int row = i >> 3; + const int vector_in_row = i & 7; + smem[i] = __ldg(reinterpret_cast( + input + static_cast(first_k + row) * padded_m + m_tile * 128) + + vector_in_row); + } + __syncthreads(); + + const uint8_t* smem_bytes = reinterpret_cast(smem); + const size_t first_output_block = static_cast(m_tile) * (padded_k >> 2) + (first_k >> 2); + for (int v = threadIdx.x; v < num_vectors; v += kColumnwiseThreadsPerBlock) { + const int k_block = v >> 5; + const int column_in_group = v & 31; + uint32_t words[4]; +#pragma unroll + for (int group = 0; group < 4; group++) { + const int column = column_in_group + 32 * group; + uint32_t word = 0u; + if (m_tile * 128 + column < original_m) { +#pragma unroll + for (int j = 0; j < 4; j++) { + const int row = k_block * 4 + j; + if (first_k + row < original_k) { + word |= static_cast(smem_bytes[row * 128 + column]) << (8 * j); + } + } + } + words[group] = word; + } + reinterpret_cast(output + (first_output_block + k_block) * 512)[column_in_group] = + make_int4(static_cast(words[0]), static_cast(words[1]), + static_cast(words[2]), static_cast(words[3])); + } +} + +__global__ void __launch_bounds__(kRowwiseThreadsPerBlock) + swizzle_rowwise_per_tile_kernel(const uint8_t* __restrict__ input, uint8_t* __restrict__ output, + int padded_k, uint32_t words_per_row_magic, int original_m, + int original_k) { + extern __shared__ uint32_t rowwise_smem[]; + swizzle_rowwise_tiles(input, output, padded_k, words_per_row_magic, original_m, original_k, + blockIdx.x, 1, rowwise_smem); +} + +__global__ void __launch_bounds__(kColumnwiseThreadsPerBlock) + swizzle_columnwise_per_tile_kernel(const uint8_t* __restrict__ input, + uint8_t* __restrict__ output, int padded_m, int padded_k, + int original_m, int original_k) { + __shared__ int4 columnwise_smem[32 * 8]; + swizzle_columnwise_chunk<32>(input, output, padded_m, padded_k, original_m, original_k, + blockIdx.x, blockIdx.y * 32, columnwise_smem); +} + +// The scales of one tensor: padded_m (a multiple of 128) by padded_k (a multiple of 4) in +// row-wise terms; for column-wise scales, m runs along the columns of the data. Scales at +// m >= original_m or k >= original_k are padding and written as zeros. +struct PerTileSwizzleTensor { + const uint8_t* input; + uint8_t* output; + int padded_m, padded_k, original_m, original_k; +}; + +struct PerTileMultiSwizzleArgs { + PerTileSwizzleTensor tensors[kMaxTensorsPerLaunch]; + int first_block[kMaxTensorsPerLaunch + 1]; + uint32_t words_per_row_magic[kMaxTensorsPerLaunch]; // row-wise only + int tiles_per_block; // row-wise only + int num_tensors; +}; + +__device__ __forceinline__ int tensor_of_block(const PerTileMultiSwizzleArgs& args, int block) { + int tensor = 0; + while (tensor + 1 < args.num_tensors && args.first_block[tensor + 1] <= block) tensor++; + return tensor; +} + +__global__ void __launch_bounds__(kRowwiseThreadsPerBlock) + multi_tensor_swizzle_rowwise_per_tile_kernel(const PerTileMultiSwizzleArgs args) { + extern __shared__ uint32_t rowwise_multi_smem[]; + const int t = tensor_of_block(args, blockIdx.x); + const int tiles_per_block = args.tiles_per_block; + const int first_tile = (blockIdx.x - args.first_block[t]) * tiles_per_block; + const int num_tiles = min(tiles_per_block, args.tensors[t].padded_m / 128 - first_tile); + swizzle_rowwise_tiles(args.tensors[t].input, args.tensors[t].output, args.tensors[t].padded_k, + args.words_per_row_magic[t], args.tensors[t].original_m, + args.tensors[t].original_k, first_tile, num_tiles, rowwise_multi_smem); +} + +template +__global__ void __launch_bounds__(kColumnwiseThreadsPerBlock) + multi_tensor_swizzle_columnwise_per_tile_kernel(const PerTileMultiSwizzleArgs args) { + __shared__ int4 columnwise_multi_smem[kChunkRows * 8]; + const int t = tensor_of_block(args, blockIdx.x); + const int block_in_tensor = blockIdx.x - args.first_block[t]; + const int num_m_tiles = args.tensors[t].padded_m / 128; + const int m_tile = block_in_tensor % num_m_tiles; + const int first_k = (block_in_tensor / num_m_tiles) * kChunkRows; + swizzle_columnwise_chunk(args.tensors[t].input, args.tensors[t].output, + args.tensors[t].padded_m, args.tensors[t].padded_k, + args.tensors[t].original_m, args.tensors[t].original_k, + m_tile, first_k, columnwise_multi_smem); +} + +inline bool is_16_byte_aligned(const void* ptr) { + return (reinterpret_cast(ptr) & 15u) == 0; +} + +inline bool per_tile_swizzle_supported(const PerTileSwizzleTensor& tensor, bool rowwise) { + if (tensor.padded_m <= 0 || tensor.padded_k <= 0 || tensor.padded_m % 128 != 0 || + tensor.padded_k % 4 != 0) { + return false; + } + if (!is_16_byte_aligned(tensor.input) || !is_16_byte_aligned(tensor.output)) return false; + if (rowwise && 128 * ((tensor.padded_k >> 2) | 1) * 4 > 48 * 1024) return false; + return true; +} + +// One kernel for up to kMaxTensorsPerLaunch tensors, all row-wise or all column-wise. Row-wise: +// the tiles per block are the smallest power of two that keeps the grid within kRowwiseMaxBlocks +// (shared memory permitting). Column-wise: 128-row chunks once 32-row chunks would exceed +// kColumnwiseMaxBlocksWith32RowChunks blocks. Returns false, without launching anything, if any +// tensor is not supported. +inline bool launch_multi_tensor_per_tile_swizzle(const PerTileSwizzleTensor* tensors, + int num_tensors, bool rowwise, + cudaStream_t stream) { + if (num_tensors <= 0 || num_tensors > kMaxTensorsPerLaunch) return false; + PerTileMultiSwizzleArgs args; + args.num_tensors = num_tensors; + args.first_block[0] = 0; + int64_t total_tiles = 0; + int max_smem_row_words = 1; + for (int i = 0; i < num_tensors; i++) { + if (!per_tile_swizzle_supported(tensors[i], rowwise)) return false; + args.tensors[i] = tensors[i]; + args.words_per_row_magic[i] = division_magic(tensors[i].padded_k >> 2); + if (rowwise) { + total_tiles += tensors[i].padded_m / 128; + max_smem_row_words = max(max_smem_row_words, (tensors[i].padded_k >> 2) | 1); + } else { + total_tiles += + static_cast(tensors[i].padded_m / 128) * ((tensors[i].padded_k + 31) / 32); + } + } + int tiles_per_block = 1, chunk_rows = 32; + if (rowwise) { + const int max_tiles_per_block = (48 * 1024) / (128 * max_smem_row_words * 4); + while (total_tiles > static_cast(kRowwiseMaxBlocks) * tiles_per_block && + tiles_per_block * 2 <= max_tiles_per_block) { + tiles_per_block *= 2; + } + } else { + if (total_tiles > kColumnwiseMaxBlocksWith32RowChunks) chunk_rows = 128; + } + args.tiles_per_block = tiles_per_block; + for (int i = 0; i < num_tensors; i++) { + const PerTileSwizzleTensor& tensor = tensors[i]; + const int blocks = + rowwise ? (tensor.padded_m / 128 + tiles_per_block - 1) / tiles_per_block + : (tensor.padded_m / 128) * ((tensor.padded_k + chunk_rows - 1) / chunk_rows); + args.first_block[i + 1] = args.first_block[i] + blocks; + } + for (int i = num_tensors + 1; i <= kMaxTensorsPerLaunch; i++) { + args.first_block[i] = args.first_block[num_tensors]; + } + const int grid = args.first_block[num_tensors]; + if (grid == 0) return true; + if (rowwise) { + multi_tensor_swizzle_rowwise_per_tile_kernel<<< + grid, kRowwiseThreadsPerBlock, tiles_per_block * 128 * max_smem_row_words * 4, stream>>>( + args); + } else if (chunk_rows == 128) { + multi_tensor_swizzle_columnwise_per_tile_kernel<128> + <<>>(args); + } else { + multi_tensor_swizzle_columnwise_per_tile_kernel<32> + <<>>(args); + } + NVTE_CHECK_CUDA(cudaGetLastError()); + return true; +} + +inline bool launch_rowwise_per_tile_swizzle(const PerTileSwizzleTensor& tensor, + cudaStream_t stream) { + if (!per_tile_swizzle_supported(tensor, true)) return false; + swizzle_rowwise_per_tile_kernel<<> 2) | 1) * 4, stream>>>( + tensor.input, tensor.output, tensor.padded_k, division_magic(tensor.padded_k >> 2), + tensor.original_m, tensor.original_k); + NVTE_CHECK_CUDA(cudaGetLastError()); + return true; +} + +inline bool launch_columnwise_per_tile_swizzle(const PerTileSwizzleTensor& tensor, + cudaStream_t stream) { + if (!per_tile_swizzle_supported(tensor, false)) return false; + dim3 grid(tensor.padded_m / 128, (tensor.padded_k + 31) / 32); + swizzle_columnwise_per_tile_kernel<<>>( + tensor.input, tensor.output, tensor.padded_m, tensor.padded_k, tensor.original_m, + tensor.original_k); + NVTE_CHECK_CUDA(cudaGetLastError()); + return true; +} + +// A single tensor uses the one-tile-per-block kernels unless its grid would be large; several +// tensors go through one multi-tensor launch. +inline bool launch_per_tile_swizzle(const PerTileSwizzleTensor* tensors, int num_tensors, + bool rowwise, cudaStream_t stream) { + const bool grid_too_large = + num_tensors == 1 && (rowwise ? tensors[0].padded_m / 128 > kRowwiseMaxBlocks + : static_cast(tensors[0].padded_m / 128) * + ((tensors[0].padded_k + 31) / 32) > + kColumnwiseMaxBlocksWith32RowChunks); + if (num_tensors == 1 && !grid_too_large) { + return rowwise ? launch_rowwise_per_tile_swizzle(tensors[0], stream) + : launch_columnwise_per_tile_swizzle(tensors[0], stream); + } + return launch_multi_tensor_per_tile_swizzle(tensors, num_tensors, rowwise, stream); +} + } // namespace void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t stream) { @@ -1099,8 +1422,17 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s const int narrow_k_slm_size = TB_DIM * num_tiles_k * SF_TILE_DIM_M * SF_TILE_DIM_K * static_cast(sizeof(int8_t)); - if (num_tiles_k < TB_DIM && - static_cast(narrow_k_slm_size) <= cuda::max_shared_memory_per_block_optin()) { + const PerTileSwizzleTensor per_tile_tensor{static_cast(input_scale_inv_ptr), + static_cast(output_scale_inv_ptr), + m, + k, + original_M, + original_K}; + if (scaling_mode == NVTE_MXFP8_1D_SCALING && use_per_tile_swizzle() && + per_tile_swizzle_supported(per_tile_tensor, true)) { + launch_per_tile_swizzle(&per_tile_tensor, 1, true, stream); + } else if (num_tiles_k < TB_DIM && static_cast(narrow_k_slm_size) <= + cuda::max_shared_memory_per_block_optin()) { // Narrow-K: batch TB_DIM M-tiles per block, fully utilizing all threads. dim3 num_blocks_narrow(DIVUP(num_tiles_m, TB_DIM)); NVTE_CHECK_CUDA( @@ -1157,8 +1489,17 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s const int narrow_m_slm_size = TB_DIM * num_tiles_m * SF_TILE_DIM_M * SF_TILE_DIM_K * static_cast(sizeof(int8_t)); - if (num_tiles_m < TB_DIM && - static_cast(narrow_m_slm_size) <= cuda::max_shared_memory_per_block_optin()) { + const PerTileSwizzleTensor per_tile_tensor{ + static_cast(input->columnwise_scale_inv.dptr), + static_cast(output->columnwise_scale_inv.dptr), + m, + k, + original_M, + original_K}; + if (use_per_tile_swizzle() && per_tile_swizzle_supported(per_tile_tensor, false)) { + launch_per_tile_swizzle(&per_tile_tensor, 1, false, stream); + } else if (num_tiles_m < TB_DIM && static_cast(narrow_m_slm_size) <= + cuda::max_shared_memory_per_block_optin()) { // Narrow-M: batch TB_DIM K-tiles per block, fully utilizing all threads. dim3 num_blocks_narrow(DIVUP(num_tiles_k, TB_DIM)); NVTE_CHECK_CUDA( @@ -1435,6 +1776,42 @@ void launch_multi_tensor_unswizzle_scaling_factors(MultiSwizzleArgs& kernel_args } } +namespace { + +// MXFP8 multi-tensor swizzle with the per-tile kernels. Returns false without launching anything +// if any tensor is not supported (the caller then uses the generic kernels, which also report +// shape errors). +bool multi_tensor_per_tile_swizzle(const std::vector& input, std::vector& output, + bool rowwise, cudaStream_t stream) { + const size_t num_tensors = input.size(); + std::vector tensors(num_tensors); + for (size_t i = 0; i < num_tensors; i++) { + if (!is_mxfp8_scaling(input[i]->scaling_mode)) return false; + const auto& in_scales = rowwise ? input[i]->scale_inv : input[i]->columnwise_scale_inv; + const auto& out_scales = rowwise ? output[i]->scale_inv : output[i]->columnwise_scale_inv; + if (in_scales.shape.size() != 2) return false; + const int padded_m = static_cast(rowwise ? in_scales.shape[0] : in_scales.shape[1]); + const int padded_k = static_cast(rowwise ? in_scales.shape[1] : in_scales.shape[0]); + if (padded_m <= 0 || padded_k <= 0 || padded_m % 128 != 0 || padded_k % 4 != 0) return false; + if (out_scales.numel() != static_cast(padded_m) * padded_k) return false; + const auto [first_dim, last_dim] = input[i]->flat_2d_dims(); + tensors[i] = {static_cast(in_scales.dptr), + static_cast(out_scales.dptr), + padded_m, + padded_k, + static_cast(rowwise ? first_dim : last_dim), + static_cast((rowwise ? last_dim : first_dim) / MXFP8_BLOCK_SIZE)}; + if (!per_tile_swizzle_supported(tensors[i], rowwise)) return false; + } + for (size_t first = 0; first < num_tensors; first += kMaxTensorsPerLaunch) { + const int count = static_cast(std::min(kMaxTensorsPerLaunch, num_tensors - first)); + launch_per_tile_swizzle(tensors.data() + first, count, rowwise, stream); + } + return true; +} + +} // namespace + void multi_tensor_swizzle_scaling_factors(const std::vector& input, std::vector& output, cudaStream_t stream, bool check_scale_inv_shapes) { @@ -1470,6 +1847,10 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, const bool rowwise_swizzle = all_has_data || all_nvfp4; const bool columnwise_swizzle = all_has_columnwise_data && !all_nvfp4; + if (!all_nvfp4 && use_per_tile_swizzle() && + multi_tensor_per_tile_swizzle(input, output, rowwise_swizzle, stream)) { + return; + } constexpr int SF_TILE_DIM_M = 128; constexpr int SF_TILE_DIM_K = 4; From 2e959f352c8e2d4b2709e6775867f2c7494656f3 Mon Sep 17 00:00:00 2001 From: Siddhartha Raman Sundara Raman Date: Mon, 12 Oct 2026 00:29:41 -0500 Subject: [PATCH 22/31] [PyTorch] Fix NVFP4 weight views of the single-group fused grouped MLP (#3662) With one group, the fused grouped MLP passes the weights to cuDNN as stacked tensors instead of per-group pointers, and two of its NVFP4 views of them were wrong: - Forward: the FC1 weight scale view divided the logical K by the scale-atom divisor, which counts packed bytes for NVFP4 (two values per byte). The view asked for twice the scales and raised. - Backward: the FC2 dgrad viewed the column-wise NVFP4 data as (out_features, in_features / 2), but it is the transposed weight, (in_features, out_features / 2), and swapped the scale atom axes, which cuDNN rejects. Build these operands with _convert_to_cudnn_grouped_gemm_tensor_format, as the FC1 dgrad does. Run the single-group test for every grouped MLP recipe, not only MXFP8. Signed-off-by: Siddhartha Raman S Co-authored-by: Claude Opus 5.5 --- tests/pytorch/test_grouped_mlp.py | 10 +++- .../pytorch/ops/fused/grouped_mlp.py | 55 +++++++++++-------- 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/tests/pytorch/test_grouped_mlp.py b/tests/pytorch/test_grouped_mlp.py index b7309aff751..88af92f0bb9 100644 --- a/tests/pytorch/test_grouped_mlp.py +++ b/tests/pytorch/test_grouped_mlp.py @@ -2054,16 +2054,20 @@ def test_grouped_mlp_fp16( activation=activation, ) + @pytest.mark.parametrize( + "quantization", [q for q in _grouped_mlp_quantization_list if q is not None] + ) @pytest.mark.parametrize("bias", (False, True)) @pytest.mark.parametrize("runtime_offsets_supported", (False, True)) - def test_grouped_mlp_single_group_mxfp8( + def test_grouped_mlp_single_group( self, monkeypatch, *, + quantization: str, bias: bool, runtime_offsets_supported: bool, ) -> None: - """Single-group GroupedLinear + ScaledSwiGLU + GroupedLinear with MXFP8.""" + """Single-group GroupedLinear + ScaledSwiGLU + GroupedLinear.""" if ( runtime_offsets_supported and not grouped_mlp_module._cudnn_frontend_supports_single_group_runtime_offsets( @@ -2080,7 +2084,7 @@ def test_grouped_mlp_single_group_mxfp8( group_size=1, bias=bias, hidden_size=128, - quantization="mxfp8", + quantization=quantization, single_grouped_weight=False, activation="scaled_swiglu", ) diff --git a/transformer_engine/pytorch/ops/fused/grouped_mlp.py b/transformer_engine/pytorch/ops/fused/grouped_mlp.py index 9fd925d32b6..108ce5311c7 100644 --- a/transformer_engine/pytorch/ops/fused/grouped_mlp.py +++ b/transformer_engine/pytorch/ops/fused/grouped_mlp.py @@ -1795,7 +1795,7 @@ def fuser_forward( fc1_w_scales = fc1_w_scales.view( 1, ceil_div(fc1_weight_shape[0], 128), - ceil_div(fc1_weight_shape[1], k_sf_divisor), + ceil_div(fc1_weight_k, k_sf_divisor), 32, 4, 4, @@ -2613,29 +2613,36 @@ def fuser_backward( fc2_weight_single._columnwise_scale_inv = original_columnwise_scale fc2_weight_single._with_gemm_swizzled_scales = original_swizzled - fc2_w_data = fc2_weight_single._columnwise_data.view(dtype=data_dtype) - fc2_w_data = fc2_w_data.view( - 1, - fc2_weight_shape[0], - fc2_weight_k, - ) - fc2_w_data = ( - fc2_w_data.permute(1, 2, 0) if use_nvfp4 else fc2_w_data.permute(2, 1, 0) - ) - fc2_w_scales = swizzled_columnwise_scale.view(dtype=scale_view_dtype) - fc2_w_scales = fc2_w_scales.view( - 1, - ceil_div(fc2_weight_shape[1], k_sf_divisor), - ceil_div(fc2_weight_shape[0], 128), - 32, - 4, - 4, - ) - fc2_w_scales = ( - fc2_w_scales.permute(3, 4, 2, 5, 1, 0) - if use_nvfp4 - else fc2_w_scales.permute(3, 4, 1, 5, 2, 0) - ) + if use_nvfp4: + # NVFP4 column-wise data is physically transposed, so it is already + # (in_features, out_features) and stays K-major. + fc2_w_data, fc2_w_scales = _convert_to_cudnn_grouped_gemm_tensor_format( + fc2_weight_single._columnwise_data, + swizzled_columnwise_scale, + data_dtype=data_dtype, + scale_dtype=scale_view_dtype, + valid_M_or_N=fc2_weight_shape[1], + k_logical=fc2_weight_shape[0], + sf_swizzled=True, + ) + else: + fc2_w_data = fc2_weight_single._columnwise_data.view(dtype=data_dtype) + fc2_w_data = fc2_w_data.view( + 1, + fc2_weight_shape[0], + fc2_weight_k, + ) + fc2_w_data = fc2_w_data.permute(2, 1, 0) + fc2_w_scales = swizzled_columnwise_scale.view(dtype=scale_view_dtype) + fc2_w_scales = fc2_w_scales.view( + 1, + ceil_div(fc2_weight_shape[1], k_sf_divisor), + ceil_div(fc2_weight_shape[0], 128), + 32, + 4, + 4, + ) + fc2_w_scales = fc2_w_scales.permute(3, 4, 1, 5, 2, 0) fc2_dactivation_kwargs["b_tensor"] = fc2_w_data fc2_dactivation_kwargs["sfb_tensor"] = fc2_w_scales else: From 81775f2cc0052d3e75608664357a8f3d3f453897 Mon Sep 17 00:00:00 2001 From: Dmitry Voropaev <144015565+v0ropaev@users.noreply.github.com> Date: Mon, 12 Oct 2026 20:13:37 +0300 Subject: [PATCH 23/31] [PyTorch] Interpolate the class name in six op error messages (#3667) Six RuntimeError messages in the fusible-op API build one string out of four fragments and leave the f off the first one, so the brace expression that names the offending subclass is emitted verbatim: {self.__class__.__name__} operation has 1 extra tensor inputs and 0 extra tensor outputs. It should override `fuser_forward` instead of `op_forward`. The two fragments after it are f-strings and do interpolate, so the counts come out right and only the class name is lost, which is the one part of the message a caller cannot work out for themselves. The same four-fragment message appears six more times in the same tree, in basic/swiglu.py, basic/activation.py and basic/grouped_linear.py, with the f in place. These six are the ones that were missed. No test: reaching these lines needs a built extension and a custom BasicOperation subclass that declares extra tensor inputs or outputs and implements op_forward rather than fuser_forward, and nothing in the tree asserts on the text. The behaviour is a property of the language, not of TE. Signed-off-by: Dmitry Voropaev --- transformer_engine/pytorch/ops/basic/add_extra_input.py | 4 ++-- transformer_engine/pytorch/ops/basic/make_extra_output.py | 4 ++-- transformer_engine/pytorch/ops/op.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/transformer_engine/pytorch/ops/basic/add_extra_input.py b/transformer_engine/pytorch/ops/basic/add_extra_input.py index 9af399f2dc6..87a4afee5a1 100644 --- a/transformer_engine/pytorch/ops/basic/add_extra_input.py +++ b/transformer_engine/pytorch/ops/basic/add_extra_input.py @@ -44,7 +44,7 @@ def __init__(self, *, in_place: bool = False): def op_forward(self, *args, **kwargs) -> None: raise RuntimeError( - "{self.__class__.__name__} operation has " + f"{self.__class__.__name__} operation has " f"{self.num_extra_inputs} extra tensor inputs " f"and {self.num_extra_outputs} extra tensor outputs. " "It overrides `fuser_forward` instead of `op_forward`." @@ -52,7 +52,7 @@ def op_forward(self, *args, **kwargs) -> None: def op_backward(self, *args, **kwargs) -> None: raise RuntimeError( - "{self.__class__.__name__} operation has " + f"{self.__class__.__name__} operation has " f"{self.num_extra_inputs} extra tensor inputs " f"and {self.num_extra_outputs} extra tensor outputs. " "It overrides `fuser_backward` instead of `op_backward`." diff --git a/transformer_engine/pytorch/ops/basic/make_extra_output.py b/transformer_engine/pytorch/ops/basic/make_extra_output.py index 47b7c6495de..dfe5c4b27d9 100644 --- a/transformer_engine/pytorch/ops/basic/make_extra_output.py +++ b/transformer_engine/pytorch/ops/basic/make_extra_output.py @@ -49,7 +49,7 @@ def __init__(self, *, in_place: bool = False): def op_forward(self, *args, **kwargs) -> None: raise RuntimeError( - "{self.__class__.__name__} operation has " + f"{self.__class__.__name__} operation has " f"{self.num_extra_inputs} extra tensor inputs " f"and {self.num_extra_outputs} extra tensor outputs. " "It overrides `fuser_forward` instead of `op_forward`." @@ -57,7 +57,7 @@ def op_forward(self, *args, **kwargs) -> None: def op_backward(self, *args, **kwargs) -> None: raise RuntimeError( - "{self.__class__.__name__} operation has " + f"{self.__class__.__name__} operation has " f"{self.num_extra_inputs} extra tensor inputs " f"and {self.num_extra_outputs} extra tensor outputs. " "It overrides `fuser_backward` instead of `op_backward`." diff --git a/transformer_engine/pytorch/ops/op.py b/transformer_engine/pytorch/ops/op.py index a4ff7503cae..0bf247e12f1 100644 --- a/transformer_engine/pytorch/ops/op.py +++ b/transformer_engine/pytorch/ops/op.py @@ -721,7 +721,7 @@ def fuser_forward( ) if self.num_extra_inputs > 0 or self.num_extra_outputs > 0: raise RuntimeError( - "{self.__class__.__name__} operation has " + f"{self.__class__.__name__} operation has " f"{self.num_extra_inputs} extra tensor inputs " f"and {self.num_extra_outputs} extra tensor outputs. " "It should override `fuser_forward` instead of `op_forward`." @@ -756,7 +756,7 @@ def fuser_backward( ) if self.num_extra_inputs > 0 or self.num_extra_outputs > 0: raise RuntimeError( - "{self.__class__.__name__} operation has " + f"{self.__class__.__name__} operation has " f"{self.num_extra_inputs} extra tensor inputs " f"and {self.num_extra_outputs} extra tensor outputs. " "It should override `fuser_backward` instead of `op_backward`." From 1c9f6f8f39d6972d8dc9bd44de7855aa740d82c5 Mon Sep 17 00:00:00 2001 From: Sudhakar Singh Date: Mon, 12 Oct 2026 15:38:31 -0700 Subject: [PATCH 24/31] [PyTorch][CP] Support D=256 THD metadata with FA4 b33 (#3540) * Update FA4 D=256 capability gates FA4 b31 added seqused_q/k forward support for the SM100 D=256 kernel, while b33 added backward support. Select FA4 by the installed release and actual backward eligibility, retain the deterministic backward guard, and exercise the released b33 dependency stack in L3. Signed-off-by: Sudhakar Singh * Cover FA4 D=256 seqused paths Protect the release boundary with focused selector cases and extend the existing no-load-balance CP test to run D=256. This verifies the existing all-gather metadata path without adding strategy-specific production plumbing. Signed-off-by: Sudhakar Singh * Integrate FA4 window sentinel handling into D=256 coverage FA4 b33 treats TE's -1 unbounded window sentinel as a literal offset, so the D=256 coverage in this PR needs the adapter from #3532 at all four FA4 entry points. Bring that final adapter change into this branch instead of leaving the new FA4 path dependent on another open PR. Keep the regression tests in test_attention_backend_selection.py alongside the focused FA4 tests. Cover both window argument forms and preserved negative offsets, compare every batch and head with a float64 reference, and run these cases under the FA4 b33 L3 lane. Signed-off-by: Sudhakar Singh * Shorten FA4 sentinel regression explanations Keep the adapter comment focused on the sentinel contract and the FA4-only boundary. Name the numerical test for its independent reference and remove the redundant all-zero assertion, which singled out one symptom already caught by the full-output comparison. Signed-off-by: Sudhakar Singh * Remove redundant FA4 float64 reference test The standalone D=128 BSHD comparison does not exercise the D=256 THD path added by this PR. Keep the direct window-sentinel regression in L3; existing attention tests provide numerical comparisons. Signed-off-by: Sudhakar Singh * Restore FA4 numerical sentinel regression The argument-only sentinel test cannot prove the adapted FA4 call produces correct attention; generic cross-backend comparisons may skip. Restore the independent reference check and keep only representative left, right, and non-sentinel cases for both window argument forms. Signed-off-by: Sudhakar Singh * Update FA4 L3 dependency and backend coverage CUTLASS DSL 4.8.0 recognizes SM107 while the previous pin does not, so keep the FA4 L3 dependency and installation guidance aligned. Running the small backend-selection module directly covers its eligibility and window-sentinel cases without a bespoke node list. Clarify that non-all-gather THD metadata depends on pad_between_seqs. Signed-off-by: Sudhakar Singh * Keep FA4 regression tests focused on model configs A dedicated D=256 model preserves separate QK and V dimensions without a one-off runner override. Exclude it from the broad CP matrix because the no-load-balance test already exercises the intended path. The L3 causal cross-backend cases cover numerical behavior, so retain the sentinel unit test and remove the redundant standalone reference test. Signed-off-by: Sudhakar Singh * Keep FA4 expectation explicit in CP test Query backend availability before deciding whether to skip the D=256 case. Supported FA4 b33 configurations should fail when FA4 is unexpectedly unavailable, while unsupported configurations still skip. Keep the general no-Flash skip separate from the D=256 assertion. Signed-off-by: Sudhakar Singh * Keep full CP model catalogs for focused tests The earlier D=256 broad-matrix exclusion kept cp_2_4 in the essential FlashAttention catalog solely for the focused no-load-balance case. That mixes model availability with test selection. Keep the full FlashAttention and FusedAttention catalogs available to the runner and focused tests, and narrow only the essential broad CP parametrizations. Non-essential broad runs continue to use the full catalog. Signed-off-by: Sudhakar Singh --------- Signed-off-by: Sudhakar Singh --- qa/L3_pytorch_FA_versions_test/test.sh | 17 +++- .../test_attention_backend_selection.py | 94 ++++++++++++++++--- .../attention/test_attention_with_cp.py | 32 +++++-- .../dot_product_attention/backends.py | 32 ++++++- .../attention/dot_product_attention/utils.py | 42 ++++++++- 5 files changed, 184 insertions(+), 33 deletions(-) diff --git a/qa/L3_pytorch_FA_versions_test/test.sh b/qa/L3_pytorch_FA_versions_test/test.sh index e7fd3189f35..266abff774d 100644 --- a/qa/L3_pytorch_FA_versions_test/test.sh +++ b/qa/L3_pytorch_FA_versions_test/test.sh @@ -36,8 +36,8 @@ export FLASH_ATTN_CUDA_ARCHS=$sm_arch CP_FA_VERSION="" if [ $sm_arch -gt 90 ] then - FA_versions=(4.0.0b11) - CP_FA_VERSION="4.0.0b11" + FA_versions=(4.0.0b33) + CP_FA_VERSION="4.0.0b33" elif [ $sm_arch -eq 90 ] then FA_versions=(3.0.0b1) @@ -66,13 +66,13 @@ do elif [[ "${fa_version}" == 4.* ]] then export NVTE_FLASH_ATTN_V4=1 - # FA4 is intentionally last in every version array. Its b11 test pin needs - # CUTLASS DSL 4.4.2, so replace the image-matched stack only for this final + # FA4 is intentionally last in every version array. CUTLASS DSL 4.8.0 supports + # SM107, so replace the image-matched stack only for this final # iteration; later iterations would otherwise need that stack restored. pip3 uninstall -y nvidia-cutlass-dsl nvidia-cutlass-dsl-libs-base \ nvidia-cutlass-dsl-libs-cu12 nvidia-cutlass-dsl-libs-cu13 \ || error_exit "Failed to isolate CUTLASS DSL for Flash Attention $fa_version" - pip3 install flash-attn-4==${fa_version} nvidia-cutlass-dsl[cu13]==4.4.2 \ + pip3 install flash-attn-4==${fa_version} nvidia-cutlass-dsl[cu13]==4.8.0 \ --no-build-isolation || error_exit "Failed to install Flash Attention $fa_version" else export NVTE_FLASH_ATTN_V3=1 @@ -135,6 +135,13 @@ do fi NVTE_TORCH_COMPILE=0 NVTE_ALLOW_UNSAFE_PICKLE_EXTRA_STATE=1 python3 -m pytest -v -s --junitxml=$XML_ATTN $TE_PATH/tests/pytorch/attention/test_attention.py || test_fail "test_attention.py (FA $fa_version)" fi + + if [[ "${fa_version}" == 4.* ]]; then + XML_BACKEND="$XML_LOG_DIR/pytest_test_attention_backend_selection_fa${fa_tag}.xml" + NVTE_TORCH_COMPILE=0 python3 -m pytest -v -s --junitxml="$XML_BACKEND" \ + "$TE_PATH/tests/pytorch/attention/test_attention_backend_selection.py" \ + || test_fail "FA4 backend-selection tests (FA $fa_version)" + fi done if [ "$RET" -ne 0 ]; then diff --git a/tests/pytorch/attention/test_attention_backend_selection.py b/tests/pytorch/attention/test_attention_backend_selection.py index c2944eece3b..bd587416a79 100644 --- a/tests/pytorch/attention/test_attention_backend_selection.py +++ b/tests/pytorch/attention/test_attention_backend_selection.py @@ -12,6 +12,7 @@ from transformer_engine.pytorch import DotProductAttention from transformer_engine.pytorch.attention.dot_product_attention import ( + backends as dpa_backends, dot_product_attention as dpa_module, utils as dpa_utils, ) @@ -42,19 +43,20 @@ def isolated_backends(monkeypatch): @pytest.mark.parametrize( - "capability,training,backward,deterministic,unfused,expect_flash", + "capability,training,backward,deterministic,head_dims,unfused,expect_flash", [ - pytest.param((12, 0), True, True, True, True, False, id="sm120-training"), - pytest.param((12, 0), False, True, True, True, False, id="sm120-eval-backward"), - pytest.param((12, 1), False, True, True, True, False, id="sm121-eval-backward"), - pytest.param((12, 0), False, False, True, True, True, id="forward-only"), - pytest.param((12, 0), True, True, False, True, True, id="nondeterministic"), - pytest.param((12, 0), True, True, True, False, False, id="no-fallback"), - pytest.param((10, 0), True, True, True, True, True, id="sm100-unaffected"), + pytest.param((12, 0), True, True, True, (128, 128), True, False, id="sm120-training"), + pytest.param((12, 0), False, True, True, (128, 128), True, False, id="sm120-eval-backward"), + pytest.param((12, 1), False, True, True, (128, 128), True, False, id="sm121-eval-backward"), + pytest.param((12, 0), False, False, True, (128, 128), True, True, id="forward-only"), + pytest.param((12, 0), True, True, False, (128, 128), True, True, id="nondeterministic"), + pytest.param((12, 0), True, True, True, (128, 128), False, False, id="no-fallback"), + pytest.param((10, 0), True, True, True, (128, 128), True, True, id="sm100-d128"), + pytest.param((10, 0), True, True, True, (256, 256), True, False, id="sm100-d256"), ], ) def test_fa4_deterministic_backend_selection( - monkeypatch, capability, training, backward, deterministic, unfused, expect_flash + monkeypatch, capability, training, backward, deterministic, head_dims, unfused, expect_flash ): monkeypatch.setenv("NVTE_UNFUSED_ATTN", str(int(unfused))) monkeypatch.setattr(dpa_utils, "get_device_compute_capability", lambda device: capability) @@ -66,8 +68,8 @@ def test_fa4_deterministic_backend_selection( device=torch.device("cuda:0"), qkv_dtype=torch.bfloat16, qkv_layout="bshd_bshd_bshd", - head_dim_qk=128, - head_dim_v=128, + head_dim_qk=head_dims[0], + head_dim_v=head_dims[1], attn_mask_type="causal", window_size=(-1, 0), deterministic=deterministic, @@ -82,6 +84,44 @@ def test_fa4_deterministic_backend_selection( assert available == [expect_flash, False, unfused] +@pytest.mark.parametrize( + "version,backward,expect_flash", + [ + pytest.param("4.0.0b30", False, False, id="b30-forward"), + pytest.param("4.0.0b31", False, True, id="b31-forward"), + pytest.param("4.0.0b32", True, False, id="b32-backward"), + pytest.param("4.0.0b33", True, True, id="b33-backward"), + ], +) +@pytest.mark.parametrize("path", ["compact", "padding", "all-gather"]) +def test_fa4_d256_seqused_backend_selection(monkeypatch, version, backward, expect_flash, path): + """FA4 D=256 eligibility follows seqused_q/k forward/backward support.""" + monkeypatch.setenv("NVTE_ALLOW_NONDETERMINISTIC_ALGO", "1") + monkeypatch.setattr(dpa_utils, "get_device_compute_capability", lambda device: (10, 0)) + monkeypatch.setattr(dpa_utils, "get_cudnn_version", lambda: (9, 26, 0)) + monkeypatch.setattr(dpa_utils.FlashAttentionUtils, "v4_is_installed", True) + monkeypatch.setattr(dpa_utils.FlashAttentionUtils, "fa4_version", PkgVersion(version)) + monkeypatch.setattr(dpa_utils.FlashAttentionUtils, "v4_validate_head_dims", None) + params = dpa_utils.AttentionParams( + device=torch.device("cuda:0"), + qkv_dtype=torch.bfloat16, + qkv_layout="thd_thd_thd", + head_dim_qk=256, + head_dim_v=256, + attn_mask_type="padding_causal", + pad_between_seqs=path == "padding", + context_parallel=path == "all-gather", + cp_comm_type="all_gather" if path == "all-gather" else "p2p", + deterministic=False, + requires_backward=backward, + ) + flash, selected_version, _, _, _, available = dpa_utils.get_attention_backend(params) + expected = path == "compact" or expect_flash + assert bool(flash) is expected + assert selected_version == (dpa_utils.FlashAttentionUtils.fa4_version if expected else None) + assert bool(available[0]) is expected + + @pytest.mark.parametrize("changed_field", ["device", "requires_backward"]) def test_thd_backend_cache_tracks_execution_context(monkeypatch, changed_field): selector = Mock(return_value=(False, None, False, None, True, [False, False, True])) @@ -161,3 +201,35 @@ def test_fa4_cached_selection_from_other_device(monkeypatch): assert selector.call_args.args[0].device == qkv[0].device assert dpa_module._attention_backends["use_unfused_attention"] assert all(t.grad is not None and torch.isfinite(t.grad).all() for t in qkv) + + +def _fa4_normalized_kwargs(kwargs): + received = {} + dpa_backends._fa4_with_none_window_sentinel(received.update)(**kwargs) + return received + + +@pytest.mark.parametrize( + "sent,expected", + [ + ({"window_size": (-1, 0)}, {"window_size": (None, 0)}), + ({"window_size": (511, -1)}, {"window_size": (511, None)}), + # Other negative offsets are not the -1 sentinel. + ({"window_size": (7, -8)}, {"window_size": (7, -8)}), + ( + {"window_size_left": -1, "window_size_right": 0}, + {"window_size_left": None, "window_size_right": 0}, + ), + ( + {"window_size_left": 0, "window_size_right": -1}, + {"window_size_left": 0, "window_size_right": None}, + ), + ( + {"window_size_left": 7, "window_size_right": -8}, + {"window_size_left": 7, "window_size_right": -8}, + ), + ], +) +def test_fa4_window_sentinel_normalization(sent, expected): + """Convert only TE's -1 sentinel for both FA4 window argument forms.""" + assert _fa4_normalized_kwargs(sent) == expected diff --git a/tests/pytorch/attention/test_attention_with_cp.py b/tests/pytorch/attention/test_attention_with_cp.py index 778c73fd40f..56055ca1694 100644 --- a/tests/pytorch/attention/test_attention_with_cp.py +++ b/tests/pytorch/attention/test_attention_with_cp.py @@ -61,6 +61,7 @@ "cp_2_1": ModelConfig(2, 4096, 12, 128, num_gqa_groups=2), # GQA "cp_2_2": ModelConfig(2, 4096, 32, 128, attn_mask_type="causal", window_size=(128, 0)), # GQA "cp_2_3": ModelConfig(2, 4096, 12, 128, num_gqa_groups=2, window_size=(512, 512)), # GQA + "cp_2_4": ModelConfig(2, 4096, 32, 256, num_gqa_groups=4, attn_mask_type="causal"), # GQA "cp_3_0": ModelConfig(2, 4096, 128, 192, attn_mask_type="causal", head_dim_v=128), # MLA "cp_3_1": ModelConfig(2, 4096, 12, 192, head_dim_v=128), # MLA "cp_3_2": ModelConfig( @@ -328,9 +329,10 @@ def _submit(pool: PoolWorker, **kwargs) -> None: dtypes = ["bf16", "fp16"] qkv_formats = ["bshd", "sbhd", "thd"] cp_comm_types = ["p2p", "all_gather", "a2a", "a2a+p2p"] +essential_model_configs_flash_attn = model_configs_flash_attn if test_essential: configs = ["cp_2_0", "cp_2_2", "cp_3_0", "cp_3_3"] - model_configs_flash_attn = {k: model_configs_flash_attn[k] for k in configs} + essential_model_configs_flash_attn = {k: model_configs_flash_attn[k] for k in configs} dtypes = ["bf16"] qkv_formats = ["sbhd", "thd"] @@ -345,7 +347,7 @@ def _submit(pool: PoolWorker, **kwargs) -> None: ) @pytest.mark.skipif(get_device_compute_capability() < (8, 0), reason="CP tests require sm80+.") @pytest.mark.parametrize("dtype", dtypes) -@pytest.mark.parametrize("model", model_configs_flash_attn.keys()) +@pytest.mark.parametrize("model", essential_model_configs_flash_attn.keys()) @pytest.mark.parametrize("qkv_format", qkv_formats) @pytest.mark.parametrize("cp_comm_type", cp_comm_types) @pytest.mark.parametrize("pad_between_seqs", [False, True]) @@ -561,6 +563,7 @@ def test_cp_with_flash_attention_softcap(cp_pool, cp_comm_type): dtypes = ["bf16", "fp16", "fp8"] qkv_formats = ["bshd", "sbhd", "thd"] cp_comm_types = ["p2p", "all_gather", "a2a", "a2a+p2p"] +essential_model_configs_fused_attn = model_configs_fused_attn if test_essential: configs = [ "cp_1_0", @@ -576,14 +579,14 @@ def test_cp_with_flash_attention_softcap(cp_pool, cp_comm_type): "cp_5_0", "cp_5_1", ] - model_configs_fused_attn = {k: model_configs_fused_attn[k] for k in configs} + essential_model_configs_fused_attn = {k: model_configs_fused_attn[k] for k in configs} dtypes = ["bf16", "fp8"] qkv_formats = ["sbhd", "thd"] @pytest.mark.skipif(get_device_compute_capability() < (8, 0), reason="CP tests require sm80+.") @pytest.mark.parametrize("dtype", dtypes) -@pytest.mark.parametrize("model", model_configs_fused_attn.keys()) +@pytest.mark.parametrize("model", essential_model_configs_fused_attn.keys()) @pytest.mark.parametrize("qkv_format", qkv_formats) @pytest.mark.parametrize("cp_comm_type", cp_comm_types) @pytest.mark.parametrize("fp8_bwd", [True, False]) @@ -788,13 +791,22 @@ def test_cp_with_fused_attention_no_load_balance(cp_pool): ) -def test_cp_with_flash_attention_no_load_balance(cp_pool): +@pytest.mark.parametrize("model", ("cp_2_0", "cp_2_4")) +def test_cp_with_flash_attention_no_load_balance(cp_pool, model): """Check the supported unpadded FlashAttention path.""" - config = copy.deepcopy(model_configs_flash_attn["cp_2_0"]) + config = copy.deepcopy(model_configs_flash_attn[model]) + fa4_expected = ( + config.head_dim_qk == 256 + and (10, 0) <= get_device_compute_capability() < (12, 0) + and fa4_enabled + and FlashAttentionUtils.v4_is_installed + and FlashAttentionUtils.fa4_version >= FlashAttentionUtils.v4_0_0_beta33 + and not _deterministic + ) config.context_parallel = True config.cp_comm_type = "all_gather" config.attn_mask_type = "padding_causal" - available_backends, _, _ = get_available_attention_backends( + available_backends, flash_backend, _ = get_available_attention_backends( config, qkv_dtype=torch.bfloat16, qkv_layout="thd_thd_thd", @@ -802,12 +814,16 @@ def test_cp_with_flash_attention_no_load_balance(cp_pool): is_training=True, deterministic=_deterministic, ) + if fa4_expected: + assert available_backends[0] and flash_backend is not None and flash_backend.major == 4 + elif config.head_dim_qk == 256: + pytest.skip("D=256 THD all-gather requires SM100/SM110 FA4 b33+ non-deterministic.") if not available_backends[0]: pytest.skip("FlashAttention is unavailable.") _submit( cp_pool(2), dtype="bf16", - model="cp_2_0", + model=model, qkv_format="thd", kernel_backend="FlashAttention", cp_comm_type="all_gather", diff --git a/transformer_engine/pytorch/attention/dot_product_attention/backends.py b/transformer_engine/pytorch/attention/dot_product_attention/backends.py index 6c4ac477781..bccb4f1e26f 100644 --- a/transformer_engine/pytorch/attention/dot_product_attention/backends.py +++ b/transformer_engine/pytorch/attention/dot_product_attention/backends.py @@ -8,6 +8,7 @@ from dataclasses import dataclass from importlib.metadata import version as get_pkg_version from importlib.metadata import PackageNotFoundError +import functools import inspect import os from typing import Any, Callable, Dict, List, Optional, Tuple, Union @@ -222,6 +223,27 @@ except (ValueError, TypeError): fa_utils.fa3_supports_softcap = False + +def _fa4_with_none_window_sentinel(func: Callable) -> Callable: + """Translate TE's ``-1`` unbounded-window sentinel to FA4's ``None``. + + Preserve other negative offsets, which can represent empty windows. Adapt the FA4 entry + points because shared FA2/FA3 callers still use ``-1``. + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): + window = kwargs.get("window_size") + if window is not None: + kwargs["window_size"] = tuple(None if bound == -1 else bound for bound in window) + for bound_name in ("window_size_left", "window_size_right"): + if kwargs.get(bound_name) == -1: + kwargs[bound_name] = None + return func(*args, **kwargs) + + return wrapper + + # Try to import Flash Attention v4 try: fa_utils.fa4_version = PkgVersion(get_pkg_version("flash-attn-4")) @@ -263,10 +285,12 @@ else: # Unlike versions 2 and 3, FlashAttention 4 registers no custom ops: it builds # its kernels through the CUTLASS DSL as it runs. Keep it an eager island. - flash_attn_func_v4 = no_torch_dynamo()(_flash_attn_func_v4) - flash_attn_varlen_func_v4 = no_torch_dynamo()(_flash_attn_varlen_func_v4) - _flash_attn_fwd_v4 = no_torch_dynamo()(_flash_attn_fwd_v4) - _flash_attn_bwd_v4 = no_torch_dynamo()(_flash_attn_bwd_v4) + flash_attn_func_v4 = no_torch_dynamo()(_fa4_with_none_window_sentinel(_flash_attn_func_v4)) + flash_attn_varlen_func_v4 = no_torch_dynamo()( + _fa4_with_none_window_sentinel(_flash_attn_varlen_func_v4) + ) + _flash_attn_fwd_v4 = no_torch_dynamo()(_fa4_with_none_window_sentinel(_flash_attn_fwd_v4)) + _flash_attn_bwd_v4 = no_torch_dynamo()(_fa4_with_none_window_sentinel(_flash_attn_bwd_v4)) fa_utils.v4_validate_head_dims = _fa4_validate_head_dims fa_utils.set_flash_attention_4_params() diff --git a/transformer_engine/pytorch/attention/dot_product_attention/utils.py b/transformer_engine/pytorch/attention/dot_product_attention/utils.py index 3e11039c1f4..a773091cd4e 100644 --- a/transformer_engine/pytorch/attention/dot_product_attention/utils.py +++ b/transformer_engine/pytorch/attention/dot_product_attention/utils.py @@ -148,11 +148,13 @@ class FlashAttentionUtils: v4_is_installed = False fa4_version = PkgVersion("0") + v4_0_0_beta31 = PkgVersion("4.0.0b31") + v4_0_0_beta33 = PkgVersion("4.0.0b33") use_v4 = False # Set by a signature probe in backends.py; fail-closed default. fa3_supports_softcap = False v4_installation_steps = """\ -pip install flash-attn-4==4.0.0b11 nvidia-cutlass-dsl[cu13]""" +pip install flash-attn-4==4.0.0b33 nvidia-cutlass-dsl[cu13]==4.8.0""" v4_warning_printed = False # Set by backends.py if FA4 is installed; calls flash_attn.cute.interface._validate_head_dims # which raises AssertionError for unsupported (head_dim, head_dim_v) combinations. @@ -1286,6 +1288,32 @@ def _is_fa3_supported(num_heads, num_gqa_groups, head_dim_qk, head_dim_v, qkv_dt cp_comm_type, ) use_flash_attention_4 = False + # SM100/SM110 D=256 gained seqused_q/k support in b31 for forward and b33 for backward. + # THD all-gather always passes this metadata; other THD paths pass it only + # when pad_between_seqs=True. + fa4_uses_seqused = qkv_format == "thd" and ( + pad_between_seqs or (context_parallel and cp_comm_type == "all_gather") + ) + if ( # pylint: disable=too-many-boolean-expressions + use_flash_attention_4 + and FlashAttentionUtils.v4_is_installed + and (10, 0) <= device_compute_capability < (12, 0) + and head_dim_qk == head_dim_v == 256 + and fa4_uses_seqused + and ( + FlashAttentionUtils.fa4_version < FlashAttentionUtils.v4_0_0_beta31 + or ( + attention_params.requires_backward + and FlashAttentionUtils.fa4_version < FlashAttentionUtils.v4_0_0_beta33 + ) + ) + ): + logger.debug( + "Disabling FlashAttention 4 for SM100/SM110 head_dim=256 with seqused_q/k: " + "version %s does not support the requested execution direction.", + FlashAttentionUtils.fa4_version, + ) + use_flash_attention_4 = False if context_parallel and ( use_flash_attention_2 or use_flash_attention_3 or use_flash_attention_4 ): @@ -1765,11 +1793,15 @@ def _is_fa3_supported(num_heads, num_gqa_groups, head_dim_qk, head_dim_v, qkv_dt ) use_flash_attention_3 = False if use_flash_attention_4 and deterministic and FlashAttentionUtils.v4_is_installed: - # eval() does not disable autograd; only forward-only calls can use this kernel. - if attention_params.requires_backward and device_compute_capability[0] == 12: + # eval() does not disable autograd. SM12x backward and SM100/SM110 D=256 backward + # reject deterministic execution, so only forward-only calls can use those kernels. + if attention_params.requires_backward and ( + device_compute_capability[0] == 12 + or ((10, 0) <= device_compute_capability < (12, 0) and head_dim_qk == head_dim_v == 256) + ): logger.debug( - "Disabling FlashAttention 4 for deterministic backward on SM12x: the CuTe" - " SM12x backward kernel does not support deterministic execution." + "Disabling FlashAttention 4 because the selected backward kernel does not " + "support deterministic execution." ) use_flash_attention_4 = False if use_fused_attention and deterministic: From 459dedc945a36abbb728479c165acc8a2c196968 Mon Sep 17 00:00:00 2001 From: Siddhartha Raman Sundara Raman Date: Mon, 12 Oct 2026 18:51:55 -0500 Subject: [PATCH 25/31] [Common][PyTorch] Fuse the MLA key/value RoPE with MXFP8 quantization (#3672) * [Common][PyTorch] Fuse the MLA key/value RoPE with MXFP8 quantization For MXFP8 attention, multi-latent attention writes its key and value in BF16 (the key/value split, with the shared key position embedding rotated and appended to every head's key) and then reads them again to quantize them. Add a kernel that reads the key/value up-projection output and the key position embedding once and writes the MXFP8 key and value directly: the same bytes as mxfp8_quantize_only applied to the RoPE'd key and the value, row-wise and column-wise, with compact scaling factors. The RoPE is that of Megatron-LM's MLA kernels, with their roundings, for DeepSeek-V3's head dimensions (128 + 64 key, 128 value) on SM 10.0+. It is exposed as nvte_fused_mla_kv_rope_mxfp8 and, for PyTorch, as fused_mla_kv_rope_mxfp8 next to mxfp8_quantize_only, whose re-wrap of the quantized 2D view in the input's 4D shape moves into a helper that both use. At [4096, 1, 128 heads] on VR200, the key/value split and quantization take 100.1 us instead of 60.8 + 75.6 us. Signed-off-by: Siddhartha Raman S Co-Authored-By: Claude Opus 5.5 * [PyTorch] Address review of the fused MLA key/value RoPE + MXFP8 kernel - Run fused_mla_kv_rope_mxfp8 on the inputs' device: check that kv, k_rope, cos and sin are on one CUDA device, guard it, and allocate key and value there. - Reject MXFP8 quantizers with 2D quantization, which the kernel does not implement. - Rename k_pos_emb to k_rope: it holds the key's RoPE dimensions, not a position embedding table. - Merge test_fused_mla_q_uproj.py and test_fused_mla_kv_rope_mxfp8.py into test_fused_mla.py, and test the device handling and the 2D quantizer check. Signed-off-by: Siddhartha Raman S Co-Authored-By: Claude Opus 5.5 --------- Signed-off-by: Siddhartha Raman S Co-authored-by: Claude Opus 5.5 --- qa/L0_pytorch_unittest/test.sh | 2 +- ...fused_mla_q_uproj.py => test_fused_mla.py} | 163 ++++++- transformer_engine/common/CMakeLists.txt | 1 + .../fused_rope/fused_mla_kv_rope_mxfp8.cu | 451 ++++++++++++++++++ .../include/transformer_engine/fused_rope.h | 41 ++ .../attention/dot_product_attention/utils.py | 148 ++++-- transformer_engine/pytorch/csrc/extensions.h | 4 + .../pytorch/csrc/extensions/apply_rope.cpp | 65 +++ .../pytorch/csrc/extensions/pybind.cpp | 4 + 9 files changed, 835 insertions(+), 44 deletions(-) rename tests/pytorch/attention/{test_fused_mla_q_uproj.py => test_fused_mla.py} (61%) create mode 100644 transformer_engine/common/fused_rope/fused_mla_kv_rope_mxfp8.cu diff --git a/qa/L0_pytorch_unittest/test.sh b/qa/L0_pytorch_unittest/test.sh index 2a486450a20..7ace265048c 100644 --- a/qa/L0_pytorch_unittest/test.sh +++ b/qa/L0_pytorch_unittest/test.sh @@ -70,7 +70,7 @@ NVTE_GDN2_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/py NVTE_GDP_TEST_REQUIRED=1 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_gdp_attention.xml $TE_PATH/tests/pytorch/attention/test_gdp_attention.py || test_fail "test_gdp_attention.py" NVTE_ALLOW_UNSAFE_PICKLE_EXTRA_STATE=1 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_attention_deterministic.xml $TE_PATH/tests/pytorch/attention/test_attention.py || test_fail "NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 test_attention.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_linear_mxfp8_attention.xml $TE_PATH/tests/pytorch/attention/test_linear_mxfp8_attention.py || test_fail "test_linear_mxfp8_attention.py" -python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_fused_mla_q_uproj.xml $TE_PATH/tests/pytorch/attention/test_fused_mla_q_uproj.py || test_fail "test_fused_mla_q_uproj.py" +python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_fused_mla.xml $TE_PATH/tests/pytorch/attention/test_fused_mla.py || test_fail "test_fused_mla.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_kv_cache.xml $TE_PATH/tests/pytorch/attention/test_kv_cache.py || test_fail "test_kv_cache.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_cu_seqlens_cache.xml $TE_PATH/tests/pytorch/attention/test_cu_seqlens_cache.py || test_fail "test_cu_seqlens_cache.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_mixed_thd_attention.xml $TE_PATH/tests/pytorch/attention/test_mixed_thd_attention.py || test_fail "test_mixed_thd_attention.py" diff --git a/tests/pytorch/attention/test_fused_mla_q_uproj.py b/tests/pytorch/attention/test_fused_mla.py similarity index 61% rename from tests/pytorch/attention/test_fused_mla_q_uproj.py rename to tests/pytorch/attention/test_fused_mla.py index 421c849a6ed..1fcd9a8e218 100644 --- a/tests/pytorch/attention/test_fused_mla_q_uproj.py +++ b/tests/pytorch/attention/test_fused_mla.py @@ -2,24 +2,39 @@ # # See LICENSE for license information. -"""Unit tests for FusedMLAQUpProjRopeQuant. +"""Unit tests for the fused multi-latent attention (MLA) kernels. + +- FusedMLAQUpProjRopeQuant: query up-projection, RoPE and MXFP8 quantization (cuDNN FE). +- fused_mla_kv_rope_mxfp8: key/value split, key RoPE and MXFP8 quantization. The reference + applies Megatron-LM's MLA key/value RoPE (the Triton kernel in mla_rope_utils) and quantizes + key and value with mxfp8_quantize_only; the fused kernel must produce the same bytes. Run: - pytest tests/pytorch/attention/test_fused_mla_q_uproj.py -v + pytest tests/pytorch/attention/test_fused_mla.py -v """ import os +import pathlib +import sys import pytest import torch -import transformer_engine.pytorch # registers transformer_engine_torch +import transformer_engine.pytorch as te import transformer_engine_torch as tex from transformer_engine.pytorch.attention import FusedMLAQUpProjFunction, FusedMLAQUpProjRopeQuant +from transformer_engine.pytorch.attention.dot_product_attention.utils import ( + fused_mla_kv_rope_mxfp8, + mxfp8_quantize_only, +) from transformer_engine.pytorch.cpp_extensions import general_gemm as _fused_general_gemm from transformer_engine.pytorch.tensor.mxfp8_tensor import MXFP8Quantizer, MXFP8Tensor -# All tests in this file require GB200/SM100 hardware (cuDNN FE fused kernel). +_current_file = pathlib.Path(__file__).resolve() +sys.path = [str(_current_file.parent)] + sys.path +from mla_rope_utils import HAVE_TRITON, apply_mla_rope_kv, build_rope_tables + +# All tests in this file require GB200/SM100 hardware. pytestmark = pytest.mark.launch_on_gb200 # DSv3 671B MLA dims @@ -307,3 +322,143 @@ def test_fused_mla_q_uproj_autograd() -> None: torch.testing.assert_close(x.grad.reshape(tokens, Q_LORA_RANK), grad_x_ref, atol=0.5, rtol=0.1) torch.testing.assert_close(gamma.grad, grad_gamma_ref, atol=0.5, rtol=0.1) torch.testing.assert_close(w_bf16.grad, grad_w_ref, atol=0.5, rtol=0.1) + + +# --- fused_mla_kv_rope_mxfp8 --------------------------------------------------------------------- + +mxfp8_available, reason_for_no_mxfp8 = te.is_mxfp8_available(return_reason=True) +sm100 = torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 10 +if not mxfp8_available: + _kv_rope_skip_reason = reason_for_no_mxfp8 +elif not sm100: + _kv_rope_skip_reason = "The fused kernel requires SM 10.0+." +elif not HAVE_TRITON: + _kv_rope_skip_reason = "The reference MLA RoPE needs Triton." +else: + _kv_rope_skip_reason = "" +requires_kv_rope_kernel = pytest.mark.skipif( + bool(_kv_rope_skip_reason), reason=_kv_rope_skip_reason +) + +HEAD_DIM_V = 128 +KV_LORA_RANK = 512 +MXFP8_BUFFERS = ("_rowwise_data", "_rowwise_scale_inv", "_columnwise_data", "_columnwise_scale_inv") + + +def _kv_quantizer(): + return MXFP8Quantizer(fp8_dtype=tex.DType.kFloat8E4M3, rowwise=True, columnwise=True) + + +def _kv_rope_inputs(s, b, h, cos_dtype, seed=1234): + torch.manual_seed(seed) + kv = (torch.randn(s, b, h, HEAD_DIM_NOPE + HEAD_DIM_V, device="cuda") * 4).to(torch.bfloat16) + # An all-zero scaling block and a value that saturates the scale of its blocks. + kv[5, 0, 0, :32] = 0 + kv[7, 0, h - 1, 3] = 3.0e4 + kv[9, b - 1, 1, HEAD_DIM_NOPE + 17] = -2.5e4 + # k_rope is a slice of the key/value down-projection output, as in MLA. + kv_down = (torch.randn(s, b, KV_LORA_RANK + HEAD_DIM_ROPE, device="cuda") * 4).to( + torch.bfloat16 + ) + k_rope = kv_down[..., KV_LORA_RANK:].unsqueeze(-2) + cos, sin = build_rope_tables(s, emb_dim=HEAD_DIM_ROPE, device="cuda") + # A magnitude scale as in YaRN, so that the products are not exact. + cos, sin = (cos * 1.2).to(cos_dtype), (sin * 1.2).to(cos_dtype) + return kv, k_rope, cos, sin + + +def _kv_rope_reference(kv, k_rope, cos, sin): + key, value = apply_mla_rope_kv(kv, k_rope, cos_table=cos, sin_table=sin) + return mxfp8_quantize_only( + [(key.contiguous(), _kv_quantizer()), (value.contiguous(), _kv_quantizer())], "sbhd" + ) + + +def _assert_identical(actual, expected, name): + assert actual.shape == expected.shape, f"{name}: shape {actual.shape} vs {expected.shape}" + for buffer in MXFP8_BUFFERS: + a, e = getattr(actual, buffer), getattr(expected, buffer) + assert a.shape == e.shape, f"{name}.{buffer}: shape {tuple(a.shape)} vs {tuple(e.shape)}" + a, e = a.view(torch.uint8), e.view(torch.uint8) + mismatches = int((a != e).sum()) + assert mismatches == 0, f"{name}.{buffer}: {mismatches} of {e.numel()} bytes differ" + + +@requires_kv_rope_kernel +@pytest.mark.parametrize( + "s, b, h", [(4096, 1, 128), (2048, 2, 64), (512, 1, 16), (256, 3, 8), (128, 2, 2)] +) +@pytest.mark.parametrize("cos_dtype", [torch.float32, torch.bfloat16], ids=["fp32", "bf16"]) +def test_matches_rope_then_quantize(s, b, h, cos_dtype): + """Same bytes as the RoPE / split kernel followed by MXFP8 quantization.""" + kv, k_rope, cos, sin = _kv_rope_inputs(s, b, h, cos_dtype) + ref_key, ref_value = _kv_rope_reference(kv, k_rope, cos, sin) + cos4, sin4 = cos.view(s, 1, 1, HEAD_DIM_ROPE), sin.view(s, 1, 1, HEAD_DIM_ROPE) + key, value = fused_mla_kv_rope_mxfp8(kv, k_rope, cos4, sin4, HEAD_DIM_V) + _assert_identical(key, ref_key, "key") + _assert_identical(value, ref_value, "value") + + +@requires_kv_rope_kernel +def test_contiguous_k_rope_and_longer_rope_tables(): + """A contiguous [s, b, emb_dim] k_rope and cos / sin with more rows than tokens.""" + s, b, h = 256, 2, 4 + kv, k_rope, _, _ = _kv_rope_inputs(s, b, h, torch.bfloat16) + cos, sin = build_rope_tables(2 * s, emb_dim=HEAD_DIM_ROPE, device="cuda") + cos, sin = cos.to(torch.bfloat16), sin.to(torch.bfloat16) + ref_key, ref_value = _kv_rope_reference(kv, k_rope, cos[:s], sin[:s]) + key, value = fused_mla_kv_rope_mxfp8(kv, k_rope.squeeze(-2).contiguous(), cos, sin, 128) + _assert_identical(key, ref_key, "key") + _assert_identical(value, ref_value, "value") + + +@requires_kv_rope_kernel +def test_non_finite_values(): + """Inf and NaN inputs give the quantizer's scaling factors and data.""" + s, b, h = 128, 1, 4 + kv, k_rope, cos, sin = _kv_rope_inputs(s, b, h, torch.bfloat16) + kv[3, 0, 0, 40] = float("inf") + kv[40, 0, 1, HEAD_DIM_NOPE + 5] = float("-inf") + kv[64, 0, 2, 100] = float("nan") + kv[96:128, 0, 3, HEAD_DIM_NOPE + 32 : HEAD_DIM_NOPE + 64] = float("nan") + ref_key, ref_value = _kv_rope_reference(kv, k_rope, cos, sin) + key, value = fused_mla_kv_rope_mxfp8(kv, k_rope, cos, sin, HEAD_DIM_V) + _assert_identical(key, ref_key, "key") + _assert_identical(value, ref_value, "value") + + +@requires_kv_rope_kernel +@pytest.mark.parametrize( + "s, h, message", [(96, 4, "multiple of 128"), (128, 3, "must be even")], ids=["s", "h"] +) +def test_unsupported_shapes(s, h, message): + kv, k_rope, cos, sin = _kv_rope_inputs(s, 1, h, torch.bfloat16) + with pytest.raises((RuntimeError, ValueError), match=message): + fused_mla_kv_rope_mxfp8(kv, k_rope, cos, sin, HEAD_DIM_V) + + +@requires_kv_rope_kernel +def test_2d_quantizer_rejected(): + """The kernel quantizes 1D blocks, so 2D MXFP8 quantizers are rejected.""" + kv, k_rope, cos, sin = _kv_rope_inputs(128, 1, 4, torch.bfloat16) + quantizer = MXFP8Quantizer( + fp8_dtype=tex.DType.kFloat8E4M3, rowwise=True, columnwise=True, with_2d_quantization=True + ) + with pytest.raises(RuntimeError, match="2D MXFP8 quantization"): + fused_mla_kv_rope_mxfp8(kv, k_rope, cos, sin, HEAD_DIM_V, quantizer, quantizer) + + +@requires_kv_rope_kernel +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="Needs two GPUs.") +def test_inputs_on_non_current_device(): + """Outputs are allocated, and the kernel runs, on the inputs' device.""" + with torch.cuda.device(1): + kv, k_rope, cos, sin = _kv_rope_inputs(128, 2, 4, torch.bfloat16) + ref_key, ref_value = _kv_rope_reference(kv, k_rope, cos, sin) + assert torch.cuda.current_device() == 0 + key, value = fused_mla_kv_rope_mxfp8(kv, k_rope, cos, sin, HEAD_DIM_V) + assert key._rowwise_data.device == kv.device and value._rowwise_data.device == kv.device + _assert_identical(key, ref_key, "key") + _assert_identical(value, ref_value, "value") + with pytest.raises(RuntimeError, match="same device"): + fused_mla_kv_rope_mxfp8(kv, k_rope.to("cuda:0"), cos, sin, HEAD_DIM_V) diff --git a/transformer_engine/common/CMakeLists.txt b/transformer_engine/common/CMakeLists.txt index ded6b6cb801..077a9a2b3a1 100644 --- a/transformer_engine/common/CMakeLists.txt +++ b/transformer_engine/common/CMakeLists.txt @@ -363,6 +363,7 @@ list(APPEND transformer_engine_cuda_arch_specific_sources cast/cast_grouped.cu cast/cast_grouped_dbias.cu cast/fused_group_requantize.cu + fused_rope/fused_mla_kv_rope_mxfp8.cu gemm/cutlass_grouped_gemm.cu hadamard_transform/group_hadamard_transform.cu hadamard_transform/graph_safe_group_hadamard_transform.cu diff --git a/transformer_engine/common/fused_rope/fused_mla_kv_rope_mxfp8.cu b/transformer_engine/common/fused_rope/fused_mla_kv_rope_mxfp8.cu new file mode 100644 index 00000000000..2b70ff79837 --- /dev/null +++ b/transformer_engine/common/fused_rope/fused_mla_kv_rope_mxfp8.cu @@ -0,0 +1,451 @@ +/************************************************************************* + * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * See LICENSE for license information. + ************************************************************************/ + +/*! \file fused_mla_kv_rope_mxfp8.cu + * \brief Multi-latent attention key/value split and key RoPE fused with MXFP8 quantization. + * + * MLA's key/value up-projection gives, per head, kv = [k_nope | v], and the key is k_nope + * followed by k_rope, the key's RoPE dimensions that all heads share, with RoPE applied. For + * MXFP8 attention, key and value are quantized row-wise (32 elements of a head) and + * column-wise (32 tokens). Otherwise the key and value are written in BF16 and read again + * by the quantization; this kernel reads kv and k_rope once and writes the MXFP8 key and + * value directly, the same bytes as the MXFP8 quantizer applied to the [s, b * h * d] views + * of the RoPE'd key and of the value. + * + * One CTA covers 32 tokens of two heads of one batch entry. It issues all of its global + * loads up front, stages the tile (RoPE applied) in shared memory, quantizes it with six + * warps row-wise and three warps column-wise, stages the results in shared memory in the + * layout of the CTA's part of the outputs and writes them with 16-byte stores. + */ + +#include +#include +#include + +#include +#include +#include + +#include "../common.h" +#include "../util/logging.h" +#include "../util/ptx.cuh" +#include "../util/ptx_arch_spec.cuh" +#include "../utils.cuh" + +namespace transformer_engine { +namespace { + +// Head dimensions: key without RoPE, RoPE part of the key, value (DeepSeek-V3's MLA). +constexpr int kNopeDim = 128; +constexpr int kRopeDim = 64; +constexpr int kHalfRopeDim = kRopeDim / 2; +constexpr int kValueDim = 128; +constexpr int kKeyDim = kNopeDim + kRopeDim; +constexpr int kKVDim = kNopeDim + kValueDim; + +constexpr int kBlock = 32; // MXFP8 scaling block, and tokens per CTA. +constexpr int kHeadsPerCTA = 2; +constexpr int kRowwiseThreads = 6 * THREADS_PER_WARP; +constexpr int kColwiseThreads = 3 * THREADS_PER_WARP; +constexpr int kThreads = kRowwiseThreads + kColwiseThreads; +constexpr int kPitch = kKeyDim + 8; // BF16 elements per staged token; the padding staggers banks. + +// Global buffers of one MXFP8 output. +struct MXFP8Output { + uint8_t *rowwise_data; + uint8_t *rowwise_scale_inv; + uint8_t *columnwise_data; + uint8_t *columnwise_scale_inv; +}; + +// A quantized tile of a tensor with head dimension D, laid out like its part of the global +// outputs: token r holds the kHeadsPerCTA * D bytes of the CTA's heads. +template +struct StagedOutput { + alignas(16) uint8_t rowwise_data[kBlock * kHeadsPerCTA * D]; + alignas(16) uint8_t columnwise_data[kBlock * kHeadsPerCTA * D]; + alignas(16) uint8_t rowwise_scale_inv[kBlock * kHeadsPerCTA * (D / kBlock)]; + alignas(16) uint8_t columnwise_scale_inv[kHeadsPerCTA * D]; +}; + +constexpr size_t kTileBytes = kHeadsPerCTA * kBlock * kPitch * sizeof(bf16); +constexpr size_t kSharedMemoryBytes = kTileBytes + sizeof(StagedOutput); + +__device__ __forceinline__ bf16 bf16_from_bits(const uint32_t bits) { + return __ushort_as_bfloat16(static_cast(bits & 0xFFFFu)); +} + +__device__ __forceinline__ uint32_t bf16_bits(const bf16 x) { return __bfloat16_as_ushort(x); } + +// Largest magnitude in a BF16 pair of maxima, as FP32 (NaNs are ignored, as by the quantizer). +__device__ __forceinline__ float bf16x2_max(const __nv_bfloat162 m) { + return fmaxf(__bfloat162float(m.x), __bfloat162float(m.y)); +} + +__device__ __forceinline__ __nv_bfloat162 abs_max(const __nv_bfloat162 m, const uint32_t pair) { + return __hmax2(m, __habs2(*reinterpret_cast(&pair))); +} + +// kBlock tokens x W BF16 elements without RoPE, token rows `stride` elements apart, loaded as +// 16-byte vectors before any of them is used. +template +struct PlainLoads { + static constexpr int kVectorsPerToken = W / 8; + static constexpr int kTasks = kBlock * kVectorsPerToken; + static constexpr int kIterations = (kTasks + kThreads - 1) / kThreads; + uint4 vectors[kIterations]; + + __device__ __forceinline__ void load(const bf16 *src, const int64_t stride) { +#pragma unroll + for (int it = 0; it < kIterations; ++it) { + const int task = threadIdx.x + it * kThreads; + if (task < kTasks) { + const int token = task / kVectorsPerToken, vector = task % kVectorsPerToken; + vectors[it] = __ldg(reinterpret_cast(src + token * stride) + vector); + } + } + } + + __device__ __forceinline__ void store(bf16 *tile) const { +#pragma unroll + for (int it = 0; it < kIterations; ++it) { + const int task = threadIdx.x + it * kThreads; + if (task < kTasks) { + const int token = task / kVectorsPerToken, vector = task % kVectorsPerToken; + *reinterpret_cast(tile + token * kPitch + 8 * vector) = vectors[it]; + } + } + } +}; + +// RoPE of k_rope for kBlock tokens, one task per (token, group of 4 rotation pairs). +template +struct RopeLoads { + using CosType = std::conditional_t; + using CosVector = std::conditional_t; // 4 values + static constexpr int kGroups = kHalfRopeDim / 4; + static constexpr int kTasks = kBlock * kGroups; + static_assert(kTasks <= kThreads); + uint4 pairs; // 4 interleaved (x1, x2) pairs + CosVector cos_left, cos_right, sin_left, sin_right; + + __device__ __forceinline__ bool active() const { return threadIdx.x < kTasks; } + + __device__ __forceinline__ void load(const bf16 *k_rope, const int64_t stride, const CosType *cos, + const CosType *sin, const int64_t first_token) { + if (!active()) return; + const int token = threadIdx.x / kGroups, group = threadIdx.x % kGroups; + pairs = __ldg(reinterpret_cast(k_rope + token * stride) + group); + const CosType *c = cos + (first_token + token) * kRopeDim; + const CosType *s = sin + (first_token + token) * kRopeDim; + cos_left = __ldg(reinterpret_cast(c) + group); + cos_right = __ldg(reinterpret_cast(c + kHalfRopeDim) + group); + sin_left = __ldg(reinterpret_cast(s) + group); + sin_right = __ldg(reinterpret_cast(s + kHalfRopeDim) + group); + } + + // Writes x_left = x1 * cos_left - x2 * sin_left and x_right = x2 * cos_right + x1 * sin_right + // to the tile's columns kNopeDim.. (all x_left, then all x_right), with the roundings of + // Megatron-LM's MLA RoPE kernels: in BF16 for BF16 cos / sin (each product and FMA rounded), + // in FP32 rounded once to BF16 for FP32 cos / sin. + __device__ __forceinline__ void store(bf16 *tile) const { + if (!active()) return; + const int token = threadIdx.x / kGroups, group = threadIdx.x % kGroups; + const uint32_t words[4] = {pairs.x, pairs.y, pairs.z, pairs.w}; + uint32_t left[4], right[4]; +#pragma unroll + for (int p = 0; p < 4; ++p) { + const bf16 x1 = bf16_from_bits(words[p]), x2 = bf16_from_bits(words[p] >> 16); + if constexpr (kCosBf16) { + const bf16 *cl = reinterpret_cast(&cos_left); + const bf16 *cr = reinterpret_cast(&cos_right); + const bf16 *sl = reinterpret_cast(&sin_left); + const bf16 *sr = reinterpret_cast(&sin_right); + left[p] = bf16_bits(__hfma(x1, cl[p], __hneg(__hmul(x2, sl[p])))); + right[p] = bf16_bits(__hfma(x1, sr[p], __hmul(x2, cr[p]))); + } else { + const float *cl = reinterpret_cast(&cos_left); + const float *cr = reinterpret_cast(&cos_right); + const float *sl = reinterpret_cast(&sin_left); + const float *sr = reinterpret_cast(&sin_right); + const float f1 = __bfloat162float(x1), f2 = __bfloat162float(x2); + left[p] = bf16_bits(__float2bfloat16_rn(__fmaf_rn(f1, cl[p], -__fmul_rn(f2, sl[p])))); + right[p] = bf16_bits(__float2bfloat16_rn(__fmaf_rn(f1, sr[p], __fmul_rn(f2, cr[p])))); + } + } + bf16 *dst = tile + token * kPitch + kNopeDim + 4 * group; + *reinterpret_cast(dst) = + make_uint2(left[0] | (left[1] << 16), left[2] | (left[3] << 16)); + *reinterpret_cast(dst + kHalfRopeDim) = + make_uint2(right[0] | (right[1] << 16), right[2] | (right[3] << 16)); + } +}; + +// Quantizes head h of the staged tile (kBlock tokens x D) into `out`, as the MXFP8 quantizer +// does: row-wise one (token, 32-element block) per task on the row-wise warps, column-wise two +// adjacent columns per task on the column-wise warps. +template +__device__ __forceinline__ void quantize_head(const bf16 *tile, StagedOutput &out, const int h) { + constexpr int kBlocksPerToken = D / kBlock; + constexpr int kTokenBytes = kHeadsPerCTA * D; + constexpr float kMaxNormRcp = Quantized_Limits::max_norm_rcp; + if (threadIdx.x < kRowwiseThreads) { + for (int task = threadIdx.x; task < kBlock * kBlocksPerToken; task += kRowwiseThreads) { + const int token = task % kBlock, block = task / kBlock; + const uint4 *src = reinterpret_cast(tile + token * kPitch + kBlock * block); + uint32_t words[16]; +#pragma unroll + for (int i = 0; i < 4; ++i) { + const uint4 u = src[i]; + words[4 * i] = u.x; + words[4 * i + 1] = u.y; + words[4 * i + 2] = u.z; + words[4 * i + 3] = u.w; + } + __nv_bfloat162 m = __habs2(*reinterpret_cast(&words[0])); +#pragma unroll + for (int i = 1; i < 16; ++i) m = abs_max(m, words[i]); + const e8m0_t biased_exponent = ptx::float_to_e8m0(bf16x2_max(m) * kMaxNormRcp); + const ptx::bf16x2 scale = ptx::exp2f_rcp_2x(biased_exponent); + uint32_t q[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) { + ptx::mul_cvt_4x(reinterpret_cast(q[i]), + reinterpret_cast(words[2 * i]), scale); + } + uint4 *dst = reinterpret_cast(out.rowwise_data + token * kTokenBytes + h * D + + kBlock * block); + dst[0] = make_uint4(q[0], q[1], q[2], q[3]); + dst[1] = make_uint4(q[4], q[5], q[6], q[7]); + out.rowwise_scale_inv[token * kHeadsPerCTA * kBlocksPerToken + h * kBlocksPerToken + block] = + biased_exponent; + } + } else { + for (int pair = threadIdx.x - kRowwiseThreads; pair < D / 2; pair += kColwiseThreads) { + // Columns 2 * pair and 2 * pair + 1 of all kBlock tokens. + const uint32_t *column = reinterpret_cast(tile) + pair; + uint32_t values[kBlock]; +#pragma unroll + for (int r = 0; r < kBlock; ++r) values[r] = column[r * (kPitch / 2)]; + __nv_bfloat162 m = __habs2(*reinterpret_cast(&values[0])); +#pragma unroll + for (int r = 1; r < kBlock; ++r) m = abs_max(m, values[r]); + const e8m0_t exponent0 = ptx::float_to_e8m0(__bfloat162float(m.x) * kMaxNormRcp); + const e8m0_t exponent1 = ptx::float_to_e8m0(__bfloat162float(m.y) * kMaxNormRcp); + const ptx::bf16x2 scale{ptx::exp2f_rcp(exponent0), ptx::exp2f_rcp(exponent1)}; +#pragma unroll + for (int r = 0; r < kBlock; r += 2) { + uint32_t q; + ptx::mul_cvt_4x(reinterpret_cast(q), + reinterpret_cast(values[r]), scale, + reinterpret_cast(values[r + 1]), scale); + uint8_t *dst = out.columnwise_data + r * kTokenBytes + h * D + 2 * pair; + *reinterpret_cast(dst) = static_cast(q); + *reinterpret_cast(dst + kTokenBytes) = static_cast(q >> 16); + } + *reinterpret_cast(out.columnwise_scale_inv + h * D + 2 * pair) = + static_cast(exponent0 | (exponent1 << 8)); + } + } +} + +// Writes the staged tile. first_row = first_token * num_columns + first_column is the tile's +// first (token, column) row of the [s * b * h, D] outputs; first_colwise_row is the matching +// row of the column-wise scales, [s / 32 * b * h, D]. +template +__device__ __forceinline__ void write_out(const StagedOutput &staged, const int64_t first_row, + const int64_t num_columns, + const int64_t first_colwise_row, const MXFP8Output &out) { + constexpr int kTokenBytes = kHeadsPerCTA * D; + constexpr int kVectorsPerToken = kTokenBytes / 16; + constexpr int kScaleBytesPerToken = kHeadsPerCTA * (D / kBlock); + static_assert(kTokenBytes % 16 == 0 && kScaleBytesPerToken % 4 == 0); + constexpr int kScaleWordsPerToken = kScaleBytesPerToken / 4; + for (int task = threadIdx.x; task < 2 * kBlock * kVectorsPerToken; task += kThreads) { + const bool columnwise = task >= kBlock * kVectorsPerToken; + const int t = task % (kBlock * kVectorsPerToken); + const int token = t / kVectorsPerToken, vector = t % kVectorsPerToken; + const uint8_t *src = + (columnwise ? staged.columnwise_data : staged.rowwise_data) + token * kTokenBytes; + uint8_t *dst = (columnwise ? out.columnwise_data : out.rowwise_data) + + (first_row + token * num_columns) * D; + reinterpret_cast(dst)[vector] = reinterpret_cast(src)[vector]; + } + for (int task = threadIdx.x; task < kBlock * kScaleWordsPerToken; task += kThreads) { + const int token = task / kScaleWordsPerToken, word = task % kScaleWordsPerToken; + uint8_t *dst = out.rowwise_scale_inv + (first_row + token * num_columns) * (D / kBlock); + reinterpret_cast(dst)[word] = reinterpret_cast( + staged.rowwise_scale_inv + token * kScaleBytesPerToken)[word]; + } + for (int task = threadIdx.x; task < kTokenBytes / 16; task += kThreads) { + reinterpret_cast(out.columnwise_scale_inv + first_colwise_row * D)[task] = + reinterpret_cast(staged.columnwise_scale_inv)[task]; + } +} + +template +__global__ void __launch_bounds__(kThreads) + fused_mla_kv_rope_mxfp8_kernel(const bf16 *__restrict__ kv, const bf16 *__restrict__ k_rope, + const int64_t k_rope_stride, + const typename RopeLoads::CosType *__restrict__ cos, + const typename RopeLoads::CosType *__restrict__ sin, + const MXFP8Output key, const MXFP8Output value, + const int num_columns, const int heads) { +#if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + extern __shared__ __align__(16) uint8_t shared_memory[]; + bf16 *tiles = reinterpret_cast(shared_memory); // [kHeadsPerCTA][kBlock][kPitch] + auto &key_out = *reinterpret_cast *>(shared_memory + kTileBytes); + auto &value_out = *reinterpret_cast *>(shared_memory + kTileBytes); + + // Column c = batch * heads + head. blockIdx.x: kHeadsPerCTA columns of one batch entry + // (kHeadsPerCTA divides heads), blockIdx.y: kBlock tokens. + const int64_t columns = num_columns; + const int64_t first_column = static_cast(blockIdx.x) * kHeadsPerCTA; + const int64_t token_block = blockIdx.y; + const int64_t first_token = token_block * kBlock; + const int64_t batch_size = columns / heads; + const int64_t batch = first_column / heads; + + PlainLoads k_nope[kHeadsPerCTA]; + PlainLoads v[kHeadsPerCTA]; + RopeLoads rope; +#pragma unroll + for (int h = 0; h < kHeadsPerCTA; ++h) { + const bf16 *src = kv + (first_token * columns + first_column + h) * kKVDim; + k_nope[h].load(src, columns * kKVDim); + v[h].load(src + kNopeDim, columns * kKVDim); + } + rope.load(k_rope + (first_token * batch_size + batch) * k_rope_stride, batch_size * k_rope_stride, + cos, sin, first_token); + + // Key: k_nope and the RoPE'd k_rope, which both heads share. +#pragma unroll + for (int h = 0; h < kHeadsPerCTA; ++h) { + k_nope[h].store(tiles + h * kBlock * kPitch); + rope.store(tiles + h * kBlock * kPitch); + } + __syncthreads(); +#pragma unroll 1 + for (int h = 0; h < kHeadsPerCTA; ++h) { + quantize_head(tiles + h * kBlock * kPitch, key_out, h); + } + __syncthreads(); + write_out(key_out, first_token * columns + first_column, columns, + token_block * columns + first_column, key); + __syncthreads(); + + // Value. +#pragma unroll + for (int h = 0; h < kHeadsPerCTA; ++h) v[h].store(tiles + h * kBlock * kPitch); + __syncthreads(); +#pragma unroll 1 + for (int h = 0; h < kHeadsPerCTA; ++h) { + quantize_head(tiles + h * kBlock * kPitch, value_out, h); + } + __syncthreads(); + write_out(value_out, first_token * columns + first_column, columns, + token_block * columns + first_column, value); +#else + NVTE_DEVICE_THREAD0_ERROR("Fused MLA key/value RoPE with MXFP8 output requires SM 10.0+."); +#endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) +} + +bool is_aligned_to(const void *ptr, const size_t alignment) { + return reinterpret_cast(ptr) % alignment == 0; +} + +MXFP8Output check_mxfp8_output(const Tensor &t, const char *name, const size_t rows, + const size_t cols) { + NVTE_CHECK(is_mxfp8_scaling(t.scaling_mode), name, " must be an MXFP8 tensor, got ", + to_string(t.scaling_mode), "."); + NVTE_CHECK(t.has_data() && t.has_columnwise_data(), name, + " needs both row-wise and column-wise data."); + NVTE_CHECK(t.data.dtype == DType::kFloat8E4M3 && t.columnwise_data.dtype == DType::kFloat8E4M3, + name, " must have E4M3 data."); + NVTE_CHECK(!t.with_gemm_swizzled_scales, name, + " must have compact (not GEMM-swizzled) scaling factors."); + const auto [first_dim, last_dim] = t.flat_2d_dims(); + NVTE_CHECK(first_dim == rows && last_dim == cols, name, " must have shape [", rows, ", ", cols, + "], got ", t.shape(), "."); + CheckOutputTensor(t, name); + MXFP8Output out{static_cast(t.data.dptr), static_cast(t.scale_inv.dptr), + static_cast(t.columnwise_data.dptr), + static_cast(t.columnwise_scale_inv.dptr)}; + NVTE_CHECK(is_aligned_to(out.rowwise_data, 16) && is_aligned_to(out.columnwise_data, 16) && + is_aligned_to(out.rowwise_scale_inv, 16) && + is_aligned_to(out.columnwise_scale_inv, 16), + name, " buffers must be 16-byte aligned."); + return out; +} + +} // namespace + +void fused_mla_kv_rope_mxfp8(const Tensor &kv, const Tensor &k_rope, const int k_rope_stride, + const Tensor &cos, const Tensor &sin, Tensor *key, Tensor *value, + cudaStream_t stream) { + NVTE_CHECK(is_supported_by_CC_100(), + "Fused MLA key/value RoPE with MXFP8 output requires SM 10.0+."); + const auto &kv_shape = kv.data.shape; + NVTE_CHECK(kv.data.dtype == DType::kBFloat16, "kv must be BF16."); + NVTE_CHECK(kv_shape.size() == 4 && kv_shape[3] == kKVDim, "kv must have shape [s, b, h, ", kKVDim, + "], got ", kv_shape, "."); + const size_t s = kv_shape[0], b = kv_shape[1], h = kv_shape[2]; + NVTE_CHECK(s % kBlock == 0, "The sequence length (", s, ") must be a multiple of ", kBlock, "."); + NVTE_CHECK(h % kHeadsPerCTA == 0, "The number of heads (", h, ") must be even."); + NVTE_CHECK(s / kBlock <= 65535 && b * h <= static_cast(std::numeric_limits::max()), + "kv shape ", kv_shape, " is too large."); + + NVTE_CHECK(k_rope.data.dtype == DType::kBFloat16, "k_rope must be BF16."); + NVTE_CHECK(k_rope.flat_first_dim() == s * b && k_rope.flat_last_dim() == kRopeDim, + "k_rope must have s * b = ", s * b, " rows of ", kRopeDim, " elements, got shape ", + k_rope.data.shape, "."); + NVTE_CHECK(k_rope_stride >= kRopeDim && k_rope_stride % 8 == 0, "The k_rope row stride (", + k_rope_stride, ") must be at least the row size and a multiple of 8 elements."); + + NVTE_CHECK(cos.data.dtype == sin.data.dtype && + (cos.data.dtype == DType::kBFloat16 || cos.data.dtype == DType::kFloat32), + "cos and sin must both be BF16 or both FP32."); + NVTE_CHECK(cos.flat_last_dim() == kRopeDim && sin.flat_last_dim() == kRopeDim && + cos.flat_first_dim() >= s && sin.flat_first_dim() >= s, + "cos and sin must have at least ", s, " rows of ", kRopeDim, " elements, got shapes ", + cos.data.shape, " and ", sin.data.shape, "."); + + const MXFP8Output key_out = check_mxfp8_output(*key, "key", s, b * h * kKeyDim); + const MXFP8Output value_out = check_mxfp8_output(*value, "value", s, b * h * kValueDim); + NVTE_CHECK(is_aligned_to(kv.data.dptr, 16) && is_aligned_to(k_rope.data.dptr, 16) && + is_aligned_to(cos.data.dptr, 16) && is_aligned_to(sin.data.dptr, 16), + "kv, k_rope, cos and sin must be 16-byte aligned."); + if (s == 0 || b == 0) { + return; + } + + const dim3 grid(b * h / kHeadsPerCTA, s / kBlock); + TRANSFORMER_ENGINE_SWITCH_CONDITION(cos.data.dtype == DType::kBFloat16, kCosBf16, { + using CosType = typename RopeLoads::CosType; + auto kernel = fused_mla_kv_rope_mxfp8_kernel; + NVTE_CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, + kSharedMemoryBytes)); + kernel<<>>( + static_cast(kv.data.dptr), static_cast(k_rope.data.dptr), + k_rope_stride, static_cast(cos.data.dptr), + static_cast(sin.data.dptr), key_out, value_out, static_cast(b * h), + static_cast(h)); + }); // NOLINT(*) + NVTE_CHECK_CUDA(cudaGetLastError()); +} + +} // namespace transformer_engine + +void nvte_fused_mla_kv_rope_mxfp8(const NVTETensor kv, const NVTETensor k_rope, + const int k_rope_stride, const NVTETensor cos, + const NVTETensor sin, NVTETensor key, NVTETensor value, + cudaStream_t stream) { + NVTE_API_CALL(nvte_fused_mla_kv_rope_mxfp8); + using namespace transformer_engine; + fused_mla_kv_rope_mxfp8(*convertNVTETensorCheck(kv), *convertNVTETensorCheck(k_rope), + k_rope_stride, *convertNVTETensorCheck(cos), *convertNVTETensorCheck(sin), + convertNVTETensorCheck(key), convertNVTETensorCheck(value), stream); +} diff --git a/transformer_engine/common/include/transformer_engine/fused_rope.h b/transformer_engine/common/include/transformer_engine/fused_rope.h index aea5256a2cb..f0e4540b7cc 100644 --- a/transformer_engine/common/include/transformer_engine/fused_rope.h +++ b/transformer_engine/common/include/transformer_engine/fused_rope.h @@ -139,6 +139,47 @@ void nvte_fused_qkv_rope_backward(const NVTETensor q_grad_out, const NVTETensor const int qkv_split_arg_list_0, const int qkv_split_arg_list_1, const int qkv_split_arg_list_2, cudaStream_t stream); +/*! \brief Split multi-latent attention's key/value, apply RoPE to the key and quantize key and + * value to MXFP8. + * + * Each head of kv holds 128 key dimensions without RoPE followed by 128 value dimensions. + * The key is those 128 dimensions followed by the 64 dimensions of k_rope with RoPE + * applied, the same for all heads. For token t, batch entry b and i < 32, with + * (x1, x2) = (k_rope[t, b, 2i], k_rope[t, b, 2i + 1]): + * + * key[t, b, h, 128 + i] = x1 * cos[t, i] - x2 * sin[t, i] + * key[t, b, h, 128 + 32 + i] = x2 * cos[t, 32 + i] + x1 * sin[t, 32 + i] + * + * This is the RoPE of Megatron-LM's MLA kernels (interleaved pairs in, the two halves out), + * with their roundings: with BF16 cos and sin, fma(x1, cos, -(x2 * sin)) and + * fma(x1, sin, x2 * cos) with every operation rounded to BF16; with FP32 cos and sin, the + * same in FP32, rounded once to BF16. + * + * key and value are written as the MXFP8 quantization of their [s, b * h * d] views, row-wise + * (32 elements of a head) and column-wise (32 tokens), with E4M3 data and compact E8M0 scaling + * factors: the bytes the MXFP8 quantizer produces for the BF16 key and value. + * + * Requires SM 10.0+, an even number of heads and a sequence length that is a multiple of 32. + * + * \param[in] kv Key/value, [s, b, h, 256] in BF16. + * \param[in] k_rope Key RoPE dimensions shared by the heads, s * b rows + * (token-major) of 64 BF16 values. + * \param[in] k_rope_stride Elements from one row of k_rope to the next, a multiple + * of 8. + * \param[in] cos Cosines, [max_s, 64] in BF16 or FP32; row t is used for + * token t. + * \param[in] sin Sines, same shape and type as cos. + * \param[in,out] key MXFP8 key of shape [s, b * h * 192], with row-wise and + * column-wise data. + * \param[in,out] value MXFP8 value of shape [s, b * h * 128], with row-wise and + * column-wise data. + * \param[in] stream CUDA stream used for the operation. + */ +void nvte_fused_mla_kv_rope_mxfp8(const NVTETensor kv, const NVTETensor k_rope, + const int k_rope_stride, const NVTETensor cos, + const NVTETensor sin, NVTETensor key, NVTETensor value, + cudaStream_t stream); + #ifdef __cplusplus } // extern "C" #endif diff --git a/transformer_engine/pytorch/attention/dot_product_attention/utils.py b/transformer_engine/pytorch/attention/dot_product_attention/utils.py index a773091cd4e..e8cbd8a34cd 100644 --- a/transformer_engine/pytorch/attention/dot_product_attention/utils.py +++ b/transformer_engine/pytorch/attention/dot_product_attention/utils.py @@ -3307,16 +3307,8 @@ def mxfp8_quantize_only(tensor_quantizer_pairs, src_format): "bshd", "sbhd", ), f"mxfp8_quantize_only only supports bshd/sbhd, got {src_format!r}." - _s_dim = {"bshd": 1, "sbhd": 0} - _d_dim = {"bshd": 3, "sbhd": 3} - fp8_tensors = [] for tensor, quantizer in tensor_quantizer_pairs: - original_shape = tensor.shape - rs_shape = list(original_shape) - rs_shape[_d_dim[src_format]] //= MXFP8_BLOCK_SCALING_SIZE - cs_shape = list(original_shape) - cs_shape[_s_dim[src_format]] //= MXFP8_BLOCK_SCALING_SIZE if src_format == "bshd": t2d = tensor.view(*tensor.shape[:2], -1) else: @@ -3325,41 +3317,119 @@ def mxfp8_quantize_only(tensor_quantizer_pairs, src_format): quantizer.optimize_for_gemm = False fp8_2d = quantizer(t2d) quantizer.optimize_for_gemm = orig_optimize - # Re-wrap with the original 4D SBHD/BSHD shape so that shape[-1] equals the per-head - # dimension (matching Q's wrapper shape) and fused_attn_bwd produces 4D dkv that - # matches key/value's expected gradient shape in _KFQuantizeKVForAttn.backward. - fp8_t = MXFP8Tensor( - shape=original_shape, - dtype=tensor.dtype, - rowwise_data=( - fp8_2d._rowwise_data.view(original_shape) - if fp8_2d._rowwise_data is not None - else None - ), - rowwise_scale_inv=( - fp8_2d._rowwise_scale_inv.view(rs_shape) - if fp8_2d._rowwise_scale_inv is not None - else None - ), - columnwise_data=( - fp8_2d._columnwise_data.view(original_shape) - if fp8_2d._columnwise_data is not None - else None - ), - columnwise_scale_inv=( - fp8_2d._columnwise_scale_inv.view(cs_shape) - if fp8_2d._columnwise_scale_inv is not None - else None - ), - quantizer=quantizer, - requires_grad=False, - fp8_dtype=fp8_2d._fp8_dtype, - with_gemm_swizzled_scales=False, + fp8_tensors.append( + _mxfp8_tensor_in_src_format(fp8_2d, tensor.shape, tensor.dtype, quantizer, src_format) ) - fp8_tensors.append(fp8_t) return fp8_tensors +def _mxfp8_tensor_in_src_format(fp8_2d, shape, dtype, quantizer, src_format): + """Wrap the MXFP8 quantization of an attention input's 2D view with the input's shape. + + Re-wrap with the original 4D SBHD/BSHD shape so that shape[-1] equals the per-head + dimension (matching Q's wrapper shape) and fused_attn_bwd produces 4D dkv that + matches key/value's expected gradient shape in _KFQuantizeKVForAttn.backward. + """ + _s_dim = {"bshd": 1, "sbhd": 0} + _d_dim = {"bshd": 3, "sbhd": 3} + rs_shape = list(shape) + rs_shape[_d_dim[src_format]] //= MXFP8_BLOCK_SCALING_SIZE + cs_shape = list(shape) + cs_shape[_s_dim[src_format]] //= MXFP8_BLOCK_SCALING_SIZE + return MXFP8Tensor( + shape=shape, + dtype=dtype, + rowwise_data=( + fp8_2d._rowwise_data.view(shape) if fp8_2d._rowwise_data is not None else None + ), + rowwise_scale_inv=( + fp8_2d._rowwise_scale_inv.view(rs_shape) + if fp8_2d._rowwise_scale_inv is not None + else None + ), + columnwise_data=( + fp8_2d._columnwise_data.view(shape) if fp8_2d._columnwise_data is not None else None + ), + columnwise_scale_inv=( + fp8_2d._columnwise_scale_inv.view(cs_shape) + if fp8_2d._columnwise_scale_inv is not None + else None + ), + quantizer=quantizer, + requires_grad=False, + fp8_dtype=fp8_2d._fp8_dtype, + with_gemm_swizzled_scales=False, + ) + + +def fused_mla_kv_rope_mxfp8( + kv, k_rope, cos, sin, v_head_dim, key_quantizer=None, value_quantizer=None +): + """MLA key/value split, key RoPE and MXFP8 quantization in one kernel. + + Splits ``kv`` into the key dimensions without RoPE and the value, appends ``k_rope`` with + RoPE applied to the key, and quantizes key and value as + :func:`mxfp8_quantize_only` (``src_format="sbhd"``) does, with byte-identical results, + without writing key and value in BF16 first. The RoPE is that of Megatron-LM's MLA + (interleaved rotation pairs in, the two halves out), with its roundings; see + ``nvte_fused_mla_kv_rope_mxfp8``. Supports the head dimensions of DeepSeek-V3 (128 key + dimensions without RoPE, 64 RoPE dimensions, 128 value dimensions) on SM 10.0+. + + Parameters + ---------- + kv : torch.Tensor + [s, b, h, k_dim + v_head_dim] BF16: per head, the key dimensions without RoPE and then + the value. ``s`` must be a multiple of 128 and ``h`` even. + k_rope : torch.Tensor + [s, b, 1, emb_dim] or [s, b, emb_dim] BF16 key RoPE dimensions shared by the heads, + rotation pairs interleaved. May be a strided view, such as a slice of a projection + output. + cos, sin : torch.Tensor + [max_s, 1, 1, emb_dim] or [max_s, emb_dim], both BF16 or both FP32; row t is used for + token t. + v_head_dim : int + Value dimensions per head in ``kv``. + key_quantizer, value_quantizer : MXFP8Quantizer, optional + E4M3 quantizers with row-wise and column-wise usage; created if not given. + + Returns + ------- + key, value : MXFP8Tensor + [s, b, h, k_dim + emb_dim] and [s, b, h, v_head_dim], data and scale_invs in SBHD + layout; NOT yet BHSD-permuted or swizzled (see :func:`mxfp8_transpose_swizzle`). + """ + s, b, h, kv_dim = kv.shape + if s % (4 * MXFP8_BLOCK_SCALING_SIZE) != 0: + raise ValueError( + "fused_mla_kv_rope_mxfp8 requires a sequence length that is a multiple of " + f"{4 * MXFP8_BLOCK_SCALING_SIZE}, got {s}." + ) + quantizers = [ + ( + quantizer + if quantizer is not None + else MXFP8Quantizer(fp8_dtype=tex.DType.kFloat8E4M3, rowwise=True, columnwise=True) + ) + for quantizer in (key_quantizer, value_quantizer) + ] + orig_optimize = [quantizer.optimize_for_gemm for quantizer in quantizers] + for quantizer in quantizers: + quantizer.optimize_for_gemm = False + try: + key_2d, value_2d = tex.fused_mla_kv_rope_mxfp8( + kv, k_rope, cos, sin, v_head_dim, quantizers[0], quantizers[1] + ) + finally: + for quantizer, optimize in zip(quantizers, orig_optimize): + quantizer.optimize_for_gemm = optimize + key_shape = (s, b, h, kv_dim - v_head_dim + k_rope.shape[-1]) + value_shape = (s, b, h, v_head_dim) + return ( + _mxfp8_tensor_in_src_format(key_2d, key_shape, kv.dtype, quantizers[0], "sbhd"), + _mxfp8_tensor_in_src_format(value_2d, value_shape, kv.dtype, quantizers[1], "sbhd"), + ) + + def mxfp8_transpose_swizzle(fp8_tensors, src_format): """Phase 2 of mxfp8_quantize_fast_path: batched BHSD-transpose + GEMM-swizzle. diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index 955763f43b1..5a4378ea328 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -570,6 +570,10 @@ at::Tensor fused_qkv_rope_backward(const at::Tensor &q_grad_out, const at::Tenso const NVTE_QKV_Format qkv_format, const bool interleaved, const int cp_size, const int cp_rank); +std::tuple fused_mla_kv_rope_mxfp8( + const at::Tensor &kv, const at::Tensor &k_rope, const at::Tensor &cos, const at::Tensor &sin, + const int64_t v_head_dim, py::handle key_quantizer, py::handle value_quantizer); + /*************************************************************************************************** * Miscellaneous **************************************************************************************************/ diff --git a/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp b/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp index 4392fa4b433..aea334972c2 100644 --- a/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp +++ b/transformer_engine/pytorch/csrc/extensions/apply_rope.cpp @@ -6,6 +6,7 @@ #include "../extensions.h" #include "common.h" +#include "pybind.h" namespace transformer_engine::pytorch { @@ -298,4 +299,68 @@ at::Tensor fused_qkv_rope_backward(const at::Tensor &q_grad_out, const at::Tenso return qkv_grad_input; } +std::tuple fused_mla_kv_rope_mxfp8( + const at::Tensor &kv, const at::Tensor &k_rope, const at::Tensor &cos, const at::Tensor &sin, + const int64_t v_head_dim, py::handle key_quantizer, py::handle value_quantizer) { + TORCH_CHECK(kv.is_cuda(), "expected kv on a CUDA device"); + const auto device = kv.device(); + TORCH_CHECK(k_rope.device() == device && cos.device() == device && sin.device() == device, + "expected kv, k_rope, cos and sin on the same device"); + at::cuda::CUDAGuard device_guard(device); + TORCH_CHECK(kv.dim() == 4, "expected kv of shape [s, b, h, k_dim + v_dim], got ", kv.sizes()); + const int64_t s = kv.size(0); + const int64_t b = kv.size(1); + const int64_t h = kv.size(2); + const int64_t emb_dim = k_rope.size(-1); + TORCH_CHECK(k_rope.dim() >= 3 && k_rope.size(0) == s && k_rope.size(1) == b && + k_rope.numel() == s * b * emb_dim, + "expected k_rope of shape [s, b, 1, emb_dim] or [s, b, emb_dim], got ", + k_rope.sizes()); + TORCH_CHECK(v_head_dim > 0 && v_head_dim < kv.size(3), "invalid v_head_dim ", v_head_dim, + " for kv of shape ", kv.sizes()); + const int64_t key_dim = kv.size(3) - v_head_dim + emb_dim; + + // k_rope is usually a slice of a wider projection output. Its rows are read in place + // when they are a fixed, 16-byte aligned stride apart. + at::Tensor k_rope_rows = k_rope; + int64_t k_rope_stride = b > 1 ? k_rope_rows.stride(1) : k_rope_rows.stride(0); + if (k_rope_rows.stride(-1) != 1 || + (b > 1 && k_rope_rows.stride(0) != b * k_rope_rows.stride(1)) || k_rope_stride < emb_dim || + k_rope_stride % 8 != 0 || reinterpret_cast(k_rope_rows.data_ptr()) % 16 != 0) { + k_rope_rows = k_rope.contiguous(); + k_rope_stride = emb_dim; + } + const auto kv_c = kv.contiguous(); + const auto cos_c = cos.contiguous(); + const auto sin_c = sin.contiguous(); + auto kv_cu = makeTransformerEngineTensor(kv_c); + auto k_rope_cu = makeTransformerEngineTensor( + k_rope_rows.data_ptr(), + std::vector{static_cast(s * b), static_cast(emb_dim)}, + GetTransformerEngineDType(k_rope_rows.scalar_type())); + auto cos_cu = makeTransformerEngineTensor(cos_c); + auto sin_cu = makeTransformerEngineTensor(sin_c); + + auto key_quantizer_cpp = convert_quantizer(key_quantizer); + auto value_quantizer_cpp = convert_quantizer(value_quantizer); + const auto *key_mxfp8 = dynamic_cast(key_quantizer_cpp.get()); + const auto *value_mxfp8 = dynamic_cast(value_quantizer_cpp.get()); + TORCH_CHECK(key_mxfp8 != nullptr && value_mxfp8 != nullptr, + "fused_mla_kv_rope_mxfp8 requires MXFP8 quantizers"); + TORCH_CHECK(!key_mxfp8->with_2d_quantization && !value_mxfp8->with_2d_quantization, + "fused_mla_kv_rope_mxfp8 does not support 2D MXFP8 quantization"); + const auto fake_dtype = GetTransformerEngineDType(kv.scalar_type()); + auto [key_cu, key_py] = key_quantizer_cpp->create_tensor( + {static_cast(s), static_cast(b * h * key_dim)}, fake_dtype, device); + auto [value_cu, value_py] = value_quantizer_cpp->create_tensor( + {static_cast(s), static_cast(b * h * v_head_dim)}, fake_dtype, device); + + NVTE_SCOPED_GIL_RELEASE({ + nvte_fused_mla_kv_rope_mxfp8(kv_cu.data(), k_rope_cu.data(), static_cast(k_rope_stride), + cos_cu.data(), sin_cu.data(), key_cu.data(), value_cu.data(), + at::cuda::getCurrentCUDAStream()); + }); + return {key_py, value_py}; +} + } // namespace transformer_engine::pytorch diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index 2046408eba8..c7f70182f36 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -589,6 +589,10 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { "Fused Apply QKV RoPE FWD", py::call_guard()); m.def("fused_qkv_rope_backward", &transformer_engine::pytorch::fused_qkv_rope_backward, "Fused Apply QKV RoPE BWD", py::call_guard()); + m.def("fused_mla_kv_rope_mxfp8", &transformer_engine::pytorch::fused_mla_kv_rope_mxfp8, + "Split MLA key/value, apply RoPE to the key and quantize both to MXFP8", py::arg("kv"), + py::arg("k_rope"), py::arg("cos"), py::arg("sin"), py::arg("v_head_dim"), + py::arg("key_quantizer"), py::arg("value_quantizer")); // fused router transformer_engine::pytorch::init_router_bindings(m); From 20e97daa877bf701b6b23a56d135cb0b952f6db4 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Mon, 12 Oct 2026 18:01:19 -0700 Subject: [PATCH 26/31] Bump nccl ep (#3677) bump nccl ep Signed-off-by: Phuong Nguyen --- 3rdparty/nccl-extensions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/nccl-extensions b/3rdparty/nccl-extensions index a9f207c2263..998d9223e00 160000 --- a/3rdparty/nccl-extensions +++ b/3rdparty/nccl-extensions @@ -1 +1 @@ -Subproject commit a9f207c2263b4af1622a1d902b3e608fe17be0eb +Subproject commit 998d9223e00c20ee515ceeccf4f0f086ee8038cd From e394b124fffa41b8932216004837a1cfe3c8a196 Mon Sep 17 00:00:00 2001 From: Tim Moon <4406448+timmoon10@users.noreply.github.com> Date: Mon, 12 Oct 2026 20:06:37 -0700 Subject: [PATCH 27/31] [PyTorch] Distributed tests set device in NCCL init and manually shut down NCCL (#3655) * Distributed tests set device in NCCL init and manually shuts down NCCL Signed-off-by: Tim Moon * Fully destroy NCCL environment when finishing distributed TE ops test Signed-off-by: Tim Moon * Apply suggestion from @timmoon10 Signed-off-by: Tim Moon <4406448+timmoon10@users.noreply.github.com> * Apply suggestion from @timmoon10 Signed-off-by: Tim Moon <4406448+timmoon10@users.noreply.github.com> --------- Signed-off-by: Tim Moon Signed-off-by: Tim Moon <4406448+timmoon10@users.noreply.github.com> --- tests/pytorch/distributed/test_fusible_ops.py | 14 ++++++++++---- .../distributed/test_parallel_cross_entropy.py | 1 + 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/pytorch/distributed/test_fusible_ops.py b/tests/pytorch/distributed/test_fusible_ops.py index 8057a75a84e..b4f3bce3a4e 100644 --- a/tests/pytorch/distributed/test_fusible_ops.py +++ b/tests/pytorch/distributed/test_fusible_ops.py @@ -1124,10 +1124,16 @@ def main() -> None: if args.compile and not args.parallel: parser.error("--compile requires --parallel") if args.parallel: - if args.compile: - run_compile_parallel_tests() - else: - run_parallel_tests() + try: + if args.compile: + run_compile_parallel_tests() + else: + run_parallel_tests() + torch.distributed.barrier(world_group()) + torch.cuda.synchronize() + finally: + if torch.distributed.is_initialized(): + torch.distributed.destroy_process_group() if __name__ == "__main__": diff --git a/tests/pytorch/distributed/test_parallel_cross_entropy.py b/tests/pytorch/distributed/test_parallel_cross_entropy.py index 16323154939..a6da464ac77 100644 --- a/tests/pytorch/distributed/test_parallel_cross_entropy.py +++ b/tests/pytorch/distributed/test_parallel_cross_entropy.py @@ -29,6 +29,7 @@ def _run_tensor_parallel(rank, world_size, init_file, label_smoothing): init_method=f"file://{init_file}", rank=rank, world_size=world_size, + device_id=device, ) try: generator = torch.Generator().manual_seed(2025) From fc48751409f2d0bd8c07260efcf9803e84eb52de Mon Sep 17 00:00:00 2001 From: Siddhartha Raman Sundara Raman Date: Mon, 12 Oct 2026 22:41:55 -0500 Subject: [PATCH 28/31] [PyTorch] Allow ZeroTensor dummy weight gradients (#3670) * [PyTorch] Allow ZeroTensor dummy weight gradients With fuse_wgrad_accumulation, the modules accumulate the weight gradient into main_grad and return a dummy weight gradient. When one backward pass uses a weight twice, autograd adds the two dummies; for an LM head shared by the main and MTP heads in Megatron-LM that is a [vocab, hidden] add whose result is discarded (0.44 ms for 129280 x 7168 on VR200). A parameter with zero_tensor_dummy_wgrad set now gets a ZeroTensor dummy, which has no storage and costs nothing to add. zero_out_wgrad keeps precedence and its zero-filled dummy; other parameters get the dummy they get today. Signed-off-by: Siddhartha Raman S Co-Authored-By: Claude Opus 5.5 * [PyTorch] Get LayerNormMLP's dummy weight gradients from get_dummy_wgrad The FC1 and FC2 branches for zero_out_wgrad, zero_tensor_dummy_wgrad and plain dummies reimplemented get_dummy_wgrad. Call it instead, as Linear, LayerNormLinear and GroupedLinear do. Signed-off-by: Siddhartha Raman S Co-Authored-By: Claude Opus 5.5 * [PyTorch] Key the cached dummy weight gradients by device get_dummy_wgrad cached its dummies by shape and dtype only, so after a backward pass on one GPU a same-shaped weight on another GPU got the first GPU's tensor, which autograd rejects ("expected device cuda:1 but got cuda:0"). This affected Linear, LayerNormLinear, ops.Linear and, since it now uses get_dummy_wgrad, LayerNormMLP. The dummies are now allocated on the current device and cached per device. Test the device of the dummy weight gradient when the same module shape runs backward on two GPUs. GroupedLinear is left out because its multi-stream GEMM already fails on a second device in the forward pass. Signed-off-by: Siddhartha Raman S Co-Authored-By: Claude Opus 5.5 --------- Signed-off-by: Siddhartha Raman S Co-authored-by: Claude Opus 5.5 --- tests/pytorch/test_sanity.py | 90 +++++++++++++++++++ transformer_engine/pytorch/module/base.py | 21 ++++- .../pytorch/module/grouped_linear.py | 2 + .../pytorch/module/layernorm_linear.py | 1 + .../pytorch/module/layernorm_mlp.py | 41 +++------ transformer_engine/pytorch/module/linear.py | 3 + transformer_engine/pytorch/ops/_common.py | 4 +- 7 files changed, 129 insertions(+), 33 deletions(-) diff --git a/tests/pytorch/test_sanity.py b/tests/pytorch/test_sanity.py index 7db3002427c..8022623d0ae 100644 --- a/tests/pytorch/test_sanity.py +++ b/tests/pytorch/test_sanity.py @@ -1186,6 +1186,96 @@ def test_sanity_gradient_accumulation_fusion(dtype, fp8_recipe, model, skip_wgra _test_sanity_e2e_gradient_accumulation_fusion(block, dtype, config, fp8_recipe, skip_wgrad) +def _wgrad_fusion_module(module: str, hidden: int, dtype: torch.dtype): + """A module that accumulates its weight gradients into ``main_grad``, and its forward.""" + kwargs = dict(params_dtype=dtype, device="cuda", fuse_wgrad_accumulation=True) + if module == "Linear": + block = Linear(hidden, hidden, bias=False, **kwargs) + elif module == "LayerNormLinear": + block = LayerNormLinear(hidden, hidden, bias=False, **kwargs) + elif module == "LayerNormMLP": + block = LayerNormMLP(hidden, 2 * hidden, bias=False, **kwargs) + elif module == "GroupedLinear": + block = GroupedLinear(2, hidden, hidden, bias=False, **kwargs) + return block, lambda x: block(x, [x.shape[0] // 2, x.shape[0] - x.shape[0] // 2]) + else: + block = te.ops.Linear( + hidden, hidden, bias=False, device="cuda", dtype=dtype, accumulate_into_main_grad=True + ) + return block, block + + +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="Needs two GPUs.") +# GroupedLinear is left out: its multi-stream cuBLAS GEMM keeps the streams of the first device +# it runs on, so its forward pass fails on a second device before any weight gradient. +@pytest.mark.parametrize("module", ["Linear", "LayerNormLinear", "LayerNormMLP", "ops.Linear"]) +@pytest.mark.parametrize("zero_out_wgrad", all_boolean) +def test_sanity_dummy_wgrad_device(module, zero_out_wgrad): + """The cached dummy weight gradient is on the weight's device when the same module shape + runs backward on one GPU and then on another.""" + dtype, hidden, tokens = torch.bfloat16, 128, 64 + for device in (0, 1): + with torch.cuda.device(device): + block, forward = _wgrad_fusion_module(module, hidden, dtype) + weights = [ + p + for name, p in block.named_parameters() + if "weight" in name and "layer_norm" not in name + ] + for p in weights: + p.main_grad = torch.zeros(p.shape, dtype=torch.float32, device="cuda") + p.grad_added_to_main_grad = False + p.zero_out_wgrad = zero_out_wgrad + x = torch.randn(tokens, hidden, dtype=dtype, device="cuda") + forward(x).backward(torch.randn(tokens, hidden, dtype=dtype, device="cuda")) + for p in weights: + assert p.grad is not None and p.grad.device == p.device + + +@pytest.mark.parametrize( + "module", ["Linear", "LayerNormLinear", "LayerNormMLP", "GroupedLinear", "ops.Linear"] +) +@pytest.mark.parametrize("zero_out_wgrad", all_boolean) +def test_sanity_zero_tensor_dummy_wgrad(module, zero_out_wgrad): + """``zero_tensor_dummy_wgrad`` makes the dummy weight gradient a ZeroTensor. + + The module runs twice before one backward pass, as an LM head shared by the main and MTP + heads does, so autograd adds the two dummy weight gradients. ``main_grad`` must not + change, and ``zero_out_wgrad`` keeps its zero-filled dummy. + """ + dtype, hidden, tokens = torch.bfloat16, 128, 64 + torch.manual_seed(seed) + block, forward = _wgrad_fusion_module(module, hidden, dtype) + weights = [ + p for name, p in block.named_parameters() if "weight" in name and "layer_norm" not in name + ] + inputs = [torch.randn(tokens, hidden, dtype=dtype, device="cuda") for _ in range(2)] + grad_outputs = [torch.randn(tokens, hidden, dtype=dtype, device="cuda") for _ in range(2)] + + main_grads = {} + for zero_tensor_dummy_wgrad in (False, True): + for p in weights: + p.main_grad = torch.zeros(p.shape, dtype=torch.float32, device="cuda") + p.grad_added_to_main_grad = False + p.zero_out_wgrad = zero_out_wgrad + p.zero_tensor_dummy_wgrad = zero_tensor_dummy_wgrad + p.grad = None + outputs = [forward(x) for x in inputs] + torch.autograd.backward(outputs, grad_outputs) + main_grads[zero_tensor_dummy_wgrad] = [p.main_grad for p in weights] + for p in weights: + assert p.grad_added_to_main_grad + assert p.grad is not None + if zero_tensor_dummy_wgrad and not zero_out_wgrad: + assert p.grad._is_zerotensor() + elif zero_out_wgrad: + assert not p.grad._is_zerotensor() + assert torch.count_nonzero(p.grad) == 0 + for reference, main_grad in zip(main_grads[False], main_grads[True]): + assert torch.count_nonzero(reference) > 0 + torch.testing.assert_close(main_grad, reference, rtol=0, atol=0) + + def test_model_multiple_cast(): a = torch.zeros((16, 16), device="cuda") m = Linear(16, 32) diff --git a/transformer_engine/pytorch/module/base.py b/transformer_engine/pytorch/module/base.py index 9ead1e73651..e8b8bc7409a 100644 --- a/transformer_engine/pytorch/module/base.py +++ b/transformer_engine/pytorch/module/base.py @@ -132,16 +132,29 @@ class UserBufferQuantizationMode(Enum): FP8 = "fp8" -def get_dummy_wgrad(shape: list, dtype: torch.dtype, zero=False) -> torch.Tensor: - """Returns a dummy tensor of given shape.""" +def get_dummy_wgrad( + shape: list, dtype: torch.dtype, zero: bool = False, zero_tensor: bool = False +) -> torch.Tensor: + """Returns a dummy tensor of given shape. + + With ``zero=True`` the cached dummy is filled with zeros. Otherwise, with + ``zero_tensor=True`` the dummy is a ZeroTensor: it has no storage and adding it costs + nothing, e.g. when autograd sums the dummy weight gradients of a weight used more than + once in one backward pass. + """ - key = (*shape, dtype) + # The dummy goes on the current device (the one the backward pass runs on), and the cache + # is keyed by it so that a dummy is never returned for a weight on another device. + device = torch.device("cuda", torch.cuda.current_device()) + if zero_tensor and not zero: + return torch._efficientzerotensor(shape, dtype=dtype, device=device) + key = (*shape, dtype, device) global _dummy_wgrads if key not in _dummy_wgrads: _dummy_wgrads[key] = torch.empty( shape, dtype=dtype, - device="cuda", + device=device, requires_grad=False, ) if zero: diff --git a/transformer_engine/pytorch/module/grouped_linear.py b/transformer_engine/pytorch/module/grouped_linear.py index 5c0dbf3ab28..62607a7c1b2 100644 --- a/transformer_engine/pytorch/module/grouped_linear.py +++ b/transformer_engine/pytorch/module/grouped_linear.py @@ -1308,6 +1308,7 @@ def handle_custom_ddp_from_mcore(weight, main_grad, wgrad): wgrad = get_dummy_wgrad( list(main_grad.shape), weight.dtype, + zero_tensor=getattr(weight, "zero_tensor_dummy_wgrad", False), ) elif ctx.fuse_wgrad_accumulation: wgrad = None @@ -1586,6 +1587,7 @@ def handle_custom_ddp_from_mcore(weight, main_grad, wgrad): wgrad = get_dummy_wgrad( list(main_grad.shape), weight.dtype, + zero_tensor=getattr(weight, "zero_tensor_dummy_wgrad", False), ) elif ctx.fuse_wgrad_accumulation: wgrad = None diff --git a/transformer_engine/pytorch/module/layernorm_linear.py b/transformer_engine/pytorch/module/layernorm_linear.py index e93d8c9641b..61b2a155449 100644 --- a/transformer_engine/pytorch/module/layernorm_linear.py +++ b/transformer_engine/pytorch/module/layernorm_linear.py @@ -1437,6 +1437,7 @@ def wgrad_gemm( wgrad = get_dummy_wgrad( list(main_grad.shape), origin_weight.dtype, + zero_tensor=getattr(origin_weight, "zero_tensor_dummy_wgrad", False), ) elif args.fuse_wgrad_accumulation: wgrad = None diff --git a/transformer_engine/pytorch/module/layernorm_mlp.py b/transformer_engine/pytorch/module/layernorm_mlp.py index ad5864d4a60..47ca3a87165 100644 --- a/transformer_engine/pytorch/module/layernorm_mlp.py +++ b/transformer_engine/pytorch/module/layernorm_mlp.py @@ -21,6 +21,7 @@ from .base import ( fill_userbuffers_buffer_for_all_gather, _ub_communicators, + get_dummy_wgrad, get_ub, get_ub_is_fp8, is_ub_initialized, @@ -2003,20 +2004,12 @@ def fc1_wgrad_gemm( fc1_weight_python_object, "grad_added_to_main_grad" ): fc1_weight_python_object.grad_added_to_main_grad = True - if getattr(fc1_weight_python_object, "zero_out_wgrad", False): - fc1_wgrad = torch.zeros( - fc1_weight_main_grad.shape, - dtype=fc1_weight_python_object.dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - else: - fc1_wgrad = torch.empty( - fc1_weight_main_grad.shape, - dtype=fc1_weight_python_object.dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) + fc1_wgrad = get_dummy_wgrad( + list(fc1_weight_main_grad.shape), + fc1_weight_python_object.dtype, + zero=getattr(fc1_weight_python_object, "zero_out_wgrad", False), + zero_tensor=getattr(fc1_weight_python_object, "zero_tensor_dummy_wgrad", False), + ) elif args.fuse_wgrad_accumulation: fc1_wgrad = None else: @@ -2028,20 +2021,12 @@ def fc1_wgrad_gemm( fc2_weight_python_object, "grad_added_to_main_grad" ): fc2_weight_python_object.grad_added_to_main_grad = True - if getattr(fc2_weight_python_object, "zero_out_wgrad", False): - fc2_wgrad = torch.zeros( - fc2_weight_main_grad.shape, - dtype=fc2_weight_python_object.dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) - else: - fc2_wgrad = torch.empty( - fc2_weight_main_grad.shape, - dtype=fc2_weight_python_object.dtype, - device=torch.cuda.current_device(), - requires_grad=False, - ) + fc2_wgrad = get_dummy_wgrad( + list(fc2_weight_main_grad.shape), + fc2_weight_python_object.dtype, + zero=getattr(fc2_weight_python_object, "zero_out_wgrad", False), + zero_tensor=getattr(fc2_weight_python_object, "zero_tensor_dummy_wgrad", False), + ) elif args.fuse_wgrad_accumulation: fc2_wgrad = None else: diff --git a/transformer_engine/pytorch/module/linear.py b/transformer_engine/pytorch/module/linear.py index 7f00e19644a..73c9503026f 100644 --- a/transformer_engine/pytorch/module/linear.py +++ b/transformer_engine/pytorch/module/linear.py @@ -1705,6 +1705,9 @@ def wgrad_gemm( wgrad = get_dummy_wgrad( wgrad_shape, origin_weight_python_object.dtype, + zero_tensor=getattr( + origin_weight_python_object, "zero_tensor_dummy_wgrad", False + ), ) elif bwd_args.fuse_wgrad_accumulation: wgrad = None diff --git a/transformer_engine/pytorch/ops/_common.py b/transformer_engine/pytorch/ops/_common.py index 50abe957e19..cf4e66023a6 100644 --- a/transformer_engine/pytorch/ops/_common.py +++ b/transformer_engine/pytorch/ops/_common.py @@ -285,7 +285,8 @@ def get_dummy_wgrads_for_params( For each parameter that exposes ``grad_added_to_main_grad``, set the flag to ``True`` and return a dummy wgrad tensor (zeroed if - ``zero_out_wgrad`` is also set on the parameter). For parameters without + ``zero_out_wgrad`` is also set on the parameter, otherwise a ZeroTensor if + ``zero_tensor_dummy_wgrad`` is set). For parameters without the flag, the corresponding entry is ``None``. The returned list has the same length and order as ``weight_params``. @@ -301,6 +302,7 @@ def get_dummy_wgrads_for_params( list(wp.size()), wp.dtype, zero=getattr(wp, "zero_out_wgrad", False), + zero_tensor=getattr(wp, "zero_tensor_dummy_wgrad", False), ) ) else: From 39a8b879dfbfea0d23bc3862dc83ec1d6f970b49 Mon Sep 17 00:00:00 2001 From: Siddhartha Raman Sundara Raman Date: Mon, 12 Oct 2026 22:42:33 -0500 Subject: [PATCH 29/31] [PyTorch] Return the argmax from parallel_cross_entropy (#3671) * [PyTorch] Return the argmax from parallel_cross_entropy The forward kernel of parallel_cross_entropy already scans every row of logits for its maximum. With the new keyword-only return_argmax=True it also returns the index of that maximum, so a caller that needs the predicted tokens, such as an accuracy metric, does not read the logits a second time. The index is the one torch.argmax returns over the unsharded logits: the first one when several logits are equal. With tensor parallelism, each rank's local index travels with the statistics in the existing all-gather (one more FP32 column, as raw bits), and the first rank that holds the global maximum provides it. Without return_argmax the kernels compile as before; with it, loss and gradient are bit-identical. Signed-off-by: Siddhartha Raman S Co-Authored-By: Claude Opus 5.5 * [PyTorch] Count NaN as the maximum in the parallel_cross_entropy argmax tl.max(..., return_indices=True) is not NaN-safe: its comparisons are false for NaN, so a block with a NaN could give an index that is neither the maximum nor the NaN ([8, -10, NaN, -10] gave 1). The argmax now follows torch.argmax: NaN counts as the maximum and the first NaN wins. Each block finds its first NaN separately and takes the maximum over the other values, in a running maximum kept apart from the loss's; with tensor parallelism each rank also shares that maximum, NaN if its shard holds a NaN. Loss and gradient are unchanged. Signed-off-by: Siddhartha Raman S Co-Authored-By: Claude Opus 5.5 * [PyTorch] Test for NaN through one helper in the cross entropy kernels pylint flags x != x as comparison-with-itself, which failed the lint of transformer_engine/common. The three NaN tests now call one _is_nan helper that keeps the comparison and disables that check on its line. Signed-off-by: Siddhartha Raman S Co-Authored-By: Claude Opus 5.5 --------- Signed-off-by: Siddhartha Raman S Co-authored-by: Claude Opus 5.5 --- .../test_parallel_cross_entropy.py | 80 ++++++++++++++++ tests/pytorch/test_cross_entropy.py | 93 +++++++++++++++++++ .../common/triton/cross_entropy.py | 88 +++++++++++++++++- transformer_engine/pytorch/cross_entropy.py | 37 ++++++-- .../pytorch/triton/cross_entropy.py | 30 +++++- 5 files changed, 313 insertions(+), 15 deletions(-) diff --git a/tests/pytorch/distributed/test_parallel_cross_entropy.py b/tests/pytorch/distributed/test_parallel_cross_entropy.py index a6da464ac77..a0dac8ca15e 100644 --- a/tests/pytorch/distributed/test_parallel_cross_entropy.py +++ b/tests/pytorch/distributed/test_parallel_cross_entropy.py @@ -89,6 +89,69 @@ def _run_tensor_parallel(rank, world_size, init_file, label_smoothing): dist.destroy_process_group() +def _run_tensor_parallel_argmax(rank, world_size, init_file, label_smoothing): + """Two-rank worker: the argmax over vocabulary shards, with and without multiple blocks.""" + + torch.cuda.set_device(rank) + device = torch.device("cuda", rank) + dist.init_process_group( + backend="nccl", + init_method=f"file://{init_file}", + rank=rank, + world_size=world_size, + ) + try: + generator = torch.Generator().manual_seed(2026) + for shape in ((2, 3, 22), (1, 4, 80000)): + vocab = shape[-1] + local_vocab = vocab // world_size + target = torch.randint(0, vocab, shape[:-1], generator=generator).to(device) + target[0, 2] = -100 + values = torch.randn(shape, generator=generator).to(device) + peak = 8.0 + values[0, 0, [3, local_vocab + 4]] = peak # equal maxima on both ranks + values[0, 1, [local_vocab + 2, vocab - 2]] = peak # equal maxima on rank 1 + values[0, 2, vocab - 1] = peak # the maximum on the last rank only + if shape[0] > 1: + # NaN counts as the maximum, as in torch.argmax: the first NaN wins. + values[1, 0, [5, local_vocab + 1]] = float("nan") # NaN on both ranks + values[1, 1, 2] = peak + values[1, 1, local_vocab + 3] = float("nan") # NaN after a larger rank-0 maximum + values[-1, -1] = 0.5 # a constant row + vocab_start = rank * local_vocab + for dtype in (torch.float32, torch.bfloat16): + global_values = values.to(dtype) + expected_argmax = torch.argmax(global_values, dim=-1) + local_values = global_values[..., vocab_start : vocab_start + local_vocab] + for reduce_loss in (False, True): + for overwrite_input in (False, True): + results = [] + for return_argmax in (False, True): + local_logits = local_values.clone().requires_grad_() + out = parallel_cross_entropy( + local_logits, + target, + label_smoothing=label_smoothing, + reduce_loss=reduce_loss, + dist_process_group=dist.group.WORLD, + overwrite_input=overwrite_input, + return_argmax=return_argmax, + ) + loss, argmax = out if return_argmax else (out, None) + loss.backward(torch.full_like(loss, 0.37)) + results.append((loss, local_logits.grad, argmax)) + (loss, grad, _), (argmax_loss, argmax_grad, argmax) = results + torch.testing.assert_close(argmax, expected_argmax, rtol=0, atol=0) + torch.testing.assert_close( + argmax_loss, loss, rtol=0, atol=0, equal_nan=True + ) + torch.testing.assert_close( + argmax_grad, grad, rtol=0, atol=0, equal_nan=True + ) + finally: + dist.destroy_process_group() + + @pytest.mark.parametrize("label_smoothing", [0.0, 0.1], ids=["plain", "smoothed"]) def test_parallel_cross_entropy_tensor_parallel(label_smoothing): """Validate tensor-parallel loss and gradients on two ranks.""" @@ -104,3 +167,20 @@ def test_parallel_cross_entropy_tensor_parallel(label_smoothing): nprocs=world_size, join=True, ) + + +@pytest.mark.parametrize("label_smoothing", [0.0, 0.1], ids=["plain", "smoothed"]) +def test_parallel_cross_entropy_tensor_parallel_argmax(label_smoothing): + """Validate the global argmax and the unchanged loss and gradients on two ranks.""" + + if torch.cuda.device_count() < 2: + pytest.skip("tensor-parallel cross entropy test requires two CUDA devices") + world_size = 2 + with tempfile.TemporaryDirectory() as temp_dir: + init_file = os.path.join(temp_dir, "distributed_init") + mp.spawn( + _run_tensor_parallel_argmax, + args=(world_size, init_file, label_smoothing), + nprocs=world_size, + join=True, + ) diff --git a/tests/pytorch/test_cross_entropy.py b/tests/pytorch/test_cross_entropy.py index 0b2d21bbbbe..f34e6666d4b 100644 --- a/tests/pytorch/test_cross_entropy.py +++ b/tests/pytorch/test_cross_entropy.py @@ -430,3 +430,96 @@ def test_parallel_cross_entropy_deprecated_input_alias(): alias_loss = parallel_cross_entropy(logits, target, _input=logits) direct_loss = parallel_cross_entropy(logits, target) torch.testing.assert_close(alias_loss, direct_loss) + + +def _logits_with_argmax_cases(shape, dtype): + """BF16-representable logits whose rows exercise ties within and across 32K-wide blocks.""" + + torch.manual_seed(7) + values = torch.randn(shape, dtype=torch.bfloat16, device="cuda").to(dtype) + vocab = shape[-1] + peak = 8.0 + values[0, 0, [5, vocab - 30000]] = peak # equal maxima in different blocks + values[0, 1, [vocab - 30000, vocab - 4]] = peak # equal maxima in later blocks + values[0, 2, [100, 7]] = peak # equal maxima in the same block + values[0, 3, vocab - 1] = peak # a strictly larger maximum in the last block + values[0, 4] = 0.5 # a constant row + return values + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16], ids=["fp32", "bf16"]) +@pytest.mark.parametrize("reduce_loss", [False, True], ids=["none", "mean"]) +@pytest.mark.parametrize("label_smoothing", [0.0, 0.1], ids=["plain", "smoothed"]) +@pytest.mark.parametrize("overwrite_input", [False, True], ids=["safe", "destructive"]) +def test_parallel_cross_entropy_return_argmax(dtype, reduce_loss, label_smoothing, overwrite_input): + """The argmax matches torch.argmax, and loss and gradient do not change with it.""" + + shape = (2, 5, 70000) + values = _logits_with_argmax_cases(shape, dtype) + target = torch.randint(0, shape[-1], shape[:-1], device="cuda") + target[1, 1] = -100 + expected_argmax = torch.argmax(values, dim=-1) + + logits = values.clone().requires_grad_() + loss = parallel_cross_entropy( + logits, target, label_smoothing, reduce_loss, overwrite_input=overwrite_input + ) + argmax_logits = values.clone().requires_grad_() + argmax_loss, argmax = parallel_cross_entropy( + argmax_logits, + target, + label_smoothing, + reduce_loss, + overwrite_input=overwrite_input, + return_argmax=True, + ) + + assert argmax.dtype == torch.int64 + assert not argmax.requires_grad + torch.testing.assert_close(argmax, expected_argmax, rtol=0, atol=0) + + external_grad = torch.randn_like(loss) + loss.backward(external_grad) + argmax_loss.backward(external_grad) + torch.testing.assert_close(argmax_loss, loss, rtol=0, atol=0) + torch.testing.assert_close(argmax_logits.grad, logits.grad, rtol=0, atol=0) + + +def test_parallel_cross_entropy_return_argmax_transposed_input(): + """Each argmax belongs to the row of the loss next to it.""" + + values = _logits_with_argmax_cases((2, 5, 70000), torch.float32).transpose(0, 1) + target = torch.randint(0, values.shape[-1], values.shape[:-1], device="cuda") + loss, argmax = parallel_cross_entropy(values, target, return_argmax=True) + assert argmax.shape == loss.shape == values.shape[:-1] + torch.testing.assert_close(argmax, torch.argmax(values, dim=-1), rtol=0, atol=0) + + +def test_parallel_cross_entropy_return_argmax_nan(): + """As with torch.argmax, a NaN logit counts as the maximum and the first NaN wins, within a + block and across blocks.""" + + nan = float("nan") + small = torch.tensor( + [ + [8.0, -10.0, nan, -10.0], + [nan, 1.0, 5.0, 2.0], + [1.0, 5.0, 2.0, nan], + [3.0, nan, 3.0, nan], + [nan, nan, nan, nan], + [float("inf"), 1.0, nan, 2.0], + ], + device="cuda", + ) + large = torch.randn(4, 70000, device="cuda") + pair = torch.tensor([nan, 50.0], device="cuda") + large[0, [40000, 10]] = pair # NaN in a later block than the maximum + large[1, [10, 40000]] = pair # NaN in an earlier block than the maximum + large[2, [40001, 40000]] = pair # NaN after the maximum in the same block + large[3, [65000, 35000]] = nan # NaN in two blocks + for values in (small, large): + for dtype in (torch.float32, torch.bfloat16): + logits = values.to(dtype).unsqueeze(0) + target = torch.zeros(logits.shape[:-1], dtype=torch.long, device="cuda") + _, argmax = parallel_cross_entropy(logits, target, return_argmax=True) + torch.testing.assert_close(argmax, torch.argmax(logits, dim=-1), rtol=0, atol=0) diff --git a/transformer_engine/common/triton/cross_entropy.py b/transformer_engine/common/triton/cross_entropy.py index 47c9a924b31..3d1fd531782 100644 --- a/transformer_engine/common/triton/cross_entropy.py +++ b/transformer_engine/common/triton/cross_entropy.py @@ -8,6 +8,41 @@ import triton.language as tl +@triton.jit +def _is_nan(x): + """NaN test that works on Triton tensors and scalars: NaN is the only value unequal to itself.""" + return x != x # pylint: disable=comparison-with-itself + + +@triton.jit +def _fold_block_argmax(x, i, offsets, n_cols, run_max, run_idx, run_nan): + """Fold one block of a row into a running argmax with torch.argmax's semantics: NaN counts + as the maximum, and the first maximum (or the first NaN) wins. + + tl.max(..., return_indices=True) is not NaN-safe (its comparisons are false for NaN, so a + block with a NaN can give a pair that is neither the maximum nor the NaN). NaN is therefore + found separately and the maximum is taken over the other values. run_max is kept apart from + the loss's running maximum, which is unchanged.""" + is_nan = _is_nan(x) + block_has_nan = tl.max(is_nan.to(tl.int32)) > 0 + first_nan = tl.min(tl.where(is_nan, offsets, n_cols)) + block_max, block_idx = tl.max( + tl.where(is_nan, float("-inf"), x), + axis=0, + return_indices=True, + return_indices_tie_break_left=True, + ) + if run_nan == 0: + if block_has_nan: + run_idx = first_nan + run_nan = 1 + elif block_max > run_max: + # Only a strictly larger maximum replaces an earlier one: first index wins. + run_idx = i + block_idx + run_max = block_max + return run_max, run_idx, run_nan + + @triton.jit def cross_entropy_forward_kernel( X_ptr, @@ -18,6 +53,7 @@ def cross_entropy_forward_kernel( Y_ptr, loss_ptr, stats_ptr, + argmax_ptr, n_non_ignore, n_cols, n_rows_1, @@ -25,9 +61,11 @@ def cross_entropy_forward_kernel( label_smoothing: tl.constexpr, COUNT_NON_IGNORE: tl.constexpr, COPY_INPUT: tl.constexpr, + COMPUTE_ARGMAX: tl.constexpr, BLOCK_SIZE: tl.constexpr, ): - """Compute single-rank loss/statistics and optionally preserve the input.""" + """Compute single-rank loss/statistics and optionally preserve the input and find the + argmax of each row.""" row = tl.program_id(0).to(tl.int64) row_0 = row // n_rows_1 @@ -43,6 +81,9 @@ def cross_entropy_forward_kernel( m = float("-inf") d = 0.0 x_sum = 0.0 + argmax = 0 + argmax_max = float("-inf") + argmax_nan = 0 for i in range(0, n_cols, BLOCK_SIZE): offsets = i + tl.arange(0, BLOCK_SIZE) mask = offsets < n_cols @@ -50,6 +91,10 @@ def cross_entropy_forward_kernel( if COPY_INPUT: tl.store(saved_input_ptr + offsets, x, mask=mask) x = x.to(tl.float32) + if COMPUTE_ARGMAX: + argmax_max, argmax, argmax_nan = _fold_block_argmax( + x, i, offsets, n_cols, argmax_max, argmax, argmax_nan + ) block_max = tl.max(x) m_new = tl.maximum(m, block_max) d = d * tl.exp(m - m_new) + tl.sum(tl.exp(x - m_new)) @@ -59,6 +104,8 @@ def cross_entropy_forward_kernel( tl.store(stats_ptr + row * 2, m) tl.store(stats_ptr + row * 2 + 1, d) + if COMPUTE_ARGMAX: + tl.store(argmax_ptr + row, argmax.to(tl.int64)) if y == ignore_idx: tl.store(loss_ptr + row, 0.0) @@ -94,6 +141,8 @@ def cross_entropy_tp_pre_kernel( COUNT_NON_IGNORE: tl.constexpr, COMPUTE_X_SUM: tl.constexpr, COPY_INPUT: tl.constexpr, + COMPUTE_ARGMAX: tl.constexpr, + LOCAL_DATA_SIZE: tl.constexpr, BLOCK_SIZE: tl.constexpr, ): """Compute the local statistics needed by tensor-parallel cross entropy.""" @@ -118,6 +167,9 @@ def cross_entropy_tp_pre_kernel( m = float("-inf") d = 0.0 x_sum = 0.0 + argmax = 0 + argmax_max = float("-inf") + argmax_nan = 0 for i in range(0, n_cols, BLOCK_SIZE): offsets = i + tl.arange(0, BLOCK_SIZE) mask = offsets < n_cols @@ -125,6 +177,10 @@ def cross_entropy_tp_pre_kernel( if COPY_INPUT: tl.store(saved_input_ptr + offsets, x, mask=mask) x = x.to(tl.float32) + if COMPUTE_ARGMAX: + argmax_max, argmax, argmax_nan = _fold_block_argmax( + x, i, offsets, n_cols, argmax_max, argmax, argmax_nan + ) block_max = tl.max(x) m_new = tl.maximum(m, block_max) d = d * tl.exp(m - m_new) + tl.sum(tl.exp(x - m_new)) @@ -132,12 +188,17 @@ def cross_entropy_tp_pre_kernel( if COMPUTE_X_SUM: x_sum += tl.sum(tl.where(mask, x, 0.0)) - local_data_ptr += row * 4 + local_data_ptr += row * LOCAL_DATA_SIZE tl.store(local_data_ptr, m) tl.store(local_data_ptr + 1, d) tl.store(local_data_ptr + 2, x_y) if COMPUTE_X_SUM: tl.store(local_data_ptr + 3, x_sum) + if COMPUTE_ARGMAX: + # The local index travels with the FP32 statistics as raw bits, followed by the local + # maximum it belongs to, NaN if the shard holds a NaN. + tl.store(local_data_ptr + 4, argmax.to(tl.float32, bitcast=True)) + tl.store(local_data_ptr + 5, tl.where(argmax_nan != 0, float("nan"), argmax_max)) @triton.jit @@ -146,27 +207,44 @@ def cross_entropy_tp_post_kernel( Y_ptr, loss_ptr, stats_ptr, + argmax_ptr, world_size, n_rows, n_cols, ignore_idx, label_smoothing: tl.constexpr, + COMPUTE_ARGMAX: tl.constexpr, + LOCAL_DATA_SIZE: tl.constexpr, ): """Combine tensor-parallel statistics and compute loss/global statistics.""" row = tl.program_id(0).to(tl.int64) - data_ptr = gathered_data_ptr + row * 4 + data_ptr = gathered_data_ptr + row * LOCAL_DATA_SIZE m = tl.load(data_ptr) d = tl.load(data_ptr + 1) x_y = tl.load(data_ptr + 2) x_sum = 0.0 if label_smoothing > 0: x_sum = tl.load(data_ptr + 3) + argmax = 0 + argmax_max = 0.0 + if COMPUTE_ARGMAX: + argmax = tl.load(data_ptr + 4).to(tl.int32, bitcast=True).to(tl.int64) + argmax_max = tl.load(data_ptr + 5) for rank_idx in range(1, world_size): - rank_data_ptr = data_ptr + rank_idx * n_rows * 4 + rank_data_ptr = data_ptr + rank_idx * n_rows * LOCAL_DATA_SIZE m_new = tl.load(rank_data_ptr) d_new = tl.load(rank_data_ptr + 1) + if COMPUTE_ARGMAX: + # Ranks hold consecutive vocabulary shards, so the first rank with the largest + # maximum (or with a NaN, which counts as the maximum) holds the global argmax. + rank_argmax_max = tl.load(rank_data_ptr + 5) + if not _is_nan(argmax_max): + if _is_nan(rank_argmax_max) or (rank_argmax_max > argmax_max): + local_argmax = tl.load(rank_data_ptr + 4).to(tl.int32, bitcast=True) + argmax = rank_idx.to(tl.int64) * n_cols + local_argmax.to(tl.int64) + argmax_max = rank_argmax_max global_m = tl.maximum(m, m_new) d = d * tl.exp(m - global_m) + d_new * tl.exp(m_new - global_m) m = global_m @@ -176,6 +254,8 @@ def cross_entropy_tp_post_kernel( tl.store(stats_ptr + row * 2, m) tl.store(stats_ptr + row * 2 + 1, d) + if COMPUTE_ARGMAX: + tl.store(argmax_ptr + row, argmax) y = tl.load(Y_ptr + row) if y == ignore_idx: diff --git a/transformer_engine/pytorch/cross_entropy.py b/transformer_engine/pytorch/cross_entropy.py index ebbf30b965a..8ae3b853801 100644 --- a/transformer_engine/pytorch/cross_entropy.py +++ b/transformer_engine/pytorch/cross_entropy.py @@ -4,7 +4,7 @@ """Cross Entropy Loss API""" -from typing import Optional +from typing import Optional, Tuple, Union import warnings import torch @@ -32,6 +32,7 @@ def forward( dist_process_group=None, ignore_idx=-100, overwrite_input=False, + return_argmax=False, ): """Compute the loss and save the input and softmax statistics for backward.""" @@ -43,6 +44,7 @@ def forward( n_non_ignore, rank, world_size, + argmax, ) = triton_cross_entropy.cross_entropy_forward( inp, target, @@ -51,6 +53,7 @@ def forward( dist_process_group, ignore_idx, overwrite_input, + return_argmax, ) tensors_to_save = (saved_input.detach(), stats, target) if reduce_loss: @@ -63,10 +66,13 @@ def forward( ctx.ignore_idx = ignore_idx ctx.overwrite_input = overwrite_input ctx.did_backward = False + if return_argmax: + ctx.mark_non_differentiable(argmax) + return loss, argmax return loss @staticmethod - def backward(ctx, grad_output): + def backward(ctx, grad_output, *_): """Reconstruct the input gradient from the tensors saved during forward.""" if ctx.did_backward: @@ -91,7 +97,7 @@ def backward(ctx, grad_output): ) if ctx.overwrite_input: torch.autograd.graph.increment_version(saved_input) - return grad_input, None, None, None, None, None, None + return grad_input, None, None, None, None, None, None, None def _validate_inputs( @@ -133,7 +139,8 @@ def _parallel_cross_entropy_overwrite_input( reduce_loss: bool, dist_process_group: Optional[torch.distributed.ProcessGroup], ignore_idx: int, -) -> torch.Tensor: + return_argmax: bool, +) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: """Run destructive cross entropy outside Torch Dynamo's compiled graph.""" return CrossEntropyFunction.apply( @@ -144,6 +151,7 @@ def _parallel_cross_entropy_overwrite_input( dist_process_group, ignore_idx, True, + return_argmax, ) @@ -157,8 +165,9 @@ def parallel_cross_entropy( is_cg_capturable: bool = False, *, overwrite_input: bool = False, + return_argmax: bool = False, _input: Optional[torch.Tensor] = None, -) -> torch.Tensor: +) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: """Cross entropy loss with optional distributed reduction. Loss and derivative calculations use FP32 arithmetic for BF16 and FP32 @@ -169,6 +178,11 @@ def parallel_cross_entropy( must not read or reuse it after starting backward. Only one backward pass is supported for each loss result. + With ``return_argmax=True``, the forward pass also finds the index of the + largest logit of each row while it scans the row for the softmax maximum, + so callers that need the predicted tokens (for example an accuracy metric) + do not have to read the logits again. + Parameters ---------- inp : torch.Tensor @@ -191,11 +205,18 @@ def parallel_cross_entropy( Allow ``inp`` to be overwritten during backward. The input must be contiguous and cannot be reused afterward. This mode is incompatible with ``torch.compile`` and will result in a graph break if used in that context. + return_argmax : bool, default = False + Also return the index of each row's largest logit in the full vocabulary, + as ``torch.argmax`` over the unsharded logits returns it: the first index + if several logits are equal. Rows with an ignored target get one too. + As with ``torch.argmax``, a NaN logit counts as the maximum: a row with + NaN logits gets the index of its first NaN (and a NaN loss). Returns ------- - torch.Tensor - The computed loss. + torch.Tensor or tuple of torch.Tensor + The computed loss. With ``return_argmax=True``, the tuple ``(loss, argmax)``, + where ``argmax`` is an int64 tensor with the shape ``inp.shape[:-1]``. """ if _input is not None: @@ -221,6 +242,7 @@ def parallel_cross_entropy( reduce_loss, dist_process_group, ignore_idx, + return_argmax, ) return CrossEntropyFunction.apply( inp, @@ -230,4 +252,5 @@ def parallel_cross_entropy( dist_process_group, ignore_idx, overwrite_input, + return_argmax, ) diff --git a/transformer_engine/pytorch/triton/cross_entropy.py b/transformer_engine/pytorch/triton/cross_entropy.py index fda28474c28..3bc586c4ece 100644 --- a/transformer_engine/pytorch/triton/cross_entropy.py +++ b/transformer_engine/pytorch/triton/cross_entropy.py @@ -29,8 +29,13 @@ def cross_entropy_forward( dist_process_group: Union[dist.ProcessGroup, None], ignore_idx: int, overwrite_input: bool, + compute_argmax: bool = False, ): - """Forward implementation that saves the input and compact softmax statistics.""" + """Forward implementation that saves the input and compact softmax statistics. + + With ``compute_argmax``, also returns the index of each row's first maximum in the + full vocabulary (otherwise None). + """ B, SQ, V = _input.shape n_rows = B * SQ @@ -45,6 +50,9 @@ def cross_entropy_forward( loss_1d = torch.empty(n_rows, dtype=torch.float32, device=_input.device) stats = torch.empty((n_rows, 2), dtype=torch.float32, device=_input.device) n_non_ignore = torch.zeros(1, dtype=torch.int64, device=_input.device) if reduce_loss else None + argmax = ( + torch.empty(n_rows, dtype=torch.int64, device=_input.device) if compute_argmax else None + ) rank = 0 if dist_process_group is None else dist.get_rank(dist_process_group) world_size = 1 if dist_process_group is None else dist.get_world_size(dist_process_group) @@ -59,6 +67,7 @@ def cross_entropy_forward( Y_ptr=target, loss_ptr=loss_1d, stats_ptr=stats, + argmax_ptr=argmax if argmax is not None else stats, n_non_ignore=n_non_ignore if n_non_ignore is not None else stats, n_cols=V, n_rows_1=SQ, @@ -66,11 +75,17 @@ def cross_entropy_forward( label_smoothing=label_smoothing, COUNT_NON_IGNORE=reduce_loss, COPY_INPUT=not overwrite_input, + COMPUTE_ARGMAX=compute_argmax, BLOCK_SIZE=BLOCK_SIZE, num_warps=32, ) else: - local_data = torch.empty((n_rows, 4), dtype=torch.float32, device=_input.device) + # Per row and rank: max, sum of exponentials, target logit, logit sum, and with + # compute_argmax the local argmax (as FP32 bits) and its value (NaN if the shard has one) + local_data_size = 6 if compute_argmax else 4 + local_data = torch.empty( + (n_rows, local_data_size), dtype=torch.float32, device=_input.device + ) cross_entropy_tp_pre_kernel[(n_rows,)]( X_ptr=_input, X_stride_0=_input.stride(0), @@ -87,11 +102,13 @@ def cross_entropy_forward( COUNT_NON_IGNORE=reduce_loss, COMPUTE_X_SUM=label_smoothing > 0, COPY_INPUT=not overwrite_input, + COMPUTE_ARGMAX=compute_argmax, + LOCAL_DATA_SIZE=local_data_size, BLOCK_SIZE=BLOCK_SIZE, num_warps=32, ) gathered_data = torch.empty( - (world_size * n_rows, 4), dtype=torch.float32, device=_input.device + (world_size * n_rows, local_data_size), dtype=torch.float32, device=_input.device ) dist.all_gather_into_tensor(gathered_data, local_data, group=dist_process_group) cross_entropy_tp_post_kernel[(n_rows,)]( @@ -99,11 +116,14 @@ def cross_entropy_forward( Y_ptr=target, loss_ptr=loss_1d, stats_ptr=stats, + argmax_ptr=argmax if argmax is not None else stats, world_size=world_size, n_rows=n_rows, n_cols=V, ignore_idx=ignore_idx, label_smoothing=label_smoothing, + COMPUTE_ARGMAX=compute_argmax, + LOCAL_DATA_SIZE=local_data_size, num_warps=1, ) @@ -111,8 +131,10 @@ def cross_entropy_forward( if reduce_loss: n_non_ignore.clamp_(min=1) loss = loss_1d.sum() / n_non_ignore + if argmax is not None: + argmax = argmax.reshape(B, SQ) - return loss, saved_input, stats, target, n_non_ignore, rank, world_size + return loss, saved_input, stats, target, n_non_ignore, rank, world_size, argmax def cross_entropy_backward( From 0c4d9610ac3b6447e90af212e70b7260ef0c4ad1 Mon Sep 17 00:00:00 2001 From: vasunvidia <108759426+vasunvidia@users.noreply.github.com> Date: Tue, 13 Oct 2026 04:09:39 -0700 Subject: [PATCH 30/31] Change to enable CuteDSL DenseGEMM for eval (#3666) * Change to enable CuteDSL DenseGEMM for eval Signed-off-by: Vasudevan Rengasamy * Add unit test for Single group grouped_gemm for no_grad and frozen_weights use cases Signed-off-by: Vasudevan Rengasamy --------- Signed-off-by: Vasudevan Rengasamy --- tests/pytorch/test_grouped_mlp.py | 125 ++++++++++++++++++ .../pytorch/ops/fused/grouped_mlp.py | 10 +- 2 files changed, 131 insertions(+), 4 deletions(-) diff --git a/tests/pytorch/test_grouped_mlp.py b/tests/pytorch/test_grouped_mlp.py index 88af92f0bb9..4ae63b44dbf 100644 --- a/tests/pytorch/test_grouped_mlp.py +++ b/tests/pytorch/test_grouped_mlp.py @@ -2089,6 +2089,131 @@ def test_grouped_mlp_single_group( activation="scaled_swiglu", ) + @pytest.mark.parametrize("mode", ("frozen_weights", "no_grad")) + @pytest.mark.parametrize("quantization", ("mxfp8", "nvfp4_rht")) + def test_grouped_mlp_single_group_no_weight_grad( + self, + *, + mode: str, + quantization: str, + hidden_size: int = 128, + tokens: int = 256, + dtype: torch.dtype = torch.bfloat16, + device: torch.device = "cuda", + ) -> None: + """Single-group fused grouped MLP when weight gradients are not needed + + Without weight gradients the FC2 input is quantized without + column-wise data, which must still run as a dense single-group + GEMM rather than a grouped cuBLAS GEMM. + """ + + maybe_skip_quantization( + quantization, dims=(tokens, hidden_size), device=device, dtype=dtype + ) + if not te.ops.fused.GroupedMLP_CuTeGEMMGLU.is_supported(): + pytest.skip("CuTe DSL fused grouped MLP is not supported") + + split_sizes = torch.tensor([tokens], dtype=torch.int, device=device) + in_shape = (tokens, hidden_size) + fc1_out_features = 2 * hidden_size + + x_ref, x_test = make_reference_and_test_tensors( + in_shape, + min=-0.25, + max=0.25, + quantization=quantization, + test_dtype=dtype, + test_device=device, + quantizer_role=QuantizerRole(tensor_type="input"), + requires_grad=(mode == "frozen_weights"), + ) + dy_ref, dy_test = make_reference_and_test_tensors( + in_shape, + min=-0.25, + max=0.25, + quantization=quantization, + test_dtype=dtype, + test_device=device, + quantizer_role=QuantizerRole(tensor_type="grad_output"), + requires_grad=False, + ) + probs_ref, probs_test = make_reference_and_test_tensors( + (tokens,), + test_dtype=dtype, + test_device=device, + requires_grad=False, + ) + fc1_w_ref, fc1_w_test = make_reference_and_test_tensors( + (fc1_out_features, hidden_size), + min=-0.25, + max=0.25, + quantization=quantization, + test_dtype=dtype, + test_device=device, + quantizer_role=QuantizerRole(tensor_type="weight"), + requires_grad=False, + ) + fc2_w_ref, fc2_w_test = make_reference_and_test_tensors( + (hidden_size, hidden_size), + min=-0.25, + max=0.25, + quantization=quantization, + test_dtype=dtype, + test_device=device, + quantizer_role=QuantizerRole(tensor_type="weight"), + requires_grad=False, + ) + + # Reference implementation + fc1_out = torch.nn.functional.linear(x_ref, fc1_w_ref) + fc1_out = fc1_out.reshape(tokens, fc1_out_features // 64, 2, 32).transpose(1, 2) + act_in1, act_in2 = fc1_out.reshape(tokens, fc1_out_features).chunk(2, dim=-1) + act_out = torch.nn.functional.silu(act_in1) * act_in2 + y_ref = torch.nn.functional.linear(act_out * probs_ref.unsqueeze(-1), fc2_w_ref) + if mode == "frozen_weights": + y_ref.backward(dy_ref) + + # Construct operations + recipe = make_recipe(quantization) + with te.quantized_model_init(enabled=True, recipe=recipe): + fc1 = te.ops.GroupedLinear( + 1, hidden_size, fc1_out_features, bias=False, device=device, dtype=dtype + ) + fc2 = te.ops.GroupedLinear( + 1, hidden_size, hidden_size, bias=False, device=device, dtype=dtype + ) + module = te.ops.Sequential(fc1, te.ops.ScaledSwiGLU(glu_interleave_size=32), fc2) + with torch.no_grad(): + fc1.weight0.copy_(fc1_w_test) + fc2.weight0.copy_(fc2_w_test) + for param in module.parameters(): + param.requires_grad_(False) + del fc1_w_test, fc2_w_test + + # Forward (and backward w.r.t. input only for frozen weights) + with te.autocast(enabled=True, recipe=recipe): + if mode == "no_grad": + with torch.no_grad(): + y_test = module(x_test, split_sizes, probs_test, split_sizes) + else: + y_test = module(x_test, split_sizes, probs_test, split_sizes) + if mode == "frozen_weights": + y_test.backward(dy_test) + + # Check that operations were fused + forward_ops = module._module_groups[0]._forward_ops + assert len(forward_ops) == 1 + assert isinstance(forward_ops[0][0], te.ops.fused.GroupedMLP_CuTeGEMMGLU) + + # Loose tols for sanity checking + tols = {"rtol": 0.125, "atol": 0.25} + assert_close(y_test, y_ref, **tols) + if mode == "frozen_weights": + assert_close_grads(x_test, x_ref, **tols) + assert fc1.weight0.grad is None + assert fc2.weight0.grad is None + @pytest.mark.skipif(not mxfp8_available, reason=reason_for_no_mxfp8) def test_single_grouped_weight_eval_preserves_columnwise_usage( self, diff --git a/transformer_engine/pytorch/ops/fused/grouped_mlp.py b/transformer_engine/pytorch/ops/fused/grouped_mlp.py index 108ce5311c7..eb336cbe6bc 100644 --- a/transformer_engine/pytorch/ops/fused/grouped_mlp.py +++ b/transformer_engine/pytorch/ops/fused/grouped_mlp.py @@ -1933,10 +1933,12 @@ def fuser_forward( fc2_bias_scale = fc2_bias_scale.to(dtype=torch.float32) fc2_out_buf = validate_or_alloc_output(output_buffer, fc2_out_shape, dtype, device) - if ( - use_dense_single_group - and grouped_fc2_x.columnwise_data is not None - and grouped_fc2_x.columnwise_scale_inv is not None + if use_dense_single_group and ( + not weight_requires_grad + or ( + grouped_fc2_x.columnwise_data is not None + and grouped_fc2_x.columnwise_scale_inv is not None + ) ): fc2_out_buf = _single_group_fc2_gemm( grouped_fc2_x, From e6afa9ddfa813e39295572385b2fc9fdc77c067b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadzi=C5=84ski?= <62263673+pggPL@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:53:38 +0200 Subject: [PATCH 31/31] [PyTorch] Refactor GroupedLinear forward and backward (#3606) * [PyTorch] Refactor GroupedLinear eager forward and backward Extract shared computation and backward state for both split-quantize and grouped-tensor execution. Preserve eager saved-tensor layout and gradient allocation so compile adapters can reuse these functions in a follow-up change. Signed-off-by: Pawel Gadzinski * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Preserve GroupedLinear behavior across the eager refactor Derive backward feature dimensions from validated forward tensors so local distributed-weight shards do not determine GEMM buffer shapes. Restore the fused input-width check and retain disabled weight-gradient stores so they can be enabled between forward and backward. Capture only the accumulator flag in deferred fused wgrad callbacks to release cast weights after dgrad. Signed-off-by: Pawel Gadzinski * [PyTorch] Preserve GroupedLinear behavior across refactor Restore the grad-output bulk-allocation and debug dbias conditions from the original implementation. Preserve quantizers selected from materialized weights and apply the common quantizer setup on both execution paths. Read GEMM accumulator settings from the saved recipe when each GEMM runs, including the default settings for high-precision backward overrides. Adapt the backward-override test to the extracted backward state holder without changing its assertions or numerical tolerances. Signed-off-by: Pawel Gadzinski * [PyTorch] Reduce GroupedLinear Python overhead Signed-off-by: Pawel Gadzinski --------- Signed-off-by: Pawel Gadzinski Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- tests/pytorch/test_backward_override.py | 9 +- .../pytorch/module/grouped_linear.py | 2553 +++++++++-------- 2 files changed, 1323 insertions(+), 1239 deletions(-) diff --git a/tests/pytorch/test_backward_override.py b/tests/pytorch/test_backward_override.py index c0acf2e6b3b..b0c56773f7b 100644 --- a/tests/pytorch/test_backward_override.py +++ b/tests/pytorch/test_backward_override.py @@ -813,21 +813,22 @@ def _run_grouped_linear_single_step_with_ctx_state( raise RuntimeError( "Output tensor has no grad_fn; cannot inspect grouped backward state." ) + state_holder = y.grad_fn.backward_objects required_attrs = ( "backward_override", "fp8", "reduce_and_update_bwd_fp8_tensors", ) - missing_attrs = [attr for attr in required_attrs if not hasattr(y.grad_fn, attr)] + missing_attrs = [attr for attr in required_attrs if not hasattr(state_holder, attr)] if missing_attrs: raise RuntimeError( "Grouped grad_fn does not expose required backward context attributes: " f"{', '.join(missing_attrs)}." ) ctx_state = ( - getattr(y.grad_fn, "backward_override"), - bool(getattr(y.grad_fn, "fp8")), - bool(getattr(y.grad_fn, "reduce_and_update_bwd_fp8_tensors")), + getattr(state_holder, "backward_override"), + bool(getattr(state_holder, "fp8")), + bool(getattr(state_holder, "reduce_and_update_bwd_fp8_tensors")), ) y.backward(dy) assert x_run.grad is not None diff --git a/transformer_engine/pytorch/module/grouped_linear.py b/transformer_engine/pytorch/module/grouped_linear.py index 62607a7c1b2..0d1712603fa 100644 --- a/transformer_engine/pytorch/module/grouped_linear.py +++ b/transformer_engine/pytorch/module/grouped_linear.py @@ -4,7 +4,8 @@ """GroupedLinear API""" -from typing import Union, Optional, Callable, Tuple, List +from dataclasses import dataclass +from typing import Any, Union, Optional, Callable, Tuple, List from itertools import chain import os import warnings @@ -153,6 +154,1116 @@ def is_module_grouped_tensor_path_supported( return False +@dataclass(slots=True) +class GroupedLinearFwdArgs: + """Forward configuration and operands.""" + + inp: torch.Tensor + weights: List[Union[torch.Tensor, QuantizedTensorStorage]] + biases: List[torch.Tensor] + + weight_workspaces: List[Union[torch.Tensor, QuantizedTensorStorage]] + out: Optional[torch.Tensor] + dgrad_out: Optional[torch.Tensor] + skip_fp8_weight_update: Optional[torch.Tensor] + m_splits_tensor: Optional[torch.Tensor] + + input_requires_grad: bool + weights_requires_grad: bool + + input_quantizers: List[Quantizer] + weight_quantizers: List[Quantizer] + output_quantizers: List[Quantizer] + grad_input_quantizers: List[Quantizer] + grad_weight_quantizers: List[Quantizer] + grad_output_quantizers: List[Quantizer] + + m_splits: Optional[List[int]] + num_gemms: int + + activation_dtype: torch.dtype + fp8: bool + fp8_calibration: bool + save_original_input: bool + backward_override: Optional[str] + fp8_recipe: Optional[Recipe] + debug: bool + + is_first_microbatch: Optional[bool] + cache_weight: bool + + use_grouped_tensor_path: bool + single_grouped_weight: bool + single_grouped_bias: bool + + use_bias: bool + fuse_wgrad_accumulation: bool + wgrad_store: Optional[Any] + cpu_offloading: bool + is_grad_enabled: bool + + +@dataclass(slots=True) +class GroupedLinearBwdArgs: + """Backward configuration and saved operands.""" + + grad_output: Optional[torch.Tensor] = None + inputmats: List[Union[torch.Tensor, QuantizedTensorStorage]] = None + weights_fp8: List[Union[torch.Tensor, QuantizedTensorStorage]] = None + saved_weights: List[Union[torch.Tensor, QuantizedTensorStorage]] = None + biases: List[torch.Tensor] = None + dgrad_out: Optional[torch.Tensor] = None + + input_quantizers: List[Quantizer] = None + weight_quantizers: List[Quantizer] = None + grad_input_quantizers: List[Quantizer] = None + grad_weight_quantizers: List[Quantizer] = None + grad_output_quantizers: List[Quantizer] = None + + m_splits: Optional[List[int]] = None + num_gemms: int = 0 + weights_shape_1: int = 0 + + use_bias: bool = False + requires_dgrad: bool = False + weights_requires_grad: bool = False + + activation_dtype: Optional[torch.dtype] = None + fp8: bool = False + backward_override: Optional[str] = None + fp8_recipe: Optional[Recipe] = None + save_original_input: bool = False + debug: bool = False + + is_first_microbatch: Optional[bool] = None + fuse_wgrad_accumulation: bool = False + wgrad_store: Optional[Any] = None + origin_weight_refs: Optional[Any] = None + origin_weights_overwrite_main_grad: bool = False + main_grad_funcs: Optional[Any] = None + + reduce_and_update_bwd_fp8_tensors: bool = False + + cpu_offloading: bool = False + + def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: + """Pull saved tensors from ``ctx`` into the fields backward consumes.""" + saved = restore_from_func_ctx(ctx) + n = self.num_gemms + self.inputmats = saved[:n] + self.weights_fp8 = saved[n : 2 * n] + self.saved_weights = saved[2 * n : 3 * n] + self.biases = saved[3 * n : 4 * n] + + +def _grouped_linear_configure_quantizers(args: GroupedLinearFwdArgs, weights) -> None: + """Configure operand usage after materializing weights.""" + if args.input_quantizers[0] is not None: + for input_quantizer in args.input_quantizers: + input_quantizer.set_usage( + rowwise=True, + columnwise=( + args.is_grad_enabled + and args.weights_requires_grad + and not args.save_original_input + and args.backward_override is None + ), + ) + columnwise_usage = args.is_grad_enabled and args.input_requires_grad + if args.backward_override is not None: + columnwise_usage = False + if not columnwise_usage: + columnwise_usage = ( + is_fp8_activation_recompute_enabled() and not in_fp8_activation_recompute_phase() + ) + if args.weight_quantizers[0] is not None and ( + not isinstance(weights[0], QuantizedTensorStorage) or args.debug + ): + for weight_quantizer in args.weight_quantizers: + weight_quantizer.set_usage(rowwise=True, columnwise=columnwise_usage) + elif isinstance(weights[0], QuantizedTensorStorage): + args.weight_quantizers = [weight._quantizer for weight in weights] + if args.output_quantizers[0] is not None: + for output_quantizer in args.output_quantizers: + output_quantizer.set_usage(rowwise=True, columnwise=False) + + +def _grouped_linear_forward_impl( + args: GroupedLinearFwdArgs, +) -> Tuple[Any, ...]: + """Compute the legacy forward and collect tensors needed by backward.""" + inp = args.inp + weights = args.weights + biases = args.biases + num_gemms = args.num_gemms + m_splits = args.m_splits + input_quantizers = args.input_quantizers + output_quantizers = args.output_quantizers + activation_dtype = args.activation_dtype + fp8 = args.fp8 + debug = args.debug + use_bias = args.use_bias + is_grad_enabled = args.is_grad_enabled + save_original_input = args.save_original_input + backward_override = args.backward_override + cpu_offloading = args.cpu_offloading + device = inp.device + weight_requires_grad = args.weights_requires_grad + + is_dist_weight = is_distributed_weight(weights[0]) + if is_dist_weight: + weights = materialize_weight_for_forward(weights) + + _grouped_linear_configure_quantizers(args, weights) + weight_quantizers = args.weight_quantizers + + # Initialize input tensors + in_features = weights[0].size(-1) + if inp.size(-1) != in_features: + raise ValueError( + f"Input tensor (shape={tuple(inp.size())}) is not compatible with " + f"weight tensor (shape={tuple(weights[0].size())})" + ) + + inp_view = inp.reshape(-1, in_features) + inputmats, _ = _split_quantization._split_quantize( + inp_view, + m_splits, + input_quantizers, + activation_dtype, + with_quantized_output=fp8 or debug, + disable_bulk_allocation=cpu_offloading, + ) + + if cpu_offloading: + start_offload(*inputmats) + + # Initialize weights + weights_fp8: list + new_workspaces = [None] * num_gemms + if fp8 or debug: + weights_fp8 = [] + update_ws = args.is_first_microbatch is None or args.is_first_microbatch + for i in range(num_gemms): + weight_fp8, new_workspaces[i] = quantize_weight( + tensor=weights[i], + quantizer=weight_quantizers[i], + workspace=args.weight_workspaces[i] if args.weight_workspaces else None, + update_workspace=update_ws, + skip_update_flag=args.skip_fp8_weight_update, + workspace_dtype=activation_dtype, + cache=args.cache_weight and not is_dist_weight, + ) + weights_fp8.append(weight_fp8) + else: + weights_fp8 = [cast_if_needed(weight, activation_dtype) for weight in weights] + + # Initialize biases + bias_dtype = activation_dtype + if fp8 and activation_dtype == torch.float32: + bias_dtype = torch.bfloat16 # FP8 GEMM only supports BF16/FP16 bias + biases = [cast_if_needed(bias, bias_dtype) for bias in biases] if use_bias else biases + # Initialize output tensor + out = _GroupedLinear._validate_or_alloc_output( + args.out, + sum(m_splits), + weights_fp8[0].size(0), + activation_dtype, + device, + ) + + # Perform GEMM + fprop_use_split_accumulator = _2X_ACC_FPROP + if fp8 and hasattr(args.fp8_recipe, "fp8_gemm_fprop"): + fprop_use_split_accumulator = args.fp8_recipe.fp8_gemm_fprop.use_split_accumulator + general_grouped_gemm( + weights_fp8, + inputmats, + [out], + output_quantizers, + activation_dtype, + single_output=True, + m_splits=m_splits, + bias=biases, + use_bias=use_bias, + use_split_accumulator=fprop_use_split_accumulator, + ) + + if args.fp8_calibration: + for i in range(num_gemms): + input_quantizers[i].calibrate(inputmats[i]) + weight_quantizers[i].calibrate(weights[i]) + + if cpu_offloading: + mark_not_offload(*weights_fp8, *weights) + + tensors_to_save = None + if is_grad_enabled: + # TODO: update after #1638 is merged. # pylint: disable=fixme + if weight_requires_grad: + if save_original_input: + inputmats = [None] * num_gemms + inputmats[0] = inp + else: + for inputmat in inputmats: + if isinstance(inputmat, QuantizedTensorStorage): + if backward_override is not None: + # In dequantized mode we should dequantize directly from + # fprop quantized layouts without retargeting usage. + inputmat.update_usage(rowwise_usage=True, columnwise_usage=False) + else: + inputmat.update_usage(rowwise_usage=False, columnwise_usage=True) + else: + inputmats = [None] * num_gemms + + # Original weights are only needed by high_precision dgrad. The weakrefs + # used for fused wgrad accumulation serve a different purpose: restoring + # Python parameter attributes without keeping the parameter alive here. + saved_weights = ( + weights + if backward_override == "high_precision" and args.input_requires_grad + else [None] * num_gemms + ) + if is_dist_weight: + # GTP: gathered workspace is transient (re-gathered in backward), don't save it. + weights_fp8 = [None] * num_gemms + saved_weights = args.weights + tensors_to_save = (*inputmats, *weights_fp8, *saved_weights, *biases) + + return out.view(-1, *inp.shape[1:-1], out.shape[-1]), new_workspaces, tensors_to_save + + +def _grouped_linear_setup_ctx( + bwd_args: GroupedLinearBwdArgs, + fwd_args: GroupedLinearFwdArgs, +) -> None: + """Populate backward arguments from the forward configuration.""" + num_gemms = fwd_args.num_gemms + weights = fwd_args.weights + bwd_args.input_quantizers = fwd_args.input_quantizers + bwd_args.weight_quantizers = fwd_args.weight_quantizers + bwd_args.grad_input_quantizers = fwd_args.grad_input_quantizers + bwd_args.grad_weight_quantizers = fwd_args.grad_weight_quantizers + bwd_args.grad_output_quantizers = fwd_args.grad_output_quantizers + + bwd_args.m_splits = fwd_args.m_splits + bwd_args.num_gemms = num_gemms + bwd_args.weights_shape_1 = fwd_args.inp.shape[-1] + + bwd_args.use_bias = fwd_args.use_bias + bwd_args.requires_dgrad = fwd_args.input_requires_grad + bwd_args.weights_requires_grad = fwd_args.weights_requires_grad + + bwd_args.activation_dtype = fwd_args.activation_dtype + bwd_args.fp8 = fwd_args.fp8 + bwd_args.backward_override = fwd_args.backward_override + bwd_args.fp8_recipe = fwd_args.fp8_recipe + bwd_args.save_original_input = fwd_args.save_original_input + bwd_args.debug = fwd_args.debug + + bwd_args.is_first_microbatch = fwd_args.is_first_microbatch + bwd_args.fuse_wgrad_accumulation = fwd_args.fuse_wgrad_accumulation + bwd_args.wgrad_store = fwd_args.wgrad_store + bwd_args.cpu_offloading = fwd_args.cpu_offloading + bwd_args.dgrad_out = fwd_args.dgrad_out + + if fwd_args.fuse_wgrad_accumulation and fwd_args.weights_requires_grad: + # Keep weakrefs to weights to preserve attributes like main_grad + # when we need to modify the weight python objects + bwd_args.origin_weight_refs = [weakref.ref(w) for w in weights] + bwd_args.origin_weights_overwrite_main_grad = getattr( + weights[0], "overwrite_main_grad", False + ) + # MCore FSDP creates main_grad lazily before backward + if hasattr(weights[0], "__fsdp_param__"): + bwd_args.main_grad_funcs = [weights[i].get_main_grad for i in range(num_gemms)] + elif is_distributed_weight(weights[0]): + bwd_args.main_grad_funcs = [weights[i].grad_buffer for i in range(num_gemms)] + else: + bwd_args.main_grad_funcs = [lambda j=i: weights[j].main_grad for i in range(num_gemms)] + + if fwd_args.backward_override is not None: + bwd_args.fp8 = False + bwd_args.debug = False + bwd_args.grad_input_quantizers = [None] * num_gemms + bwd_args.grad_weight_quantizers = [None] * num_gemms + bwd_args.grad_output_quantizers = [None] * num_gemms + + +def _finish_wgrad(weight, main_grad, wgrad, fuse_wgrad_accumulation): + """Return the gradient expected by autograd or MCore's main-grad hooks.""" + if not fuse_wgrad_accumulation: + return wgrad + if not hasattr(weight, "grad_added_to_main_grad"): + return None + weight.grad_added_to_main_grad = True + return get_dummy_wgrad( + list(main_grad.shape), + weight.dtype, + zero=getattr(weight, "zero_out_wgrad", False), + zero_tensor=getattr(weight, "zero_tensor_dummy_wgrad", False), + ) + + +def _grouped_linear_backward_impl( + args: GroupedLinearBwdArgs, +) -> Tuple[Optional[torch.Tensor], List[Optional[torch.Tensor]], List[Optional[torch.Tensor]]]: + """Backward implementation for the grouped linear layer. + + Caller must have populated ``args.grad_output`` and run + ``args.setup_saved_tensors(ctx)`` before invocation. Returns + ``(dgrad, wgrad_list, grad_biases)``. + """ + grad_output = args.grad_output + num_gemms = args.num_gemms + m_splits = args.m_splits + inputmats = args.inputmats + weights = args.weights_fp8 + saved_weights = args.saved_weights + biases = args.biases + device = grad_output.device + in_features = args.weights_shape_1 + dgrad = None + + # Restore from weakrefs to get original weight python objects + # (preserves attributes like main_grad, grad_added_to_main_grad, etc.) + # Only needed when fuse_wgrad_accumulation is enabled. + origin_weights = [None] * num_gemms + main_grads = [None] * num_gemms + is_dist_weight = is_distributed_weight(saved_weights[0]) + if is_dist_weight: + origin_weights = saved_weights + if args.fuse_wgrad_accumulation and args.weights_requires_grad: + main_grads = [main_grad_func() for main_grad_func in args.main_grad_funcs] + elif args.fuse_wgrad_accumulation and args.weights_requires_grad: + origin_weight_refs = args.origin_weight_refs + args.origin_weight_refs = None + origin_weights = [ref() if ref is not None else None for ref in origin_weight_refs] + assert all( + w is not None for w in origin_weights + ), "weight was removed while fuse_wgrad_accumulation=True" + main_grads = [main_grad_func() for main_grad_func in args.main_grad_funcs] + for origin_weight, main_grad in zip(origin_weights, main_grads): + if main_grad is not None: + origin_weight.main_grad = main_grad + + # Preprocess grad output + out_features = grad_output.shape[-1] + grad_output_view = grad_output.contiguous().view(-1, out_features) + grad_output_reference = args.grad_output_quantizers[0] + if args.fp8 and isinstance(grad_output_reference, HybridQuantizer): + # Usage is a runtime decision, not part of generation validation. + # Apply it uniformly so dispatch can read the first parent without + # rescanning every expert. + for grad_output_quantizer in args.grad_output_quantizers: + grad_output_quantizer.set_usage( + rowwise=args.requires_dgrad, + columnwise=args.weights_requires_grad, + ) + grad_output, grad_biases = _split_quantization._split_quantize( + grad_output_view, + m_splits, + args.grad_output_quantizers, + args.activation_dtype, + with_quantized_output=args.fp8 or args.debug, + compute_dbias=(args.fp8 or args.debug) and args.use_bias, + disable_bulk_allocation=( + args.cpu_offloading + and isinstance(grad_output_reference, HybridQuantizer) + and not _split_quantization._uses_identity_quantizer(grad_output_reference) + ), + ) + if grad_biases is None: + grad_biases = [None] * num_gemms + + if is_dist_weight: + accumulate_wgrad_into_param_main_grad = False + elif args.is_first_microbatch is not None: + accumulate_wgrad_into_param_main_grad = ( + args.fuse_wgrad_accumulation and not args.is_first_microbatch + ) + else: + accumulate_wgrad_into_param_main_grad = args.fuse_wgrad_accumulation + + if is_dist_weight: + weights = materialize_weight_for_backward(origin_weights) + + if args.requires_dgrad: + dgrad_use_split_accumulator = _2X_ACC_DGRAD + if (args.fp8 or args.debug) and hasattr(args.fp8_recipe, "fp8_gemm_dgrad"): + dgrad_use_split_accumulator = args.fp8_recipe.fp8_gemm_dgrad.use_split_accumulator + dgrad = _GroupedLinear._validate_or_alloc_output( + args.dgrad_out, + sum(m_splits), + in_features, + args.activation_dtype, + device, + ) + weights_for_dgrad = weights + if args.backward_override == "dequantized": + weights_for_dgrad = [ + _GroupedLinear._maybe_dequantize(weight, args.activation_dtype) + for weight in weights + ] + elif args.backward_override == "high_precision": + weights_for_dgrad = [ + _GroupedLinear._maybe_dequantize(weight, args.activation_dtype) + for weight in saved_weights + ] + elif is_dist_weight and args.fp8: + weights_for_dgrad = [] + for idx, weight in enumerate(weights): + if not isinstance(weight, QuantizedTensorStorage): + quantizer = args.weight_quantizers[idx] + quantizer.set_usage(rowwise=True, columnwise=True) + weight = quantizer(weight) + weights_for_dgrad.append(weight) + # Make sure weights are available in column-wise format + # for dgrad computation. + for weight in weights_for_dgrad: + if isinstance(weight, QuantizedTensorStorage): + weight.update_usage(columnwise_usage=True) + general_grouped_gemm( + weights_for_dgrad, + grad_output, + [dgrad], + args.grad_input_quantizers, + args.activation_dtype, + single_output=True, + layout="NN", + m_splits=m_splits, + grad=True, + use_split_accumulator=dgrad_use_split_accumulator, + ) + + if args.weights_requires_grad: + if ( + is_dist_weight + and args.wgrad_store is not None + and args.wgrad_store.delay_wgrad_compute() + ): + raise RuntimeError( + "distributed-weight GroupedLinear requires delay_wgrad_compute=False." + ) + wgrad_use_split_accumulator = _2X_ACC_WGRAD + if args.fp8 and hasattr(args.fp8_recipe, "fp8_gemm_wgrad"): + wgrad_use_split_accumulator = args.fp8_recipe.fp8_gemm_wgrad.use_split_accumulator + if args.fuse_wgrad_accumulation: + wgrad_list = main_grads + else: + wgrad_packed = torch.empty( + num_gemms, + out_features, + in_features, + dtype=args.activation_dtype, + device=device, + ) + wgrad_list = [wgrad_packed[i] for i in range(num_gemms)] + if is_dist_weight: + # Gathered weights are no longer needed after dgrad GEMM. + del weights + + if args.save_original_input: + inp = inputmats[0] + inp_view = inp.reshape(-1, in_features) + if args.input_quantizers[0] is not None: + for input_quantizer in args.input_quantizers: + if isinstance( + input_quantizer, + (Float8Quantizer, Float8CurrentScalingQuantizer), + ): + input_quantizer.set_usage(rowwise=True, columnwise=True) + else: + input_quantizer.set_usage(rowwise=False, columnwise=True) + inputmats, _ = _split_quantization._split_quantize( + inp_view, + m_splits, + with_quantized_output=args.fp8 or args.debug, + quantizers=args.input_quantizers, + activation_dtype=args.activation_dtype, + disable_bulk_allocation=args.cpu_offloading, + ) + elif args.backward_override == "dequantized": + inputmats = [ + _GroupedLinear._maybe_dequantize(inputmat, args.activation_dtype) + for inputmat in inputmats + ] + grouped_gemm_wgrad = functools.partial( + general_grouped_gemm, + quantization_params=args.grad_weight_quantizers, + out_dtype=args.activation_dtype, + layout="NT", + grad=True, + m_splits=m_splits, + use_bias=args.use_bias if grad_biases[0] is None else None, + bias=biases, + use_split_accumulator=wgrad_use_split_accumulator, + accumulate=( + accumulate_wgrad_into_param_main_grad + if not is_dist_weight and not args.origin_weights_overwrite_main_grad + else False + ), + ) + # WGRAD + if args.wgrad_store is not None and args.wgrad_store.delay_wgrad_compute(): + args.wgrad_store.put([inputmats, grad_output, wgrad_list], grouped_gemm_wgrad) + else: + _, grad_biases_, _ = grouped_gemm_wgrad(inputmats, grad_output, wgrad_list) + + for i in range(num_gemms): + if grad_biases[i] is None: + grad_biases[i] = grad_biases_[i] + del grad_biases_ + + clear_tensor_data(*inputmats) + + if is_dist_weight: + wgrad_list = finalize_weight_grads(origin_weights, wgrad_list) + else: + wgrad_list = [ + _finish_wgrad(weight, main_grad, wgrad, args.fuse_wgrad_accumulation) + for weight, main_grad, wgrad in zip(origin_weights, main_grads, wgrad_list) + ] + else: + wgrad_list = [None] * num_gemms + + if not args.use_bias or ( + args.wgrad_store is not None and args.wgrad_store.delay_wgrad_compute() and not args.fp8 + ): + grad_biases = [None] * num_gemms + + dgrad_out = None + if args.requires_dgrad: + # Input shape rederived from grad_output (out.shape == (*inp.shape[:-1], out_features)). + dgrad_out = dgrad.view(*args.grad_output.shape[:-1], in_features) + return (dgrad_out, wgrad_list, list(grad_biases)) + + +@dataclass(slots=True) +class GroupedLinearGraphSafeBwdArgs: + """Backward configuration and saved grouped operands.""" + + grad_output: Optional[torch.Tensor] = None + inputmat: Any = None + weights_fp8: List[Union[torch.Tensor, QuantizedTensorStorage]] = None + m_splits_tensor: Optional[torch.Tensor] = None + base_split_offsets: Optional[torch.Tensor] = None + input_tensor_offsets: Optional[torch.Tensor] = None + output_tensor_offsets: Optional[torch.Tensor] = None + dgrad_out: Optional[torch.Tensor] = None + input_quantizers: List[Quantizer] = None + weight_quantizers: List[Quantizer] = None + grad_output_quantizers: List[Quantizer] = None + is_dist_weight: bool = False + num_gemms: int = 0 + in_features: int = 0 + out_features: int = 0 + activation_dtype: Optional[torch.dtype] = None + fp8: bool = False + use_bias: bool = False + requires_dgrad: bool = False + weights_requires_grad: bool = False + save_original_input: bool = False + single_grouped_weight: bool = False + single_grouped_bias: bool = False + is_first_microbatch: Optional[bool] = None + fp8_recipe: Optional[Recipe] = None + fuse_wgrad_accumulation: bool = False + origin_weight_refs: Optional[Any] = None + origin_weights_overwrite_main_grad: bool = False + main_grad_funcs: Optional[Any] = None + wgrad_store: Optional[WeightGradStore] = None + reduce_and_update_bwd_fp8_tensors: bool = False + + def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: + """Restore the grouped operands and split metadata.""" + saved = restore_from_func_ctx(ctx) + n = 1 if self.single_grouped_weight else self.num_gemms + self.inputmat = saved[0] + self.weights_fp8 = saved[1 : 1 + n] + ( + self.m_splits_tensor, + self.base_split_offsets, + self.input_tensor_offsets, + self.output_tensor_offsets, + ) = saved[1 + n :] + + +def _grouped_linear_graph_safe_forward(args: GroupedLinearFwdArgs) -> Tuple[Any, ...]: + """Compute the grouped forward and collect tensors needed by backward.""" + inp = args.inp + use_bias = args.use_bias + is_first_microbatch = args.is_first_microbatch + fp8 = args.fp8 + input_quantizers = args.input_quantizers + activation_dtype = args.activation_dtype + is_grad_enabled = args.is_grad_enabled + weight_workspaces = args.weight_workspaces + cache_weight = args.cache_weight + skip_fp8_weight_update = args.skip_fp8_weight_update + save_original_input = args.save_original_input + single_grouped_weight = args.single_grouped_weight + single_grouped_bias = args.single_grouped_bias + weights = args.weights + is_dist_weight = is_distributed_weight(weights[0]) + if is_dist_weight: + weights = materialize_weight_for_forward(weights) + _grouped_linear_configure_quantizers(args, weights) + weight_quantizers = args.weight_quantizers + biases = args.biases + out = args.out + m_splits = args.m_splits_tensor + num_gemms = len(m_splits) + device = inp.device + weight_shape = weights[0].shape + in_features = weight_shape[-1] + out_features = weight_shape[-2] + if inp.size(-1) != in_features: + raise ValueError( + f"Input tensor (shape={tuple(inp.size())}) is not compatible with " + f"weight tensor (shape={tuple(weights[0].size())})" + ) + weight_requires_grad = args.weights_requires_grad + save_original_input = save_original_input and weight_requires_grad + + split_sizes, ( + base_split_offsets, + input_tensor_offsets, + output_tensor_offsets, + ) = tex.splits_to_offsets_multi( + m_splits, + device, + strides=[1, in_features, out_features], + include_leading_zero=[True, True, True], + dtypes=[torch.int64, torch.int64, torch.int64], + bulk_allocate=True, + ) + + inp_view = inp.reshape(-1, in_features) + x = cast_if_needed(inp_view, activation_dtype) + if fp8: + input_quantizer = input_quantizers[0] + input_quantizer.set_usage( + rowwise=True, + columnwise=(is_grad_enabled and weight_requires_grad and not save_original_input), + ) + input_quantizer.optimize_for_gemm = True + grouped_x = tex.group_quantize( + x, + input_quantizer, + num_gemms, + split_sizes, + tensor_offsets=input_tensor_offsets, + ) + else: + grouped_x = _GroupedLinear._make_grouped_tensor( + x, + num_gemms=num_gemms, + split_sizes=split_sizes, + tensor_offsets=input_tensor_offsets, + last_dim=in_features, + dtype=activation_dtype, + ) + + columnwise_usage = is_grad_enabled and args.input_requires_grad + weights_for_gemm, new_workspaces = _GroupedLinear._prepare_weights_for_grouped_tensor_gemm( + weights, + weight_quantizers, + weight_workspaces, + num_gemms=num_gemms, + single_grouped_weight=single_grouped_weight, + with_quantized_compute=fp8, + columnwise_usage=columnwise_usage, + activation_dtype=activation_dtype, + is_first_microbatch=is_first_microbatch, + skip_fp8_weight_update=skip_fp8_weight_update, + cache_weight=cache_weight and not is_dist_weight, + ) + + out = _GroupedLinear._validate_or_alloc_output( + out, + x.size(0), + out_features, + activation_dtype, + device, + ) + grouped_out = _GroupedLinear._make_grouped_tensor( + out, + num_gemms=num_gemms, + split_sizes=split_sizes, + tensor_offsets=output_tensor_offsets, + last_dim=out_features, + dtype=activation_dtype, + ) + + grouped_bias = None + if use_bias: + grouped_bias = _GroupedLinear._prepare_bias_for_grouped_tensor_gemm( + biases, + single_grouped_bias=single_grouped_bias, + num_gemms=num_gemms, + out_features=out_features, + dtype=activation_dtype, + ) + + fprop_use_split_accumulator = _2X_ACC_FPROP + if fp8 and hasattr(args.fp8_recipe, "fp8_gemm_fprop"): + fprop_use_split_accumulator = args.fp8_recipe.fp8_gemm_fprop.use_split_accumulator + general_grouped_gemm_for_grouped_tensor( + weights_for_gemm, + grouped_x, + grouped_out, + layout="TN", + bias=grouped_bias, + use_split_accumulator=fprop_use_split_accumulator, + ) + + tensors_to_save = None + if is_grad_enabled: + input_to_save = grouped_x + if weight_requires_grad: + if save_original_input: + # Save the high-precision input and reconstruct the grouped columnwise + # operand in backward instead of retaining a second quantized copy. + input_to_save = inp + elif fp8 and grouped_x.columnwise_data is not None: + # Wgrad only consumes the columnwise representation. + grouped_x.rowwise_data = None + grouped_x.scale_inv = None + else: + input_to_save = None + + weights_to_save = [weights_for_gemm] if single_grouped_weight else weights_for_gemm + if not args.input_requires_grad: + weights_to_save = [None] * len(weights_to_save) + + if is_dist_weight: + weights_to_save = list(args.weights) + + # Megatron-LM paged stashing uses this marker to identify the dynamic activation + # buffers among the tensors saved by the GroupedLinear autograd function. The + # operation-fuser grouped MLP applies the same marker to its saved activations. + mark_grouped_tensor(input_to_save) + tensors_to_save = ( + input_to_save, + *weights_to_save, + split_sizes, + base_split_offsets, + input_tensor_offsets, + output_tensor_offsets, + ) + return out.view(-1, *inp.shape[1:-1], out.shape[-1]), new_workspaces, tensors_to_save + + +def _grouped_linear_graph_safe_setup( + bwd_args: GroupedLinearGraphSafeBwdArgs, + fwd_args: GroupedLinearFwdArgs, + out_features: int, +) -> None: + """Populate grouped backward arguments from the forward configuration.""" + bwd_args.input_quantizers = fwd_args.input_quantizers + bwd_args.weight_quantizers = fwd_args.weight_quantizers + bwd_args.is_dist_weight = is_distributed_weight(fwd_args.weights[0]) + bwd_args.grad_output_quantizers = fwd_args.grad_output_quantizers + bwd_args.m_splits_tensor = fwd_args.m_splits_tensor + bwd_args.num_gemms = fwd_args.num_gemms + bwd_args.activation_dtype = fwd_args.activation_dtype + bwd_args.fp8 = fwd_args.fp8 + bwd_args.use_bias = fwd_args.use_bias + bwd_args.single_grouped_weight = fwd_args.single_grouped_weight + bwd_args.single_grouped_bias = fwd_args.single_grouped_bias + bwd_args.is_first_microbatch = fwd_args.is_first_microbatch + bwd_args.fp8_recipe = fwd_args.fp8_recipe + bwd_args.fuse_wgrad_accumulation = fwd_args.fuse_wgrad_accumulation + bwd_args.wgrad_store = fwd_args.wgrad_store + bwd_args.dgrad_out = fwd_args.dgrad_out + bwd_args.in_features = fwd_args.inp.shape[-1] + bwd_args.out_features = out_features + bwd_args.requires_dgrad = fwd_args.input_requires_grad + bwd_args.weights_requires_grad = fwd_args.weights_requires_grad + bwd_args.save_original_input = fwd_args.save_original_input and fwd_args.weights_requires_grad + if fwd_args.fuse_wgrad_accumulation and fwd_args.weights_requires_grad: + weights = fwd_args.weights + bwd_args.origin_weight_refs = [weakref.ref(w) for w in weights] + bwd_args.origin_weights_overwrite_main_grad = getattr( + weights[0], "overwrite_main_grad", False + ) + if hasattr(weights[0], "__fsdp_param__"): + bwd_args.main_grad_funcs = [weight.get_main_grad for weight in weights] + elif bwd_args.is_dist_weight: + bwd_args.main_grad_funcs = [weight.grad_buffer for weight in weights] + else: + bwd_args.main_grad_funcs = [ + lambda j=i: weights[j].main_grad for i in range(len(weights)) + ] + + +def _grouped_linear_graph_safe_backward( + args: GroupedLinearGraphSafeBwdArgs, +) -> Tuple[Optional[torch.Tensor], List[Optional[torch.Tensor]], List[Optional[torch.Tensor]]]: + """Compute gradients with the grouped-tensor kernels.""" + grad_output = args.grad_output + N = args.num_gemms + saved_input = args.inputmat + weight_tensors = args.weights_fp8 + weights_for_gemm = weight_tensors[0] if args.single_grouped_weight else weight_tensors + split_sizes = args.m_splits_tensor + base_split_offsets = args.base_split_offsets + input_tensor_offsets = args.input_tensor_offsets + output_tensor_offsets = args.output_tensor_offsets + if args.save_original_input: + x = cast_if_needed( + saved_input.reshape(-1, args.in_features), + args.activation_dtype, + ) + if args.fp8: + input_quantizer = args.input_quantizers[0] + input_quantizer.set_usage(rowwise=False, columnwise=True) + input_quantizer.optimize_for_gemm = True + grouped_x = tex.group_quantize(x, input_quantizer, N, split_sizes) + else: + grouped_x = _GroupedLinear._make_grouped_tensor( + x, + num_gemms=N, + split_sizes=split_sizes, + tensor_offsets=input_tensor_offsets, + last_dim=args.in_features, + dtype=args.activation_dtype, + ) + else: + grouped_x = saved_input + + num_weight_args = 1 if args.single_grouped_weight else N + origin_weights = [None] * num_weight_args + main_grads = [None] * num_weight_args + is_dist_weight = args.is_dist_weight + if is_dist_weight: + origin_weights = list(weight_tensors) + if args.fuse_wgrad_accumulation and args.weights_requires_grad: + main_grads = [main_grad_func() for main_grad_func in args.main_grad_funcs] + weight_tensors = materialize_weight_for_backward(origin_weights) + weights_for_gemm = None + if args.requires_dgrad: + weights_for_gemm, _ = _GroupedLinear._prepare_weights_for_grouped_tensor_gemm( + weight_tensors, + args.weight_quantizers, + None, + num_gemms=N, + single_grouped_weight=args.single_grouped_weight, + with_quantized_compute=args.fp8, + columnwise_usage=True, + activation_dtype=args.activation_dtype, + is_first_microbatch=None, + skip_fp8_weight_update=None, + cache_weight=False, + ) + elif args.fuse_wgrad_accumulation and args.weights_requires_grad: + origin_weight_refs = args.origin_weight_refs + args.origin_weight_refs = None + origin_weights = [ref() if ref is not None else None for ref in origin_weight_refs] + assert all( + w is not None for w in origin_weights + ), "weight was removed while fuse_wgrad_accumulation=True" + main_grads = [main_grad_func() for main_grad_func in args.main_grad_funcs] + for origin_weight, main_grad in zip(origin_weights, main_grads): + if main_grad is not None: + origin_weight.main_grad = main_grad + + grad_output_view = grad_output.contiguous().view(-1, grad_output.shape[-1]) + dy_2d = cast_if_needed(grad_output_view, args.activation_dtype) + dbias_packed = None + if args.fp8: + grad_output_quantizer = args.grad_output_quantizers[0] + grad_output_quantizer.set_usage( + rowwise=args.requires_dgrad, + columnwise=args.weights_requires_grad, + ) + grad_output_quantizer.optimize_for_gemm = True + # The grouped FP8 block-scaling bgrad kernel computes dbias in the rowwise + # pass, so the fusion needs rowwise output (i.e. dgrad required). + fuse_bgrad = isinstance(grad_output_quantizer, MXFP8Quantizer) or ( + isinstance(grad_output_quantizer, Float8BlockQuantizer) and args.requires_dgrad + ) + if args.use_bias and fuse_bgrad: + grouped_dy, dbias_packed = tex.bgrad_group_quantize( + dy_2d, + grad_output_quantizer, + N, + split_sizes, + tensor_offsets=output_tensor_offsets, + ) + else: + grouped_dy = tex.group_quantize( + dy_2d, + grad_output_quantizer, + N, + split_sizes, + tensor_offsets=output_tensor_offsets, + ) + else: + grouped_dy = _GroupedLinear._make_grouped_tensor( + dy_2d, + num_gemms=N, + split_sizes=split_sizes, + tensor_offsets=output_tensor_offsets, + last_dim=args.out_features, + dtype=args.activation_dtype, + ) + + if args.use_bias: + if dbias_packed is None: + dbias_packed = compute_grouped_dbias(dy_2d, base_split_offsets, N) + if args.single_grouped_bias: + grad_bias_args = [dbias_packed.to(dtype=args.activation_dtype)] + else: + grad_bias_args = [dbias_packed[i].to(dtype=args.activation_dtype) for i in range(N)] + else: + num_bias_args = 1 if args.single_grouped_bias else N + grad_bias_args = [None] * num_bias_args + + dgrad = None + if args.requires_dgrad: + dgrad_use_split_accumulator = _2X_ACC_DGRAD + if args.fp8 and hasattr(args.fp8_recipe, "fp8_gemm_dgrad"): + dgrad_use_split_accumulator = args.fp8_recipe.fp8_gemm_dgrad.use_split_accumulator + for weight in weight_tensors: + if isinstance(weight, QuantizedTensorStorage): + weight.update_usage(columnwise_usage=True) + dgrad = _GroupedLinear._validate_or_alloc_output( + args.dgrad_out, + dy_2d.size(0), + args.in_features, + args.activation_dtype, + grad_output.device, + ) + grouped_dgrad = _GroupedLinear._make_grouped_tensor( + dgrad, + num_gemms=N, + split_sizes=split_sizes, + tensor_offsets=input_tensor_offsets, + last_dim=args.in_features, + dtype=args.activation_dtype, + ) + general_grouped_gemm_for_grouped_tensor( + weights_for_gemm, + grouped_dy, + grouped_dgrad, + layout="NN", + use_split_accumulator=dgrad_use_split_accumulator, + ) + + if is_dist_weight: + accumulate_wgrad_into_param_main_grad = False + elif args.is_first_microbatch is not None: + accumulate_wgrad_into_param_main_grad = ( + args.fuse_wgrad_accumulation and not args.is_first_microbatch + ) + else: + accumulate_wgrad_into_param_main_grad = args.fuse_wgrad_accumulation + + if args.weights_requires_grad: + if ( + is_dist_weight + and args.wgrad_store is not None + and args.wgrad_store.delay_wgrad_compute() + ): + raise RuntimeError( + "distributed-weight GroupedLinear requires delay_wgrad_compute=False." + ) + wgrad_use_split_accumulator = _2X_ACC_WGRAD + if args.fp8 and hasattr(args.fp8_recipe, "fp8_gemm_wgrad"): + wgrad_use_split_accumulator = args.fp8_recipe.fp8_gemm_wgrad.use_split_accumulator + if args.fuse_wgrad_accumulation: + if args.single_grouped_weight: + main_grad = main_grads[0] + grouped_wgrad = GroupedTensor.make_grouped_tensor_from_rowwise_data( + num_tensors=N, + tensor_shape=(args.out_features, args.in_features), + rowwise_data=main_grad.view(-1), + dtype=main_grad.dtype, + ) + wgrad_output = grouped_wgrad + wgrad_list = [main_grad] + else: + wgrad_output = main_grads + wgrad_list = main_grads + else: + if args.single_grouped_weight: + grouped_wgrad = GroupedTensor.make_grouped_tensor_with_shapes( + num_tensors=N, + shapes=[(args.out_features, args.in_features)] * N, + quantizer=None, + device=grad_output.device, + dtype=args.activation_dtype, + ) + wgrad_output = grouped_wgrad + wgrad_list = [ + grouped_wgrad.rowwise_data.view(N, args.out_features, args.in_features) + ] + else: + wgrad_packed = torch.empty( + N, + args.out_features, + args.in_features, + dtype=args.activation_dtype, + device=grad_output.device, + ) + wgrad_output = [wgrad_packed[i] for i in range(N)] + wgrad_list = wgrad_output + + accumulate = ( + accumulate_wgrad_into_param_main_grad + if not getattr(args, "origin_weights_overwrite_main_grad", False) + else False + ) + + def grouped_gemm_wgrad(inputmats, grad_output_mats, grad_weights): + general_grouped_gemm_for_grouped_tensor( + inputmats, + grad_output_mats, + grad_weights, + layout="NT", + use_split_accumulator=wgrad_use_split_accumulator, + accumulate=accumulate, + ) + return None, [None] * N, None + + if args.wgrad_store is not None and args.wgrad_store.delay_wgrad_compute(): + args.wgrad_store.put([grouped_x, grouped_dy, wgrad_output], grouped_gemm_wgrad) + else: + grouped_gemm_wgrad(grouped_x, grouped_dy, wgrad_output) + + if is_dist_weight: + wgrad_list = finalize_weight_grads(origin_weights, wgrad_list) + else: + wgrad_list = [ + _finish_wgrad(weight, main_grad, wgrad, args.fuse_wgrad_accumulation) + for weight, main_grad, wgrad in zip(origin_weights, main_grads, wgrad_list) + ] + else: + wgrad_list = [None] * num_weight_args + + if dgrad is not None: + dgrad = dgrad.view(*grad_output.shape[:-1], args.in_features) + return dgrad, wgrad_list, grad_bias_args + + +@no_torch_dynamo() +def _grouped_linear_eager( + inp: torch.Tensor, + m_splits: torch.Tensor, + fwd_args: GroupedLinearFwdArgs, + weights_and_biases: Tuple[torch.Tensor, ...], + is_grad_enabled: bool, +) -> Tuple[torch.Tensor, list]: + """Run ``_GroupedLinear`` eagerly, bypassing Dynamo.""" + if fwd_args.m_splits is None and not fwd_args.use_grouped_tensor_path: + fwd_args.m_splits = tuple(m_splits.tolist()) + if is_grad_enabled: + return _GroupedLinear.apply(inp, fwd_args, *weights_and_biases) + return _GroupedLinear.forward(None, inp, fwd_args, *weights_and_biases) + + class _GroupedLinear(torch.autograd.Function): """GroupedLinear semi-top level module Calls custom cuda extensions. @@ -406,937 +1517,65 @@ def _prepare_bias_for_grouped_tensor_gemm( ) @staticmethod - def _forward_grouped_tensor( - ctx, - *, - inp: torch.Tensor, - m_splits: torch.Tensor, - use_bias: bool, - is_first_microbatch: Optional[bool], - fp8: bool, - wgrad_store: Optional[WeightGradStore], - input_quantizers: List[Optional[Quantizer]], - weight_quantizers: List[Optional[Quantizer]], - grad_input_quantizers: List[Optional[Quantizer]], - grad_weight_quantizers: List[Optional[Quantizer]], - grad_output_quantizers: List[Optional[Quantizer]], - fuse_wgrad_accumulation: bool, - activation_dtype: torch.dtype, - is_grad_enabled: bool, - weight_workspaces: List[Optional[QuantizedTensorStorage]], - cache_weight: bool, - skip_fp8_weight_update: Optional[torch.Tensor], - save_original_input: bool, - single_grouped_weight: bool, - single_grouped_bias: bool, - weights: Tuple[torch.Tensor, ...], - biases: Tuple[torch.Tensor, ...], - origin_weights: Tuple[torch.Tensor, ...], - is_dist_weight: bool, - out: Optional[torch.Tensor] = None, - dgrad_out: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, list]: - """Forward path backed by GroupedTensor + cuBLASLt grouped GEMM. - - ``weights`` feed the GEMM; for a distributed weight they are transient gathered - copies. ``origin_weights`` are the sharded parameters, which own ``grad_buffer`` - and survive into backward. - """ - num_gemms = len(m_splits) - device = inp.device - in_features = weights[0].size(-1) - out_features = weights[0].size(-2) - # Read from the parameters: the protocol does not require a materialized copy to - # carry requires_grad, and a False here would silently skip the whole wgrad path. - weight_requires_grad = (origin_weights if is_dist_weight else weights)[0].requires_grad - save_original_input = save_original_input and weight_requires_grad - - split_sizes, ( - base_split_offsets, - input_tensor_offsets, - output_tensor_offsets, - ) = tex.splits_to_offsets_multi( - m_splits, - device, - strides=[1, in_features, out_features], - include_leading_zero=[True, True, True], - dtypes=[torch.int64, torch.int64, torch.int64], - bulk_allocate=True, - ) - - inp_view = inp.reshape(-1, in_features) - x = cast_if_needed(inp_view, activation_dtype) - if fp8: - input_quantizer = input_quantizers[0] - input_quantizer.set_usage( - rowwise=True, - columnwise=(is_grad_enabled and weight_requires_grad and not save_original_input), - ) - input_quantizer.optimize_for_gemm = True - grouped_x = tex.group_quantize( - x, - input_quantizer, - num_gemms, - split_sizes, - tensor_offsets=input_tensor_offsets, - ) - else: - grouped_x = _GroupedLinear._make_grouped_tensor( - x, - num_gemms=num_gemms, - split_sizes=split_sizes, - tensor_offsets=input_tensor_offsets, - last_dim=in_features, - dtype=activation_dtype, - ) - - columnwise_usage = is_grad_enabled and inp.requires_grad - weights_for_gemm, new_workspaces = _GroupedLinear._prepare_weights_for_grouped_tensor_gemm( - weights, - weight_quantizers, - weight_workspaces, - num_gemms=num_gemms, - single_grouped_weight=single_grouped_weight, - with_quantized_compute=fp8, - columnwise_usage=columnwise_usage, - activation_dtype=activation_dtype, - is_first_microbatch=is_first_microbatch, - skip_fp8_weight_update=skip_fp8_weight_update, - # Never cache a gathered copy: the workspace would pin the unsharded weight for - # the module's lifetime, which is the memory sharding exists to avoid. - cache_weight=cache_weight and not is_dist_weight, - ) - - out = _GroupedLinear._validate_or_alloc_output( - out, - x.size(0), - out_features, - activation_dtype, - device, - ) - grouped_out = _GroupedLinear._make_grouped_tensor( - out, - num_gemms=num_gemms, - split_sizes=split_sizes, - tensor_offsets=output_tensor_offsets, - last_dim=out_features, - dtype=activation_dtype, - ) - - grouped_bias = None - if use_bias: - grouped_bias = _GroupedLinear._prepare_bias_for_grouped_tensor_gemm( - biases, - single_grouped_bias=single_grouped_bias, - num_gemms=num_gemms, - out_features=out_features, - dtype=activation_dtype, - ) - - use_split_accumulator = _2X_ACC_FPROP - if fp8: - recipe = FP8GlobalStateManager.get_fp8_recipe() - if hasattr(recipe, "fp8_gemm_fprop"): - use_split_accumulator = recipe.fp8_gemm_fprop.use_split_accumulator - - general_grouped_gemm_for_grouped_tensor( - weights_for_gemm, - grouped_x, - grouped_out, - layout="TN", - bias=grouped_bias, - use_split_accumulator=use_split_accumulator, - ) - - if is_grad_enabled: - input_to_save = grouped_x - if weight_requires_grad: - if save_original_input: - # Save the high-precision input and reconstruct the grouped columnwise - # operand in backward instead of retaining a second quantized copy. - input_to_save = inp - elif fp8 and grouped_x.columnwise_data is not None: - # Wgrad only consumes the columnwise representation. - grouped_x.rowwise_data = None - grouped_x.scale_inv = None - else: - input_to_save = None - - weights_to_save = [weights_for_gemm] if single_grouped_weight else weights_for_gemm - if not inp.requires_grad: - weights_to_save = [None] * len(weights_to_save) - if is_dist_weight: - # Gathered copies are transient, so save the shards and re-gather in backward. - weights_to_save = list(origin_weights) - - # Megatron-LM paged stashing uses this marker to identify the dynamic activation - # buffers among the tensors saved by the GroupedLinear autograd function. The - # operation-fuser grouped MLP applies the same marker to its saved activations. - mark_grouped_tensor(input_to_save) - tensors_to_save, tensor_objects = prepare_for_saving( - input_to_save, - *weights_to_save, - split_sizes, - base_split_offsets, - input_tensor_offsets, - output_tensor_offsets, - ) + def _forward_grouped_tensor(ctx, fwd_args: GroupedLinearFwdArgs) -> Tuple[torch.Tensor, list]: + """Run the grouped forward and persist its autograd state.""" + out, new_workspaces, saved = _grouped_linear_graph_safe_forward(fwd_args) + if ctx is not None: + bwd_args = GroupedLinearGraphSafeBwdArgs() + _grouped_linear_graph_safe_setup(bwd_args, fwd_args, out.shape[-1]) + tensors_to_save, tensor_objects = prepare_for_saving(*saved) ctx.save_for_backward(*tensors_to_save) ctx.tensor_objects = tensor_objects - - ctx.grouped_tensor_supported = True - ctx.weight_quantizers = weight_quantizers - ctx.weights_shape_0 = out_features - ctx.weights_shape_1 = in_features - ctx.grad_input_quantizers = grad_input_quantizers - ctx.grad_output_quantizers = grad_output_quantizers - ctx.grad_weight_quantizers = grad_weight_quantizers - ctx.weights_requires_grad = weight_requires_grad - ctx.single_grouped_weight = single_grouped_weight - ctx.single_grouped_bias = single_grouped_bias - ctx.is_dist_weight = is_dist_weight - ctx.fp8_for_weight_prep = fp8 - if fuse_wgrad_accumulation and ctx.weights_requires_grad: - # Weakref the parameters, not the gathered copies: those can be dead by backward. - ref_weights = origin_weights if is_dist_weight else weights - ctx.origin_weight_refs = [weakref.ref(w) for w in ref_weights] - ctx.origin_weights_overwrite_main_grad = getattr( - ref_weights[0], "overwrite_main_grad", False - ) - if hasattr(weights[0], "__fsdp_param__"): - ctx.main_grad_funcs = [weight.get_main_grad for weight in weights] - elif is_dist_weight: - # main_grad is shard-shaped; the unsharded wgrad goes to grad_buffer, - # which finalize_weight_grads reduce-scatters. - ctx.main_grad_funcs = [w.grad_buffer for w in origin_weights] - else: - ctx.main_grad_funcs = [ - lambda j=i: weights[j].main_grad for i in range(len(weights)) - ] - ctx.device = device - ctx.dgrad_out = dgrad_out - ctx.m_splits = None - ctx.num_gemms = num_gemms - ctx.activation_dtype = activation_dtype - ctx.fp8 = fp8 - ctx.fp8_recipe = FP8GlobalStateManager.get_fp8_recipe() if fp8 else None - ctx.backward_override = None - ctx.fuse_wgrad_accumulation = fuse_wgrad_accumulation - ctx.cpu_offloading = False - ctx.is_first_microbatch = is_first_microbatch - ctx.use_bias = use_bias - ctx.inp_shape = inp.shape - ctx.requires_dgrad = inp.requires_grad - ctx.reduce_and_update_bwd_fp8_tensors = False - if ctx.fp8 and requires_grad(inp, weights[0], biases[0]): - ctx.reduce_and_update_bwd_fp8_tensors = ( - ctx.reduce_and_update_bwd_fp8_tensors - or FP8GlobalStateManager.is_first_fp8_module() + ctx.use_grouped_tensor_path = True + ctx.inp_shape = fwd_args.inp.shape + ctx.backward_objects = bwd_args + if fwd_args.fp8 and requires_grad( + fwd_args.inp, fwd_args.weights[0], fwd_args.biases[0] + ): + bwd_args.reduce_and_update_bwd_fp8_tensors = ( + FP8GlobalStateManager.is_first_fp8_module() ) - ctx.wgrad_store = wgrad_store - ctx.debug = False - ctx.save_original_input = save_original_input - ctx.input_quantizers = input_quantizers + return out, new_workspaces - return out.view(-1, *inp.shape[1:-1], out.shape[-1]), new_workspaces - - # pylint: disable=keyword-arg-before-vararg @staticmethod def forward( ctx, inp: torch.Tensor, - m_splits: torch.Tensor, - non_tensor_args: Tuple, - out: Optional[torch.Tensor], - dgrad_out: Optional[torch.Tensor], + fwd_args: GroupedLinearFwdArgs, *weights_and_biases, ) -> Tuple[torch.Tensor, list]: - # pylint: disable=missing-function-docstring + """Forward pass: compute grouped linear output and set up autograd context. - # Reduce number of arguments to autograd function in order - # to reduce CPU overhead due to pytorch arg checking. - ( - use_bias, - is_first_microbatch, - fp8, - fp8_calibration, - wgrad_store, - input_quantizers, - weight_quantizers, - output_quantizers, - grad_input_quantizers, - grad_weight_quantizers, - grad_output_quantizers, - fuse_wgrad_accumulation, - cpu_offloading, - sequence_parallel, - activation_dtype, - is_grad_enabled, - weight_workspaces, - cache_weight, - skip_fp8_weight_update, - save_original_input, - debug, - single_grouped_weight, - single_grouped_bias, - use_grouped_tensor, - ) = non_tensor_args - recipe = FP8GlobalStateManager.get_fp8_recipe() if fp8 else None - backward_override = recipe.backward_override if recipe is not None else None - if backward_override == "high_precision": - save_original_input = True - elif backward_override == "dequantized": - save_original_input = False - - num_gemms = len(m_splits) - num_weight_args = 1 if single_grouped_weight else num_gemms - num_bias_args = 1 if single_grouped_bias else num_gemms - weights = weights_and_biases[:num_weight_args] - biases = weights_and_biases[num_weight_args : num_weight_args + num_bias_args] - device = inp.device - weight_requires_grad = weights[0].requires_grad - - origin_weights = weights - is_dist_weight = is_distributed_weight(weights[0]) - if is_dist_weight: - weights = materialize_weight_for_forward(weights) - - backward_needs_input = is_grad_enabled and weight_requires_grad - if backward_override is None and save_original_input and backward_needs_input: - if recipe is not None and recipe.delayed(): - raise ValueError("DelayedScaling recipe is not supported with save_original_input") - - # Megatron-Core may enable this automatically to reuse an activation - # already retained by an upstream operation. Built-in recipes guarantee - # group homogeneity, while CustomRecipe generations are validated once, - # so runtime safety can be determined from expert 0. - if recipe is not None: - input_quantizer = input_quantizers[0] - if isinstance(input_quantizer, Float8Quantizer): - warnings.warn( - "save_original_input is incompatible with delayed-scaling quantizers " - "(Float8Quantizer). Disabling save_original_input for this module.", - stacklevel=2, - ) - save_original_input = False - elif not can_reconstruct_wgrad_input_from_original(input_quantizer): - warnings.warn( - "Ignoring save_original_input=True because the input quantizer cannot " - "safely reconstruct the backward operand from the original input " - f"({input_quantizer}).", - stacklevel=2, - ) - save_original_input = False - - # Configure quantizers - if input_quantizers[0] is not None: - for input_quantizer in input_quantizers: - input_quantizer.set_usage( - rowwise=True, - columnwise=( - is_grad_enabled - and weight_requires_grad - and not save_original_input - and backward_override is None - ), - ) - columnwise_usage = is_grad_enabled and inp.requires_grad - if backward_override is not None: - columnwise_usage = False - if not columnwise_usage: - columnwise_usage = ( - is_fp8_activation_recompute_enabled() - and not in_fp8_activation_recompute_phase() - ) - # No need to set the quantizer states if weight is already quantized - # for debug mode we create quantizer every iteration, thus we need to set the quantizer states - if weight_quantizers[0] is not None and ( - not isinstance(weights[0], QuantizedTensorStorage) or debug - ): - for weight_quantizer in weight_quantizers: - weight_quantizer.set_usage(rowwise=True, columnwise=columnwise_usage) - elif isinstance(weights[0], QuantizedTensorStorage): - # If weights are already quantized, no need to set quantizer states - weight_quantizers = [weight._quantizer for weight in weights] - if output_quantizers[0] is not None: - for output_quantizer in output_quantizers: - output_quantizer.set_usage(rowwise=True, columnwise=False) - - # Initialize input tensors - in_features = weights[0].size(-1) - if inp.size(-1) != in_features: - raise ValueError( - f"Input tensor (shape={tuple(inp.size())}) is not compatible with " - f"weight tensor (shape={tuple(weights[0].size())})" - ) - - grouped_tensor_supported = False - if use_grouped_tensor and not ( - fp8_calibration - or debug - or cpu_offloading - or any(q is not None for q in output_quantizers) - ): - if ( - fp8 - and recipe.float8_block_scaling() - and (10, 0) <= get_device_compute_capability() <= (11, 0) - ): - raise RuntimeError( - "use_grouped_tensor=True does not support the FP8 block-scaling recipe on " - "Blackwell GPUs: the native grouped FP8 block-scaling path is Hopper-only. " - "Set use_grouped_tensor=False, or unset " - "NVTE_GROUPED_LINEAR_USE_FUSED_GROUPED_GEMM if it enabled this path, to use " - "the MXFP8-emulated path on Blackwell." - ) - grouped_tensor_supported = is_module_grouped_tensor_path_supported( - recipe, - activation_dtype, - ) - if ( - use_grouped_tensor - and not grouped_tensor_supported - and (single_grouped_weight or single_grouped_bias) - ): - raise RuntimeError( - "Single grouped parameters require the native grouped-tensor path, but the active " - "device, cuBLASLt version, quantization recipe, or GroupedLinear feature " - "configuration does not support it. Disable single_grouped_weight and " - "single_grouped_bias to allow the split-quantize fallback." - ) - if grouped_tensor_supported: - if m_splits.device.type != "cuda": - raise ValueError( - "The native grouped_tensor path requires CUDA m_splits. Pass a CUDA int64 " - "tensor, or set use_grouped_tensor=False." - ) - return _GroupedLinear._forward_grouped_tensor( - ctx, - inp=inp, - m_splits=m_splits, - use_bias=use_bias, - is_first_microbatch=is_first_microbatch, - fp8=fp8, - wgrad_store=wgrad_store, - input_quantizers=input_quantizers, - weight_quantizers=weight_quantizers, - grad_input_quantizers=grad_input_quantizers, - grad_weight_quantizers=grad_weight_quantizers, - grad_output_quantizers=grad_output_quantizers, - fuse_wgrad_accumulation=fuse_wgrad_accumulation, - activation_dtype=activation_dtype, - is_grad_enabled=is_grad_enabled, - weight_workspaces=weight_workspaces, - cache_weight=cache_weight, - skip_fp8_weight_update=skip_fp8_weight_update, - save_original_input=save_original_input, - single_grouped_weight=single_grouped_weight, - single_grouped_bias=single_grouped_bias, - weights=weights, - biases=biases, - origin_weights=origin_weights, - is_dist_weight=is_dist_weight, - out=out, - dgrad_out=dgrad_out, - ) - - # Convert splits to list of ints for compatibility with split functions - m_splits = m_splits.tolist() - - inp_view = inp.reshape(-1, in_features) - # Disable bulk allocation when CPU offloading is active: offloading skips small - # tensors (like scales), but bulk allocation shares storage across all tensors, - # so if scales can't be offloaded, nothing in the group can be offloaded. - inputmats, _ = _split_quantization._split_quantize( - inp_view, - m_splits, - input_quantizers, - activation_dtype, - with_quantized_output=fp8 or debug, - disable_bulk_allocation=cpu_offloading, - ) - - if cpu_offloading: - start_offload(*inputmats) - - # Initialize weights - weights_fp8: list - new_workspaces = [None] * num_gemms - if fp8 or debug: - weights_fp8 = [] - update_ws = is_first_microbatch is None or is_first_microbatch - for i in range(num_gemms): - weight_fp8, new_workspaces[i] = quantize_weight( - tensor=weights[i], - quantizer=weight_quantizers[i], - workspace=weight_workspaces[i] if weight_workspaces else None, - update_workspace=update_ws, - skip_update_flag=skip_fp8_weight_update, - workspace_dtype=activation_dtype, - # Never cache a gathered copy: the workspace would pin the unsharded - # weight for the module's lifetime. - cache=cache_weight and not is_dist_weight, - ) - weights_fp8.append(weight_fp8) - - else: - weights_fp8 = [cast_if_needed(weight, activation_dtype) for weight in weights] - - # Initialize biases - bias_dtype = activation_dtype - if fp8 and activation_dtype == torch.float32: - bias_dtype = torch.bfloat16 # FP8 GEMM only supports BF16/FP16 bias - biases = [cast_if_needed(bias, bias_dtype) for bias in biases] if use_bias else biases - # Initialize output tensor - out = _GroupedLinear._validate_or_alloc_output( - out, - sum(m_splits), - weights_fp8[0].size(0), - activation_dtype, - device, - ) - - # Choose whether to use split accumulator - use_split_accumulator = _2X_ACC_FPROP - if fp8: - recipe = FP8GlobalStateManager.get_fp8_recipe() - if hasattr(recipe, "fp8_gemm_fprop"): - use_split_accumulator = recipe.fp8_gemm_fprop.use_split_accumulator - - # Perform GEMM - general_grouped_gemm( - weights_fp8, - inputmats, - [out], - output_quantizers, - activation_dtype, - single_output=True, - m_splits=m_splits, - bias=biases, - use_bias=use_bias, - use_split_accumulator=use_split_accumulator, - ) - - if fp8_calibration: - for i in range(num_gemms): - input_quantizers[i].calibrate(inputmats[i]) - weight_quantizers[i].calibrate(weights[i]) - - if cpu_offloading: - mark_not_offload(*weights_fp8, *weights) - - if is_grad_enabled: - ctx.grouped_tensor_supported = False - ctx.weight_quantizers = weight_quantizers - ctx.weights_shape_1 = weights[0].shape[1] - - # TODO: update after #1638 is merged. # pylint: disable=fixme - if weight_requires_grad: - if save_original_input: - inputmats = [None] * num_gemms - inputmats[0] = inp - else: - for inputmat in inputmats: - if isinstance(inputmat, QuantizedTensorStorage): - if backward_override is not None: - # In dequantized mode we should dequantize directly from - # fprop quantized layouts without retargeting usage. - inputmat.update_usage(rowwise_usage=True, columnwise_usage=False) - else: - inputmat.update_usage(rowwise_usage=False, columnwise_usage=True) - else: - inputmats = [None] * num_gemms - - # Original weights are only needed by high_precision dgrad. The weakrefs - # used for fused wgrad accumulation serve a different purpose: restoring - # Python parameter attributes without keeping the parameter alive here. - saved_weights = ( - weights - if backward_override == "high_precision" and inp.requires_grad - else [None] * num_gemms - ) - if is_dist_weight: - # GTP: gathered workspace is transient (re-gathered in backward), don't save it. - weights_fp8 = [None] * num_gemms - saved_weights = origin_weights - tensors_to_save, tensor_objects = prepare_for_saving( - *inputmats, - *weights_fp8, - *saved_weights, - *biases, - ) + ``inp`` and the weights / biases are positional Tensor arguments so + autograd tracks them; they are immediately re-attached to ``fwd_args`` + so every downstream helper can be invoked with a single argument. + """ + num_gemms = fwd_args.num_gemms + fwd_args.inp = inp + num_weight_args = 1 if fwd_args.single_grouped_weight else num_gemms + fwd_args.weights = list(weights_and_biases[:num_weight_args]) + fwd_args.biases = list(weights_and_biases[num_weight_args:]) + + if fwd_args.use_grouped_tensor_path: + return _GroupedLinear._forward_grouped_tensor(ctx, fwd_args) + + out, new_workspaces, saved = _grouped_linear_forward_impl(fwd_args) + if ctx is not None: + ctx.use_grouped_tensor_path = False + bwd_args = GroupedLinearBwdArgs() + _grouped_linear_setup_ctx(bwd_args, fwd_args) + tensors_to_save, tensor_objects = prepare_for_saving(*saved) ctx.save_for_backward(*tensors_to_save) ctx.tensor_objects = tensor_objects - - ctx.grad_input_quantizers = grad_input_quantizers - ctx.grad_output_quantizers = grad_output_quantizers - ctx.grad_weight_quantizers = grad_weight_quantizers - - # weight_requires_grad was read from the parameters, before materialization - # replaced ``weights`` with gathered copies that need not carry requires_grad. - ctx.weights_requires_grad = weight_requires_grad - if fuse_wgrad_accumulation and ctx.weights_requires_grad: - # Keep weakrefs to weights to preserve attributes like main_grad - # when we need to modify the weight python objects. Target the parameters: - # the gathered copies are transient and carry none of these attributes. - ref_weights = origin_weights if is_dist_weight else weights - ctx.origin_weight_refs = [weakref.ref(w) for w in ref_weights] - ctx.origin_weights_overwrite_main_grad = getattr( - ref_weights[0], "overwrite_main_grad", False - ) - # This check is needed to ensure that main_grad is not created - # during the forward pass when using MCore FSDP as it creates - # the main_grad buffer lazily before backprop - if hasattr(weights[0], "__fsdp_param__"): - # MCore FSDP creates main_grad lazily before backward - ctx.main_grad_funcs = [weights[i].get_main_grad for i in range(num_gemms)] - elif is_dist_weight: - ctx.main_grad_funcs = [origin_weights[i].grad_buffer for i in range(num_gemms)] - else: - ctx.main_grad_funcs = [ - lambda j=i: weights[j].main_grad for i in range(num_gemms) - ] - ctx.device = device - ctx.output_quantizers = output_quantizers - ctx.m_splits = m_splits - ctx.num_gemms = num_gemms - ctx.activation_dtype = activation_dtype - ctx.fp8 = fp8 - ctx.fp8_recipe = FP8GlobalStateManager.get_fp8_recipe() if fp8 else None - ctx.backward_override = backward_override - ctx.fuse_wgrad_accumulation = fuse_wgrad_accumulation - ctx.cpu_offloading = cpu_offloading - ctx.is_first_microbatch = is_first_microbatch - ctx.use_bias = use_bias - ctx.sequence_parallel = sequence_parallel - ctx.inp_shape = inp.shape - ctx.requires_dgrad = inp.requires_grad - ctx.reduce_and_update_bwd_fp8_tensors = False - if ctx.fp8 and requires_grad(inp, weights[0], biases[0]): - ctx.reduce_and_update_bwd_fp8_tensors = ( - ctx.reduce_and_update_bwd_fp8_tensors - or FP8GlobalStateManager.is_first_fp8_module() - ) - ctx.wgrad_store = wgrad_store - ctx.debug = debug - ctx.save_original_input = save_original_input - ctx.input_quantizers = input_quantizers - ctx.dgrad_out = dgrad_out - - # backward overrides - if backward_override is not None: - ctx.fp8 = False - ctx.debug = False - ctx.ub_overlap_ag = False - ctx.ub_overlap_rs_dgrad = False - ctx.ub_bulk_dgrad = False - ctx.ub_bulk_wgrad = False - ctx.grad_input_quantizers = [None] * num_gemms - ctx.grad_weight_quantizers = [None] * num_gemms - ctx.grad_output_quantizers = [None] * num_gemms - ctx.reduce_and_update_bwd_fp8_tensors = False - - # [*, in_features] -> [*, out_features] except first dimension changes for SP - return out.view(-1, *inp.shape[1:-1], out.shape[-1]), new_workspaces - - @staticmethod - def _backward_grouped_tensor( - ctx, - grad_output: torch.Tensor, - ) -> Tuple[Union[torch.Tensor, None], ...]: - """Backward path paired with ``_forward_grouped_tensor``.""" - saved_tensors = restore_from_func_ctx(ctx) - N = ctx.num_gemms - saved_input = saved_tensors[0] - if ctx.single_grouped_weight: - weights_for_gemm = saved_tensors[1] - weight_tensors = [weights_for_gemm] - split_sizes = saved_tensors[2] - base_split_offsets = saved_tensors[3] - input_tensor_offsets = saved_tensors[4] - output_tensor_offsets = saved_tensors[5] - else: - weight_tensors = saved_tensors[1 : 1 + N] - weights_for_gemm = weight_tensors - split_sizes = saved_tensors[1 + N] - base_split_offsets = saved_tensors[2 + N] - input_tensor_offsets = saved_tensors[3 + N] - output_tensor_offsets = saved_tensors[4 + N] - - if ctx.save_original_input: - x = cast_if_needed( - saved_input.reshape(-1, ctx.weights_shape_1), - ctx.activation_dtype, - ) - if ctx.fp8: - input_quantizer = ctx.input_quantizers[0] - input_quantizer.set_usage(rowwise=False, columnwise=True) - input_quantizer.optimize_for_gemm = True - grouped_x = tex.group_quantize(x, input_quantizer, N, split_sizes) - else: - grouped_x = _GroupedLinear._make_grouped_tensor( - x, - num_gemms=N, - split_sizes=split_sizes, - tensor_offsets=input_tensor_offsets, - last_dim=ctx.weights_shape_1, - dtype=ctx.activation_dtype, - ) - else: - grouped_x = saved_input - - num_weight_args = 1 if ctx.single_grouped_weight else N - origin_weights = [None] * num_weight_args - main_grads = [None] * num_weight_args - is_dist_weight = getattr(ctx, "is_dist_weight", False) - if is_dist_weight: - # Forward saved the shards, not the gathered copies. - origin_weights = list(weight_tensors) - if ctx.fuse_wgrad_accumulation and ctx.weights_requires_grad: - main_grads = [main_grad_func() for main_grad_func in ctx.main_grad_funcs] - # Re-gather even without dgrad: the call advances the implementer's prefetch chain - # for the next layer's backward. Only the operand rebuild is dgrad-only. - weight_tensors = materialize_weight_for_backward(origin_weights) - weights_for_gemm = None - if ctx.requires_dgrad: - # cache_weight=False: the gathered copy must not outlive this backward. - weights_for_gemm, _ = _GroupedLinear._prepare_weights_for_grouped_tensor_gemm( - weight_tensors, - ctx.weight_quantizers, - None, - num_gemms=N, - single_grouped_weight=ctx.single_grouped_weight, - with_quantized_compute=ctx.fp8_for_weight_prep, - columnwise_usage=True, - activation_dtype=ctx.activation_dtype, - is_first_microbatch=None, - skip_fp8_weight_update=None, - cache_weight=False, - ) - elif ctx.fuse_wgrad_accumulation and ctx.weights_requires_grad: - origin_weight_refs = ctx.origin_weight_refs - ctx.origin_weight_refs = None - origin_weights = [ref() if ref is not None else None for ref in origin_weight_refs] - assert all( - w is not None for w in origin_weights - ), "weight was removed while fuse_wgrad_accumulation=True" - main_grads = [main_grad_func() for main_grad_func in ctx.main_grad_funcs] - for origin_weight, main_grad in zip(origin_weights, main_grads): - if main_grad is not None: - origin_weight.main_grad = main_grad - - grad_output_view = grad_output.contiguous().view(-1, grad_output.shape[-1]) - dy_2d = cast_if_needed(grad_output_view, ctx.activation_dtype) - dbias_packed = None - if ctx.fp8: - grad_output_quantizer = ctx.grad_output_quantizers[0] - grad_output_quantizer.set_usage( - rowwise=ctx.requires_dgrad, - columnwise=ctx.weights_requires_grad, - ) - grad_output_quantizer.optimize_for_gemm = True - # The grouped FP8 block-scaling bgrad kernel computes dbias in the rowwise - # pass, so the fusion needs rowwise output (i.e. dgrad required). - fuse_bgrad = isinstance(grad_output_quantizer, MXFP8Quantizer) or ( - isinstance(grad_output_quantizer, Float8BlockQuantizer) and ctx.requires_dgrad - ) - if ctx.use_bias and fuse_bgrad: - grouped_dy, dbias_packed = tex.bgrad_group_quantize( - dy_2d, - grad_output_quantizer, - N, - split_sizes, - tensor_offsets=output_tensor_offsets, - ) - else: - grouped_dy = tex.group_quantize( - dy_2d, - grad_output_quantizer, - N, - split_sizes, - tensor_offsets=output_tensor_offsets, + ctx.backward_objects = bwd_args + if fwd_args.fp8 and requires_grad(inp, fwd_args.weights[0], fwd_args.biases[0]): + bwd_args.reduce_and_update_bwd_fp8_tensors = ( + FP8GlobalStateManager.is_first_fp8_module() ) - else: - grouped_dy = _GroupedLinear._make_grouped_tensor( - dy_2d, - num_gemms=N, - split_sizes=split_sizes, - tensor_offsets=output_tensor_offsets, - last_dim=ctx.weights_shape_0, - dtype=ctx.activation_dtype, - ) + if fwd_args.backward_override is not None: + bwd_args.reduce_and_update_bwd_fp8_tensors = False - if ctx.use_bias: - if dbias_packed is None: - dbias_packed = compute_grouped_dbias(dy_2d, base_split_offsets, N) - if ctx.single_grouped_bias: - grad_bias_args = [dbias_packed.to(dtype=ctx.activation_dtype)] - else: - grad_bias_args = [dbias_packed[i].to(dtype=ctx.activation_dtype) for i in range(N)] - else: - num_bias_args = 1 if ctx.single_grouped_bias else N - grad_bias_args = [None] * num_bias_args - - dgrad = None - if ctx.requires_dgrad: - dgrad_gemm_use_split_accumulator = _2X_ACC_DGRAD - if ctx.fp8: - recipe = ctx.fp8_recipe - if hasattr(recipe, "fp8_gemm_dgrad"): - dgrad_gemm_use_split_accumulator = recipe.fp8_gemm_dgrad.use_split_accumulator - for weight in weight_tensors: - if isinstance(weight, QuantizedTensorStorage): - weight.update_usage(columnwise_usage=True) - dgrad = _GroupedLinear._validate_or_alloc_output( - ctx.dgrad_out, - dy_2d.size(0), - ctx.weights_shape_1, - ctx.activation_dtype, - ctx.device, - ) - grouped_dgrad = _GroupedLinear._make_grouped_tensor( - dgrad, - num_gemms=N, - split_sizes=split_sizes, - tensor_offsets=input_tensor_offsets, - last_dim=ctx.weights_shape_1, - dtype=ctx.activation_dtype, - ) - general_grouped_gemm_for_grouped_tensor( - weights_for_gemm, - grouped_dy, - grouped_dgrad, - layout="NN", - use_split_accumulator=dgrad_gemm_use_split_accumulator, - ) - - if is_dist_weight: - # grad_buffer is per-step scratch, so overwrite it instead of accumulating. - accumulate_wgrad_into_param_main_grad = False - elif ctx.is_first_microbatch is not None: - accumulate_wgrad_into_param_main_grad = ( - ctx.fuse_wgrad_accumulation and not ctx.is_first_microbatch - ) - else: - accumulate_wgrad_into_param_main_grad = ctx.fuse_wgrad_accumulation - - if ctx.weights_requires_grad: - if ( - is_dist_weight - and ctx.wgrad_store is not None - and ctx.wgrad_store.delay_wgrad_compute() - ): - raise RuntimeError( - "distributed-weight GroupedLinear requires delay_wgrad_compute=False." - ) - wgrad_gemm_use_split_accumulator = _2X_ACC_WGRAD - if ctx.fp8: - recipe = ctx.fp8_recipe - if hasattr(recipe, "fp8_gemm_wgrad"): - wgrad_gemm_use_split_accumulator = recipe.fp8_gemm_wgrad.use_split_accumulator - if ctx.fuse_wgrad_accumulation: - if ctx.single_grouped_weight: - main_grad = main_grads[0] - grouped_wgrad = GroupedTensor.make_grouped_tensor_from_rowwise_data( - num_tensors=N, - tensor_shape=(ctx.weights_shape_0, ctx.weights_shape_1), - rowwise_data=main_grad.view(-1), - dtype=main_grad.dtype, - ) - wgrad_output = grouped_wgrad - wgrad_list = [main_grad] - else: - wgrad_output = main_grads - wgrad_list = main_grads - else: - if ctx.single_grouped_weight: - grouped_wgrad = GroupedTensor.make_grouped_tensor_with_shapes( - num_tensors=N, - shapes=[(ctx.weights_shape_0, ctx.weights_shape_1)] * N, - quantizer=None, - device=ctx.device, - dtype=ctx.activation_dtype, - ) - wgrad_output = grouped_wgrad - wgrad_list = [ - grouped_wgrad.rowwise_data.view(N, ctx.weights_shape_0, ctx.weights_shape_1) - ] - else: - wgrad_packed = torch.empty( - N, - ctx.weights_shape_0, - ctx.weights_shape_1, - dtype=ctx.activation_dtype, - device=ctx.device, - ) - wgrad_output = [wgrad_packed[i] for i in range(N)] - wgrad_list = wgrad_output - - accumulate = ( - accumulate_wgrad_into_param_main_grad - if not getattr(ctx, "origin_weights_overwrite_main_grad", False) - else False - ) - - def grouped_gemm_wgrad(inputmats, grad_output_mats, grad_weights): - general_grouped_gemm_for_grouped_tensor( - inputmats, - grad_output_mats, - grad_weights, - layout="NT", - use_split_accumulator=wgrad_gemm_use_split_accumulator, - accumulate=accumulate, - ) - return None, [None] * N, None - - if ctx.wgrad_store is not None and ctx.wgrad_store.delay_wgrad_compute(): - ctx.wgrad_store.put([grouped_x, grouped_dy, wgrad_output], grouped_gemm_wgrad) - else: - grouped_gemm_wgrad(grouped_x, grouped_dy, wgrad_output) - - def handle_custom_ddp_from_mcore(weight, main_grad, wgrad): - if ctx.weights_requires_grad: - if ctx.fuse_wgrad_accumulation and hasattr(weight, "grad_added_to_main_grad"): - weight.grad_added_to_main_grad = True - if getattr(weight, "zero_out_wgrad", False): - wgrad = get_dummy_wgrad( - list(main_grad.shape), - weight.dtype, - zero=True, - ) - else: - wgrad = get_dummy_wgrad( - list(main_grad.shape), - weight.dtype, - zero_tensor=getattr(weight, "zero_tensor_dummy_wgrad", False), - ) - elif ctx.fuse_wgrad_accumulation: - wgrad = None - else: - wgrad = None - return wgrad - - if is_dist_weight: - wgrad_list = finalize_weight_grads(origin_weights, wgrad_list) - else: - wgrad_list = [ - handle_custom_ddp_from_mcore(weight, main_grad, wgrad) - for weight, main_grad, wgrad in zip(origin_weights, main_grads, wgrad_list) - ] - else: - wgrad_list = [None] * num_weight_args - - if ctx.reduce_and_update_bwd_fp8_tensors: - FP8GlobalStateManager.reduce_and_update_fp8_tensors(forward=False) - return ( - dgrad.view(ctx.inp_shape) if ctx.requires_dgrad else None, - None, # m_splits - None, # non_tensor_args - None, # out - None, # dgrad_out - *wgrad_list, - *grad_bias_args, - ) + return out, new_workspaces @staticmethod def backward( @@ -1344,285 +1583,23 @@ def backward( ) -> Tuple[Union[torch.Tensor, None], ...]: # pylint: disable=missing-function-docstring with get_nvtx_range_context("_GroupedLinear_backward"): - if ctx.grouped_tensor_supported: - return _GroupedLinear._backward_grouped_tensor(ctx, grad_output) - - saved_tensors = restore_from_func_ctx(ctx) - N = ctx.num_gemms - inputmats = saved_tensors[:N] - weights = saved_tensors[N : 2 * N] - saved_weights = saved_tensors[2 * N : 3 * N] - biases = saved_tensors[3 * N : 4 * N] - - # Restore from weakrefs to get original weight python objects - # (preserves attributes like main_grad, grad_added_to_main_grad, etc.) - # Only needed when fuse_wgrad_accumulation is enabled. - origin_weights = [None] * N - main_grads = [None] * N - is_dist_weight = is_distributed_weight(saved_weights[0]) - if is_dist_weight: - origin_weights = saved_weights - if ctx.fuse_wgrad_accumulation and ctx.weights_requires_grad: - main_grads = [main_grad_func() for main_grad_func in ctx.main_grad_funcs] - elif ctx.fuse_wgrad_accumulation and ctx.weights_requires_grad: - origin_weight_refs = ctx.origin_weight_refs - ctx.origin_weight_refs = None - origin_weights = [ref() if ref is not None else None for ref in origin_weight_refs] - assert all( - w is not None for w in origin_weights - ), "weight was removed while fuse_wgrad_accumulation=True" - main_grads = [main_grad_func() for main_grad_func in ctx.main_grad_funcs] - for origin_weight, main_grad in zip(origin_weights, main_grads): - if main_grad is not None: - origin_weight.main_grad = main_grad - - # Preprocess grad output - grad_output_view = grad_output.contiguous().view(-1, grad_output.shape[-1]) - grad_output_reference = ctx.grad_output_quantizers[0] - if ctx.fp8 and isinstance(grad_output_reference, HybridQuantizer): - # Usage is a runtime decision, not part of generation validation. - # Apply it uniformly so dispatch can read the first parent without - # rescanning every expert. - for grad_output_quantizer in ctx.grad_output_quantizers: - grad_output_quantizer.set_usage( - rowwise=ctx.requires_dgrad, - columnwise=ctx.weights_requires_grad, - ) - grad_output, grad_biases = _split_quantization._split_quantize( - grad_output_view, - ctx.m_splits, - ctx.grad_output_quantizers, - ctx.activation_dtype, - with_quantized_output=ctx.fp8 or ctx.debug, - compute_dbias=(ctx.fp8 or ctx.debug) and ctx.use_bias, - disable_bulk_allocation=( - ctx.cpu_offloading - and isinstance(grad_output_reference, HybridQuantizer) - and not _split_quantization._uses_identity_quantizer(grad_output_reference) - ), - ) - if grad_biases is None: - grad_biases = [None] * N - - if is_dist_weight: - accumulate_wgrad_into_param_main_grad = False - elif ctx.is_first_microbatch is not None: - accumulate_wgrad_into_param_main_grad = ( - ctx.fuse_wgrad_accumulation and not ctx.is_first_microbatch - ) - else: - accumulate_wgrad_into_param_main_grad = ctx.fuse_wgrad_accumulation - - if is_dist_weight: - weights = materialize_weight_for_backward(origin_weights) - - if ctx.requires_dgrad: - dgrad_gemm_use_split_accumulator = _2X_ACC_DGRAD - if ctx.fp8 or ctx.debug: - recipe = ctx.fp8_recipe - if hasattr(recipe, "fp8_gemm_dgrad"): - dgrad_gemm_use_split_accumulator = ( - recipe.fp8_gemm_dgrad.use_split_accumulator - ) - dgrad = _GroupedLinear._validate_or_alloc_output( - ctx.dgrad_out, - sum(ctx.m_splits), - ctx.weights_shape_1, - ctx.activation_dtype, - ctx.device, - ) - weights_for_dgrad = weights - if ctx.backward_override == "dequantized": - weights_for_dgrad = [ - ( - weight.dequantize(dtype=ctx.activation_dtype) - if isinstance(weight, QuantizedTensorStorage) - else cast_if_needed(weight, ctx.activation_dtype) - ) - for weight in weights - ] - elif ctx.backward_override == "high_precision": - weights_for_dgrad = [ - ( - weight.dequantize(dtype=ctx.activation_dtype) - if isinstance(weight, QuantizedTensorStorage) - else cast_if_needed(weight, ctx.activation_dtype) - ) - for weight in saved_weights - ] - elif is_dist_weight and ctx.fp8: - # Re-gathering drops the quantization forward applied: a bf16 shard comes - # back bf16, which this GEMM cannot pair with fp8 grad_output. - weights_for_dgrad = [] - for idx, weight in enumerate(weights): - if not isinstance(weight, QuantizedTensorStorage): - quantizer = ctx.weight_quantizers[idx] - quantizer.set_usage(rowwise=True, columnwise=True) - weight = quantizer(weight) - weights_for_dgrad.append(weight) - # Make sure weights are available in column-wise format - # for dgrad computation. - for weight in weights_for_dgrad: - if isinstance(weight, QuantizedTensorStorage): - weight.update_usage(columnwise_usage=True) - general_grouped_gemm( - weights_for_dgrad, - grad_output, - [dgrad], - ctx.grad_input_quantizers, - ctx.activation_dtype, - single_output=True, - layout="NN", - m_splits=ctx.m_splits, - grad=True, - use_split_accumulator=dgrad_gemm_use_split_accumulator, - ) - - if ctx.weights_requires_grad: - if ( - is_dist_weight - and ctx.wgrad_store is not None - and ctx.wgrad_store.delay_wgrad_compute() - ): - raise RuntimeError( - "distributed-weight GroupedLinear requires delay_wgrad_compute=False." - ) - wgrad_gemm_use_split_accumulator = _2X_ACC_WGRAD - if ctx.fp8: - recipe = ctx.fp8_recipe - if hasattr(recipe, "fp8_gemm_wgrad"): - wgrad_gemm_use_split_accumulator = ( - recipe.fp8_gemm_wgrad.use_split_accumulator - ) - if ctx.fuse_wgrad_accumulation: - wgrad_list = main_grads - else: - wgrad_packed = torch.empty( - ctx.num_gemms, - *weights[0].size(), - dtype=ctx.activation_dtype, - device=ctx.device, - ) - wgrad_list = [wgrad_packed[i] for i in range(ctx.num_gemms)] - if is_dist_weight: - # Gathered weights are no longer needed after dgrad GEMM. - del weights - - if ctx.save_original_input: - inp = inputmats[0] - in_features = inp.shape[-1] - inp_view = inp.reshape(-1, in_features) - if ctx.input_quantizers[0] is not None: - for input_quantizer in ctx.input_quantizers: - if isinstance( - input_quantizer, - (Float8Quantizer, Float8CurrentScalingQuantizer), - ): - input_quantizer.set_usage(rowwise=True, columnwise=True) - else: - input_quantizer.set_usage(rowwise=False, columnwise=True) - inputmats, _ = _split_quantization._split_quantize( - inp_view, - ctx.m_splits, - ctx.input_quantizers, - ctx.activation_dtype, - with_quantized_output=ctx.fp8 or ctx.debug, - disable_bulk_allocation=ctx.cpu_offloading, - ) - elif ctx.backward_override == "dequantized": - inputmats_dequant = [] - for inputmat in inputmats: - if isinstance(inputmat, QuantizedTensorStorage): - inputmats_dequant.append( - inputmat.dequantize(dtype=ctx.activation_dtype) - ) - else: - inputmats_dequant.append(cast_if_needed(inputmat, ctx.activation_dtype)) - inputmats = inputmats_dequant - grouped_gemm_wgrad = functools.partial( - general_grouped_gemm, - quantization_params=ctx.grad_weight_quantizers, - out_dtype=ctx.activation_dtype, - layout="NT", - grad=True, - m_splits=ctx.m_splits, - use_bias=ctx.use_bias if grad_biases[0] is None else None, - bias=biases, - use_split_accumulator=wgrad_gemm_use_split_accumulator, - accumulate=( - accumulate_wgrad_into_param_main_grad - if not is_dist_weight - and not getattr(ctx, "origin_weights_overwrite_main_grad", False) - else False - ), - ) - # WGRAD - if ctx.wgrad_store is not None and ctx.wgrad_store.delay_wgrad_compute(): - ctx.wgrad_store.put([inputmats, grad_output, wgrad_list], grouped_gemm_wgrad) - else: - _, grad_biases_, _ = grouped_gemm_wgrad(inputmats, grad_output, wgrad_list) - - for i in range(ctx.num_gemms): - if grad_biases[i] is None: - grad_biases[i] = grad_biases_[i] - del grad_biases_ - - # Deallocate input tensor - clear_tensor_data(*inputmats) - - def handle_custom_ddp_from_mcore(weight, main_grad, wgrad): - if ctx.weights_requires_grad: - # Handle custom DDP from mcore. - if ctx.fuse_wgrad_accumulation and hasattr( - weight, "grad_added_to_main_grad" - ): - weight.grad_added_to_main_grad = True - if getattr(weight, "zero_out_wgrad", False): - wgrad = get_dummy_wgrad( - list(main_grad.shape), - weight.dtype, - zero=True, - ) - else: - wgrad = get_dummy_wgrad( - list(main_grad.shape), - weight.dtype, - zero_tensor=getattr(weight, "zero_tensor_dummy_wgrad", False), - ) - elif ctx.fuse_wgrad_accumulation: - wgrad = None - else: - wgrad = None - return wgrad - - if is_dist_weight: - wgrad_list = finalize_weight_grads(origin_weights, wgrad_list) - else: - wgrad_list = [ - handle_custom_ddp_from_mcore(weight, main_grad, wgrad) - for weight, main_grad, wgrad in zip(origin_weights, main_grads, wgrad_list) - ] + bwd_args = ctx.backward_objects + bwd_args.grad_output = grad_output + bwd_args.setup_saved_tensors(ctx) + if ctx.use_grouped_tensor_path: + dgrad, wgrad_list, grad_biases = _grouped_linear_graph_safe_backward(bwd_args) + if dgrad is not None: + dgrad = dgrad.view(ctx.inp_shape) else: - wgrad_list = [None] * ctx.num_gemms + dgrad, wgrad_list, grad_biases = _grouped_linear_backward_impl(bwd_args) + reduce_and_update_bwd_fp8_tensors = bwd_args.reduce_and_update_bwd_fp8_tensors + # Release saved state after either backward path. + ctx.backward_objects = None + del bwd_args - if not ctx.use_bias or ( - ctx.wgrad_store is not None - and ctx.wgrad_store.delay_wgrad_compute() - and not ctx.fp8 - ): - grad_biases = [None] * ctx.num_gemms - - if ctx.reduce_and_update_bwd_fp8_tensors: + if reduce_and_update_bwd_fp8_tensors: FP8GlobalStateManager.reduce_and_update_fp8_tensors(forward=False) - return ( - dgrad.view(ctx.inp_shape) if ctx.requires_dgrad else None, - None, # m_splits - None, # non_tensor_args - None, # out - None, # dgrad_out - *wgrad_list, - *grad_biases, - ) + return (dgrad, None, *wgrad_list, *grad_biases) class GroupedLinear(TransformerEngineBaseModule): @@ -2137,7 +2114,7 @@ def _remap_grouped_weight_state_dict_keys(self, state_dict, prefix: str) -> None if not has_grouped_weight and has_per_gemm_weights: per_gemm_weights = [state_dict.pop(key) for key in per_gemm_weight_keys] per_gemm_weights = [ - weight.dequantize() if isinstance(weight, QuantizedTensorStorage) else weight + (weight.dequantize() if isinstance(weight, QuantizedTensorStorage) else weight) for weight in per_gemm_weights ] state_dict[grouped_weight_key] = torch.stack(per_gemm_weights, dim=0) @@ -2220,14 +2197,27 @@ def load_state_dict(self, state_dict, strict: bool = True, assign: bool = False) return super().load_state_dict(state_dict_copy, strict=strict, assign=assign) def _load_from_state_dict( - self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs + self, + state_dict, + prefix, + local_metadata, + strict, + missing_keys, + unexpected_keys, + error_msgs, ): """Load state, including compatibility across grouped-weight checkpoint formats.""" self._remap_grouped_weight_state_dict_keys(state_dict, prefix) self._remap_grouped_bias_state_dict_keys(state_dict, prefix) super()._load_from_state_dict( - state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs + state_dict, + prefix, + local_metadata, + strict, + missing_keys, + unexpected_keys, + error_msgs, ) @no_torch_dynamo() @@ -2346,13 +2336,6 @@ def forward( for q in weight_quantizers: q.optimize_for_gemm = optimize_for_gemm - if is_grad_enabled: - linear_fn = _GroupedLinear.apply - autograd_ctx = [] - else: - linear_fn = _GroupedLinear.forward - autograd_ctx = [None] - cache_weight = is_first_microbatch is not None if self.single_grouped_weight: weight_workspaces = [self._fp8_workspaces.get("weight")] if cache_weight else [None] @@ -2363,41 +2346,141 @@ def forward( else [None] * num_gemms ) - non_tensor_args = ( - self.apply_bias, - is_first_microbatch, - self.fp8, - self.fp8_calibration, - self.wgrad_store, - input_quantizers, - weight_quantizers, - output_quantizers, - grad_input_quantizers, - grad_weight_quantizers, - grad_output_quantizers, - self.fuse_wgrad_accumulation, - is_cpu_offload_enabled(), - self.sequence_parallel, - self.activation_dtype, - is_grad_enabled, - weight_workspaces, - cache_weight, - skip_fp8_weight_update, - self.save_original_input, - debug, - self.single_grouped_weight, - use_grouped_bias, - self.use_grouped_tensor, + weight_requires_grad = weight_tensors[0].requires_grad + if self.fp8: + _recipe = FP8GlobalStateManager.get_fp8_recipe() + backward_override = _recipe.backward_override + else: + _recipe = None + backward_override = None + + save_original_input = self.save_original_input + if backward_override == "high_precision": + save_original_input = True + elif backward_override == "dequantized": + save_original_input = False + backward_needs_input = is_grad_enabled and weight_requires_grad + if backward_override is None and save_original_input and backward_needs_input: + if _recipe is not None and _recipe.delayed(): + raise ValueError( + "DelayedScaling recipe is not supported with save_original_input" + ) + + # Megatron-Core may enable this automatically to reuse an activation + # already retained by an upstream operation. Built-in recipes guarantee + # group homogeneity, while CustomRecipe generations are validated once, + # so runtime safety can be determined from expert 0. + if _recipe is not None: + input_quantizer = input_quantizers[0] + if isinstance(input_quantizer, Float8Quantizer): + warnings.warn( + "save_original_input is incompatible with delayed-scaling quantizers " + "(Float8Quantizer). Disabling save_original_input for this module.", + stacklevel=2, + ) + save_original_input = False + elif not can_reconstruct_wgrad_input_from_original(input_quantizer): + warnings.warn( + "Ignoring save_original_input=True because the input quantizer cannot " + "safely reconstruct the backward operand from the original input " + f"({input_quantizer}).", + stacklevel=2, + ) + save_original_input = False + + cpu_offloading = is_cpu_offload_enabled() + use_grouped_tensor_path = False + if self.use_grouped_tensor and not ( + self.fp8_calibration + or debug + or cpu_offloading + or any(q is not None for q in output_quantizers) + ): + if ( + self.fp8 + and _recipe.float8_block_scaling() + and (10, 0) <= get_device_compute_capability() <= (11, 0) + ): + raise RuntimeError( + "use_grouped_tensor=True does not support the FP8 block-scaling recipe on" + " Blackwell GPUs: the native grouped FP8 block-scaling path is Hopper-only." + " Set use_grouped_tensor=False, or unset" + " NVTE_GROUPED_LINEAR_USE_FUSED_GROUPED_GEMM if it enabled this path, to" + " use the MXFP8-emulated path on Blackwell." + ) + use_grouped_tensor_path = is_module_grouped_tensor_path_supported( + _recipe, + self.activation_dtype, + ) + if ( + self.use_grouped_tensor + and not use_grouped_tensor_path + and (self.single_grouped_weight or use_grouped_bias) + ): + raise RuntimeError( + "Single grouped parameters require the native grouped-tensor path, but the" + " active device, cuBLASLt version, quantization recipe, or GroupedLinear" + " feature configuration does not support it. Disable single_grouped_weight and" + " single_grouped_bias to allow the split-quantize fallback." + ) + if use_grouped_tensor_path: + if m_splits.device.type != "cuda": + raise ValueError( + "The native grouped_tensor path requires CUDA m_splits. Pass a CUDA int64 " + "tensor, or set use_grouped_tensor=False." + ) + fwd_args = GroupedLinearFwdArgs( + # tensors + inp=inp, + weights=weight_tensors, + biases=bias_tensors, + weight_workspaces=weight_workspaces, + out=out, + dgrad_out=dgrad_out, + skip_fp8_weight_update=skip_fp8_weight_update, + m_splits_tensor=m_splits, + # requires_grad flags + input_requires_grad=inp.requires_grad, + weights_requires_grad=weight_requires_grad, + # quantizers + input_quantizers=input_quantizers, + weight_quantizers=weight_quantizers, + output_quantizers=output_quantizers, + grad_input_quantizers=grad_input_quantizers, + grad_weight_quantizers=grad_weight_quantizers, + grad_output_quantizers=grad_output_quantizers, + # split geometry + m_splits=None, + num_gemms=num_gemms, + # numerical / dtype config + activation_dtype=self.activation_dtype, + fp8=self.fp8, + fp8_calibration=self.fp8_calibration, + save_original_input=save_original_input, + backward_override=backward_override, + fp8_recipe=_recipe, + debug=debug, + # weight-workspace caching + is_first_microbatch=is_first_microbatch, + cache_weight=cache_weight, + # fused GroupedTensor path + use_grouped_tensor_path=use_grouped_tensor_path, + single_grouped_weight=self.single_grouped_weight, + single_grouped_bias=use_grouped_bias, + # misc + use_bias=self.apply_bias, + fuse_wgrad_accumulation=self.fuse_wgrad_accumulation, + wgrad_store=self.wgrad_store, + cpu_offloading=cpu_offloading, + is_grad_enabled=is_grad_enabled, ) - out, new_workspaces = linear_fn( - *autograd_ctx, + + out, new_workspaces = _grouped_linear_eager( inp, m_splits, - non_tensor_args, - out, - dgrad_out, - *weight_tensors, - *bias_tensors, + fwd_args, + (*weight_tensors, *bias_tensors), + is_grad_enabled, ) if cache_weight: