From 83641e8ede8633153f1fb1fe2c105b67bd9a4115 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 29 Oct 2026 13:07:03 +0100 Subject: [PATCH 01/12] [PyTorch] Add torch.compile support for LayerNormLinear and LayerNormMLP Register LayerNorm module forward and backward custom ops with matching fake tensor metadata, shared eligibility checks, and compiled weight caches. Extend the existing compile tests to cover the LayerNorm modules. Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 912 ++++++++++++++---- .../dot_product_attention/backends.py | 6 +- transformer_engine/pytorch/csrc/extensions.h | 3 + .../pytorch/csrc/extensions/normalization.cpp | 23 +- .../pytorch/csrc/extensions/pybind.cpp | 2 + .../pytorch/dynamo/custom_op.py | 42 +- transformer_engine/pytorch/dynamo/fake.py | 56 ++ transformer_engine/pytorch/module/_common.py | 83 +- .../pytorch/module/layernorm_linear.py | 499 +++++++++- .../pytorch/module/layernorm_mlp.py | 620 +++++++++++- transformer_engine/pytorch/module/linear.py | 166 +--- 11 files changed, 1978 insertions(+), 434 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 6868de7ef18..66c925a25a2 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -22,6 +22,7 @@ except ImportError: # pragma: no cover counters = None from torch._subclasses.fake_tensor import FakeTensor, FakeTensorMode +from torch._dynamo.testing import CompileCounterWithBackend try: from torch._opaque_base import OpaqueBaseMeta @@ -45,6 +46,11 @@ from transformer_engine.common import recipe from transformer_engine.pytorch.constants import FP8FwdTensorIdx, FP8BwdTensorIdx from transformer_engine.pytorch.module.base import TransformerEngineBaseModule +from transformer_engine.pytorch.module import ( + linear as linear_module, + layernorm_linear as layernorm_linear_module, + layernorm_mlp as layernorm_mlp_module, +) from transformer_engine.pytorch.quantization import FP8GlobalStateManager, QuantizerRole from transformer_engine.pytorch.ops.basic.basic_linear import BasicLinear from transformer_engine.pytorch.ops.basic.bias import BiasBwdArgs @@ -60,7 +66,7 @@ Quantizer, ) from transformer_engine.pytorch.dynamo import TensorSpec, to_tensor_spec -from transformer_engine.pytorch.dynamo.fake import fake_bgrad +from transformer_engine.pytorch.dynamo.fake import fake_bgrad, fake_norm from transformer_engine.pytorch import ( is_fp8_available, is_mxfp8_available, @@ -129,7 +135,8 @@ def nvfp4_4over6(): if mxfp8_available: _all_recipes.append(recipe.MXFP8BlockScaling()) if nvfp4_available: - _all_recipes.append(recipe.NVFP4BlockScaling()) + # Disable stochastic rounding for exact eager/compiled comparisons. + _all_recipes.append(recipe.NVFP4BlockScaling(disable_stochastic_rounding=True)) _all_recipes.append(nvfp4_4over6()) _all_recipes.append(nvfp4_row_scaled()) @@ -175,30 +182,75 @@ def _assert_no_cudagraph_skips(enabled: bool): _EAGER_ATOL, _EAGER_RTOL = 0.0, 0.0 -def _assert_close_eager_compiled(fn, compiled, model, base): - """Run ``fn`` eagerly and ``compiled`` on identical inputs; assert the - forward output and the input / weight / bias gradients match.""" +def _flatten_outputs(out): + return list(out) if isinstance(out, (tuple, list)) else [out] + + +def _assert_close_eager_compiled(fn, compiled, model, base, *, dynamic_batch=False): + """Run ``fn`` eagerly and ``compiled`` on identical inputs; assert every + forward output and the input / parameter gradients match.""" inp_eager = base.detach().clone().requires_grad_(True) model.zero_grad(set_to_none=True) - out_eager = fn(inp_eager) - out_eager.sum().backward() - ref_out = out_eager.detach().clone() - ref_wgrad = model.weight.grad.detach().clone() + outs_eager = _flatten_outputs(fn(inp_eager)) + sum(o.sum() for o in outs_eager).backward() + ref_outs = [o.detach().clone() for o in outs_eager] ref_igrad = inp_eager.grad.detach().clone() - # bias=False keeps a 0-element ``bias`` around; grad is None then. - ref_bgrad = model.bias.grad.detach().clone() if model.bias.grad is not None else None + ref_pgrads = { + name: p.grad.detach().clone() for name, p in model.named_parameters() if p.grad is not None + } inp_compiled = base.detach().clone().requires_grad_(True) + if dynamic_batch: + torch._dynamo.mark_dynamic(inp_compiled, 0) model.zero_grad(set_to_none=True) # Clone before a later cuda-graph replay overwrites the static output buffer. - out_compiled = compiled(inp_compiled).clone() - out_compiled.sum().backward() + outs_compiled = [o.clone() for o in _flatten_outputs(compiled(inp_compiled))] + sum(o.sum() for o in outs_compiled).backward() - torch.testing.assert_close(out_compiled, ref_out, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) + assert len(outs_compiled) == len(ref_outs) + for out, ref in zip(outs_compiled, ref_outs): + torch.testing.assert_close(out, ref, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) torch.testing.assert_close(inp_compiled.grad, ref_igrad, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) - torch.testing.assert_close(model.weight.grad, ref_wgrad, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) - if ref_bgrad is not None: - torch.testing.assert_close(model.bias.grad, ref_bgrad, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) + pgrads = {name: p.grad for name, p in model.named_parameters() if p.grad is not None} + assert pgrads.keys() == ref_pgrads.keys() + for name, ref in ref_pgrads.items(): + torch.testing.assert_close( + pgrads[name], ref, atol=_EAGER_ATOL, rtol=_EAGER_RTOL, msg=f"grad mismatch: {name}" + ) + + +def _run_module_compile_test(model, fp8_recipe, compile_mode, in_features, dtype=torch.bfloat16): + device = "cuda" + + def fn(inp): + if fp8_recipe is None: + return model(inp) + with te.autocast(recipe=fp8_recipe): + return model(inp) + + torch._dynamo.reset() + if compile_mode == "reduce-overhead": + warm = torch.randn(32, in_features, dtype=dtype, device=device, requires_grad=True) + outs = _flatten_outputs(fn(warm)) + sum(o.sum() for o in outs).backward() + model.zero_grad(set_to_none=True) + compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) + + n_iters = 3 if compile_mode == "reduce-overhead" else 1 + with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): + for _ in range(n_iters): + base = torch.randn(32, in_features, dtype=dtype, device=device) + _assert_close_eager_compiled(fn, compiled, model, base) + + +_LINEAR_MODULES = ["Linear", "LayerNormLinear", "LayerNormMLP"] + + +def _make_module(module, *, dtype=torch.bfloat16, device="cuda", **kwargs): + """Small module with dimensions suitable for FP8 GEMMs.""" + return getattr(te, module)( + 64, 128 if module == "LayerNormMLP" else 32, params_dtype=dtype, device=device, **kwargs + ) # --------------------------------------------------------------------------- @@ -1898,6 +1950,75 @@ def fn(x): assert out.shape == () +@pytest.mark.parametrize("with_optional_shape", [False, True]) +def test_custom_op_symbolic_shape(with_optional_shape): + """Shape arguments stay dynamic across the custom-op boundary.""" + from transformer_engine.pytorch.dynamo.custom_op import register_custom_op_with_autograd + + @dataclasses.dataclass + class ShapeArgs: + input_: torch.Tensor + shape: torch.Size + optional_shape: torch.Size | None + + @dataclasses.dataclass + class ShapeBwdArgs: + grad_output: torch.Tensor | None = None + shape: torch.Size | None = None + + def setup_grad_outputs(self, grads): + (self.grad_output,) = grads + + def impl(args): + assert isinstance(args.shape, torch.Size) + shape = args.shape if args.optional_shape is None else args.optional_shape + assert isinstance(shape, torch.Size) + return args.input_.reshape(shape).clone(), (), None + + def fake_impl(args): + shape = args.shape if args.optional_shape is None else args.optional_shape + return TensorSpec(shape=shape, dtype=args.input_.dtype, device=args.input_.device), (), None + + def setup_context(bwd_args, fwd_args, *_): + bwd_args.shape = fwd_args.input_.shape + return () + + def backward_impl(args): + return (args.grad_output.reshape(args.shape).clone(),) + + def backward_fake(args): + grad = args.grad_output + return (TensorSpec(shape=args.shape, dtype=grad.dtype, device=grad.device),) + + op = register_custom_op_with_autograd( + op_name=f"test_symbolic_shape_{with_optional_shape}", + input_tensors_for_grad=["input_"], + fwd_arg_type=ShapeArgs, + fwd_impl=impl, + fwd_fake_impl=fake_impl, + setup_context=setup_context, + bwd_arg_type=ShapeBwdArgs, + bwd_impl=backward_impl, + bwd_fake_impl=backward_fake, + ) + assert op is not None + + def fn(inp): + shape = torch.Size((inp.shape[0], 2, 4)) + return op(ShapeArgs(inp, shape, shape if with_optional_shape else None)) + + counter = CompileCounterWithBackend("aot_eager") + compiled = torch.compile(fn, fullgraph=True, backend=counter) + for batch in (2, 3, 5): + inp = torch.randn(batch, 8, requires_grad=True) + torch._dynamo.mark_dynamic(inp, 0) + out = compiled(inp) + torch.testing.assert_close(out, inp.reshape(batch, 2, 4)) + out.sum().backward() + torch.testing.assert_close(inp.grad, torch.ones_like(inp)) + assert counter.frame_count == 1 + + def test_to_tensor_spec_plain(): """``to_tensor_spec`` describes a plain tensor.""" t = torch.empty(2, 3, dtype=torch.float32) @@ -1909,11 +2030,19 @@ def test_to_tensor_spec_plain(): @pytest.mark.parametrize("factory, shape", _SPEC_QUANTIZERS) -def test_to_tensor_spec_quantized(factory, shape): +@pytest.mark.parametrize("swizzled", [False, True]) +def test_to_tensor_spec_quantized(factory, shape, swizzled): """``to_tensor_spec`` round-trips a quantized tensor back into a spec.""" q = factory() + has_scale_layout = isinstance(q, (MXFP8Quantizer, NVFP4Quantizer)) + if has_scale_layout: + q.optimize_for_gemm = not swizzled tensor = TensorSpec( - shape=shape, dtype=torch.bfloat16, quantizer=q, device=torch.device("cpu") + shape=shape, + dtype=torch.bfloat16, + quantizer=q, + device=torch.device("cpu"), + with_gemm_swizzled_scales=swizzled if has_scale_layout else None, ).create_tensor() spec = to_tensor_spec(tensor) @@ -1923,13 +2052,45 @@ def test_to_tensor_spec_quantized(factory, shape): # Same buffer layout as the original tensor. assert spec.inner_names() == tuple(q.inner_tensor_specs(shape)) # Rebuilding from the derived spec matches the original tensor's structure. - assert _signature(spec.create_tensor(), spec.inner_names()) == _signature( - tensor, spec.inner_names() - ) + rebuilt = spec.create_tensor() + assert _signature(rebuilt, spec.inner_names()) == _signature(tensor, spec.inner_names()) + if has_scale_layout: + assert tensor._with_gemm_swizzled_scales == swizzled + assert spec.with_gemm_swizzled_scales == swizzled + assert rebuilt._with_gemm_swizzled_scales == swizzled + assert spec.quantizer.optimize_for_gemm == q.optimize_for_gemm == (not swizzled) + + +@pytest.mark.parametrize("normalization", ["LayerNorm", "RMSNorm"]) +@pytest.mark.parametrize("with_2d_quantization", [False, True]) +@pytest.mark.parametrize("optimize_for_gemm", [False, True]) +@pytest.mark.parametrize( + "use_cudnn,shape,compact", + [ + (True, (128, 128), True), + (True, (2, 64, 128), True), + (True, (64, 128), False), + (False, (128, 128), False), + ], +) +def test_fake_norm_mxfp8_scale_layout( + monkeypatch, normalization, with_2d_quantization, optimize_for_gemm, use_cudnn, shape, compact +): + """RMSNorm's 2D MXFP8 fallback preserves the quantizer's scale layout.""" + monkeypatch.setenv("NVTE_NORM_FWD_USE_CUDNN", str(int(use_cudnn))) + quantizer = MXFP8Quantizer(tex.DType.kFloat8E4M3, with_2d_quantization=with_2d_quantization) + quantizer.optimize_for_gemm = optimize_for_gemm + inp = TensorSpec(shape=shape, dtype=torch.bfloat16, device=torch.device("cpu")) + out, _, _ = fake_norm(inp, quantizer, normalization=normalization) + compact = compact and (normalization == "LayerNorm" or not with_2d_quantization) + expected = optimize_for_gemm and not compact + assert out.create_metadata()["nontensor_kwargs"]["with_gemm_swizzled_scales"] == expected + assert out.create_tensor()._with_gemm_swizzled_scales == expected + assert quantizer.optimize_for_gemm == optimize_for_gemm # --------------------------------------------------------------------------- -# te.Linear +# te.Linear / te.LayerNormLinear / te.LayerNormMLP # --------------------------------------------------------------------------- @@ -1940,17 +2101,48 @@ def test_to_tensor_spec_quantized(factory, shape): [None, *_all_recipes], ids=lambda r: "bf16" if r is None else type(r).__name__, ) -def test_te_linear_compiles(fp8_recipe, compile_mode): - """ - torch.compile(fullgraph=True) of ``te.Linear`` under every built-in - recipe (plus the bf16-only baseline with no autocast), for both the default - backend and ``mode="reduce-overhead"`` (CUDA-graph trees). - """ - dtype = torch.bfloat16 - device = "cuda" +@pytest.mark.parametrize("module", _LINEAR_MODULES) +def test_te_modules_compile(module, fp8_recipe, compile_mode): + """Fullgraph forward/backward agrees with eager across modules, recipes and modes.""" + if module == "LayerNormMLP" and getattr(fp8_recipe, "backward_override", None) is not None: + pytest.skip("LayerNormMLP does not support backward_override, including in eager mode") + model = _make_module(module) + _run_module_compile_test(model, fp8_recipe, compile_mode, 64) + + +_non_2d_cases = [ + pytest.param((64,), None, id="rank1-bf16"), + pytest.param((2, 4, 4, 64), None, id="rank4-bf16"), + *( + pytest.param( + (2, 4, 16, 128) if isinstance(r, recipe.Float8BlockScaling) else (2, 4, 4, 64), + r, + id=f"rank4-{recipe_id(r)}", + ) + for r in _all_recipes + ), + *( + pytest.param((2, 4, 8, 64), r, id=f"rank4-aligned-{recipe_id(r)}") + for r in _all_recipes + if isinstance(r, recipe.NVFP4BlockScaling) and not r.disable_rht + ), +] - # FP8 GEMMs require leading dimensions divisible by 16. - model = te.Linear(64, 32, params_dtype=dtype, device=device) + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("compile_mode", _compile_modes) +@pytest.mark.parametrize("module", _LINEAR_MODULES) +@pytest.mark.parametrize("input_shape, fp8_recipe", _non_2d_cases) +def test_te_modules_compile_non_2d_input(input_shape, fp8_recipe, module, compile_mode): + """Compiled modules preserve non-matrix activation shapes and gradients.""" + if module == "LayerNormMLP" and getattr(fp8_recipe, "backward_override", None) is not None: + pytest.skip("LayerNormMLP does not support backward_override, including in eager mode") + dtype = torch.bfloat16 + if isinstance(fp8_recipe, recipe.Float8BlockScaling): + out_features = 256 if module == "LayerNormMLP" else 128 + model = getattr(te, module)(128, out_features, params_dtype=dtype, device="cuda") + else: + model = _make_module(module, dtype=dtype) def fn(inp): if fp8_recipe is None: @@ -1962,17 +2154,15 @@ def fn(inp): if compile_mode == "reduce-overhead": _cudagraph_warmup( fn, - torch.randn(32, 64, dtype=dtype, device=device, requires_grad=True), + torch.randn(input_shape, dtype=dtype, device="cuda", requires_grad=True), backward=True, ) model.zero_grad(set_to_none=True) compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) - # Iterate a few times so reduce-overhead actually replays a captured graph. - n_iters = 3 if compile_mode == "reduce-overhead" else 1 with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): - for _ in range(n_iters): - base = torch.randn(32, 64, dtype=dtype, device=device) + for _ in range(3 if compile_mode == "reduce-overhead" else 1): + base = torch.randn(input_shape, dtype=dtype, device="cuda") _assert_close_eager_compiled(fn, compiled, model, base) @@ -1987,19 +2177,22 @@ def fn(inp): + [ pytest.param(fp8_recipe, shape, id=f"{name}-{shape}") for name, fp8_recipe in ( + ("bf16", None), + ("mxfp8", recipe.MXFP8BlockScaling()), ("nvfp4", recipe.NVFP4BlockScaling()), ("nvfp4-no-rht", recipe.NVFP4BlockScaling(disable_rht=True)), ) - for shape in ((1, 128, 128), (2, 4, 16, 128)) + for shape in ((1, 128, 128), (128, 1, 128), (2, 4, 16, 128)) ], ) def test_te_linear_compiles_non_2d_input(fp8_recipe, input_shape, compile_mode): """Compiled Linear preserves non-matrix activation shapes in forward and backward.""" if isinstance(fp8_recipe, recipe.NVFP4BlockScaling) and not nvfp4_available: pytest.skip(reason_for_no_nvfp4) + if isinstance(fp8_recipe, recipe.MXFP8BlockScaling) and not mxfp8_available: + pytest.skip(reason_for_no_mxfp8) dtype = torch.bfloat16 - out_features = 32 if fp8_recipe is None else 128 - model = te.Linear(input_shape[-1], out_features, params_dtype=dtype, device="cuda") + model = te.Linear(input_shape[-1], 128, params_dtype=dtype, device="cuda") def fn(inp): if fp8_recipe is None: @@ -2026,37 +2219,14 @@ def fn(inp): @pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") @pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) @pytest.mark.parametrize("compile_mode", _compile_modes) -def test_te_linear_compile_with_quantized_fp8_weight(compile_mode): - """torch.compile of Linear with the weight initialized as an FP8 tensor - (exercises the wrapper op's ``register_torch_dispatch`` input flattening).""" - dtype = torch.bfloat16 - device = "cuda" +@pytest.mark.parametrize("module", ["Linear", "LayerNormLinear"]) +def test_te_modules_compile_with_quantized_fp8_weight(module, compile_mode): + """FP8 primary weights cross the wrapper-op boundary in both compile modes.""" fp8_recipe = recipe.Float8CurrentScaling() - with te.quantized_model_init(enabled=True, recipe=fp8_recipe): - model = te.Linear(64, 32, params_dtype=dtype, device=device) - + model = _make_module(module) assert isinstance(model.weight, te.Float8Tensor) - - def fn(inp): - with te.autocast(recipe=fp8_recipe): - return model(inp) - - torch._dynamo.reset() - if compile_mode == "reduce-overhead": - _cudagraph_warmup( - fn, - torch.randn(32, 64, dtype=dtype, device=device, requires_grad=True), - backward=True, - ) - model.zero_grad(set_to_none=True) - compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) - - n_iters = 3 if compile_mode == "reduce-overhead" else 1 - with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): - for _ in range(n_iters): - base = torch.randn(32, 64, dtype=dtype, device=device) - _assert_close_eager_compiled(fn, compiled, model, base) + _run_module_compile_test(model, fp8_recipe, compile_mode, 64) @pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") @@ -2105,7 +2275,7 @@ def fn(inp): ) -# Configs rejected by LinearFwdArgs.compile_unsupported_reason() that a +# Configs rejected by the modules' early fallback checks that a # single-GPU unit test can construct. Distributed-only reasons (fsdp_group, # DistributedWeight) and CPU offloading need machinery this file doesn't have; # delayed scaling is a hard error (check_recipe_support), tested separately. @@ -2114,58 +2284,67 @@ def fn(inp): # output crossing the graph-break boundary gets a plain-tensor tangent); # "no_grad" = forward under no_grad. _FALLBACK_CASES = [ - "fp8_output_differentiable", - "fuse_wgrad_accumulation", - "delayed_wgrad", - "quantized_input", + (module, case) + for module in _LINEAR_MODULES + for case in ("fuse_wgrad_accumulation", "delayed_wgrad") +] + [ + ("Linear", "fp8_output_differentiable"), + ("Linear", "quantized_input"), + ("LayerNormMLP", "checkpoint"), ] -def _fallback_case(case, dtype, device): - """Build ``(model, fn, mode, post_backward, reason)`` for one case.""" +def _fallback_case(module, case, dtype, device): + """Build ``(model, fn, mode, post_backward, reason)`` for one fallback case.""" model_kwargs = {} if case == "fuse_wgrad_accumulation": model_kwargs["fuse_wgrad_accumulation"] = True elif case == "delayed_wgrad": model_kwargs["delay_wgrad_compute"] = True - model = te.Linear(64, 32, params_dtype=dtype, device=device, **model_kwargs) - - if case == "fp8_output_differentiable": - fp8_recipe = recipe.Float8CurrentScaling() + elif case == "checkpoint": + model_kwargs["checkpoint"] = True + model = _make_module(module, dtype=dtype, device=device, **model_kwargs) + fp8_recipe = recipe.Float8CurrentScaling() - def fn(inp): - with te.autocast(recipe=fp8_recipe): + def fn(inp): + # Preserve the original Linear BF16 accumulation/delayed-wgrad cases; + # the LayerNorm cases exercise those paths under FP8. + use_fp8 = module != "Linear" or case in ("fp8_output_differentiable", "quantized_input") + with te.autocast(enabled=use_fp8, recipe=fp8_recipe): + if case == "fp8_output_differentiable": return model(inp, fp8_output=True).dequantize() + return model(inp) + if case == "fp8_output_differentiable": return model, fn, "fwd_grad", None, "differentiable fp8_output=True" if case == "fuse_wgrad_accumulation": - model.weight.main_grad = torch.zeros_like(model.weight, dtype=torch.float32) - return model, model, "bwd", None, "fuse_wgrad_accumulation" + weights = ( + (model.fc1_weight, model.fc2_weight) if module == "LayerNormMLP" else (model.weight,) + ) + for weight in weights: + weight.main_grad = torch.zeros_like(weight, dtype=torch.float32) + return model, fn, "bwd", None, "fuse_wgrad_accumulation" if case == "delayed_wgrad": - return model, model, "bwd", model.backward_dw, "delayed wgrad compute" + return model, fn, "bwd", model.backward_dw, "delayed wgrad compute" if case == "quantized_input": - fp8_recipe = recipe.Float8CurrentScaling() - - def fn(inp): - with te.autocast(recipe=fp8_recipe): - return model(inp) - return model, fn, "no_grad", None, "a quantized input tensor" + if case == "checkpoint": + return model, fn, "bwd", None, "activation checkpointing" raise ValueError(case) @pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") @pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) -@pytest.mark.parametrize("case", _FALLBACK_CASES) -def test_te_linear_compile_eager_fallback(case): +@pytest.mark.parametrize("module,case", _FALLBACK_CASES) +def test_te_modules_compile_eager_fallback(module, case): """Configs unsupported on the compiled custom-op path must fall back to eager under ``torch.compile`` -- warning + numerics identical to eager -- and graph-break with the explicit reason under ``fullgraph=True``.""" dtype, device = torch.bfloat16, "cuda" torch.manual_seed(0) - model_ref, fn_ref, mode, post_bwd_ref, _ = _fallback_case(case, dtype, device) + model_ref, fn_ref, mode, post_bwd_ref, _ = _fallback_case(module, case, dtype, device) torch.manual_seed(0) - model, fn, _, post_bwd, reason = _fallback_case(case, dtype, device) + model, fn, _, post_bwd, reason = _fallback_case(module, case, dtype, device) def make_inp(): torch.manual_seed(1) @@ -2194,20 +2373,27 @@ def make_inp(): if post_bwd is not None: post_bwd() torch.testing.assert_close(inp.grad, inp_ref.grad, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) - # The fallback must also preserve the stateful side effects that define - # these cases (main_grad accumulation, delayed wgrad), not just inp.grad. - if getattr(model_ref.weight, "main_grad", None) is not None: - assert model.weight.grad is None - torch.testing.assert_close( - model.weight.main_grad, - model_ref.weight.main_grad, - atol=_EAGER_ATOL, - rtol=_EAGER_RTOL, - ) - else: - torch.testing.assert_close( - model.weight.grad, model_ref.weight.grad, atol=_EAGER_ATOL, rtol=_EAGER_RTOL - ) + # Check both gradients and stateful accumulation for every parameter. + ref_params = dict(model_ref.named_parameters()) + for name, param in model.named_parameters(): + ref_param = ref_params[name] + if getattr(ref_param, "main_grad", None) is not None: + assert param.grad is None + torch.testing.assert_close( + param.main_grad, + ref_param.main_grad, + atol=_EAGER_ATOL, + rtol=_EAGER_RTOL, + msg=f"main_grad mismatch: {name}", + ) + else: + torch.testing.assert_close( + param.grad, + ref_param.grad, + atol=_EAGER_ATOL, + rtol=_EAGER_RTOL, + msg=f"grad mismatch: {name}", + ) torch.testing.assert_close(out.detach(), out_ref.detach(), atol=_EAGER_ATOL, rtol=_EAGER_RTOL) torch._dynamo.reset() @@ -2264,14 +2450,17 @@ def fn(inp): ], ids=recipe_id, ) -def test_te_linear_compile_is_first_microbatch( - fp8_recipe, compile_mode, deferred_backward, monkeypatch +@pytest.mark.parametrize("module", _LINEAR_MODULES) +def test_te_modules_compile_is_first_microbatch( + module, fp8_recipe, compile_mode, deferred_backward, monkeypatch ): """Reuse cached weights across microbatches and refresh after optimizer updates.""" dtype, device = torch.bfloat16, "cuda" - model = te.Linear(128, 128, params_dtype=dtype, device=device) - ref_model = te.Linear(128, 128, params_dtype=dtype, device=device) + model = getattr(te, module)(128, 128, params_dtype=dtype, device=device) + ref_model = getattr(te, module)(128, 128, params_dtype=dtype, device=device) ref_model.load_state_dict(model.state_dict()) + cache_names = ("fc1_weight", "fc2_weight") if module == "LayerNormMLP" else ("weight",) + cached_workspaces = {} def fn(inp, is_first): with te.autocast(recipe=fp8_recipe): @@ -2315,17 +2504,20 @@ def replay(graph): out = compiled(inp, is_first).clone() torch.testing.assert_close(out, out_ref, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) - workspace = model._fp8_workspaces["weight"] - if is_first: - cached_workspace = workspace - else: - assert workspace is cached_workspace - torch.testing.assert_close( - workspace.dequantize(), - ref_model._fp8_workspaces["weight"].dequantize(), - atol=_EAGER_ATOL, - rtol=_EAGER_RTOL, - ) + for name in cache_names: + workspace = model._fp8_workspaces[name] + if is_first: + if name in cached_workspaces: + assert workspace is not cached_workspaces[name] + cached_workspaces[name] = workspace + else: + assert workspace is cached_workspaces[name] + torch.testing.assert_close( + workspace.dequantize(), + ref_model._fp8_workspaces[name].dequantize(), + atol=_EAGER_ATOL, + rtol=_EAGER_RTOL, + ) pending.append((out, out_ref, inp, inp_ref)) if not deferred_backward or microbatch == 2: @@ -2346,7 +2538,7 @@ def replay(graph): for param, ref_param in zip(model.parameters(), ref_model.parameters()): param.add_(param.grad, alpha=-0.001) ref_param.add_(ref_param.grad, alpha=-0.001) - del cached_workspace, workspace, out, out_ref, result, ref_result + del workspace, out, out_ref, result, ref_result torch._dynamo.reset() @@ -2403,74 +2595,363 @@ def train_step(): @pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") -def test_te_linear_dynamic_shapes(): - """torch.compile of ``te.Linear`` with a ``mark_dynamic`` batch dimension: - one graph must serve all batch sizes -- no recompiles -- and match eager - numerically. - - Only the leading (batch/sequence) dims may be dynamic; the last dim is - fixed by the weight's ``in_features``. - """ - dtype = torch.bfloat16 - device = "cuda" - in_features, out_features = 64, 32 - model = te.Linear(in_features, out_features, params_dtype=dtype, device=device) +@pytest.mark.parametrize( + "module,bias_gelu_fusion", + [ + ("Linear", False), + ("LayerNormLinear", False), + ("LayerNormMLP", False), + ("LayerNormMLP", True), + ], +) +def test_te_modules_dynamic_shapes(module, bias_gelu_fusion): + """Batch sizes share the warmed-up outer graph and match eager outputs/gradients.""" + dtype, device = torch.bfloat16, "cuda" + model = _make_module(module, dtype=dtype, device=device) + if module == "LayerNormMLP": + model.bias_gelu_nvfusion = bias_gelu_fusion def fn(inp): return model(inp) torch._dynamo.reset() - compiled = torch.compile(fn, fullgraph=True) - - batch_sizes = [16, 32, 48] - - # Two warmup calls: the second absorbs the one-time recompile from module - # attributes lazily created during call one (e.g. the cached ``is_fsdp2``). + # Count the outer graph independently of lazy compilation of fused helpers. + compile_counter = CompileCounterWithBackend("inductor") + compiled = torch.compile(fn, fullgraph=True, backend=compile_counter) for _ in range(2): - warm = torch.randn(batch_sizes[0], in_features, dtype=dtype, device=device) + warm = torch.randn(16, 64, dtype=dtype, device=device) torch._dynamo.mark_dynamic(warm, 0) compiled(warm.requires_grad_(True)).sum().backward() model.zero_grad(set_to_none=True) - unique_graphs_baseline = _dynamo_counter("stats", "unique_graphs") - if not unique_graphs_baseline: - warnings.warn("unique_graphs counter is stale; skipping the recompile check") + frames_after_warmup = compile_counter.frame_count + assert frames_after_warmup > 0 - for batch in batch_sizes: - inp = torch.randn(batch, in_features, dtype=dtype, device=device, requires_grad=True) - # Mark batch dim as dynamic so Dynamo traces once and reuses across batch sizes. - torch._dynamo.mark_dynamic(inp, 0) - out = compiled(inp) - assert out.shape == (batch, out_features), f"wrong output shape for batch={batch}" - out.sum().backward() - assert inp.grad is not None, f"no input gradient for batch={batch}" - assert inp.grad.shape == inp.shape, f"wrong grad shape for batch={batch}" + for batch in (16, 32, 48): + base = torch.randn(batch, 64, dtype=dtype, device=device) + _assert_close_eager_compiled(fn, compiled, model, base, dynamic_batch=True) - # Verify numerics against eager on each distinct batch size. - inp_eager = inp.detach().clone().requires_grad_(True) - model.zero_grad(set_to_none=True) - out_eager = model(inp_eager) - out_eager.sum().backward() - torch.testing.assert_close( - out.detach(), - out_eager.detach(), - atol=_EAGER_ATOL, - rtol=_EAGER_RTOL, - msg=f"forward mismatch at batch={batch}", - ) - torch.testing.assert_close( - inp.grad, - inp_eager.grad, - atol=_EAGER_ATOL, - rtol=_EAGER_RTOL, - msg=f"dgrad mismatch at batch={batch}", + assert compile_counter.frame_count == frames_after_warmup + + +# --------------------------------------------------------------------------- +# Module fake/real metadata and LayerNorm variants +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("module", ["LayerNormLinear", "LayerNormMLP"]) +@pytest.mark.parametrize("loss_source", ["output", "normalization", "both"]) +@pytest.mark.parametrize("fp8_recipe", [None, recipe.Float8CurrentScaling()]) +def test_te_layernorm_output_gradients(module, loss_source, fp8_recipe): + """Route distinct gradients from each public output to the correct backward field.""" + if fp8_recipe is not None and not fp8_available: + pytest.skip(reason_for_no_fp8) + model = _make_module(module, return_layernorm_output=True) + + def fn(inp): + if fp8_recipe is None: + output, norm_output = model(inp) + else: + with te.autocast(recipe=fp8_recipe): + output, norm_output = model(inp) + if loss_source == "output": + return output + if loss_source == "normalization": + return norm_output + return output, 0.25 * norm_output + + torch._dynamo.reset() + compiled = torch.compile(fn, fullgraph=True) + base = torch.randn(32, 64, dtype=torch.bfloat16, device="cuda") + _assert_close_eager_compiled(fn, compiled, model, base) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize( + "case", ["linear", "layernorm_linear", "mlp_bias_gelu", "mlp_gemm_gelu", "mlp_swiglu"] +) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("bias", [False, True]) +@pytest.mark.parametrize("fp8_recipe", [None, recipe.Float8CurrentScaling()]) +def test_te_modules_fake_matches_real(monkeypatch, case, dtype, bias, fp8_recipe): + """Compare fake outputs, saved buffers and gradients against the eager kernels.""" + from transformer_engine.pytorch.dynamo.custom_op import _parse_arg_type, _spec_view + + if fp8_recipe is not None and not fp8_available: + pytest.skip(reason_for_no_fp8) + if fp8_recipe is not None and case == "mlp_gemm_gelu": + if dtype == torch.float32 or (dtype == torch.float16 and not bias): + pytest.skip( + "cuBLAS FP8 GELU requires matching output and auxiliary dtypes (BF16 without bias)" + ) + module_name = case if case in ("linear", "layernorm_linear") else "layernorm_mlp" + module = { + "linear": linear_module, + "layernorm_linear": layernorm_linear_module, + "layernorm_mlp": layernorm_mlp_module, + }[module_name] + checked = [] + + def assert_metadata(spec, real): + if isinstance(spec, TensorSpec): + assert real is not None + assert tuple(spec.shape) == tuple(real.shape) + assert spec.dtype == to_tensor_spec(real).dtype + assert spec.device == real.device + names = ( + tuple(real.__tensor_flatten__()[0]) + if hasattr(real, "__tensor_flatten__") + else ("data",) + ) + assert spec.inner_names() == names + for name, fake_inner in zip(names, spec.create_inner_tensors()): + real_inner = real if name == "data" else getattr(real, name) + assert fake_inner.shape == real_inner.shape + assert fake_inner.dtype == real_inner.dtype + assert fake_inner.stride() == real_inner.stride() + elif isinstance(spec, tuple): + assert isinstance(real, tuple) and len(spec) == len(real) + for fake_value, real_value in zip(spec, real): + assert_metadata(fake_value, real_value) + elif isinstance(spec, dict): + # No manual FSDP or checkpointing here; [] and None both denote no shards. + for key in spec.keys() | real.keys(): + if key in ("fsdp_shapes", "checkpoint"): + assert not spec.get(key) and not real.get(key) + else: + assert spec[key] == real[key] + else: + assert spec == real + + def checked_impl(direction): + real_impl = getattr(module, f"_{module_name}_{direction}_impl") + fake_impl = getattr(module, f"_{module_name}_{direction}_fake") + + def wrapper(args): + spec_args = _spec_view(args, _parse_arg_type(type(args)).tensor_field_names()) + # Fake execution may change quantizer usage; leave the real invocation untouched. + spec_args = dataclasses.replace( + spec_args, + **{ + field.name: value.copy() + for field in dataclasses.fields(spec_args) + if isinstance(value := getattr(spec_args, field.name), Quantizer) + }, + ) + expected = fake_impl(spec_args) + actual = real_impl(args) + assert_metadata(expected, actual) + checked.append(direction) + return actual + + return wrapper + + for direction in ("forward", "backward"): + monkeypatch.setattr(module, f"_{module_name}_{direction}_impl", checked_impl(direction)) + kwargs = dict(params_dtype=dtype, device="cuda", bias=bias) + if case != "linear": + kwargs["return_layernorm_output"] = True + if case == "linear": + model = te.Linear(64, 32, **kwargs) + elif case == "layernorm_linear": + model = te.LayerNormLinear(64, 32, **kwargs) + else: + model = te.LayerNormMLP( + 64, 128, activation="swiglu" if case == "mlp_swiglu" else "gelu", **kwargs ) + model.bias_gelu_nvfusion = case == "mlp_bias_gelu" + model.gemm_gelu_fusion = case == "mlp_gemm_gelu" + inp = torch.randn(2, 16, 64, dtype=dtype, device="cuda", requires_grad=True) + with te.autocast(enabled=fp8_recipe is not None, recipe=fp8_recipe): + outs = _flatten_outputs(model(inp)) + sum(out.sum() for out in outs).backward() + assert checked == ["forward", "backward"] - if unique_graphs_baseline: - unique_graphs_after = _dynamo_counter("stats", "unique_graphs") - assert unique_graphs_after == unique_graphs_baseline, ( - "Unexpected recompilation(s) across different batch sizes: " - f"{unique_graphs_after - unique_graphs_baseline} extra graph(s) compiled" + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +@pytest.mark.parametrize( + "dtype,bias", [(torch.bfloat16, False), (torch.bfloat16, True), (torch.float16, True)] +) +@pytest.mark.parametrize("compile_mode", _compile_modes) +def test_te_layernorm_mlp_compile_fp8_gemm_gelu(dtype, bias, compile_mode): + """Fused FP8 GEMM-GELU must preserve the auxiliary tensor dtype through backward.""" + model = te.LayerNormMLP(64, 128, params_dtype=dtype, device="cuda", bias=bias) + model.gemm_gelu_fusion = True + _run_module_compile_test(model, recipe.Float8CurrentScaling(), compile_mode, 64, dtype=dtype) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize( + "quantization,disable_rht_fusion", + [("mxfp8", False), ("mxfp8_2d", False), ("nvfp4", False), ("nvfp4", True)], +) +@pytest.mark.parametrize("module", ["LayerNormLinear", "LayerNormMLP"]) +@pytest.mark.parametrize("normalization", ["LayerNorm", "RMSNorm"]) +@pytest.mark.parametrize( + "use_cudnn,input_shape,return_layernorm_output", + [ + (False, (128, 128), False), + (True, (128, 128), False), + (True, (64, 128), False), + (True, (2, 64, 128), False), + (True, (128, 128), True), + ], +) +def test_te_layernorm_compile_scale_layout( + monkeypatch, + module, + normalization, + use_cudnn, + input_shape, + return_layernorm_output, + quantization, + disable_rht_fusion, +): + """Saved normalization scales retain their layout through compiled backward.""" + if quantization.startswith("mxfp8") and not mxfp8_available: + pytest.skip(reason_for_no_mxfp8) + if quantization == "nvfp4" and not nvfp4_available: + pytest.skip(reason_for_no_nvfp4) + monkeypatch.setenv("NVTE_NORM_FWD_USE_CUDNN", str(int(use_cudnn))) + monkeypatch.setenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", str(int(disable_rht_fusion))) + model = getattr(te, module)( + 128, + 128, + params_dtype=torch.bfloat16, + device="cuda", + normalization=normalization, + return_layernorm_output=return_layernorm_output, + ) + if quantization == "mxfp8_2d": + + def qfactory(role): + return MXFP8Quantizer( + tex.DType.kFloat8E4M3, + with_2d_quantization=role is not None and role.tensor_type == "input", + ) + + fp8_recipe = recipe.CustomRecipe(qfactory=qfactory) + elif quantization == "mxfp8": + fp8_recipe = recipe.MXFP8BlockScaling() + else: + fp8_recipe = recipe.NVFP4BlockScaling(disable_stochastic_rounding=True) + + def fn(inp): + with te.autocast(recipe=fp8_recipe): + return model(inp) + + torch._dynamo.reset() + compiled = torch.compile(fn, fullgraph=True) + base = torch.randn(input_shape, dtype=torch.bfloat16, device="cuda") + _assert_close_eager_compiled(fn, compiled, model, base) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("sequence_parallel", [False, True]) +@pytest.mark.parametrize("gathered", [False, True]) +@pytest.mark.parametrize("input_shape", [(128, 128), (2, 64, 128)]) +def test_te_layernorm_mlp_fake_gathered_output( + monkeypatch, sequence_parallel, gathered, input_shape +): + """The normalization gather is independent of set_parallel_mode.""" + from transformer_engine.pytorch.dynamo.custom_op import _parse_arg_type, _spec_view + + module = layernorm_mlp_module + real_impl = module._layernorm_mlp_forward_impl + checked = [] + + def checked_impl(args): + args = dataclasses.replace( + args, + sequence_parallel=sequence_parallel, + tp_size=2, + return_layernorm_output_gathered=gathered, ) + spec_args = _spec_view(args, _parse_arg_type(type(args)).tensor_field_names()) + expected = module._layernorm_mlp_forward_fake(spec_args) + actual = real_impl(args) + for spec, tensor in zip(expected[:2], actual[:2]): + assert tuple(spec.shape) == tuple(tensor.shape) + checked.append(True) + return actual + + # Emulate a two-rank gather while exercising the real normalization and GEMMs. + monkeypatch.setattr(module, "get_distributed_world_size", lambda group: 2) + monkeypatch.setattr( + module, + "gather_along_first_dim", + lambda tensor, group, quantizer=None: (torch.cat([tensor, tensor]), None), + ) + monkeypatch.setattr(module, "_layernorm_mlp_forward_impl", checked_impl) + model = te.LayerNormMLP( + 128, + 128, + params_dtype=torch.bfloat16, + device="cuda", + set_parallel_mode=False, + return_layernorm_output=True, + ) + with torch.no_grad(): + model(torch.randn(input_shape, dtype=torch.bfloat16, device="cuda")) + assert checked == [True] + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("normalization", ["LayerNorm", "RMSNorm"]) +@pytest.mark.parametrize("return_layernorm_output", [False, True]) +@pytest.mark.parametrize("zero_centered_gamma", [False, True]) +def test_te_layernorm_linear_compile_variants( + normalization, return_layernorm_output, zero_centered_gamma +): + """Norm-type / returned-norm-output / zero-centered-gamma variants of + ``te.LayerNormLinear`` under torch.compile, bf16 and FP8 current scaling.""" + model = te.LayerNormLinear( + 64, + 32, + params_dtype=torch.bfloat16, + device="cuda", + normalization=normalization, + return_layernorm_output=return_layernorm_output, + zero_centered_gamma=zero_centered_gamma, + ) + _run_module_compile_test(model, None, "default", 64) + if fp8_available: + _run_module_compile_test(model, recipe.Float8CurrentScaling(), "default", 64) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("activation", ["gelu", "swiglu", "relu", "qgeglu"]) +@pytest.mark.parametrize("normalization", ["LayerNorm", "RMSNorm"]) +@pytest.mark.parametrize("return_layernorm_output", [False, True]) +def test_te_layernorm_mlp_compile_variants(activation, normalization, return_layernorm_output): + """Activation / norm-type / returned-norm-output variants of + ``te.LayerNormMLP`` under torch.compile, bf16 and FP8 current scaling.""" + model = te.LayerNormMLP( + 64, + 128, + params_dtype=torch.bfloat16, + device="cuda", + activation=activation, + normalization=normalization, + return_layernorm_output=return_layernorm_output, + ) + _run_module_compile_test(model, None, "default", 64) + if fp8_available: + _run_module_compile_test(model, recipe.Float8CurrentScaling(), "default", 64) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("bias", [True, False]) +def test_te_layernorm_mlp_compile_no_bias_or_frozen(bias): + """``te.LayerNormMLP`` without biases, and with frozen FC1 weight (exercises + the ``None`` saved-tensor / grad slots).""" + model = te.LayerNormMLP(64, 128, params_dtype=torch.bfloat16, device="cuda", bias=bias) + _run_module_compile_test(model, None, "default", 64) + model.fc1_weight.requires_grad_(False) + _run_module_compile_test(model, None, "default", 64) + if fp8_available: + _run_module_compile_test(model, recipe.Float8CurrentScaling(), "default", 64) # --------------------------------------------------------------------------- # @@ -2912,21 +3393,20 @@ def test_te_ops_bias_compile_with_fp8_input(backward): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize( - "dtype,quantization,shape,disable_rht_fusion,case,grads", + "dtype,quantization,shape,disable_rht_fusion", [ - (dtype, quantization, (2, 16, 32), False, case, grads) - for dtype in (torch.float16, torch.bfloat16) - for quantization in (None, "fp8") - for case in ("linear", "bias", "pair", "unfused") - for grads in ("all", "input", "weight", "bias", "none") - ] - + [ - (torch.bfloat16, "nvfp4", (2, 16, 128), False, "pair", "all"), - (torch.bfloat16, "nvfp4", (64, 128), False, "pair", "all"), - (torch.bfloat16, "nvfp4", (64, 128), True, "pair", "all"), - (torch.bfloat16, "nvfp4-no-rht", (2, 16, 128), False, "pair", "all"), + (torch.float16, None, (2, 16, 32), False), + (torch.bfloat16, None, (2, 16, 32), False), + (torch.float16, "fp8", (2, 16, 32), False), + (torch.bfloat16, "fp8", (2, 16, 32), False), + (torch.bfloat16, "nvfp4", (2, 16, 128), False), + (torch.bfloat16, "nvfp4", (64, 128), False), + (torch.bfloat16, "nvfp4", (64, 128), True), + (torch.bfloat16, "nvfp4-no-rht", (2, 16, 128), False), ], ) +@pytest.mark.parametrize("case", ["linear", "bias", "pair", "unfused"]) +@pytest.mark.parametrize("grads", ["all", "input", "weight", "bias", "none"]) def test_te_ops_linear_bias_compile( dtype, quantization, shape, disable_rht_fusion, case, grads, monkeypatch ): @@ -3058,13 +3538,29 @@ def unexpected_allocation(*args, **kwargs): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize("training", [False, True]) +@pytest.mark.parametrize("initial_device", ["cuda", "cpu", "meta"]) @pytest.mark.parametrize( - "initial_device,quantization", - [("cuda", None), ("cpu", None), ("meta", None), ("cuda", "nvfp4"), ("cuda", "nvfp4-no-rht")], + "quantization", + [ + None, + "fp8", + "fp8-block", + "mxfp8", + "nvfp4", + "nvfp4-no-rht", + "nvfp4-4over6", + "nvfp4-row-scaled", + ], ) def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device, quantization): """A fresh process prevents earlier GEMMs from hiding lazy workspace allocation.""" - if quantization is not None and not nvfp4_available: + if quantization == "fp8" and not fp8_available: + pytest.skip(reason_for_no_fp8) + if quantization == "fp8-block" and not fp8_block_scaling_available: + pytest.skip(reason_for_no_fp8_block_scaling) + if quantization == "mxfp8" and not mxfp8_available: + pytest.skip(reason_for_no_mxfp8) + if quantization is not None and quantization.startswith("nvfp4") and not nvfp4_available: pytest.skip(reason_for_no_nvfp4) script = textwrap.dedent( """ @@ -3079,17 +3575,27 @@ def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device, quantizati training, initial_device, quantization = sys.argv[1:] training = training == "True" - is_nvfp4 = quantization != "None" - quant_recipe = ( - recipe.NVFP4BlockScaling( - disable_rht=quantization == "nvfp4-no-rht", + is_nvfp4 = quantization.startswith("nvfp4") + if quantization == "None": + quant_recipe = None + elif is_nvfp4: + row_scaled = quantization == "nvfp4-row-scaled" + quant_recipe = recipe.NVFP4BlockScaling( + disable_rht=quantization != "nvfp4", disable_stochastic_rounding=True, + disable_2d_quantization=row_scaled, + row_scaled_activation=row_scaled, + nvfp4_4over6="all" if quantization == "nvfp4-4over6" else "none", + backward_override="dequantized" if row_scaled else None, ) - if is_nvfp4 - else None - ) - in_features, out_features = (128, 128) if is_nvfp4 else (32, 64) - batch_size = 32 + else: + quant_recipe = { + "fp8": recipe.Float8CurrentScaling, + "fp8-block": recipe.Float8BlockScaling, + "mxfp8": recipe.MXFP8BlockScaling, + }[quantization]() + in_features, out_features = (128, 128) if quant_recipe is not None else (32, 64) + batch_size = 128 if isinstance(quant_recipe, recipe.Float8BlockScaling) else 32 eager_tols = {"rtol": 0, "atol": 0} if is_nvfp4 else {} assert get_cublas_workspace.cache_info().currsize == 0 linear = te.ops.BasicLinear( diff --git a/transformer_engine/pytorch/attention/dot_product_attention/backends.py b/transformer_engine/pytorch/attention/dot_product_attention/backends.py index 2345a1808a6..4ceade26060 100644 --- a/transformer_engine/pytorch/attention/dot_product_attention/backends.py +++ b/transformer_engine/pytorch/attention/dot_product_attention/backends.py @@ -11,7 +11,7 @@ import functools import inspect import os -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union import warnings import logging from packaging.version import Version as PkgVersion @@ -1662,6 +1662,10 @@ class FusedAttnBwdArgs: dO_quantizer: Optional[Any] = None dP_quantizer: Optional[Any] = None + def setup_grad_outputs(self, grads: Sequence[Any]) -> None: + """Assign the output gradient, ignoring auxiliary outputs.""" + self.grad_output = grads[0] + def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: """Pull saved tensors from ``ctx`` into the fields backward consumes.""" ( diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index 88dfb808420..324231fc5f2 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -340,6 +340,9 @@ py::tuple scaled_dsrelu(const at::Tensor &grad, const at::Tensor &input, * LayerNorm **************************************************************************************************/ +std::optional mxfp8_norm_fused_alignment(bool rmsnorm, bool with_2d_quantization, + bool use_cudnn); + std::vector layernorm_bwd(const at::Tensor &dz, const at::Tensor &x, const at::Tensor &mu, const at::Tensor &rsigma, const at::Tensor &gamma, const int sm_margin, diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index e93c01026a9..cc990576081 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -10,6 +10,14 @@ namespace transformer_engine::pytorch { +std::optional mxfp8_norm_fused_alignment(bool rmsnorm, bool with_2d_quantization, + bool use_cudnn) { + if (rmsnorm ? with_2d_quantization : !use_cudnn) { + return std::nullopt; + } + return 128; +} + std::vector layernorm_bwd(const at::Tensor &dz, const at::Tensor &x, const at::Tensor &mu, const at::Tensor &rsigma, const at::Tensor &gamma, const int sm_margin, @@ -106,9 +114,11 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe if (quantizer.is_none() || IsFloat8Quantizers(quantizer.ptr())) { impl = Impl::FULLY_FUSED; } else if (IsMXFP8Quantizers(quantizer.ptr())) { - if (transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN") && outer_size % 128 == 0 && - inner_size % 128 == 0) { - // cuDNN MXFP8 kernel requires full 128x128 tiles + const auto mxfp8_quantizer_cpp = static_cast(quantizer_cpp.get()); + const auto alignment = + mxfp8_norm_fused_alignment(false, mxfp8_quantizer_cpp->with_2d_quantization, + transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")); + if (alignment && outer_size % *alignment == 0 && inner_size % *alignment == 0) { impl = Impl::FULLY_FUSED; } } else if (detail::IsFloat8CurrentScalingQuantizers(quantizer.ptr()) && @@ -350,9 +360,10 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w } else if (IsMXFP8Quantizers(quantizer.ptr())) { auto mxfp8_quantizer_cpp = dynamic_cast(quantizer_cpp.get()); NVTE_CHECK(mxfp8_quantizer_cpp != nullptr, "Could not cast to MXFP8 quantizer"); - // The fused MXFP8 kernels quantize 1D blocks and require full 128x128 tiles - if (!mxfp8_quantizer_cpp->with_2d_quantization && outer_size % 128 == 0 && - inner_size % 128 == 0) { + const auto alignment = + mxfp8_norm_fused_alignment(true, mxfp8_quantizer_cpp->with_2d_quantization, + transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")); + if (alignment && outer_size % *alignment == 0 && inner_size % *alignment == 0) { if (transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")) { impl = Impl::FUSED_NORM_QUANT_UNSWIZZLED; } else { diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index ebcdd17b07c..a5a5a025c75 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -402,6 +402,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { py::call_guard()); // Other granular functions + m.def("mxfp8_norm_fused_alignment", &transformer_engine::pytorch::mxfp8_norm_fused_alignment, + py::arg("rmsnorm"), py::arg("with_2d_quantization"), py::arg("use_cudnn")); m.def("layernorm_fwd", &transformer_engine::pytorch::layernorm_fwd, "LayerNorm", py::arg("input"), py::arg("weight"), py::arg("bias"), py::arg("eps"), py::arg("ln_out"), py::arg("quantizer"), py::arg("otype"), py::arg("sm_margin"), py::arg("zero_centered_gamma")); diff --git a/transformer_engine/pytorch/dynamo/custom_op.py b/transformer_engine/pytorch/dynamo/custom_op.py index 1ed84017ecd..5a784abf332 100644 --- a/transformer_engine/pytorch/dynamo/custom_op.py +++ b/transformer_engine/pytorch/dynamo/custom_op.py @@ -18,7 +18,7 @@ keeps its saved-tensor and context-metadata contract. A ``torch.library`` custom op is narrower: it only accepts flat schema slots -(tensors plus opaque objects) and returns a flat ``Tensor[]``. +(tensors, symbolic shapes and opaque objects) and returns a flat ``Tensor[]``. Bridging the two takes three parts (below): a parsed per-op *arg plan* maps the args dataclass onto the op's input slots; a per-trace *output plan*, parsed from @@ -37,7 +37,8 @@ * ``TENSOR_OR_QUANTIZED`` -- a field that may be a plain tensor, a bare quantized storage, or ``None``: three slots (the tensor, its flat inner buffers, and a ``__kind__`` tag) so a quantized tensor crosses as its buffers. - * ``SIMPLE`` -- every remaining simple value (scalars, enums, sizes, + * ``SHAPE`` -- a ``torch.Size`` / ``Optional[torch.Size]``: one ``SymInt[]`` slot. + * ``SIMPLE`` -- every remaining simple value (scalars, enums, quantizers -- value-opaque constants baked into the graph -- and nested collections of them), gathered into one shared ``OpaqueValueBundle`` slot. * ``PROCESS_GROUP`` -- rides in the shared bundle too, as its c10d registry @@ -66,8 +67,8 @@ forward state + ``ctx_attrs`` (e.g. saved-tensor aliases) and return the tensors to persist; the plan's output ranges are stashed on ``ctx``; * on ``backward()`` the incoming flat grads are sliced per user output from the - stashed plan (a ``grad_outputs`` field on the backward args receives the - whole tuple; otherwise ``grad_output`` receives the first output's grad), + stashed plan and passed in forward-output order to the backward + container's ``setup_grad_outputs`` method, the container's optional ``setup_saved_tensors`` hook restores the saved tensors, then the *backward op* runs the real ``bwd_impl`` and returns the flat grads (``bwd_fake_impl`` is its data-free fake). @@ -397,6 +398,7 @@ class _FieldKind(Enum): TENSOR = "tensor" # one ``Tensor`` / ``Tensor?`` slot TENSOR_OR_QUANTIZED = "tensor_or_quantized" # 3 slots: tensor / inner / meta + SHAPE = "shape" # one ``SymInt[]`` / ``SymInt[]?`` slot PROCESS_GROUP = "process_group" # c10d group name inside the shared bundle SIMPLE = "simple" # value carried verbatim inside the shared bundle UNSUPPORTED = "unsupported" # no slots; only a trivial value may cross @@ -475,6 +477,9 @@ def _is_simple_annot(annot: Any) -> bool: if get_origin(annot) in (tuple, list): inner = [a for a in get_args(annot) if a is not Ellipsis] return bool(inner) and all(_is_simple_annot(a) for a in inner) + if get_origin(annot) is dict: + args = get_args(annot) + return len(args) == 2 and args[0] is str and (args[1] is Any or _is_simple_annot(args[1])) return False @@ -491,6 +496,9 @@ def _parse_field(name: str, annot: Any) -> _FieldPlan: if stripped is torch.Tensor: slot = _SlotSpec(name, "Tensor?" if is_optional else "Tensor") return _FieldPlan(name, _FieldKind.TENSOR, (slot,)) + if stripped is torch.Size: + slot = _SlotSpec(name, "SymInt[]?" if is_optional else "SymInt[]") + return _FieldPlan(name, _FieldKind.SHAPE, (slot,)) # A union mixing tensor types with anything else is a malformed signature # (e.g. a bare quantized-storage Optional, or Tensor | int): reject it at # registration instead of silently degrading to an unsupported field. @@ -631,7 +639,7 @@ def pack(self, obj: Any) -> Dict[str, Any]: for field in self.fields: value = getattr(obj, field.name, None) match field.kind: - case _FieldKind.TENSOR: + case _FieldKind.TENSOR | _FieldKind.SHAPE: slots[field.slots[0].name] = value case _FieldKind.TENSOR_OR_QUANTIZED: _pack_tensor_or_quantized(field, value, slots) @@ -667,6 +675,9 @@ def unpack(self, slots: Dict[str, Any]) -> Any: match field.kind: case _FieldKind.TENSOR: kwargs[field.name] = slots[field.slots[0].name] + case _FieldKind.SHAPE: + value = slots[field.slots[0].name] + kwargs[field.name] = None if value is None else torch.Size(value) case _FieldKind.TENSOR_OR_QUANTIZED: kwargs[field.name] = _unpack_tensor_or_quantized(field, slots) case _FieldKind.PROCESS_GROUP: @@ -1029,12 +1040,11 @@ def _register_autograd_for_op( the saved tuple + ``ctx_attrs`` to the module's ``setup_context`` and stashes the plan on ``ctx`` so backward can slice its grads per user output. """ - bwd_takes_grad_tuple = any(f.name == "grad_outputs" for f in bwd_plan.fields) + + tensor_list_positions = tuple(offset + 1 for offset in fwd_plan.tensor_or_quantized_offsets()) def _setup_context(ctx, inputs, output): - ctx.fwd_tensor_list_lengths = { - i: len(value) for i, value in enumerate(inputs) if isinstance(value, list) - } + ctx.fwd_tensor_list_lengths = {i: len(inputs[i]) for i in tensor_list_positions} fwd_obj = fwd_plan.unpack(dict(zip(fwd_plan.slot_names, inputs))) spec_obj = _spec_view(fwd_obj, fwd_plan.tensor_field_names()) @@ -1069,10 +1079,7 @@ def _autograd_backward(ctx, *grad_outputs): ctx.tensor_objects = None user_grads = _slice_user_grads(ctx.output_ranges, grad_outputs[0]) ctx.output_ranges = None - if bwd_takes_grad_tuple: - bwd_obj.grad_outputs = tuple(user_grads) - else: - bwd_obj.grad_output = user_grads[0] + bwd_obj.setup_grad_outputs(user_grads) kwargs = bwd_plan.pack(bwd_obj) bwd_args_flat = [kwargs[name] for name in bwd_plan.slot_names] grads = [_decode_none(g) for g in bwd_op(*bwd_args_flat)] @@ -1359,6 +1366,9 @@ def register_custom_op_with_autograd( non-differentiable input). * ``bwd_fake_impl(bwd_args)`` -- data-free twin of ``bwd_impl`` returning :class:`TensorSpec` grads. + * ``bwd_arg_type.setup_grad_outputs(grads)`` -- assign incoming gradients + to the container's fields. Receives one entry per forward user output, + in return order, including auxiliary outputs such as weight workspaces. * ``bwd_arg_type.setup_saved_tensors(ctx)`` -- optional hook; skipped if absent. @@ -1367,10 +1377,8 @@ def register_custom_op_with_autograd( forward state and returns the tensors to persist; the framework saves them via ``ctx.save_for_backward``. Before ``bwd_impl`` runs, the framework restores them into the container's tensor fields through the - ``setup_saved_tensors`` hook and sets the incoming gradient directly -- - into a ``grad_outputs`` field (tuple, one grad per user output) if - ``bwd_arg_type`` declares one, else into ``grad_output`` (the first user - output's grad) -- so ``bwd_impl`` receives a fully-populated + ``setup_saved_tensors`` hook and passes the incoming gradients to + ``setup_grad_outputs``, so ``bwd_impl`` receives a fully-populated ``bwd_arg_type``. Registration touches experimental ``torch.library`` / opaque-object APIs diff --git a/transformer_engine/pytorch/dynamo/fake.py b/transformer_engine/pytorch/dynamo/fake.py index 5f856d62214..9b70526fadc 100644 --- a/transformer_engine/pytorch/dynamo/fake.py +++ b/transformer_engine/pytorch/dynamo/fake.py @@ -14,6 +14,7 @@ from .tensor_spec import TensorSpec from ..quantized_tensor import Quantizer +from ..tensor.mxfp8_tensor import MXFP8Quantizer from ..tensor.nvfp4_tensor import NVFP4Quantizer from ..utils import get_device_compute_capability @@ -92,3 +93,58 @@ def fake_bgrad( ) ) return db, dx + + +def fake_activation( + input_: TensorSpec, + quantizer: Optional[Quantizer] = None, + *, + gated: bool = False, + dtype: Optional[torch.dtype] = None, + fused_quantization: bool = True, +) -> TensorSpec: + """Describe an activation output, including optional fused quantization.""" + shape = (*input_.shape[:-1], input_.shape[-1] // 2 if gated else input_.shape[-1]) + dtype = dtype or input_.dtype + if fused_quantization: + return _make_fused_quantization_output_spec(shape, dtype, input_.device, quantizer) + return TensorSpec(shape=shape, dtype=dtype, device=input_.device, quantizer=quantizer) + + +@torch.compiler.assume_constant_result +def _mxfp8_norm_fused_alignment( + normalization: str, with_2d_quantization: bool, use_cudnn: bool +) -> Optional[int]: + return tex.mxfp8_norm_fused_alignment( + normalization == "RMSNorm", with_2d_quantization, use_cudnn + ) + + +def fake_norm( + input_: TensorSpec, + quantizer: Optional[Quantizer] = None, + *, + normalization: str = "LayerNorm", + dtype: Optional[torch.dtype] = None, +) -> tuple[TensorSpec, Optional[TensorSpec], TensorSpec]: + """Describe newly allocated norm output, mean and reciprocal standard deviation.""" + if normalization not in ("LayerNorm", "RMSNorm"): + raise ValueError(f"Unsupported normalization: {normalization}") + out = _make_fused_quantization_output_spec( + input_.shape, dtype or input_.dtype, input_.device, quantizer + ) + rows, cols = math.prod(input_.shape[:-1]), input_.shape[-1] + if isinstance(quantizer, MXFP8Quantizer): + use_cudnn = bool(int(os.getenv("NVTE_NORM_FWD_USE_CUDNN", "0") or "0")) + alignment = _mxfp8_norm_fused_alignment( + normalization, quantizer.with_2d_quantization, use_cudnn + ) + if use_cudnn and alignment is not None and rows % alignment == 0 and cols % alignment == 0: + out.with_gemm_swizzled_scales = False + mean = ( + TensorSpec(shape=(rows,), dtype=torch.float32, device=input_.device) + if normalization == "LayerNorm" + else None + ) + rsigma = TensorSpec(shape=(rows,), dtype=torch.float32, device=input_.device) + return out, mean, rsigma diff --git a/transformer_engine/pytorch/module/_common.py b/transformer_engine/pytorch/module/_common.py index 3d04882698a..fd1e9b067b9 100644 --- a/transformer_engine/pytorch/module/_common.py +++ b/transformer_engine/pytorch/module/_common.py @@ -6,19 +6,33 @@ import dataclasses import queue -from typing import Any, Callable, List, Optional, Tuple, Union +from typing import Any, Callable, List, Optional, Sequence, Tuple, Union import torch from .. import cpp_extensions as tex from ..constants import TE_DType from ..distributed import in_fp8_activation_recompute_phase +from ..dynamo import TensorSpec, is_value_opaque_quantizer from ..export import is_in_onnx_export_mode from ..quantization import FP8GlobalStateManager +from ..quantized_tensor import Quantizer +from ..tensor.float8_blockwise_tensor import Float8BlockQuantizer from ..tensor.hybrid_tensor import HybridQuantizer from ..utils import get_default_init_method +def compile_unsupported_quantizer_reason( + quantizers: Sequence[Optional[Quantizer]], +) -> Optional[str]: + """Return a fallback reason if a quantizer cannot cross the custom-op boundary.""" + for quantizer in quantizers: + # Delayed-scaling and unregistered custom-recipe quantizers are not value-opaque. + if quantizer is not None and not is_value_opaque_quantizer(quantizer): + return "a quantizer not registered as a torch.compile value-opaque type" + return None + + def sum_bias_grad(tensor: torch.Tensor) -> torch.Tensor: """Reduce all token dimensions, preserving the final feature dimension.""" if tensor.ndim == 1: @@ -343,3 +357,70 @@ def check_fp8_reduce_and_update(restore_first_module: bool = False) -> bool: if restore_first_module or in_fp8_activation_recompute_phase(): qstate.is_first_fp8_module = first_fp8_module return result + + +def get_output_first_dim_size(input_first_dim_size: int, args: Any) -> int: + """Compute the output's first dimension size from the input's. + + Sequence parallelism gathers this dimension in column-parallel mode and + scatters it in row-parallel mode. ``args`` provides ``sequence_parallel``, + ``parallel_mode``, and ``tp_size``. + """ + if not args.sequence_parallel: + return input_first_dim_size + if args.parallel_mode == "column": + return input_first_dim_size * args.tp_size + if args.parallel_mode == "row": + return input_first_dim_size // args.tp_size + return input_first_dim_size + + +def fake_workspace_valid(workspace: TensorSpec, quantizer: Optional[Quantizer]) -> bool: + """Spec-level mirror of ``_is_weight_workspace_valid``.""" + if quantizer is None: + return True + if ( + isinstance(workspace.quantizer, Float8BlockQuantizer) + and workspace.quantizer.block_scaling_dim == 2 + and workspace.quantizer.rowwise_usage + ): + # 2D block scaling can reconstruct columnwise buffers from rowwise data. + return True + required = TensorSpec( + shape=workspace.shape, dtype=workspace.dtype, quantizer=quantizer, device=workspace.device + ).inner_names() + return set(required) <= set(workspace.inner_names()) + + +def fake_quantized_weight( + weight: TensorSpec, + quantizer: Optional[Quantizer], + workspace: Optional[TensorSpec], + *, + activation_dtype: torch.dtype, + cache_weight: bool, +) -> Tuple[TensorSpec, Optional[TensorSpec], Optional[str]]: + """Spec-level mirror of ``quantize_weight``: ``(weightmat, new_workspace, + save_alias)`` where ``save_alias`` names the forward input / output the + saved weight aliases (``None`` when it is a fresh tensor).""" + if weight.is_quantized: + return weight, None, "weight" + if workspace is not None and not fake_workspace_valid(workspace, quantizer): + workspace = None + if workspace is not None: + weightmat = dataclasses.replace(workspace) + weightmat.update_usage(rowwise_usage=True) + return weightmat, None, "weight_workspace" + weightmat = TensorSpec( + shape=tuple(weight.shape), + dtype=activation_dtype, + quantizer=quantizer, + device=weight.device, + ) + new_workspace = None + if cache_weight: + if weightmat.quantizer is not None: + weightmat.quantizer.internal = False + new_workspace = weightmat + weightmat.update_usage(rowwise_usage=True) + return weightmat, new_workspace, "new_weight_workspace" if cache_weight else None diff --git a/transformer_engine/pytorch/module/layernorm_linear.py b/transformer_engine/pytorch/module/layernorm_linear.py index b4a91b2a3ea..a61afe9f154 100644 --- a/transformer_engine/pytorch/module/layernorm_linear.py +++ b/transformer_engine/pytorch/module/layernorm_linear.py @@ -6,8 +6,8 @@ import os import warnings import weakref -from dataclasses import dataclass -from typing import Any, Callable, Dict, Optional, Tuple, Union, List +from dataclasses import dataclass, replace as dataclass_replace +from typing import Sequence, Any, Callable, Dict, Optional, Tuple, Union, List import torch from torch.nn import init @@ -44,6 +44,9 @@ nvtx_range_push, needs_quantized_gemm, get_nvtx_range_context, + warn_compile_eager_fallback, + warn_if_compile_disabled, + check_gemm_dims, ) from ..distributed import ( set_tensor_model_parallel_attributes, @@ -65,9 +68,12 @@ from ..jit import no_torch_dynamo from ..graph import is_graph_capturing from ._common import ( + compile_unsupported_quantizer_reason, apply_normalization, check_fp8_reduce_and_update, + fake_workspace_valid, noop_cat, + get_output_first_dim_size, set_quantizer_amax_reduction_group, set_quantizer_usage_for_wgrad_all_gather, WeightGradStore, @@ -79,9 +85,15 @@ prepare_for_saving, restore_from_func_ctx, ) -from ..dynamo import TensorOrQuantized +from ..dynamo.fake import fake_norm +from ..dynamo import ( + TensorSpec, + TensorOrQuantized, + register_custom_op_with_autograd, +) from ...debug.pytorch.debug_state import TEDebugState from ..tensor.mxfp8_tensor import MXFP8Quantizer +from ..tensor.float8_blockwise_tensor import Float8BlockQuantizer from ..tensor.hybrid_tensor import HybridQuantizer from ..tensor.identity_tensor import IdentityQuantizer from ..cpu_offload import ( @@ -279,6 +291,10 @@ class LayerNormLinearBwdArgs: # --- Per-backward scratch state (populated inside the backward impl) --- ub_obj_gradout: Optional[Any] = None + def setup_grad_outputs(self, grads: Sequence[Any]) -> None: + """Unpack gradients in forward-output order, ignoring weight workspaces.""" + self.grad_output, self.grad_ln_out, _ = grads + def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: """Pull saved tensors from ``ctx`` into the fields backward consumes.""" ( @@ -1466,6 +1482,353 @@ def wgrad_gemm( ) +def _layernorm_linear_forward_fake( + args: LayerNormLinearFwdArgs, +) -> Tuple[ + TensorSpec, + Optional[TensorSpec], + Optional[TensorSpec], + Optional[Tuple[Any, ...]], + Optional[Dict], +]: + """Shape/metadata-only twin of :func:`_layernorm_linear_forward_impl` for + torch.compile, returning ``TensorSpec`` descriptors for the outputs and + saved tensors instead of allocating real data.""" + if args.fsdp_group is not None and args.is_grad_enabled: + raise NotImplementedError( + "Compile-time LayerNormLinear forward does not support manual TE FSDP " + "(fsdp_group is not None); use FSDP2 or MCore FSDP." + ) + + weight_quantizer = args.weight_quantizer + fp8_or_debug = args.fp8 or args.debug + with_input_all_gather = args.parallel_mode == "column" and args.sequence_parallel + backward_needs_input = args.is_grad_enabled and args.weight_requires_grad + device = args.inp.device + + out_features, in_features = args.weight.shape + inp_leading = args.inp.shape[0] if len(args.inp.shape) > 1 else 1 + inputmat_aliases_inp = args.inp.dtype == args.activation_dtype + ln_weight_aliases = args.ln_weight.dtype == args.activation_dtype + + # Norm output quantizer usage -- mirrors the impl; the ln_out spec must be + # taken now, before the later rowwise-only usage for the all-gather. + if args.fp8: + if args.input_quantizer is None: + raise ValueError("Missing quantizer for input tensor") + args.input_quantizer.set_usage( + rowwise=True, + columnwise=backward_needs_input and args.backward_override is None, + ) + if with_input_all_gather and args.input_quantizer.supports_only_rowwise_all_gather(): + args.input_quantizer.set_usage(columnwise=False) + custom = is_custom(args.input_quantizer) + hybrid = isinstance(args.input_quantizer, HybridQuantizer) + identity = isinstance(args.input_quantizer, IdentityQuantizer) + with_quantized_norm = ( + args.fp8 + and not args.debug + and not args.return_layernorm_output + and not args.return_layernorm_output_gathered + and args.backward_override is None + and not custom + and not hybrid + and not identity + ) + # A custom quantizer's ln_out stays in high precision on the plain + # all-gather path (only the gathered copy is quantized). + ln_out_quantized = fp8_or_debug and not ( + with_input_all_gather and not args.return_layernorm_output_gathered and custom + ) + ln_out_spec, mu, rsigma = fake_norm( + args.inp, + args.input_quantizer if with_quantized_norm else None, + normalization=args.normalization, + dtype=args.activation_dtype, + ) + if ln_out_quantized and not with_quantized_norm: + ln_out_spec = TensorSpec( + shape=ln_out_spec.shape, + dtype=args.activation_dtype, + quantizer=args.input_quantizer, + device=device, + ) + if with_input_all_gather and fp8_or_debug: + args.input_quantizer.set_usage(rowwise=True, columnwise=False) + # ``ln_out_return`` is the high-precision norm output (or its all-gathered copy). + ln_out_return_is_total = with_input_all_gather and args.return_layernorm_output_gathered + # Whether the quantized ln_out is a different object from the returned one. + ln_out_rebound = ln_out_quantized and not with_quantized_norm + + # ------------------------------------------------------ + # Weight pipeline -- mirror ``quantize_weight`` / ``cast_if_needed``. + # ------------------------------------------------------ + new_weight_workspace = None + workspace = None # args.weight_workspace after validation + weightmat = None + weightmat_is_storage = False + weightmat_aliases_weight = False + is_weight_param_quantized = False + if fp8_or_debug: + is_weight_param_quantized = args.weight.is_quantized + if is_weight_param_quantized and not args.debug: + weight_quantizer = args.weight.quantizer + elif weight_quantizer is not None: + weight_quantizer.set_usage( + rowwise=True, + columnwise=args.is_grad_enabled + and not args.is_fsdp2 + and args.backward_override is None, + ) + + if args.weight.is_quantized: + # Primary-quantized args.weight: the impl reuses it as ``weightmat``. + weightmat = args.weight + weightmat_is_storage = True + weightmat_aliases_weight = True + else: + weightmat_is_storage = True + workspace = args.weight_workspace + if workspace is not None and not fake_workspace_valid(workspace, weight_quantizer): + # quantize_weight drops a stale workspace and builds a new one. + workspace = None + if workspace is not None: + # Copy, so the ``update_usage`` below stays off the input spec. + weightmat = dataclass_replace(workspace) + else: + weightmat = TensorSpec( + shape=tuple(args.weight.shape), + dtype=args.activation_dtype, + quantizer=weight_quantizer, + device=args.weight.device, + ) + if args.cache_weight: + # Persistent cache entries are wrappers, not bare storages. + if weightmat.quantizer is not None: + weightmat.quantizer.internal = False + new_weight_workspace = weightmat + weightmat.update_usage(rowwise_usage=True) + else: + weightmat_aliases_weight = args.weight.dtype == args.activation_dtype + weightmat = TensorSpec( + shape=tuple(args.weight.shape), dtype=args.activation_dtype, device=args.weight.device + ) + + # Bias cast: cuBLAS has no FP8 GEMM with FP32 args.bias. + bias_dtype = args.activation_dtype + if fp8_or_debug and args.activation_dtype == torch.float32: + bias_dtype = torch.bfloat16 + bias_aliases = args.bias is not None and args.bias.dtype == bias_dtype + + if args.output_quantizer is not None: + args.output_quantizer.set_usage(rowwise=True, columnwise=False) + + # ------------------------------------------------------ + # Outputs: y = norm(x) @ w^T and the optional norm output. + # ------------------------------------------------------ + requires_grad = args.is_grad_enabled and args.any_requires_grad() + out = TensorSpec( + shape=( + (out_features,) + if len(args.inp.shape) == 1 + else ( + get_output_first_dim_size(inp_leading, args), + *tuple(args.inp.shape[1:-1]), + out_features, + ) + ), + dtype=args.activation_dtype, + quantizer=args.output_quantizer, + requires_grad=requires_grad, + device=device, + ) + ln_out_for_return = None + if args.return_layernorm_output: + ln_leading = inp_leading + if args.return_layernorm_output_gathered and with_input_all_gather: + ln_leading = inp_leading * args.tp_size + ln_out_for_return = TensorSpec( + shape=( + (ln_leading, *tuple(args.inp.shape[1:])) + if len(args.inp.shape) > 1 + else (in_features,) + ), + dtype=args.activation_dtype, + requires_grad=requires_grad, + device=device, + ) + + # ------------------------------------------------------ + # Backward state -- saved-tensor layout + # (inputmat, wt_save, saved_weight, args.bias, args.ln_weight, ln_out, mu, rsigma). + # ------------------------------------------------------ + tensors_to_save_from_forward = None + ctx_attrs = None + if args.is_grad_enabled: + ln_out_needs_gather = args.weight_requires_grad and with_input_all_gather + + # Slot 5 -- ``ln_out_to_save``. + ln_out_alias = None + ln_out_to_save = None + if args.backward_override == "high_precision": + # ``ln_out_hp`` is taken before ln_out may be dropped, so it is always saved. + ln_out_to_save = TensorSpec( + shape=tuple(args.inp.shape), dtype=args.activation_dtype, device=device + ) + if args.return_layernorm_output and not ln_out_return_is_total: + ln_out_alias = "ln_out" + elif args.weight_requires_grad or args.return_layernorm_output: + ln_out_to_save = ln_out_spec + if ( + backward_needs_input + and args.backward_override is None + and ln_out_quantized + and ( + isinstance(args.input_quantizer, (MXFP8Quantizer, Float8BlockQuantizer)) + or not ln_out_needs_gather + ) + ): + ln_out_to_save.update_usage(rowwise_usage=False) + if args.return_layernorm_output and not ln_out_return_is_total and not ln_out_rebound: + ln_out_alias = "ln_out" + + # Slot 1 -- ``wt_save``, with the impl's alias dedup. + wt_alias = None + wt_save = None + if weightmat_aliases_weight: + wt_alias = "weight" + elif args.is_fsdp2: + pass # FSDP2 re-quantizes from the gathered args.weight in backward. + elif weightmat_is_storage and new_weight_workspace is not None: + wt_alias = "new_weight_workspace" + elif weightmat_is_storage and workspace is not None: + wt_alias = "weight_workspace" + elif weightmat_is_storage: + wt_save = weightmat + else: + wt_save = TensorSpec( + shape=tuple(args.weight.shape), + dtype=args.activation_dtype, + device=args.weight.device, + ) + + saved_tensor_aliases = ( + "inp" if inputmat_aliases_inp else None, + wt_alias, + "weight", + "bias" if bias_aliases else None, + "ln_weight" if ln_weight_aliases else None, + ln_out_alias, + None, + None, + ) + tensors_to_save_from_forward = ( + ( + None + if inputmat_aliases_inp + else TensorSpec( + shape=tuple(args.inp.shape), dtype=args.activation_dtype, device=device + ) + ), + wt_save, + None, + ( + None + if (args.bias is None or bias_aliases) + else TensorSpec(shape=tuple(args.bias.shape), dtype=bias_dtype, device=device) + ), + ( + None + if ln_weight_aliases + else TensorSpec( + shape=tuple(args.ln_weight.shape), dtype=args.activation_dtype, device=device + ) + ), + None if ln_out_alias is not None else ln_out_to_save, + mu, + rsigma, + ) + ctx_attrs = { + "fsdp_shapes": [], + "saved_tensor_aliases": saved_tensor_aliases, + "is_weight_param_quantized": is_weight_param_quantized, + "ln_out_needs_gather": ln_out_needs_gather, + } + + return out, ln_out_for_return, new_weight_workspace, tensors_to_save_from_forward, ctx_attrs + + +def _layernorm_linear_backward_fake( + args: LayerNormLinearBwdArgs, +) -> Tuple[Optional[TensorSpec], ...]: + """Allocation-free fake of :func:`_layernorm_linear_backward_impl` on + ``TensorSpec``. Returns ``(dgrad, dgamma, dbeta, wgrad, grad_bias)`` specs; + TP/SP gather/scatter happens inside the eager op, so the specs carry + rank-local shapes.""" + if args.fsdp_group is not None: + raise NotImplementedError( + "Fake LayerNormLinear backward does not support manual TE FSDP " + "(fsdp_group is not None); use FSDP2 or MCore FSDP." + ) + + weight = args.saved_weight + out_dtype = args.activation_dtype + out_features, in_features = weight.shape + device = args.grad_output.device + + # Mirrors the impl; affects dgrad's buffer layout. + if args.grad_input_quantizer is not None: + args.grad_input_quantizer.set_usage(rowwise=True, columnwise=False) + + dgrad = None + if args.requires_dgrad: + dgrad = TensorSpec( + shape=args.inp_shape, + dtype=out_dtype, + device=device, + ) + + # The norm backward always runs; its grads take the saved (cast) ln_weight dtype. + dgamma = TensorSpec(shape=(in_features,), dtype=args.ln_weight.dtype, device=device) + dbeta = None + if args.normalization == "LayerNorm": + dbeta = TensorSpec(shape=(in_features,), dtype=args.ln_weight.dtype, device=device) + + wgrad = None + # Under fuse_wgrad_accumulation the grad goes into main_grad in place. + if args.requires_wgrad and not args.fuse_wgrad_accumulation: + wgrad = TensorSpec( + shape=(out_features, in_features), + dtype=out_dtype, + quantizer=args.grad_weight_quantizer, + device=weight.device, + ) + + grad_bias = None + # FP8 backward computes bgrad in grad_output_preprocess whenever bias is + # used; in high precision it is fused into the wgrad GEMM, so it only + # exists when wgrad runs. + fp8_bwd = args.fp8 and args.backward_override is None + if args.use_bias and (args.requires_wgrad or fp8_bwd): + grad_bias = TensorSpec(shape=(out_features,), dtype=out_dtype, device=device) + + return dgrad, dgamma, dbeta, wgrad, grad_bias + + +# Custom op used under ``torch.compile``. +_layernorm_linear_op = register_custom_op_with_autograd( + op_name="layernorm_linear", + input_tensors_for_grad=["inp", "ln_weight", "ln_bias", "weight", "bias"], + fwd_arg_type=LayerNormLinearFwdArgs, + fwd_impl=_layernorm_linear_forward_impl, + fwd_fake_impl=_layernorm_linear_forward_fake, + setup_context=_layernorm_linear_setup_ctx, + bwd_arg_type=LayerNormLinearBwdArgs, + bwd_impl=_layernorm_linear_backward_impl, + bwd_fake_impl=_layernorm_linear_backward_fake, +) + + class _LayerNormLinear(torch.autograd.Function): """LayerNormLinear semi-top level module Calls custom cuda extensions. @@ -1530,8 +1893,7 @@ def backward( ) -> Tuple[Union[torch.Tensor, None], ...]: """Backward pass: compute gradients and reduce FP8 scaling factors.""" bwd_args: LayerNormLinearBwdArgs = ctx.backward_objects - bwd_args.grad_output = grad_output - bwd_args.grad_ln_out = grad_ln_out + bwd_args.setup_grad_outputs((grad_output, grad_ln_out, _grad_weight_workspace)) bwd_args.setup_saved_tensors(ctx) nvtx_label = "transformer_engine._LayerNormLinear.backward" if bwd_args.ub_name is not None: @@ -1556,6 +1918,22 @@ def backward( ) +@no_torch_dynamo() +def _layernorm_linear_eager( + inp: torch.Tensor, + ln_weight: torch.Tensor, + ln_bias: Optional[torch.Tensor], + weight: torch.Tensor, + bias: Optional[torch.Tensor], + fwd_args: LayerNormLinearFwdArgs, + is_grad_enabled: bool, +) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]: + """Run ``_LayerNormLinear`` eagerly, bypassing Dynamo.""" + if is_grad_enabled: + return _LayerNormLinear.apply(inp, ln_weight, ln_bias, weight, bias, fwd_args) + return _LayerNormLinear.forward(None, inp, ln_weight, ln_bias, weight, bias, fwd_args) + + class LayerNormLinear(TransformerEngineBaseModule): r""" Applies layer normalization followed by linear transformation to the incoming data. @@ -2023,7 +2401,6 @@ def reset_parameters(self, defer_init=False): elif self.parallel_mode == "column": set_tensor_model_parallel_attributes(getattr(self, bias), True, 0, 1) - @no_torch_dynamo() def forward( self, inp: torch.Tensor, @@ -2051,6 +2428,13 @@ def forward( * it also allows skipping gradient accumulation during the first microbatch (since it is the first gradient being produced) + + Under ``torch.compile``, the FP8 weight cache is replaced on + the first microbatch and reused on subsequent microbatches. + ``fuse_wgrad_accumulation=True`` remains unsupported. + With ``mode="reduce-overhead"``, call + ``torch.compiler.cudagraph_mark_step_begin()`` once before + each minibatch to keep the cache live across its microbatches. """ is_grad_enabled = torch.is_grad_enabled() @@ -2106,10 +2490,19 @@ def forward( weight_quantizer, weight_tensor ) + use_compiled_op = torch.compiler.is_compiling() and _layernorm_linear_op is not None + if _layernorm_linear_op is None and torch.compiler.is_compiling(): + warn_if_compile_disabled() + if use_compiled_op: + # Process groups cross the op boundary separately from quantizers. + for quantizer in (input_quantizer, grad_output_quantizer): + if getattr(quantizer, "amax_reduction_group", None) is not None: + set_quantizer_amax_reduction_group(quantizer, None) + cache_name = None if (is_first_microbatch is None or self.is_fsdp2) else "weight" - weight_workspace = ( - self._fp8_workspaces.get(cache_name) if cache_name is not None else None - ) + weight_workspace = None + if cache_name is not None and not (use_compiled_op and is_first_microbatch): + weight_workspace = self._fp8_workspaces.get(cache_name) dgrad_use_split_accumulator = _2X_ACC_DGRAD wgrad_use_split_accumulator = _2X_ACC_WGRAD @@ -2141,6 +2534,25 @@ def forward( linear_bias_tensor = ( bias_tensor if (self.apply_bias and not self.gemm_bias_unfused_add) else None ) + if use_compiled_op: + fallback_reason = self._compile_eager_fallback_reason( + inp, + skip_fp8_weight_update, + fp8_output, + fp8_grad, + is_grad_enabled, + debug, + quantizers, + ) + if fallback_reason is not None: + warn_compile_eager_fallback(fallback_reason) + torch._dynamo.graph_break( + msg=f"te.LayerNormLinear falling back to eager: {fallback_reason}" + ) + use_compiled_op = False + + wgrad_store = None if use_compiled_op else self.wgrad_store + fwd_args = LayerNormLinearFwdArgs( # tensors inp=inp, @@ -2210,30 +2622,24 @@ def forward( is_fsdp2=self.is_fsdp2, # weight-grad scheduling fuse_wgrad_accumulation=self.fuse_wgrad_accumulation, - wgrad_store=self.wgrad_store, + wgrad_store=wgrad_store, # misc cpu_offloading=is_cpu_offload_enabled(), is_grad_enabled=is_grad_enabled, ) - if is_grad_enabled: - out, ln_out, new_weight_workspace = _LayerNormLinear.apply( - inp, - self.layer_norm_weight, - self.layer_norm_bias, - weight_tensor, - linear_bias_tensor, - fwd_args, - ) + if use_compiled_op: + check_gemm_dims(inp, weight_tensor, self.fp8) + out, ln_out, new_weight_workspace = _layernorm_linear_op(fwd_args) else: - out, ln_out, new_weight_workspace = _LayerNormLinear.forward( - None, + out, ln_out, new_weight_workspace = _layernorm_linear_eager( inp, self.layer_norm_weight, self.layer_norm_bias, weight_tensor, linear_bias_tensor, fwd_args, + is_grad_enabled, ) if new_weight_workspace is not None and cache_name is not None: @@ -2300,6 +2706,57 @@ def _get_debug_quantizers(self, fp8_output, fp8_grad, is_grad_enabled): for name, q in zip(names, original_quantizers) ) + def _compile_eager_fallback_reason( + self, + inp: torch.Tensor, + skip_fp8_weight_update: Optional[torch.Tensor], + fp8_output: bool, + fp8_grad: bool, + is_grad_enabled: bool, + debug: bool, + quantizers: Sequence[Optional[Quantizer]], + ) -> Optional[str]: + """Return a fallback reason after preparing the forward quantizers.""" + if debug: + return "debug instrumentation (nvidia-dlfw-inspect)" + weight_tensor, bias_tensor = self._get_weight_and_bias_tensors() + if is_distributed_weight(weight_tensor): + return "a DistributedWeight (custom weight parallelism, e.g. GTP)" + if isinstance(inp, (QuantizedTensor, QuantizedTensorStorage)): + return "a quantized input tensor" + if self.fsdp_group is not None: + return "manual TE FSDP (fsdp_group); use FSDP2 or MCore FSDP" + any_requires_grad = ( + inp.requires_grad + or self.layer_norm_weight.requires_grad + or (self.layer_norm_bias is not None and self.layer_norm_bias.requires_grad) + or weight_tensor.requires_grad + or (bias_tensor is not None and bias_tensor.requires_grad) + ) + if fp8_output and is_grad_enabled and any_requires_grad: + return "differentiable fp8_output=True" + if is_cpu_offload_enabled(): + return "CPU activation offloading" + # A queued store can be enabled between forward and backward. + if self.wgrad_store is not None and ( + self.wgrad_store.context is not None or self.wgrad_store.delay_wgrad_compute() + ): + return "delayed wgrad compute (wgrad_store)" + if self.fuse_wgrad_accumulation: + return "fuse_wgrad_accumulation (main_grad)" + fp8 = FP8GlobalStateManager.is_fp8_enabled() + needs_dgrad = is_grad_enabled and inp.requires_grad + if ( + fp8 + and fp8_grad + and needs_dgrad + and not (self.ub_overlap_rs_dgrad or self.ub_bulk_wgrad) + ): + return "a quantized input grad (fp8_grad=True)" + if fp8 and skip_fp8_weight_update is not None: + return "GPU-controlled FP8 weight cache updates" + return compile_unsupported_quantizer_reason(quantizers) + def _get_weight_and_bias_tensors(self): # Get concatenated weight and bias tensors unfused_weights = self._get_weight_tensors() diff --git a/transformer_engine/pytorch/module/layernorm_mlp.py b/transformer_engine/pytorch/module/layernorm_mlp.py index ba15f1c7bd1..d22fe30ad2a 100644 --- a/transformer_engine/pytorch/module/layernorm_mlp.py +++ b/transformer_engine/pytorch/module/layernorm_mlp.py @@ -50,6 +50,8 @@ clear_tensor_data, needs_quantized_gemm, get_nvtx_range_context, + warn_compile_eager_fallback, + warn_if_compile_disabled, ) from ..distributed import ( set_tensor_model_parallel_attributes, @@ -73,9 +75,11 @@ from ..tensor.hybrid_tensor import HybridQuantizer from ..tensor.identity_tensor import IdentityQuantizer from ._common import ( + compile_unsupported_quantizer_reason, sum_bias_grad, apply_normalization, check_fp8_reduce_and_update, + fake_quantized_weight, set_quantizer_amax_reduction_group, set_quantizer_usage_for_wgrad_all_gather, WeightGradStore, @@ -87,12 +91,18 @@ mark_activation_offload, ) from ..quantized_tensor import ( + QuantizedTensor, QuantizedTensorStorage, Quantizer, prepare_for_saving, restore_from_func_ctx, ) -from ..dynamo import TensorOrQuantized +from ..dynamo.fake import fake_activation, fake_norm +from ..dynamo import ( + TensorSpec, + TensorOrQuantized, + register_custom_op_with_autograd, +) from ..cpp_extensions import ( general_gemm, ) @@ -102,8 +112,10 @@ __all__ = ["LayerNormMLP"] -def _get_act_func_supported_list(recipe: Optional[Recipe] = None): - if recipe is None: +def _get_act_func_supported_list( + recipe: Optional[Recipe] = None, dbias_fusion: Optional[bool] = None +): + if dbias_fusion is False or (dbias_fusion is None and recipe is None): # bf16 (recipe is None): return { "gelu": (tex.gelu, tex.dgelu, None), @@ -119,7 +131,7 @@ def _get_act_func_supported_list(recipe: Optional[Recipe] = None): "swiglu": (tex.swiglu, tex.dswiglu, None), "clamped_swiglu": (tex.clamped_swiglu, tex.clamped_dswiglu, None), } - if recipe.delayed() or recipe.mxfp8(): + if dbias_fusion is True or recipe.delayed() or recipe.mxfp8(): # Delayed scaling, fusion supported list: [tex.dbias_dgelu, tex.dbias_drelu, tex.dbias_dqgelu, tex.dbias_dsrelu] # MXFP8: [tex.dbias_dgelu, tex.dbias_drelu, tex.dbias_dqgelu, tex.dbias_dsrelu] return { @@ -162,18 +174,26 @@ def _get_act_func_supported_list(recipe: Optional[Recipe] = None): raise NotImplementedError(f"Unhandled recipe type {recipe}") -def _act_func(activation: str, recipe: Optional[Recipe] = None): +def _act_func( + activation: str, recipe: Optional[Recipe] = None, dbias_fusion: Optional[bool] = None +): # based on each quantization mode, we have different kernel fusion supported: # bf16 (recipe is None): [tex.dbias_dgelu, tex.dbias_drelu, tex.dbias_dqgelu, tex.dbias_dsrelu] # Delayed scaling, fusion supported list: [tex.dbias_dgelu, tex.dbias_drelu, tex.dbias_dqgelu, tex.dbias_dsrelu] # MXFP8: [tex.dbias_dgelu, tex.dbias_drelu, tex.dbias_dqgelu, tex.dbias_dsrelu] # Per-tensor current scaling or fp8 blockwise scaling: [] - funcs = _get_act_func_supported_list(recipe) + funcs = _get_act_func_supported_list(recipe, dbias_fusion) if activation not in funcs: raise NotImplementedError("Activation type " + activation + " is not supported!") return funcs[activation] +# Activations whose output halves the last dim (gated linear units). +_GATED_ACTIVATIONS = frozenset( + {"geglu", "glu", "qgeglu", "reglu", "sreglu", "swiglu", "clamped_swiglu"} +) + + @dataclass(slots=True) class LayerNormMLPFwdArgs: """Single-argument bag for the forward path of :class:`_LayerNormMLP`.""" @@ -236,6 +256,10 @@ class LayerNormMLPFwdArgs: backward_override: Optional[str] dgrad_use_split_accumulator: bool wgrad_use_split_accumulator: bool + # Recipe properties the backward needs (the recipe itself can't cross the op boundary). + recipe_float8_block_scaling: bool + recipe_custom: bool + recipe_dbias_dact_fusion: bool debug: bool # --- Weight-workspace caching --- @@ -356,7 +380,9 @@ class LayerNormMLPBwdArgs: # --- Numerical / dtype config --- activation_dtype: Optional[torch.dtype] = None fp8: bool = False - fp8_recipe: Optional[Any] = None + recipe_float8_block_scaling: bool = False + recipe_custom: bool = False + recipe_dbias_dact_fusion: bool = False dgrad_use_split_accumulator: bool = _2X_ACC_DGRAD wgrad_use_split_accumulator: bool = _2X_ACC_WGRAD backward_override: Optional[str] = None @@ -400,6 +426,10 @@ class LayerNormMLPBwdArgs: # --- Per-backward scratch state (populated inside the backward impl) --- ub_obj_gradout: Optional[Any] = None + def setup_grad_outputs(self, grads: Sequence[Any]) -> None: + """Unpack gradients in forward-output order, ignoring weight workspaces.""" + self.grad_output, self.grad_ln_out, _, _ = grads + def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: """Pull saved tensors from ``ctx`` into the fields backward consumes.""" self.set_saved_tensors(restore_from_func_ctx(ctx)) @@ -1168,7 +1198,9 @@ def _layernorm_mlp_setup_ctx( # Numerical / dtype config bwd_args.activation_dtype = fwd_args.activation_dtype bwd_args.fp8 = fp8 - bwd_args.fp8_recipe = FP8GlobalStateManager.get_fp8_recipe() if fp8 else None + bwd_args.recipe_float8_block_scaling = fwd_args.recipe_float8_block_scaling + bwd_args.recipe_custom = fwd_args.recipe_custom + bwd_args.recipe_dbias_dact_fusion = fwd_args.recipe_dbias_dact_fusion bwd_args.dgrad_use_split_accumulator = fwd_args.dgrad_use_split_accumulator bwd_args.wgrad_use_split_accumulator = fwd_args.wgrad_use_split_accumulator bwd_args.backward_override = fwd_args.backward_override @@ -1630,7 +1662,7 @@ def _layernorm_mlp_backward_impl( # Whether to set grad arg in general_gemm grad_arg = True - if args.fp8 and args.fp8_recipe.float8_block_scaling(): + if args.fp8 and args.recipe_float8_block_scaling: grad_arg = False # Arguments to include in wgrad GEMM closure @@ -1676,7 +1708,7 @@ def fc2_wgrad_gemm( # Update grad bias if needed if fc2_bias_grad is None: - if args.fp8 and args.fp8_recipe.float8_block_scaling() and fc2_bias is not None: + if args.fp8 and args.recipe_float8_block_scaling and fc2_bias is not None: # BGRAD not fused with GEMM for float8 blockwise gemm. fc2_bias_grad_ = sum_bias_grad(act_out) fc2_bias_grad = fc2_bias_grad_ @@ -1709,12 +1741,13 @@ def fc2_wgrad_gemm( fc1_bias_grad = sum_bias_grad(dact) dact = args.fc1_grad_output_quantizer(dact) elif ( - _act_func(args.activation, args.fp8_recipe if args.fp8 else None)[2] is not None + _act_func(args.activation, dbias_fusion=args.fp8 and args.recipe_dbias_dact_fusion)[2] + is not None and args.fp8 ): # Fusion: gemm, bias + gelu + quantize dbias_dact_quantize_func = _act_func( - args.activation, args.fp8_recipe if args.fp8 else None + args.activation, dbias_fusion=args.fp8 and args.recipe_dbias_dact_fusion )[2] fc1_bias_grad, dact = dbias_dact_quantize_func( fc2_dgrad, @@ -1726,7 +1759,7 @@ def fc2_wgrad_gemm( # Fusion: gemm + gelu, if not fc2_dgrad_gemm_gelu_fusion: activation_func_bwd = _act_func( - args.activation, args.fp8_recipe if args.fp8 else None + args.activation, dbias_fusion=args.fp8 and args.recipe_dbias_dact_fusion )[1] dact = activation_func_bwd( fc2_dgrad, fc1_out.to(args.activation_dtype), None, **act_params @@ -1739,7 +1772,7 @@ def fc2_wgrad_gemm( args.fc1_grad_output_quantizer, (Float8BlockQuantizer, IdentityQuantizer), ) - or args.fp8_recipe.custom() + or args.recipe_custom ): fc1_bias_grad = sum_bias_grad(dact) dact = args.fc1_grad_output_quantizer(dact) @@ -2071,6 +2104,420 @@ def fc1_wgrad_gemm( ) +def _layernorm_mlp_forward_fake( + args: LayerNormMLPFwdArgs, +) -> Tuple[ + TensorSpec, + Optional[TensorSpec], + Optional[TensorSpec], + Optional[TensorSpec], + Optional[Tuple[Any, ...]], + Optional[Dict], +]: + """Shape/metadata-only twin of :func:`_layernorm_mlp_forward_impl` for + torch.compile (no args.activation checkpointing: that path falls back to eager).""" + if args.fsdp_group is not None and args.is_grad_enabled: + raise NotImplementedError( + "Compile-time LayerNormMLP forward does not support manual TE FSDP " + "(fsdp_group is not None); use FSDP2 or MCore FSDP." + ) + if args.checkpoint and args.is_grad_enabled: + raise NotImplementedError( + "Compile-time LayerNormMLP forward does not support args.activation checkpointing" + ) + + fc1_weight_quantizer = args.fc1_weight_quantizer + fc2_weight_quantizer = args.fc2_weight_quantizer + fp8_or_debug = args.fp8 or args.debug + bias_gelu_fusion = args.bias_gelu_fusion + gemm_gelu_fusion = args.gemm_gelu_fusion + device = args.inp.device + + in_features = args.ln_weight.shape[0] + fc1_out_features = args.fc1_weight.shape[0] + fc2_out_features = args.fc2_weight.shape[0] + inp_leading = args.inp.shape[0] if len(args.inp.shape) > 1 else 1 + gemm_leading_shape = ( + (inp_leading * args.tp_size, *args.inp.shape[1:-1]) + if args.sequence_parallel + else tuple(args.inp.shape[:-1]) + ) + inputmat_aliases_inp = args.inp.dtype == args.activation_dtype + ln_weight_aliases = args.ln_weight.dtype == args.activation_dtype + + backwards_needs_fc1_input = args.fc1_weight_requires_grad and args.is_grad_enabled + + # Norm output quantizer usage -- mirrors the impl; the ln_out spec must be + # taken now, before the later rowwise-only usage for the all-gather. + if args.fp8: + if args.fc1_input_quantizer is None: + raise ValueError("Missing quantizer for FC1 input tensor") + args.fc1_input_quantizer.set_usage(rowwise=True, columnwise=backwards_needs_fc1_input) + if args.sequence_parallel and args.fc1_input_quantizer.supports_only_rowwise_all_gather(): + args.fc1_input_quantizer.set_usage(columnwise=False) + custom = is_custom(args.fc1_input_quantizer) + hybrid = isinstance(args.fc1_input_quantizer, HybridQuantizer) + identity = isinstance(args.fc1_input_quantizer, IdentityQuantizer) + with_quantized_norm = ( + args.fp8 + and not args.debug + and not args.return_layernorm_output + and not args.return_layernorm_output_gathered + and not custom + and not hybrid + and not identity + ) + # A custom quantizer's ln_out stays in high precision on the plain + # all-gather path (only the gathered copy is quantized). + ln_out_quantized = fp8_or_debug and not ( + args.sequence_parallel and not args.return_layernorm_output_gathered and custom + ) + ln_out, mu, rsigma = fake_norm( + args.inp, + args.fc1_input_quantizer if with_quantized_norm else None, + normalization=args.normalization, + dtype=args.activation_dtype, + ) + if ln_out_quantized and not with_quantized_norm: + ln_out = TensorSpec( + shape=ln_out.shape, + dtype=args.activation_dtype, + quantizer=args.fc1_input_quantizer, + device=device, + ) + if args.sequence_parallel and fp8_or_debug: + args.fc1_input_quantizer.set_usage(rowwise=True, columnwise=False) + ln_out_return_is_total = args.sequence_parallel and args.return_layernorm_output_gathered + ln_out_rebound = ln_out_quantized and not with_quantized_norm + + # ------------------------------------------------------ + # Weights -- mirror ``quantize_weight`` / ``cast_if_needed``. + # ------------------------------------------------------ + new_fc1_weight_workspace = None + new_fc2_weight_workspace = None + if fp8_or_debug: + if args.fc1_weight.is_quantized and not args.debug: + fc1_weight_quantizer = args.fc1_weight.quantizer + elif fc1_weight_quantizer is not None: + fc1_weight_quantizer.set_usage( + rowwise=True, columnwise=args.is_grad_enabled and not args.is_fsdp2 + ) + if args.fc2_weight.is_quantized and not args.debug: + fc2_weight_quantizer = args.fc2_weight.quantizer + elif fc2_weight_quantizer is not None: + fc2_weight_quantizer.set_usage( + rowwise=True, columnwise=args.is_grad_enabled and not args.is_fsdp2 + ) + fc1_weight_final, new_fc1_weight_workspace, fc1_wt_alias = fake_quantized_weight( + args.fc1_weight, + fc1_weight_quantizer, + args.fc1_weight_workspace, + activation_dtype=args.activation_dtype, + cache_weight=args.cache_weight, + ) + fc2_weight_final, new_fc2_weight_workspace, fc2_wt_alias = fake_quantized_weight( + args.fc2_weight, + fc2_weight_quantizer, + args.fc2_weight_workspace, + activation_dtype=args.activation_dtype, + cache_weight=args.cache_weight, + ) + fc1_wt_alias = ( + None if fc1_wt_alias is None else fc1_wt_alias.replace("weight", "fc1_weight") + ) + fc2_wt_alias = ( + None if fc2_wt_alias is None else fc2_wt_alias.replace("weight", "fc2_weight") + ) + else: + fc1_weight_final = TensorSpec( + shape=tuple(args.fc1_weight.shape), + dtype=args.activation_dtype, + device=args.fc1_weight.device, + ) + fc2_weight_final = TensorSpec( + shape=tuple(args.fc2_weight.shape), + dtype=args.activation_dtype, + device=args.fc2_weight.device, + ) + fc1_wt_alias = "fc1_weight" if args.fc1_weight.dtype == args.activation_dtype else None + fc2_wt_alias = "fc2_weight" if args.fc2_weight.dtype == args.activation_dtype else None + + # Bias cast: cuBLAS has no FP8 GEMM with FP32 bias. + bias_dtype = args.activation_dtype + if fp8_or_debug and args.activation_dtype == torch.float32: + bias_dtype = torch.bfloat16 + fc1_bias_aliases = args.fc1_bias is not None and args.fc1_bias.dtype == bias_dtype + fc2_bias_aliases = args.fc2_bias is not None and args.fc2_bias.dtype == bias_dtype + + # ------------------------------------------------------ + # FC1 GEMM + args.activation (see the impl for the fusion rules). + # ------------------------------------------------------ + if args.activation != "gelu": + gemm_gelu_fusion = bias_gelu_fusion = False + else: + if not args.fp8: + gemm_gelu_fusion = True + if gemm_gelu_fusion and bias_gelu_fusion: + gemm_gelu_fusion = False + if args.debug: + gemm_gelu_fusion = False + fc1_out = None + fc1_out_without_bias = None + if bias_gelu_fusion: + fc1_out_without_bias = TensorSpec( + shape=(*gemm_leading_shape, fc1_out_features), + dtype=args.activation_dtype, + device=device, + ) + else: + # generic_gemm stores the pre-GELU auxiliary output in the bias dtype + # for low-precision inputs, defaulting to BF16 when there is no bias. + fc1_out_dtype = args.activation_dtype + if gemm_gelu_fusion and args.fp8: + fc1_out_dtype = bias_dtype if args.fc1_bias is not None else torch.bfloat16 + fc1_out = TensorSpec( + shape=(*gemm_leading_shape, fc1_out_features), + dtype=fc1_out_dtype, + device=device, + ) + act_out = fake_activation( + fc1_out_without_bias if bias_gelu_fusion else fc1_out, + args.fc2_input_quantizer if fp8_or_debug else None, + gated=args.activation in _GATED_ACTIVATIONS, + dtype=args.activation_dtype, + fused_quantization=( + args.fp8 + and not args.debug + and not gemm_gelu_fusion + and not args.recipe_float8_block_scaling + and not args.recipe_custom + ), + ) + if args.fc2_output_quantizer is not None: + args.fc2_output_quantizer.set_usage(rowwise=True, columnwise=False) + + # ------------------------------------------------------ + # Outputs. + # ------------------------------------------------------ + requires_grad = args.is_grad_enabled and args.any_requires_grad() + out_leading = inp_leading + if args.sequence_parallel and not args.set_parallel_mode: + out_leading = inp_leading * args.tp_size + fc2_out = TensorSpec( + shape=( + (fc2_out_features,) + if len(args.inp.shape) == 1 + else (out_leading, *tuple(args.inp.shape[1:-1]), fc2_out_features) + ), + dtype=args.activation_dtype, + quantizer=args.fc2_output_quantizer, + requires_grad=requires_grad, + device=device, + ) + ln_out_for_return = None + if args.return_layernorm_output: + ln_leading = inp_leading + if ln_out_return_is_total: + ln_leading = inp_leading * args.tp_size + ln_out_for_return = TensorSpec( + shape=( + (ln_leading, *tuple(args.inp.shape[1:])) + if len(args.inp.shape) > 1 + else (in_features,) + ), + dtype=args.activation_dtype, + requires_grad=requires_grad, + device=device, + ) + + # ------------------------------------------------------ + # Backward state -- saved-tensor layout (see the impl). + # ------------------------------------------------------ + tensors_to_save_from_forward = None + ctx_attrs = None + if args.is_grad_enabled: + if not args.fc1_weight_requires_grad: + ln_out = None + if not args.fc2_weight_requires_grad: + act_out = None + fc1_wt_save = fc1_weight_final + fc2_wt_save = fc2_weight_final + if args.is_fsdp2: + if fc1_wt_alias != "fc1_weight": + fc1_wt_save, fc1_wt_alias = None, None + if fc2_wt_alias != "fc2_weight": + fc2_wt_save, fc2_wt_alias = None, None + ln_out_alias = ( + "ln_out" + if args.return_layernorm_output + and ln_out is not None + and not ln_out_return_is_total + and not ln_out_rebound + else None + ) + saved_tensor_aliases = ( + "inp" if inputmat_aliases_inp else None, + "ln_weight" if ln_weight_aliases else None, + ln_out_alias, + fc1_wt_alias, + "fc1_weight", + "fc1_bias" if fc1_bias_aliases else None, + None, + None, + None, + fc2_wt_alias, + "fc2_weight", + "fc2_bias" if fc2_bias_aliases else None, + None, + None, + ) + saved = ( + TensorSpec(shape=tuple(args.inp.shape), dtype=args.activation_dtype, device=device), + TensorSpec( + shape=tuple(args.ln_weight.shape), dtype=args.activation_dtype, device=device + ), + ln_out, + fc1_wt_save, + None, + ( + None + if args.fc1_bias is None + else TensorSpec(shape=tuple(args.fc1_bias.shape), dtype=bias_dtype, device=device) + ), + fc1_out, + fc1_out_without_bias, + act_out, + fc2_wt_save, + None, + ( + None + if args.fc2_bias is None + else TensorSpec(shape=tuple(args.fc2_bias.shape), dtype=bias_dtype, device=device) + ), + mu, + rsigma, + ) + tensors_to_save_from_forward = tuple( + None if alias is not None else spec for alias, spec in zip(saved_tensor_aliases, saved) + ) + ctx_attrs = { + "saved_tensor_aliases": saved_tensor_aliases, + "fsdp_shapes": None, + "is_recomputation": False, + } + + return ( + fc2_out, + ln_out_for_return, + new_fc1_weight_workspace, + new_fc2_weight_workspace, + tensors_to_save_from_forward, + ctx_attrs, + ) + + +def _layernorm_mlp_backward_fake( + args: LayerNormMLPBwdArgs, +) -> Tuple[Optional[TensorSpec], ...]: + """Allocation-free fake of :func:`_layernorm_mlp_backward_impl` on + ``TensorSpec``. Returns ``(dgrad, dgamma, dbeta, fc1_wgrad, fc1_bias_grad, + fc2_wgrad, fc2_bias_grad)`` specs with rank-local shapes.""" + if args.fsdp_group is not None: + raise NotImplementedError( + "Fake LayerNormMLP backward does not support manual TE FSDP " + "(fsdp_group is not None); use FSDP2 or MCore FSDP." + ) + out_dtype = args.activation_dtype + device = args.grad_output.device + in_features = args.ln_weight.shape[-1] + fc1_out_features = args.fc1_weight.shape[0] + fc2_out_features, act_features = args.fc2_weight.shape + + if args.fc1_grad_output_quantizer is not None: + args.fc1_grad_output_quantizer.set_usage(rowwise=True, columnwise=True) + + dgrad = None + if args.requires_dgrad: + inp_leading = args.grad_output.shape[0] + if args.sequence_parallel and not args.set_parallel_mode: + inp_leading = inp_leading // args.tp_size + dgrad = TensorSpec( + shape=( + (in_features,) + if len(args.grad_output.shape) == 1 + else (inp_leading, *args.grad_output.shape[1:-1], in_features) + ), + dtype=out_dtype, + device=device, + ) + + # The norm backward always runs; its grads take the saved (cast) ln_weight dtype. + dgamma = TensorSpec(shape=(in_features,), dtype=args.ln_weight.dtype, device=device) + dbeta = None + if args.normalization == "LayerNorm": + dbeta = TensorSpec(shape=(in_features,), dtype=args.ln_weight.dtype, device=device) + + fc1_wgrad = None + if args.fc1_weight_requires_grad and not args.fuse_wgrad_accumulation: + fc1_wgrad = TensorSpec( + shape=(fc1_out_features, in_features), + dtype=out_dtype, + quantizer=args.fc1_grad_weight_quantizer, + device=device, + ) + fc1_bias_grad = None + # FP8, debug and the fused bias-gelu path always produce it; in high + # precision it is fused into the FC1 wgrad GEMM or summed when only the + # bias needs a grad. + if args.fc1_bias is not None and any( + ( + args.fp8, + args.debug, + args.bias_gelu_fusion, + args.fc1_weight_requires_grad, + args.fc1_bias_requires_grad, + ) + ): + fc1_bias_grad = TensorSpec(shape=(fc1_out_features,), dtype=out_dtype, device=device) + + fc2_wgrad = None + if args.fc2_weight_requires_grad and not args.fuse_wgrad_accumulation: + fc2_wgrad = TensorSpec( + shape=(fc2_out_features, act_features), + dtype=out_dtype, + quantizer=args.fc2_grad_weight_quantizer, + device=device, + ) + fc2_bias_grad = None + fp8_bwd = args.fp8 and args.backward_override is None + if args.use_bias and (args.fc2_weight_requires_grad or fp8_bwd): + fc2_bias_grad = TensorSpec(shape=(fc2_out_features,), dtype=out_dtype, device=device) + + return dgrad, dgamma, dbeta, fc1_wgrad, fc1_bias_grad, fc2_wgrad, fc2_bias_grad + + +# Custom op used under ``torch.compile``. +_layernorm_mlp_op = register_custom_op_with_autograd( + op_name="layernorm_mlp", + input_tensors_for_grad=[ + "inp", + "ln_weight", + "ln_bias", + "fc1_weight", + "fc1_bias", + "fc2_weight", + "fc2_bias", + ], + fwd_arg_type=LayerNormMLPFwdArgs, + fwd_impl=_layernorm_mlp_forward_impl, + fwd_fake_impl=_layernorm_mlp_forward_fake, + setup_context=_layernorm_mlp_setup_ctx, + bwd_arg_type=LayerNormMLPBwdArgs, + bwd_impl=_layernorm_mlp_backward_impl, + bwd_fake_impl=_layernorm_mlp_backward_fake, +) + + class _LayerNormMLP(torch.autograd.Function): """LayerNormMLP semi-top level module Calls custom cuda extensions. @@ -2141,8 +2588,9 @@ def backward( ) -> Tuple[Union[torch.Tensor, None], ...]: """Backward pass: compute gradients and reduce FP8 scaling factors.""" bwd_args: LayerNormMLPBwdArgs = ctx.backward_objects - bwd_args.grad_output = grad_output - bwd_args.grad_ln_out = grad_ln_out + bwd_args.setup_grad_outputs( + (grad_output, grad_ln_out, _grad_fc1_weight_workspace, _grad_fc2_weight_workspace) + ) with get_nvtx_range_context("_LayerNormMLP_backward"): _layernorm_mlp_recompute(bwd_args, ctx) ( @@ -2173,6 +2621,18 @@ def backward( ) +@no_torch_dynamo() +def _layernorm_mlp_eager( + tensors: Tuple[Optional[torch.Tensor], ...], + fwd_args: LayerNormMLPFwdArgs, + is_grad_enabled: bool, +) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor]]: + """Run ``_LayerNormMLP`` eagerly, bypassing Dynamo.""" + if is_grad_enabled: + return _LayerNormMLP.apply(*tensors, fwd_args) + return _LayerNormMLP.forward(None, *tensors, fwd_args) + + class LayerNormMLP(TransformerEngineBaseModule): r""" Applies layer normalization on the input followed by the MLP module, consisting of @@ -2603,7 +3063,6 @@ def reset_parameters(self, defer_init=False): if self.set_parallel_mode: setattr(self.fc2_bias, "sequence_parallel", self.sequence_parallel) - @no_torch_dynamo() def forward( self, inp: torch.Tensor, @@ -2629,6 +3088,13 @@ def forward( * it also allows skipping gradient accumulation during the first microbatch (since it is the first gradient being produced) + + Under ``torch.compile``, the FP8 weight cache is replaced on + the first microbatch and reused on subsequent microbatches. + ``fuse_wgrad_accumulation=True`` remains unsupported. + With ``mode="reduce-overhead"``, call + ``torch.compiler.cudagraph_mark_step_begin()`` once before + each minibatch to keep the cache live across its microbatches. """ is_grad_enabled = torch.is_grad_enabled() @@ -2703,21 +3169,32 @@ def forward( if self.bias_gelu_nvfusion and not use_reentrant_activation_recompute(): self.fast_setattr("bias_gelu_nvfusion", False) + use_compiled_op = torch.compiler.is_compiling() and _layernorm_mlp_op is not None + if _layernorm_mlp_op is None and torch.compiler.is_compiling(): + warn_if_compile_disabled() + if use_compiled_op: + # Process groups cross the op boundary separately from quantizers. + for quantizer in (fc1_input_quantizer, fc2_grad_output_quantizer): + if getattr(quantizer, "amax_reduction_group", None) is not None: + set_quantizer_amax_reduction_group(quantizer, None) + cache_name_fc1 = ( None if (is_first_microbatch is None or self.is_fsdp2) else "fc1_weight" ) cache_name_fc2 = ( None if (is_first_microbatch is None or self.is_fsdp2) else "fc2_weight" ) - fc1_weight_workspace = ( - self._fp8_workspaces.get(cache_name_fc1) if cache_name_fc1 is not None else None - ) - fc2_weight_workspace = ( - self._fp8_workspaces.get(cache_name_fc2) if cache_name_fc2 is not None else None - ) + fc1_weight_workspace = None + fc2_weight_workspace = None + if cache_name_fc1 is not None and not (use_compiled_op and is_first_microbatch): + fc1_weight_workspace = self._fp8_workspaces.get(cache_name_fc1) + fc2_weight_workspace = self._fp8_workspaces.get(cache_name_fc2) dgrad_use_split_accumulator = _2X_ACC_DGRAD wgrad_use_split_accumulator = _2X_ACC_WGRAD + recipe_float8_block_scaling = False + recipe_custom = False + recipe_dbias_dact_fusion = False if self.fp8: _recipe = FP8GlobalStateManager.get_fp8_recipe() backward_override = _recipe.backward_override @@ -2725,6 +3202,9 @@ def forward( dgrad_use_split_accumulator = _recipe.fp8_gemm_dgrad.use_split_accumulator if hasattr(_recipe, "fp8_gemm_wgrad"): wgrad_use_split_accumulator = _recipe.fp8_gemm_wgrad.use_split_accumulator + recipe_float8_block_scaling = _recipe.float8_block_scaling() + recipe_custom = _recipe.custom() + recipe_dbias_dact_fusion = _recipe.delayed() or _recipe.mxfp8() else: backward_override = None @@ -2744,6 +3224,24 @@ def forward( fc2_bias_tensor = ( fc2_bias if self.apply_bias and not self.gemm_bias_unfused_add else None ) + if use_compiled_op: + fallback_reason = self._compile_eager_fallback_reason( + inp, + skip_fp8_weight_update, + fp8_output, + is_grad_enabled, + debug, + quantizers, + ) + if fallback_reason is not None: + warn_compile_eager_fallback(fallback_reason) + torch._dynamo.graph_break( + msg=f"te.LayerNormMLP falling back to eager: {fallback_reason}" + ) + use_compiled_op = False + + wgrad_store = None if use_compiled_op else self.wgrad_store + fwd_args = LayerNormMLPFwdArgs( # tensors inp=inp, @@ -2804,6 +3302,9 @@ def forward( backward_override=backward_override, dgrad_use_split_accumulator=dgrad_use_split_accumulator, wgrad_use_split_accumulator=wgrad_use_split_accumulator, + recipe_float8_block_scaling=recipe_float8_block_scaling, + recipe_custom=recipe_custom, + recipe_dbias_dact_fusion=recipe_dbias_dact_fusion, debug=debug, # weight-workspace caching is_first_microbatch=is_first_microbatch, @@ -2827,7 +3328,7 @@ def forward( is_fsdp2=self.is_fsdp2, # weight-grad scheduling fuse_wgrad_accumulation=self.fuse_wgrad_accumulation, - wgrad_store=self.wgrad_store, + wgrad_store=wgrad_store, # activation checkpointing checkpoint=self.checkpoint, fp8_meta=self.fp8_meta if self.checkpoint else None, @@ -2837,28 +3338,21 @@ def forward( is_grad_enabled=is_grad_enabled, ) - if is_grad_enabled: - out, ln_out, new_fc1_ws, new_fc2_ws = _LayerNormMLP.apply( - inp, - self.layer_norm_weight, - self.layer_norm_bias, - fc1_weight, - fc1_bias, - fc2_weight, - fc2_bias_tensor, - fwd_args, - ) + if use_compiled_op: + out, ln_out, new_fc1_ws, new_fc2_ws = _layernorm_mlp_op(fwd_args) else: - out, ln_out, new_fc1_ws, new_fc2_ws = _LayerNormMLP.forward( - None, - inp, - self.layer_norm_weight, - self.layer_norm_bias, - fc1_weight, - fc1_bias, - fc2_weight, - fc2_bias_tensor, + out, ln_out, new_fc1_ws, new_fc2_ws = _layernorm_mlp_eager( + ( + inp, + self.layer_norm_weight, + self.layer_norm_bias, + fc1_weight, + fc1_bias, + fc2_weight, + fc2_bias_tensor, + ), fwd_args, + is_grad_enabled, ) if new_fc1_ws is not None and cache_name_fc1 is not None: @@ -2884,6 +3378,44 @@ def forward( return out, ln_out return out + def _compile_eager_fallback_reason( + self, + inp: torch.Tensor, + skip_fp8_weight_update: Optional[torch.Tensor], + fp8_output: bool, + is_grad_enabled: bool, + debug: bool, + quantizers: Sequence[Optional[Quantizer]], + ) -> Optional[str]: + """Return a fallback reason after preparing the forward quantizers.""" + if debug: + return "debug instrumentation (nvidia-dlfw-inspect)" + if self.checkpoint and is_grad_enabled: + return "activation checkpointing (checkpoint=True)" + if isinstance(inp, (QuantizedTensor, QuantizedTensorStorage)): + return "a quantized input tensor" + if self.fsdp_group is not None: + return "manual TE FSDP (fsdp_group); use FSDP2 or MCore FSDP" + if ( + fp8_output + and is_grad_enabled + and (inp.requires_grad or any(p.requires_grad for p in self.parameters())) + ): + return "differentiable fp8_output=True" + if is_cpu_offload_enabled(): + return "CPU activation offloading" + # A queued store can be enabled between forward and backward. + if self.wgrad_store is not None and ( + self.wgrad_store.context is not None or self.wgrad_store.delay_wgrad_compute() + ): + return "delayed wgrad compute (wgrad_store)" + if self.fuse_wgrad_accumulation: + return "fuse_wgrad_accumulation (main_grad)" + fp8 = FP8GlobalStateManager.is_fp8_enabled() + if fp8 and skip_fp8_weight_update is not None: + return "GPU-controlled FP8 weight cache updates" + return compile_unsupported_quantizer_reason(quantizers) + def _get_quantizers(self, fp8_output, is_grad_enabled): if self.fp8: self._warn_missing_output_quantizer_role(fp8_output, False) diff --git a/transformer_engine/pytorch/module/linear.py b/transformer_engine/pytorch/module/linear.py index 082e88a2c76..8cd4cf0d1e3 100644 --- a/transformer_engine/pytorch/module/linear.py +++ b/transformer_engine/pytorch/module/linear.py @@ -5,7 +5,7 @@ """Linear API""" from dataclasses import dataclass, replace as dataclass_replace -from typing import Any, Callable, Dict, Optional, Tuple, Union, List +from typing import Sequence, Any, Callable, Dict, Optional, Tuple, Union, List import warnings import weakref @@ -32,8 +32,11 @@ _2X_ACC_WGRAD, ) from ._common import ( + compile_unsupported_quantizer_reason, can_reconstruct_wgrad_input_from_original, check_fp8_reduce_and_update, + fake_workspace_valid, + get_output_first_dim_size, noop_cat, set_quantizer_amax_reduction_group, set_quantizer_usage_for_wgrad_all_gather, @@ -90,7 +93,6 @@ TensorSpec, TensorOrQuantized, register_custom_op_with_autograd, - is_value_opaque_quantizer, ) from ..tensor.float8_tensor import Float8CurrentScalingQuantizer, Float8Quantizer from ..tensor.mxfp8_tensor import MXFP8Quantizer @@ -182,54 +184,6 @@ class LinearFwdArgs: cpu_offloading: bool is_grad_enabled: bool - def compile_unsupported_reason(self) -> Optional[str]: - """Reason this config can't use the torch.compile custom-op path (else None).""" - if self.debug: - return "debug instrumentation (nvidia-dlfw-inspect)" - if is_distributed_weight(self.weight): - return "a DistributedWeight (custom weight parallelism, e.g. GTP)" - if isinstance(self.inp, (QuantizedTensor, QuantizedTensorStorage)): - return "a quantized input tensor" - if self.fsdp_group is not None: - return "manual TE FSDP (fsdp_group); use FSDP2 or MCore FSDP" - if ( - self.fp8_output - and self.is_grad_enabled - and (self.input_requires_grad or self.weight_requires_grad or self.bias_requires_grad) - ): - return "differentiable fp8_output=True" - if self.cpu_offloading: - return "CPU activation offloading" - if self.wgrad_store is not None: - # Non-None only when delayed wgrad compute is on (see Linear.forward). - return "delayed wgrad compute (wgrad_store)" - if ( - self.grad_input_quantizer is not None - and self.is_grad_enabled - and self.input_requires_grad - and not (self.ub_overlap_rs_dgrad or self.ub_bulk_wgrad) - ): - # A quantized dgrad can't cross the op boundary: grads are packed - # one plain Tensor[] slot each (_pack_bwd_result). - return "a quantized input grad (fp8_grad=True)" - if self.fp8 and self.skip_fp8_weight_update is not None: - return "GPU-controlled FP8 weight cache updates" - if self.fuse_wgrad_accumulation: - return "fuse_wgrad_accumulation (main_grad)" - for quantizer in ( - self.input_quantizer, - self.weight_quantizer, - self.output_quantizer, - self.grad_input_quantizer, - self.grad_weight_quantizer, - self.grad_output_quantizer, - ): - # e.g. delayed-scaling Float8Quantizer and unregistered custom-recipe - # quantizers are not value-opaque and can't cross the custom-op boundary. - if quantizer is not None and not is_value_opaque_quantizer(quantizer): - return "a quantizer not registered as a torch.compile value-opaque type" - return None - @dataclass(slots=True) class LinearBwdArgs: @@ -304,6 +258,10 @@ class LinearBwdArgs: # --- Per-backward scratch state (populated inside _linear_backward_impl) --- ub_obj_gradout: Optional[Any] = None + def setup_grad_outputs(self, grads: Sequence[Any]) -> None: + """Unpack gradients in forward-output order, ignoring weight workspaces.""" + self.grad_output, _ = grads + def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: """Pull saved tensors from ``ctx`` into the fields backward consumes.""" ( @@ -316,40 +274,6 @@ def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: ) # pylint: disable=unbalanced-tuple-unpacking -def _out_leading_from_inp(leading: int, args: Union[LinearFwdArgs, LinearBwdArgs]) -> int: - """Output's leading (sequence) dim from the input's: sequence parallelism - gathers it (column-parallel) or scatters it (row-parallel).""" - if not args.sequence_parallel: - return leading - if args.parallel_mode == "column": - return leading * args.tp_size - if args.parallel_mode == "row": - return leading // args.tp_size - return leading - - -def _inp_leading_from_out(leading: int, args: Union[LinearFwdArgs, LinearBwdArgs]) -> int: - """Inverse of :func:`_out_leading_from_inp`.""" - if not args.sequence_parallel: - return leading - if args.parallel_mode == "column": - return leading // args.tp_size - if args.parallel_mode == "row": - return leading * args.tp_size - return leading - - -def _fake_workspace_valid(workspace: TensorSpec, quantizer: Optional[Quantizer]) -> bool: - """Spec-level mirror of ``_is_weight_workspace_valid``: the cached workspace - must already hold every inner buffer the quantizer's current usage needs.""" - if quantizer is None: - return True - required = TensorSpec( - shape=workspace.shape, dtype=workspace.dtype, quantizer=quantizer, device=workspace.device - ).inner_names() - return set(required) <= set(workspace.inner_names()) - - def _linear_forward_impl( args: LinearFwdArgs, ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple], Optional[Dict]]: @@ -862,7 +786,7 @@ def _linear_forward_fake( else: weightmat_is_storage = True workspace = args.weight_workspace - if workspace is not None and not _fake_workspace_valid(workspace, weight_quantizer): + if workspace is not None and not fake_workspace_valid(workspace, weight_quantizer): # quantize_weight drops a stale workspace and builds a new one. workspace = None if workspace is not None: @@ -894,7 +818,7 @@ def _linear_forward_fake( # Output tensor: y = x @ w^T (quantized iff an output quantizer is set). # ------------------------------------------------------ inp_leading = inp.shape[0] if len(inp.shape) > 1 else 1 - out_leading = _out_leading_from_inp(inp_leading, args) + out_leading = get_output_first_dim_size(inp_leading, args) out = TensorSpec( shape=( (out_features,) @@ -1021,10 +945,7 @@ def _linear_setup_ctx( bwd_args.use_bias = bias is not None bwd_args.requires_dgrad = fwd_args.input_requires_grad bwd_args.requires_wgrad = fwd_args.weight_requires_grad - # Don't store inp_shape in the value bundle: under torch.compile(dynamic=True) - # inp.shape contains SymInt dims which are not hashable in OpaqueValueBundle. - # The backward reconstructs inp_shape from grad_output + weight + SP config. - bwd_args.inp_shape = None + bwd_args.inp_shape = inp.shape # Numerical / dtype config bwd_args.activation_dtype = fwd_args.activation_dtype @@ -1176,16 +1097,6 @@ def _linear_backward_impl(args: LinearBwdArgs) -> Tuple[Union[torch.Tensor, None ) nvtx_range_pop(f"{nvtx_label}.fsdp_gather") - # Reconstruct inp_shape when not stored (compiled mode with dynamic shapes). - if bwd_args.inp_shape is None: - in_features = saved_weight.shape[-1] - inp_leading = _inp_leading_from_out(grad_output.shape[0], bwd_args) - bwd_args.inp_shape = ( - torch.Size([in_features]) - if grad_output.ndim == 1 - else torch.Size([inp_leading, *grad_output.shape[1:-1], in_features]) - ) - # Configure Userbuffers communication (comm+GEMM overlap) bwd_args.ub_obj_gradout = None ub_obj_dgrad = None @@ -1769,20 +1680,13 @@ def _linear_backward_fake( dgrad = None if args.requires_dgrad: - # Input shape rederived from grad_output + SP config (inp_shape is not - # stored: torch.Size with SymInt cannot cross in OpaqueValueBundle). - dgrad_leading = _inp_leading_from_out(args.grad_output.shape[0], args) # Under UB reduce-scatter or bulk-wgrad overlap the returned dgrad is a # plain tensor; the quantizer only feeds the comm buffer. dgrad_quantizer = ( None if (args.ub_overlap_rs_dgrad or args.ub_bulk_wgrad) else args.grad_input_quantizer ) dgrad = TensorSpec( - shape=( - (in_features,) - if len(args.grad_output.shape) == 1 - else (dgrad_leading, *args.grad_output.shape[1:-1], in_features) - ), + shape=args.inp_shape, dtype=out_dtype, quantizer=dgrad_quantizer, device=args.grad_output.device, @@ -1890,7 +1794,7 @@ def backward( ) -> Tuple[Union[torch.Tensor, None], ...]: """Backward pass: compute gradients and reduce FP8 scaling factors.""" bwd_args: LinearBwdArgs = ctx.backward_objects - bwd_args.grad_output = grad_output + bwd_args.setup_grad_outputs((grad_output, _grad_weight_workspace)) bwd_args.setup_saved_tensors(ctx) nvtx_label = "transformer_engine._Linear.backward" if bwd_args.ub_name is not None: @@ -2418,16 +2322,6 @@ def forward( if get_ub_is_fp8(self.ub_name + "_dgrad", FP8GlobalStateManager.is_fp8_enabled()): fp8_grad = True - if torch.compiler.is_compiling() and _linear_op is not None: - reason = self._compile_eager_fallback_reason( - inp, skip_fp8_weight_update, fp8_output, fp8_grad, is_grad_enabled, debug - ) - if reason is not None: - # A break inside the try/finally below would skip the whole frame. - warn_compile_eager_fallback(reason) - torch._dynamo.graph_break(msg=f"te.Linear falling back to eager: {reason}") - return self._forward_eager_fallback(inp, is_first_microbatch, fp8_output, fp8_grad) - inp = self.prepare_forward(inp, allow_non_contiguous=isinstance(inp, QuantizedTensor)) try: weight_tensor, bias_tensor = self._get_weight_and_bias_tensors() @@ -2570,8 +2464,15 @@ def forward( ) if use_compiled_op: - # Safety net for quantizer-dependent conditions only. - fallback_reason = fwd_args.compile_unsupported_reason() + fallback_reason = self._compile_eager_fallback_reason( + inp, + skip_fp8_weight_update, + fp8_output, + fp8_grad, + is_grad_enabled, + debug, + quantizers, + ) if fallback_reason is not None: warn_compile_eager_fallback(fallback_reason) torch._dynamo.graph_break( @@ -2670,9 +2571,9 @@ def _compile_eager_fallback_reason( fp8_grad: bool, is_grad_enabled: bool, debug: bool, + quantizers: Sequence[Optional[Quantizer]], ) -> Optional[str]: - """Why this call can't use the compiled op (else None), decided before - prepare_forward. Quantizer checks stay in compile_unsupported_reason.""" + """Return a fallback reason after preparing the forward quantizers.""" if debug: return "debug instrumentation (nvidia-dlfw-inspect)" weight_tensor, bias_tensor = self._get_weight_and_bias_tensors() @@ -2706,24 +2607,7 @@ def _compile_eager_fallback_reason( return "a quantized input grad (fp8_grad=True)" if fp8 and skip_fp8_weight_update is not None: return "GPU-controlled FP8 weight cache updates" - return None - - @torch._dynamo.disable - def _forward_eager_fallback( - self, - inp: torch.Tensor, - is_first_microbatch: Optional[bool], - fp8_output: bool, - fp8_grad: bool, - ) -> Union[torch.Tensor, Tuple[torch.Tensor, ...]]: - """Re-run forward outside Dynamo (unsupported-config fallback).""" - return Linear.forward( - self, - inp, - is_first_microbatch=is_first_microbatch, - fp8_output=fp8_output, - fp8_grad=fp8_grad, - ) + return compile_unsupported_quantizer_reason(quantizers) def _get_weight_and_bias_tensors(self) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: # Get concatenated weight and bias tensors From 3fda6d9863881d3018b01d3ce8e92dcb505a7a6d Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Fri, 30 Oct 2026 13:18:34 +0100 Subject: [PATCH 02/12] Fix captured FP8 state and share normalization output layout policy Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 14 ++++-- transformer_engine/pytorch/csrc/extensions.h | 4 +- .../pytorch/csrc/extensions/normalization.cpp | 43 ++++++++++++------- .../pytorch/csrc/extensions/pybind.cpp | 5 ++- transformer_engine/pytorch/dynamo/fake.py | 26 ++++++----- .../pytorch/module/layernorm_linear.py | 13 +++--- .../pytorch/module/layernorm_mlp.py | 22 +++++----- transformer_engine/pytorch/module/linear.py | 34 ++++++++------- 8 files changed, 93 insertions(+), 68 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 66c925a25a2..d7a3890b66b 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -2076,13 +2076,13 @@ def test_to_tensor_spec_quantized(factory, shape, swizzled): def test_fake_norm_mxfp8_scale_layout( monkeypatch, normalization, with_2d_quantization, optimize_for_gemm, use_cudnn, shape, compact ): - """RMSNorm's 2D MXFP8 fallback preserves the quantizer's scale layout.""" + """2D MXFP8 fallback preserves the quantizer's scale layout.""" monkeypatch.setenv("NVTE_NORM_FWD_USE_CUDNN", str(int(use_cudnn))) quantizer = MXFP8Quantizer(tex.DType.kFloat8E4M3, with_2d_quantization=with_2d_quantization) quantizer.optimize_for_gemm = optimize_for_gemm inp = TensorSpec(shape=shape, dtype=torch.bfloat16, device=torch.device("cpu")) out, _, _ = fake_norm(inp, quantizer, normalization=normalization) - compact = compact and (normalization == "LayerNorm" or not with_2d_quantization) + compact = compact and not with_2d_quantization expected = optimize_for_gemm and not compact assert out.create_metadata()["nontensor_kwargs"]["with_gemm_swizzled_scales"] == expected assert out.create_tensor()._with_gemm_swizzled_scales == expected @@ -2098,7 +2098,15 @@ def test_fake_norm_mxfp8_scale_layout( @pytest.mark.parametrize("compile_mode", _compile_modes) @pytest.mark.parametrize( "fp8_recipe", - [None, *_all_recipes], + [ + None, + *_all_recipes, + recipe.CustomRecipe( + qfactory=lambda _role: Float8CurrentScalingQuantizer( + tex.DType.kFloat8E4M3, device="cuda" + ) + ), + ], ids=lambda r: "bf16" if r is None else type(r).__name__, ) @pytest.mark.parametrize("module", _LINEAR_MODULES) diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index 324231fc5f2..dbcd851593c 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -340,8 +340,8 @@ py::tuple scaled_dsrelu(const at::Tensor &grad, const at::Tensor &input, * LayerNorm **************************************************************************************************/ -std::optional mxfp8_norm_fused_alignment(bool rmsnorm, bool with_2d_quantization, - bool use_cudnn); +std::optional> normalization_fwd_output_layout( + const std::string &normalization, py::handle quantizer, bool use_cudnn); std::vector layernorm_bwd(const at::Tensor &dz, const at::Tensor &x, const at::Tensor &mu, const at::Tensor &rsigma, diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index cc990576081..e1083df3048 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -10,12 +10,30 @@ namespace transformer_engine::pytorch { -std::optional mxfp8_norm_fused_alignment(bool rmsnorm, bool with_2d_quantization, - bool use_cudnn) { - if (rmsnorm ? with_2d_quantization : !use_cudnn) { +namespace { + +std::optional> normalization_output_layout(const std::string &normalization, + const Quantizer &quantizer, + bool use_cudnn) { + NVTE_CHECK(normalization == "LayerNorm" || normalization == "RMSNorm", + "Unsupported normalization: ", normalization); + const auto *mxfp8 = dynamic_cast(&quantizer); + if (mxfp8 == nullptr || mxfp8->with_2d_quantization || + (normalization == "LayerNorm" && !use_cudnn)) { return std::nullopt; } - return 128; + return std::pair{128, !use_cudnn && quantizer.optimize_for_gemm}; +} + +} // namespace + +std::optional> normalization_fwd_output_layout( + const std::string &normalization, py::handle quantizer, bool use_cudnn) { + init_extension(); + if (!detail::IsMXFP8Quantizers(quantizer.ptr())) { + return std::nullopt; + } + return normalization_output_layout(normalization, *convert_quantizer(quantizer), use_cudnn); } std::vector layernorm_bwd(const at::Tensor &dz, const at::Tensor &x, @@ -114,11 +132,9 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe if (quantizer.is_none() || IsFloat8Quantizers(quantizer.ptr())) { impl = Impl::FULLY_FUSED; } else if (IsMXFP8Quantizers(quantizer.ptr())) { - const auto mxfp8_quantizer_cpp = static_cast(quantizer_cpp.get()); - const auto alignment = - mxfp8_norm_fused_alignment(false, mxfp8_quantizer_cpp->with_2d_quantization, - transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")); - if (alignment && outer_size % *alignment == 0 && inner_size % *alignment == 0) { + const auto layout = normalization_output_layout( + "LayerNorm", *quantizer_cpp, transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")); + if (layout && outer_size % layout->first == 0 && inner_size % layout->first == 0) { impl = Impl::FULLY_FUSED; } } else if (detail::IsFloat8CurrentScalingQuantizers(quantizer.ptr()) && @@ -358,12 +374,9 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w if (quantizer.is_none() || IsFloat8Quantizers(quantizer.ptr())) { impl = Impl::FULLY_FUSED; } else if (IsMXFP8Quantizers(quantizer.ptr())) { - auto mxfp8_quantizer_cpp = dynamic_cast(quantizer_cpp.get()); - NVTE_CHECK(mxfp8_quantizer_cpp != nullptr, "Could not cast to MXFP8 quantizer"); - const auto alignment = - mxfp8_norm_fused_alignment(true, mxfp8_quantizer_cpp->with_2d_quantization, - transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")); - if (alignment && outer_size % *alignment == 0 && inner_size % *alignment == 0) { + const auto layout = normalization_output_layout( + "RMSNorm", *quantizer_cpp, transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")); + if (layout && outer_size % layout->first == 0 && inner_size % layout->first == 0) { if (transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")) { impl = Impl::FUSED_NORM_QUANT_UNSWIZZLED; } else { diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index a5a5a025c75..6dc36b4c78a 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -402,8 +402,9 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { py::call_guard()); // Other granular functions - m.def("mxfp8_norm_fused_alignment", &transformer_engine::pytorch::mxfp8_norm_fused_alignment, - py::arg("rmsnorm"), py::arg("with_2d_quantization"), py::arg("use_cudnn")); + m.def("normalization_fwd_output_layout", + &transformer_engine::pytorch::normalization_fwd_output_layout, py::arg("normalization"), + py::arg("quantizer"), py::arg("use_cudnn")); m.def("layernorm_fwd", &transformer_engine::pytorch::layernorm_fwd, "LayerNorm", py::arg("input"), py::arg("weight"), py::arg("bias"), py::arg("eps"), py::arg("ln_out"), py::arg("quantizer"), py::arg("otype"), py::arg("sm_margin"), py::arg("zero_centered_gamma")); diff --git a/transformer_engine/pytorch/dynamo/fake.py b/transformer_engine/pytorch/dynamo/fake.py index 9b70526fadc..29c177f33d0 100644 --- a/transformer_engine/pytorch/dynamo/fake.py +++ b/transformer_engine/pytorch/dynamo/fake.py @@ -14,7 +14,6 @@ from .tensor_spec import TensorSpec from ..quantized_tensor import Quantizer -from ..tensor.mxfp8_tensor import MXFP8Quantizer from ..tensor.nvfp4_tensor import NVFP4Quantizer from ..utils import get_device_compute_capability @@ -112,12 +111,11 @@ def fake_activation( @torch.compiler.assume_constant_result -def _mxfp8_norm_fused_alignment( - normalization: str, with_2d_quantization: bool, use_cudnn: bool -) -> Optional[int]: - return tex.mxfp8_norm_fused_alignment( - normalization == "RMSNorm", with_2d_quantization, use_cudnn - ) +def _normalization_fwd_output_layout( + normalization: str, quantizer: Quantizer, use_cudnn: bool +) -> Optional[tuple[int, bool]]: + """Return the alignment and scale-layout override for a normalization output.""" + return tex.normalization_fwd_output_layout(normalization, quantizer, use_cudnn) def fake_norm( @@ -134,13 +132,13 @@ def fake_norm( input_.shape, dtype or input_.dtype, input_.device, quantizer ) rows, cols = math.prod(input_.shape[:-1]), input_.shape[-1] - if isinstance(quantizer, MXFP8Quantizer): - use_cudnn = bool(int(os.getenv("NVTE_NORM_FWD_USE_CUDNN", "0") or "0")) - alignment = _mxfp8_norm_fused_alignment( - normalization, quantizer.with_2d_quantization, use_cudnn - ) - if use_cudnn and alignment is not None and rows % alignment == 0 and cols % alignment == 0: - out.with_gemm_swizzled_scales = False + if quantizer is not None: + use_cudnn = bool(int(os.environ.get("NVTE_NORM_FWD_USE_CUDNN", "0") or "0")) + layout = _normalization_fwd_output_layout(normalization, quantizer, use_cudnn) + if layout is not None: + alignment, with_gemm_swizzled_scales = layout + if rows % alignment == 0 and cols % alignment == 0: + out.with_gemm_swizzled_scales = with_gemm_swizzled_scales mean = ( TensorSpec(shape=(rows,), dtype=torch.float32, device=input_.device) if normalization == "LayerNorm" diff --git a/transformer_engine/pytorch/module/layernorm_linear.py b/transformer_engine/pytorch/module/layernorm_linear.py index a61afe9f154..d9e3925a2b7 100644 --- a/transformer_engine/pytorch/module/layernorm_linear.py +++ b/transformer_engine/pytorch/module/layernorm_linear.py @@ -156,6 +156,7 @@ class LayerNormLinearFwdArgs: fp8: bool fp8_calibration: bool backward_override: Optional[str] + fprop_use_split_accumulator: bool dgrad_use_split_accumulator: bool wgrad_use_split_accumulator: bool debug: bool @@ -379,7 +380,7 @@ def _layernorm_linear_forward_impl( ln_bias = cast_if_needed(ln_bias, activation_dtype) nvtx_range_pop(f"{nvtx_label}.norm_input_cast") - if is_cpu_offload_enabled(): + if cpu_offloading: start_offload(inputmat) tp_world_size = get_distributed_world_size(tp_group) @@ -561,11 +562,7 @@ def _layernorm_linear_forward_impl( weight_quantizer.calibrate(weight) # Choose whether to use GEMM kernel with split accumulator - use_split_accumulator = _2X_ACC_FPROP - if fp8: - recipe = FP8GlobalStateManager.get_fp8_recipe() - if hasattr(recipe, "fp8_gemm_fprop"): - use_split_accumulator = recipe.fp8_gemm_fprop.use_split_accumulator + use_split_accumulator = args.fprop_use_split_accumulator # Configure output quantizer if output_quantizer is not None: @@ -2504,11 +2501,14 @@ def forward( if cache_name is not None and not (use_compiled_op and is_first_microbatch): weight_workspace = self._fp8_workspaces.get(cache_name) + fprop_use_split_accumulator = _2X_ACC_FPROP dgrad_use_split_accumulator = _2X_ACC_DGRAD wgrad_use_split_accumulator = _2X_ACC_WGRAD if self.fp8: _recipe = FP8GlobalStateManager.get_fp8_recipe() backward_override = _recipe.backward_override + if hasattr(_recipe, "fp8_gemm_fprop"): + fprop_use_split_accumulator = _recipe.fp8_gemm_fprop.use_split_accumulator if hasattr(_recipe, "fp8_gemm_dgrad"): dgrad_use_split_accumulator = _recipe.fp8_gemm_dgrad.use_split_accumulator if hasattr(_recipe, "fp8_gemm_wgrad"): @@ -2595,6 +2595,7 @@ def forward( fp8=self.fp8, fp8_calibration=self.fp8_calibration, backward_override=backward_override, + fprop_use_split_accumulator=fprop_use_split_accumulator, dgrad_use_split_accumulator=dgrad_use_split_accumulator, wgrad_use_split_accumulator=wgrad_use_split_accumulator, debug=debug, diff --git a/transformer_engine/pytorch/module/layernorm_mlp.py b/transformer_engine/pytorch/module/layernorm_mlp.py index d22fe30ad2a..622b2818b7e 100644 --- a/transformer_engine/pytorch/module/layernorm_mlp.py +++ b/transformer_engine/pytorch/module/layernorm_mlp.py @@ -254,9 +254,10 @@ class LayerNormMLPFwdArgs: fp8: bool fp8_calibration: bool backward_override: Optional[str] + fprop_use_split_accumulator: bool dgrad_use_split_accumulator: bool wgrad_use_split_accumulator: bool - # Recipe properties the backward needs (the recipe itself can't cross the op boundary). + # The recipe itself can't cross the op boundary. recipe_float8_block_scaling: bool recipe_custom: bool recipe_dbias_dact_fusion: bool @@ -570,7 +571,7 @@ def _layernorm_mlp_forward_impl( assert_dim_for_fp8_exec(inputmat, fc1_weight, fc2_weight) activation_func = _act_func( - activation, FP8GlobalStateManager.get_fp8_recipe() if fp8 else None + activation, dbias_fusion=fp8 and args.recipe_dbias_dact_fusion )[0] # Cast for native AMP @@ -578,7 +579,7 @@ def _layernorm_mlp_forward_impl( ln_weight_cast = cast_if_needed(ln_weight, activation_dtype) if ln_bias is not None: ln_bias = cast_if_needed(ln_bias, activation_dtype) - if is_cpu_offload_enabled(): + if cpu_offloading: start_offload(inputmat) tp_world_size = get_distributed_world_size(tp_group) @@ -597,11 +598,7 @@ def _layernorm_mlp_forward_impl( ub_overlap_rs = ub_overlap_rs and is_grad_enabled # Choose whether to use GEMM kernel with split accumulator - use_split_accumulator = _2X_ACC_FPROP - if fp8: - recipe = FP8GlobalStateManager.get_fp8_recipe() - if hasattr(recipe, "fp8_gemm_fprop"): - use_split_accumulator = recipe.fp8_gemm_fprop.use_split_accumulator + use_split_accumulator = args.fprop_use_split_accumulator # Configure quantizer for norm output if fp8: @@ -847,12 +844,11 @@ def _layernorm_mlp_forward_impl( else: fc1_out, *_ = fc1_outputs if fp8: - recipe = FP8GlobalStateManager.get_fp8_recipe() - if recipe.float8_block_scaling(): + if args.recipe_float8_block_scaling: # tex.quantize does not support GELU fusion for blockwise act_out = activation_func(fc1_out, None, **act_params) act_out = tex.quantize(act_out, fc2_input_quantizer) - elif recipe.custom(): + elif args.recipe_custom: # tex.quantize does not support custom quantizers act_out = activation_func(fc1_out, None, **act_params) act_out = fc2_input_quantizer(act_out) @@ -3190,6 +3186,7 @@ def forward( fc1_weight_workspace = self._fp8_workspaces.get(cache_name_fc1) fc2_weight_workspace = self._fp8_workspaces.get(cache_name_fc2) + fprop_use_split_accumulator = _2X_ACC_FPROP dgrad_use_split_accumulator = _2X_ACC_DGRAD wgrad_use_split_accumulator = _2X_ACC_WGRAD recipe_float8_block_scaling = False @@ -3198,6 +3195,8 @@ def forward( if self.fp8: _recipe = FP8GlobalStateManager.get_fp8_recipe() backward_override = _recipe.backward_override + if hasattr(_recipe, "fp8_gemm_fprop"): + fprop_use_split_accumulator = _recipe.fp8_gemm_fprop.use_split_accumulator if hasattr(_recipe, "fp8_gemm_dgrad"): dgrad_use_split_accumulator = _recipe.fp8_gemm_dgrad.use_split_accumulator if hasattr(_recipe, "fp8_gemm_wgrad"): @@ -3300,6 +3299,7 @@ def forward( fp8=self.fp8, fp8_calibration=self.fp8_calibration, backward_override=backward_override, + fprop_use_split_accumulator=fprop_use_split_accumulator, dgrad_use_split_accumulator=dgrad_use_split_accumulator, wgrad_use_split_accumulator=wgrad_use_split_accumulator, recipe_float8_block_scaling=recipe_float8_block_scaling, diff --git a/transformer_engine/pytorch/module/linear.py b/transformer_engine/pytorch/module/linear.py index 8cd4cf0d1e3..0b73a4de17d 100644 --- a/transformer_engine/pytorch/module/linear.py +++ b/transformer_engine/pytorch/module/linear.py @@ -143,8 +143,11 @@ class LinearFwdArgs: fp8_output: bool save_original_input: bool backward_override: Optional[str] + fprop_use_split_accumulator: bool dgrad_use_split_accumulator: bool wgrad_use_split_accumulator: bool + recipe_custom: bool + fp8_recompute_forward: bool custom: bool debug: bool @@ -324,7 +327,7 @@ def _linear_forward_impl( # to reuse the high-precision DPA output saved by attention backward. Validate # the resolved quantizer since this is not necessarily an informed user opt-in. if isinstance(input_quantizer, Float8Quantizer): - if FP8GlobalStateManager.get_fp8_recipe().custom(): + if args.recipe_custom: warnings.warn( "save_original_input is incompatible with delayed-scaling quantizers " "(Float8Quantizer). Disabling save_original_input for this module.", @@ -448,7 +451,7 @@ def _linear_forward_impl( inputmat = cast_if_needed(inp, activation_dtype) # Cast for AMP inputmat_total = inputmat - if is_cpu_offload_enabled(): + if cpu_offloading: start_offload(inputmat) nvtx_range_pop(f"{nvtx_label}.input_cast_comm") # ------------------------------------------------------ @@ -477,10 +480,7 @@ def _linear_forward_impl( if backward_override is not None: columnwise_usage = False if not columnwise_usage: - columnwise_usage = ( - is_fp8_activation_recompute_enabled() - and not in_fp8_activation_recompute_phase() - ) + columnwise_usage = args.fp8_recompute_forward weight_quantizer.set_usage(rowwise=True, columnwise=columnwise_usage) elif isinstance(weight, QuantizedTensor): weight_quantizer = weight._quantizer @@ -519,11 +519,7 @@ def _linear_forward_impl( weight_quantizer.calibrate(weight) # Choose whether to use GEMM kernel with split accumulator - use_split_accumulator = _2X_ACC_FPROP - if fp8: - recipe = FP8GlobalStateManager.get_fp8_recipe() - if hasattr(recipe, "fp8_gemm_fprop"): - use_split_accumulator = recipe.fp8_gemm_fprop.use_split_accumulator + use_split_accumulator = args.fprop_use_split_accumulator # Configure output quantizer if output_quantizer is not None: @@ -770,10 +766,7 @@ def _linear_forward_fake( if args.backward_override is not None: columnwise_usage = False if not columnwise_usage: - columnwise_usage = ( - is_fp8_activation_recompute_enabled() - and not in_fp8_activation_recompute_phase() - ) + columnwise_usage = args.fp8_recompute_forward weight_quantizer.set_usage(rowwise=True, columnwise=columnwise_usage) elif weight.is_quantized: weight_quantizer = weight.quantizer @@ -2364,15 +2357,20 @@ def forward( if cache_name is not None and not (use_compiled_op and is_first_microbatch): weight_workspace = self._fp8_workspaces.get(cache_name) + fprop_use_split_accumulator = _2X_ACC_FPROP dgrad_use_split_accumulator = _2X_ACC_DGRAD wgrad_use_split_accumulator = _2X_ACC_WGRAD + recipe_custom = False if self.fp8: _recipe = FP8GlobalStateManager.get_fp8_recipe() backward_override = _recipe.backward_override + if hasattr(_recipe, "fp8_gemm_fprop"): + fprop_use_split_accumulator = _recipe.fp8_gemm_fprop.use_split_accumulator if hasattr(_recipe, "fp8_gemm_dgrad"): dgrad_use_split_accumulator = _recipe.fp8_gemm_dgrad.use_split_accumulator if hasattr(_recipe, "fp8_gemm_wgrad"): wgrad_use_split_accumulator = _recipe.fp8_gemm_wgrad.use_split_accumulator + recipe_custom = _recipe.custom() else: backward_override = None custom = is_custom(input_quantizer) or is_custom(weight_quantizer) @@ -2428,8 +2426,14 @@ def forward( fp8_output=fp8_output, save_original_input=self.save_original_input, backward_override=backward_override, + fprop_use_split_accumulator=fprop_use_split_accumulator, dgrad_use_split_accumulator=dgrad_use_split_accumulator, wgrad_use_split_accumulator=wgrad_use_split_accumulator, + recipe_custom=recipe_custom, + fp8_recompute_forward=( + is_fp8_activation_recompute_enabled() + and not in_fp8_activation_recompute_phase() + ), custom=custom, debug=debug, # weight-workspace caching From 0a7f8bf4a305aea48459f6def6ac76751369636d Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Fri, 30 Oct 2026 13:42:50 +0100 Subject: [PATCH 03/12] Gate custom compile recipe on FP8 support Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index d7a3890b66b..b4c82ee4b48 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -2098,15 +2098,18 @@ def test_fake_norm_mxfp8_scale_layout( @pytest.mark.parametrize("compile_mode", _compile_modes) @pytest.mark.parametrize( "fp8_recipe", - [ - None, - *_all_recipes, - recipe.CustomRecipe( - qfactory=lambda _role: Float8CurrentScalingQuantizer( - tex.DType.kFloat8E4M3, device="cuda" + [None, *_all_recipes] + + ( + [ + recipe.CustomRecipe( + qfactory=lambda _role: Float8CurrentScalingQuantizer( + tex.DType.kFloat8E4M3, device="cuda" + ) ) - ), - ], + ] + if fp8_available + else [] + ), ids=lambda r: "bf16" if r is None else type(r).__name__, ) @pytest.mark.parametrize("module", _LINEAR_MODULES) From 3add45caa70b633b0192270410f51cb10cfac891 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:44:12 +0000 Subject: [PATCH 04/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- transformer_engine/pytorch/module/layernorm_mlp.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/transformer_engine/pytorch/module/layernorm_mlp.py b/transformer_engine/pytorch/module/layernorm_mlp.py index 622b2818b7e..3cad214627d 100644 --- a/transformer_engine/pytorch/module/layernorm_mlp.py +++ b/transformer_engine/pytorch/module/layernorm_mlp.py @@ -570,9 +570,7 @@ def _layernorm_mlp_forward_impl( if fp8: assert_dim_for_fp8_exec(inputmat, fc1_weight, fc2_weight) - activation_func = _act_func( - activation, dbias_fusion=fp8 and args.recipe_dbias_dact_fusion - )[0] + activation_func = _act_func(activation, dbias_fusion=fp8 and args.recipe_dbias_dact_fusion)[0] # Cast for native AMP inputmat = cast_if_needed(inputmat, activation_dtype) From 9108ab557b828741f1209124a8a2f76582ad6fa1 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Fri, 30 Oct 2026 13:59:08 +0100 Subject: [PATCH 05/12] Clarify normalization scale layout override naming Signed-off-by: Pawel Gadzinski --- transformer_engine/pytorch/csrc/extensions.h | 2 +- .../pytorch/csrc/extensions/normalization.cpp | 15 ++++++++------- .../pytorch/csrc/extensions/pybind.cpp | 4 ++-- transformer_engine/pytorch/dynamo/fake.py | 8 ++++---- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index dbcd851593c..1924c9c081f 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -340,7 +340,7 @@ py::tuple scaled_dsrelu(const at::Tensor &grad, const at::Tensor &input, * LayerNorm **************************************************************************************************/ -std::optional> normalization_fwd_output_layout( +std::optional> normalization_scale_layout_override( const std::string &normalization, py::handle quantizer, bool use_cudnn); std::vector layernorm_bwd(const at::Tensor &dz, const at::Tensor &x, diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index e1083df3048..de2ed37420c 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -12,9 +12,9 @@ namespace transformer_engine::pytorch { namespace { -std::optional> normalization_output_layout(const std::string &normalization, - const Quantizer &quantizer, - bool use_cudnn) { +// (row/column alignment, swizzled scales); nullopt keeps the quantizer's default layout. +std::optional> normalization_scale_layout_override_impl( + const std::string &normalization, const Quantizer &quantizer, bool use_cudnn) { NVTE_CHECK(normalization == "LayerNorm" || normalization == "RMSNorm", "Unsupported normalization: ", normalization); const auto *mxfp8 = dynamic_cast(&quantizer); @@ -27,13 +27,14 @@ std::optional> normalization_output_layout(const std::st } // namespace -std::optional> normalization_fwd_output_layout( +std::optional> normalization_scale_layout_override( const std::string &normalization, py::handle quantizer, bool use_cudnn) { init_extension(); if (!detail::IsMXFP8Quantizers(quantizer.ptr())) { return std::nullopt; } - return normalization_output_layout(normalization, *convert_quantizer(quantizer), use_cudnn); + return normalization_scale_layout_override_impl(normalization, *convert_quantizer(quantizer), + use_cudnn); } std::vector layernorm_bwd(const at::Tensor &dz, const at::Tensor &x, @@ -132,7 +133,7 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe if (quantizer.is_none() || IsFloat8Quantizers(quantizer.ptr())) { impl = Impl::FULLY_FUSED; } else if (IsMXFP8Quantizers(quantizer.ptr())) { - const auto layout = normalization_output_layout( + const auto layout = normalization_scale_layout_override_impl( "LayerNorm", *quantizer_cpp, transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")); if (layout && outer_size % layout->first == 0 && inner_size % layout->first == 0) { impl = Impl::FULLY_FUSED; @@ -374,7 +375,7 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w if (quantizer.is_none() || IsFloat8Quantizers(quantizer.ptr())) { impl = Impl::FULLY_FUSED; } else if (IsMXFP8Quantizers(quantizer.ptr())) { - const auto layout = normalization_output_layout( + const auto layout = normalization_scale_layout_override_impl( "RMSNorm", *quantizer_cpp, transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")); if (layout && outer_size % layout->first == 0 && inner_size % layout->first == 0) { if (transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")) { diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index 6dc36b4c78a..3aa55833dc5 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -402,8 +402,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { py::call_guard()); // Other granular functions - m.def("normalization_fwd_output_layout", - &transformer_engine::pytorch::normalization_fwd_output_layout, py::arg("normalization"), + m.def("normalization_scale_layout_override", + &transformer_engine::pytorch::normalization_scale_layout_override, py::arg("normalization"), py::arg("quantizer"), py::arg("use_cudnn")); m.def("layernorm_fwd", &transformer_engine::pytorch::layernorm_fwd, "LayerNorm", py::arg("input"), py::arg("weight"), py::arg("bias"), py::arg("eps"), py::arg("ln_out"), py::arg("quantizer"), diff --git a/transformer_engine/pytorch/dynamo/fake.py b/transformer_engine/pytorch/dynamo/fake.py index 29c177f33d0..d9b22f57501 100644 --- a/transformer_engine/pytorch/dynamo/fake.py +++ b/transformer_engine/pytorch/dynamo/fake.py @@ -111,11 +111,11 @@ def fake_activation( @torch.compiler.assume_constant_result -def _normalization_fwd_output_layout( +def _normalization_scale_layout_override( normalization: str, quantizer: Quantizer, use_cudnn: bool ) -> Optional[tuple[int, bool]]: - """Return the alignment and scale-layout override for a normalization output.""" - return tex.normalization_fwd_output_layout(normalization, quantizer, use_cudnn) + """Return (alignment, swizzled scales), or None to keep the quantizer's layout.""" + return tex.normalization_scale_layout_override(normalization, quantizer, use_cudnn) def fake_norm( @@ -134,7 +134,7 @@ def fake_norm( rows, cols = math.prod(input_.shape[:-1]), input_.shape[-1] if quantizer is not None: use_cudnn = bool(int(os.environ.get("NVTE_NORM_FWD_USE_CUDNN", "0") or "0")) - layout = _normalization_fwd_output_layout(normalization, quantizer, use_cudnn) + layout = _normalization_scale_layout_override(normalization, quantizer, use_cudnn) if layout is not None: alignment, with_gemm_swizzled_scales = layout if rows % alignment == 0 and cols % alignment == 0: From 267b0d4f68c4e7f31fc92669031c89b8347ffcb1 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Fri, 30 Oct 2026 14:14:07 +0100 Subject: [PATCH 06/12] Use normalization_scale_layout and explain returned metadata Signed-off-by: Pawel Gadzinski --- transformer_engine/pytorch/csrc/extensions.h | 5 +++-- .../pytorch/csrc/extensions/normalization.cpp | 18 ++++++++++-------- .../pytorch/csrc/extensions/pybind.cpp | 5 ++--- transformer_engine/pytorch/dynamo/fake.py | 6 +++--- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index 1924c9c081f..2221f014bae 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -340,8 +340,9 @@ py::tuple scaled_dsrelu(const at::Tensor &grad, const at::Tensor &input, * LayerNorm **************************************************************************************************/ -std::optional> normalization_scale_layout_override( - const std::string &normalization, py::handle quantizer, bool use_cudnn); +std::optional> normalization_scale_layout(const std::string &normalization, + py::handle quantizer, + bool use_cudnn); std::vector layernorm_bwd(const at::Tensor &dz, const at::Tensor &x, const at::Tensor &mu, const at::Tensor &rsigma, diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index de2ed37420c..a32899fd47d 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -12,8 +12,10 @@ namespace transformer_engine::pytorch { namespace { -// (row/column alignment, swizzled scales); nullopt keeps the quantizer's default layout. -std::optional> normalization_scale_layout_override_impl( +// Return (row/column alignment, GEMM-swizzled scales); apply only to aligned shapes. +// cuDNN emits unswizzled MXFP8 scales even with optimize_for_gemm=true. +// nullopt keeps the quantizer's default layout. +std::optional> normalization_scale_layout_impl( const std::string &normalization, const Quantizer &quantizer, bool use_cudnn) { NVTE_CHECK(normalization == "LayerNorm" || normalization == "RMSNorm", "Unsupported normalization: ", normalization); @@ -27,14 +29,14 @@ std::optional> normalization_scale_layout_override_impl( } // namespace -std::optional> normalization_scale_layout_override( - const std::string &normalization, py::handle quantizer, bool use_cudnn) { +std::optional> normalization_scale_layout(const std::string &normalization, + py::handle quantizer, + bool use_cudnn) { init_extension(); if (!detail::IsMXFP8Quantizers(quantizer.ptr())) { return std::nullopt; } - return normalization_scale_layout_override_impl(normalization, *convert_quantizer(quantizer), - use_cudnn); + return normalization_scale_layout_impl(normalization, *convert_quantizer(quantizer), use_cudnn); } std::vector layernorm_bwd(const at::Tensor &dz, const at::Tensor &x, @@ -133,7 +135,7 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe if (quantizer.is_none() || IsFloat8Quantizers(quantizer.ptr())) { impl = Impl::FULLY_FUSED; } else if (IsMXFP8Quantizers(quantizer.ptr())) { - const auto layout = normalization_scale_layout_override_impl( + const auto layout = normalization_scale_layout_impl( "LayerNorm", *quantizer_cpp, transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")); if (layout && outer_size % layout->first == 0 && inner_size % layout->first == 0) { impl = Impl::FULLY_FUSED; @@ -375,7 +377,7 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w if (quantizer.is_none() || IsFloat8Quantizers(quantizer.ptr())) { impl = Impl::FULLY_FUSED; } else if (IsMXFP8Quantizers(quantizer.ptr())) { - const auto layout = normalization_scale_layout_override_impl( + const auto layout = normalization_scale_layout_impl( "RMSNorm", *quantizer_cpp, transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")); if (layout && outer_size % layout->first == 0 && inner_size % layout->first == 0) { if (transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")) { diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index 3aa55833dc5..9d72aa8b5ed 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -402,9 +402,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { py::call_guard()); // Other granular functions - m.def("normalization_scale_layout_override", - &transformer_engine::pytorch::normalization_scale_layout_override, py::arg("normalization"), - py::arg("quantizer"), py::arg("use_cudnn")); + m.def("normalization_scale_layout", &transformer_engine::pytorch::normalization_scale_layout, + py::arg("normalization"), py::arg("quantizer"), py::arg("use_cudnn")); m.def("layernorm_fwd", &transformer_engine::pytorch::layernorm_fwd, "LayerNorm", py::arg("input"), py::arg("weight"), py::arg("bias"), py::arg("eps"), py::arg("ln_out"), py::arg("quantizer"), py::arg("otype"), py::arg("sm_margin"), py::arg("zero_centered_gamma")); diff --git a/transformer_engine/pytorch/dynamo/fake.py b/transformer_engine/pytorch/dynamo/fake.py index d9b22f57501..0ced4d17c88 100644 --- a/transformer_engine/pytorch/dynamo/fake.py +++ b/transformer_engine/pytorch/dynamo/fake.py @@ -111,11 +111,11 @@ def fake_activation( @torch.compiler.assume_constant_result -def _normalization_scale_layout_override( +def _normalization_scale_layout( normalization: str, quantizer: Quantizer, use_cudnn: bool ) -> Optional[tuple[int, bool]]: """Return (alignment, swizzled scales), or None to keep the quantizer's layout.""" - return tex.normalization_scale_layout_override(normalization, quantizer, use_cudnn) + return tex.normalization_scale_layout(normalization, quantizer, use_cudnn) def fake_norm( @@ -134,7 +134,7 @@ def fake_norm( rows, cols = math.prod(input_.shape[:-1]), input_.shape[-1] if quantizer is not None: use_cudnn = bool(int(os.environ.get("NVTE_NORM_FWD_USE_CUDNN", "0") or "0")) - layout = _normalization_scale_layout_override(normalization, quantizer, use_cudnn) + layout = _normalization_scale_layout(normalization, quantizer, use_cudnn) if layout is not None: alignment, with_gemm_swizzled_scales = layout if rows % alignment == 0 and cols % alignment == 0: From 267639c90f6bc492a0b65bd31781ab5b61cfef02 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 2 Nov 2026 19:20:53 +0100 Subject: [PATCH 07/12] [PyTorch] Honor optimize_for_gemm in quantized outputs Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_quantized_tensor.py | 102 ++++++++++++++++++ tests/pytorch/test_torch_compile.py | 3 +- transformer_engine/pytorch/csrc/common.h | 3 + .../pytorch/csrc/extensions/activation.cpp | 3 + .../pytorch/csrc/extensions/bias.cpp | 5 + .../pytorch/csrc/extensions/cast.cpp | 16 +-- .../pytorch/csrc/extensions/gemm.cpp | 1 + .../pytorch/csrc/extensions/normalization.cpp | 23 ++-- .../pytorch/csrc/extensions/swizzle.cpp | 18 +++- transformer_engine/pytorch/dynamo/fake.py | 72 +------------ .../pytorch/quantized_tensor.py | 3 + 11 files changed, 160 insertions(+), 89 deletions(-) diff --git a/tests/pytorch/test_quantized_tensor.py b/tests/pytorch/test_quantized_tensor.py index 6cd1bd403b1..b4280df0333 100644 --- a/tests/pytorch/test_quantized_tensor.py +++ b/tests/pytorch/test_quantized_tensor.py @@ -8,6 +8,7 @@ import pytest import torch +import transformer_engine_torch as tex import transformer_engine.common.recipe import transformer_engine.pytorch as te @@ -1233,3 +1234,104 @@ def test_restore_real_quantized_storage(monkeypatch, compiled_metadata): assert restored[0] is x and restored[2] is None assert isinstance(tail, tuple) and tail[0] is x torch.testing.assert_close(restored[1].dequantize(), expected, rtol=0, atol=0) + + +@pytest.mark.parametrize( + "quantization,shape", + [ + pytest.param( + "mxfp8", + (128, 128), + marks=pytest.mark.skipif(not mxfp8_available, reason=reason_for_no_mxfp8), + ), + *[ + pytest.param( + name, + shape, + marks=pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4), + ) + for name, shape in [ + ("nvfp4", (32, 128)), + ("nvfp4-rht", (32, 128)), + ("nvfp4-rht", (128, 128)), + ("nvfp4-2d", (128, 128)), + ("nvfp4_row_scaled", (128, 128)), + ("nvfp4_4over6", (32, 128)), + ] + ], + ], +) +@pytest.mark.parametrize("optimize_for_gemm", [False, True]) +@pytest.mark.parametrize( + "operation", + [ + "quantize", + "multi_tensor_quantize", + "split_quantize", + "gelu", + "dgelu", + "scaled_srelu", + "dbias_dgelu", + "bgrad", + "layernorm", + "rmsnorm", + ], +) +def test_native_quantization_scale_layout( + quantization, shape, optimize_for_gemm, operation, monkeypatch +): + """Every producer honors the quantizer's requested scale layout.""" + monkeypatch.setenv("NVTE_NORM_FWD_USE_CUDNN", "1") + if quantization in ("nvfp4-rht", "nvfp4-2d"): + quantizer = NVFP4Quantizer( + with_rht=quantization == "nvfp4-rht", + with_post_rht_amax=quantization == "nvfp4-rht", + with_2d_quantization=quantization == "nvfp4-2d", + stochastic_rounding=False, + with_random_sign_mask=False, + ) + else: + quantizer = make_quantizer(quantization) + inp = torch.randn(shape, dtype=torch.bfloat16, device="cuda") + grad = torch.randn_like(inp) + weight = torch.randn(shape[-1], dtype=inp.dtype, device=inp.device) + bias = torch.randn_like(weight) + scales = torch.rand(shape[0], dtype=torch.float32, device=inp.device) + + def run(out=None): + if operation == "quantize": + return quantizer(inp) + if operation == "multi_tensor_quantize": + return tex.multi_tensor_quantize([inp], [quantizer])[0] + if operation == "split_quantize": + return tex.split_quantize(inp, [shape[0]], [quantizer])[0] + if operation == "gelu": + return tex.gelu(inp, quantizer) + if operation == "dgelu": + return tex.dgelu(grad, inp, quantizer) + if operation == "scaled_srelu": + return tex.scaled_srelu(inp, scales, quantizer) + if operation == "dbias_dgelu": + return tex.dbias_dgelu(grad, inp, quantizer)[1] + if operation == "bgrad": + return tex.bgrad_quantize(inp, quantizer)[1] + if operation == "layernorm": + return tex.layernorm_fwd( + inp, weight, bias, 1e-5, out, quantizer, te.DType.kBFloat16, 0, False + )[0] + if operation == "rmsnorm": + return tex.rmsnorm_fwd(inp, weight, 1e-5, out, quantizer, te.DType.kBFloat16, 0, False)[ + 0 + ] + raise AssertionError(operation) + + quantizer.optimize_for_gemm = False + expected = run().dequantize() + quantizer.optimize_for_gemm = optimize_for_gemm + actual = run() + assert actual._with_gemm_swizzled_scales == optimize_for_gemm + torch.testing.assert_close(actual.dequantize(), expected, atol=0, rtol=0) + if operation in ("layernorm", "rmsnorm"): + assert run(actual) is actual + assert actual._with_gemm_swizzled_scales == optimize_for_gemm + torch.testing.assert_close(actual.dequantize(), expected, atol=0, rtol=0) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 6868de7ef18..e9ab4df8221 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -2970,7 +2970,8 @@ def bgrad(inp): previous_graph_count = len(graphs) dx, db = compiled(inp) expected_db, expected_dx = tex.bgrad_quantize(inp, quantizer) - assert dx._with_gemm_swizzled_scales == expected_dx._with_gemm_swizzled_scales + assert dx._with_gemm_swizzled_scales == optimize_for_gemm + assert expected_dx._with_gemm_swizzled_scales == optimize_for_gemm torch.testing.assert_close(db, expected_db, atol=0, rtol=0) torch.testing.assert_close(dx.dequantize(), expected_dx.dequantize(), atol=0, rtol=0) torch.testing.assert_close( diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index 9a3c6314054..c3ecc35eb1e 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -135,6 +135,9 @@ class Quantizer { virtual void quantize(const TensorWrapper& input, TensorWrapper& out, const std::optional& noop_flag = std::nullopt) = 0; + /*! @brief Apply the requested GEMM scale layout to a computed tensor. */ + void optimize_tensor_for_gemm(TensorWrapper& tensor, py::handle tensor_py) const; + virtual ~Quantizer() = default; DType dtype = DType::kNumTypes; diff --git a/transformer_engine/pytorch/csrc/extensions/activation.cpp b/transformer_engine/pytorch/csrc/extensions/activation.cpp index 86419f0d3ed..cdd56e2e4a7 100644 --- a/transformer_engine/pytorch/csrc/extensions/activation.cpp +++ b/transformer_engine/pytorch/csrc/extensions/activation.cpp @@ -125,6 +125,7 @@ py::object activation_helper(const at::Tensor& input, py::handle quantizer, int NVTE_ERROR("Invalid activation implementation (", static_cast(impl), ")"); } + quantizer_cpp->optimize_tensor_for_gemm(out_nvte, out_py); return out_py; } @@ -241,6 +242,7 @@ py::object dactivation_helper(const at::Tensor& grad_output, const at::Tensor& i NVTE_ERROR("Invalid activation implementation (", static_cast(impl), ")"); } + quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); return grad_input_py; } } // namespace @@ -438,6 +440,7 @@ py::object maybe_quantize(const at::Tensor& tensor, py::handle quantizer) { auto fake_dtype = GetTransformerEngineDType(tensor.scalar_type()); auto [out_nvte, out_py] = quantizer_cpp->create_tensor(shape, fake_dtype); quantizer_cpp->quantize(tensor_nvte, out_nvte); + quantizer_cpp->optimize_tensor_for_gemm(out_nvte, out_py); return out_py; } diff --git a/transformer_engine/pytorch/csrc/extensions/bias.cpp b/transformer_engine/pytorch/csrc/extensions/bias.cpp index 2dbeb965a1a..3631b1641d2 100644 --- a/transformer_engine/pytorch/csrc/extensions/bias.cpp +++ b/transformer_engine/pytorch/csrc/extensions/bias.cpp @@ -67,6 +67,7 @@ std::vector bgrad_quantize(const at::Tensor &grad_output, py::handle // Trivial impl if tensors are empty if (product(shape) == 0) { grad_bias_torch.zero_(); + quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } @@ -91,6 +92,7 @@ std::vector bgrad_quantize(const at::Tensor &grad_output, py::handle if (!with_fused_kernel) { sum_bias_grad(grad_bias_torch, grad_output_torch); quantizer_cpp->quantize(grad_output_nvte, grad_input_nvte); + quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } @@ -116,6 +118,7 @@ std::vector bgrad_quantize(const at::Tensor &grad_output, py::handle workspace_nvte.data(), stream); }); + quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } @@ -149,6 +152,7 @@ std::vector dact_dbias( // Return immediately if tensors are empty if (product(output_shape) == 0) { grad_bias_torch.zero_(); + quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } @@ -259,6 +263,7 @@ std::vector dact_dbias( NVTE_ERROR("Invalid implementation"); } + quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } diff --git a/transformer_engine/pytorch/csrc/extensions/cast.cpp b/transformer_engine/pytorch/csrc/extensions/cast.cpp index 3d61dc525f8..226f7c92b04 100644 --- a/transformer_engine/pytorch/csrc/extensions/cast.cpp +++ b/transformer_engine/pytorch/csrc/extensions/cast.cpp @@ -81,11 +81,7 @@ py::object quantize(const at::Tensor &tensor, py::handle quantizer, const py::ob // Perform quantization quantizer_cpp->quantize(input_cpp, output_cpp, noop_flag_cpp); - // Post-quantize swizzle for quantizers whose kernel does not bake - // the GEMM-swizzled scale layout in directly - if (quantizer_cpp->optimize_for_gemm && !output_cpp.get_with_gemm_swizzled_scales()) { - inplace_swizzle_scale_for_gemm(output_py); - } + quantizer_cpp->optimize_tensor_for_gemm(output_cpp, output_py); return output_py; } @@ -128,9 +124,7 @@ py::object nvfp4_quantize_with_amax(const at::Tensor &tensor, py::handle quantiz } nvfp4_quantizer_cpp->quantize_impl(input_cpp, output_cpp, std::nullopt, false); - if (quantizer_cpp->optimize_for_gemm && !output_cpp.get_with_gemm_swizzled_scales()) { - inplace_swizzle_scale_for_gemm(output_py); - } + quantizer_cpp->optimize_tensor_for_gemm(output_cpp, output_py); return output_py; } @@ -1165,6 +1159,9 @@ std::vector multi_tensor_quantize(const std::vector &ten // Perform multi-tensor quantization multi_tensor_quantize_impl(input_cpp_list, quantizer_list, quantizer_cpp_list, output_cpp_list); + for (size_t i = 0; i < output_cpp_list.size(); ++i) { + quantizer_cpp_list[i]->optimize_tensor_for_gemm(output_cpp_list[i], output_py_list[i]); + } return output_py_list; } @@ -2265,6 +2262,9 @@ std::vector split_quantize(const at::Tensor &tensor, multi_tensor_quantize_impl(input_list, quantizer_list, quantizer_cpp_list, output_cpp_list); } + for (size_t i = 0; i < output_cpp_list.size(); ++i) { + quantizer_cpp_list[i]->optimize_tensor_for_gemm(output_cpp_list[i], output_py_list[i]); + } return output_py_list; } diff --git a/transformer_engine/pytorch/csrc/extensions/gemm.cpp b/transformer_engine/pytorch/csrc/extensions/gemm.cpp index c3484802246..124230b02bf 100644 --- a/transformer_engine/pytorch/csrc/extensions/gemm.cpp +++ b/transformer_engine/pytorch/csrc/extensions/gemm.cpp @@ -424,6 +424,7 @@ std::vector gemm(py::handle A, bool transa, py::handle B, bool trans // Quantize the output std::unique_ptr my_quantizer = convert_quantizer(quantizer); my_quantizer->quantize(unquantized_D_tensor, D_tensor); + my_quantizer->optimize_tensor_for_gemm(D_tensor, D); } // Pack outputs std::vector out; diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index e93c01026a9..fae0315253d 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -135,14 +135,14 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe // Output tensor TensorWrapper out_nvte; if (out.is_none()) { - if (impl == Impl::FULLY_FUSED) { - // FP8 has no special logic to optimize for GEMM, MXFP8 cuDNN - // kernel does not support GEMM swizzled scales - quantizer_cpp->optimize_for_gemm = false; - } std::tie(out_nvte, out) = quantizer_cpp->create_tensor(shape, out_dtype); } else { - out_nvte = makeTransformerEngineTensor(out, quantizer); + std::tie(out_nvte, out) = quantizer_cpp->convert_and_update_tensor(out); + } + if (impl == Impl::FULLY_FUSED && IsMXFP8Quantizers(quantizer.ptr())) { + // cuDNN writes compact scales, including into preallocated outputs. + out_nvte.set_with_gemm_swizzled_scales(false); + out.attr("_with_gemm_swizzled_scales") = false; } // Construct unquantized output tensor if needed @@ -213,6 +213,7 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe } } + quantizer_cpp->optimize_tensor_for_gemm(out_nvte, out); return {out, py::cast(mu_py), py::cast(rsigma_py)}; } @@ -384,12 +385,13 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w // Output tensor TensorWrapper out_nvte; if (out.is_none()) { - if (impl == Impl::FUSED_NORM_QUANT_UNSWIZZLED) { - quantizer_cpp->optimize_for_gemm = false; - } std::tie(out_nvte, out) = quantizer_cpp->create_tensor(shape, out_dtype); } else { - out_nvte = makeTransformerEngineTensor(out, quantizer); + std::tie(out_nvte, out) = quantizer_cpp->convert_and_update_tensor(out); + } + if (impl == Impl::FUSED_NORM_QUANT_UNSWIZZLED) { + out_nvte.set_with_gemm_swizzled_scales(false); + out.attr("_with_gemm_swizzled_scales") = false; } // Construct unquantized output tensor if needed @@ -458,6 +460,7 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w } } + quantizer_cpp->optimize_tensor_for_gemm(out_nvte, out); return {out, py::none(), py::cast(rsigma_py)}; } diff --git a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp index 03eec7d80fd..052c06a8080 100644 --- a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp +++ b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp @@ -727,10 +727,9 @@ void inplace_multi_tensor_swizzle_scales_for_gemm_unchecked(std::vector Optional[tuple[int, int]]: - return tex.nvfp4_gemm_swizzled_scales_alignment( - optimize_for_gemm, - with_rht, - with_2d_quantization, - row_scaled_nvfp4, - nvfp4_use_4over6, - sm_arch, - disable_rht_cast_fusion, - ) - - -def _make_fused_quantization_output_spec( - shape: tuple[int, ...], - dtype: torch.dtype, - device: Optional[torch.device], - quantizer: Optional[Quantizer], -) -> TensorSpec: - """Describe a native output without a separate post-quantize swizzle.""" - spec = TensorSpec(shape=tuple(shape), dtype=dtype, device=device, quantizer=quantizer) - quantizer = spec.quantizer - if not isinstance(quantizer, NVFP4Quantizer): - return spec - major, minor = get_device_compute_capability(device) - # Direct environ access lets Dynamo guard changes to the flag. - disable_rht_cast_fusion = bool( - int(os.environ.get("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", "0") or "0") - ) - alignment = _nvfp4_gemm_swizzled_scales_alignment( - quantizer.optimize_for_gemm, - quantizer.with_rht, - quantizer.with_2d_quantization, - quantizer.row_scaled_nvfp4, - quantizer.nvfp4_use_4over6, - major * 10 + minor, - disable_rht_cast_fusion, - ) - # Keep divisibility checks visible to Dynamo without specializing the dimensions. - rows, cols = math.prod(shape[:-1]), shape[-1] - if alignment is None: - spec.with_gemm_swizzled_scales = False - else: - row_alignment, col_alignment = alignment - spec.with_gemm_swizzled_scales = bool( - rows % row_alignment == 0 and cols % col_alignment == 0 - ) - return spec def fake_bgrad( @@ -87,8 +22,11 @@ def fake_bgrad( dx = ( grad_output if quantizer is None - else _make_fused_quantization_output_spec( - grad_output.shape, grad_output.dtype, grad_output.device, quantizer + else TensorSpec( + shape=grad_output.shape, + dtype=grad_output.dtype, + device=grad_output.device, + quantizer=quantizer, ) ) return db, dx diff --git a/transformer_engine/pytorch/quantized_tensor.py b/transformer_engine/pytorch/quantized_tensor.py index 204e703e1d9..c486bfc7586 100644 --- a/transformer_engine/pytorch/quantized_tensor.py +++ b/transformer_engine/pytorch/quantized_tensor.py @@ -450,6 +450,9 @@ class Quantizer(abc.ABC): """Whether to solely optimize for matrix multiplication + MXFP8 and NVFP4 outputs use GEMM-swizzled scales even when a separate + swizzle is needed after quantization. + The resulting quantized tensors are not guaranteed to support any operation other than matrix multiplication. Use with care since this is likely to break communication, checkpointing, and many From 466ef8903dc99978bfa214f802a9d571f728f778 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 2 Nov 2026 19:32:11 +0100 Subject: [PATCH 08/12] Remove the unused scale-layout binding and refine regression coverage Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_quantized_tensor.py | 9 +++++++-- tests/pytorch/test_torch_compile.py | 6 +++++- transformer_engine/pytorch/csrc/extensions/pybind.cpp | 5 ----- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/pytorch/test_quantized_tensor.py b/tests/pytorch/test_quantized_tensor.py index b4280df0333..e6bcad04692 100644 --- a/tests/pytorch/test_quantized_tensor.py +++ b/tests/pytorch/test_quantized_tensor.py @@ -8,10 +8,10 @@ import pytest import torch -import transformer_engine_torch as tex import transformer_engine.common.recipe import transformer_engine.pytorch as te +import transformer_engine_torch as tex from transformer_engine.pytorch import ( Float8Quantizer, Float8CurrentScalingQuantizer, @@ -1304,7 +1304,12 @@ def run(out=None): if operation == "multi_tensor_quantize": return tex.multi_tensor_quantize([inp], [quantizer])[0] if operation == "split_quantize": - return tex.split_quantize(inp, [shape[0]], [quantizer])[0] + return tex.split_quantize( + inp, + [shape[0]], + [quantizer], + disable_bulk_allocation=shape[0] % 64 != 0 or quantization == "nvfp4-2d", + )[0] if operation == "gelu": return tex.gelu(inp, quantizer) if operation == "dgelu": diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index e9ab4df8221..6dea6836187 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -2984,7 +2984,11 @@ def bgrad(inp): inp = torch.randn(shape, device="cuda", dtype=torch.bfloat16) _, expected_dx = tex.bgrad_quantize(inp, quantizer) _, dx_spec = fake_bgrad(to_tensor_spec(inp), quantizer) - assert dx_spec.with_gemm_swizzled_scales == expected_dx._with_gemm_swizzled_scales + metadata = dx_spec.create_metadata() + assert ( + metadata["nontensor_kwargs"]["with_gemm_swizzled_scales"] + == expected_dx._with_gemm_swizzled_scales + ) for name, (inner_shape, inner_dtype) in dx_spec.quantizer.inner_tensor_specs(shape).items(): inner = getattr(expected_dx, name) assert inner.shape == inner_shape diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index c852d4bac1e..895b9a96e87 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -213,11 +213,6 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { py::arg("output") = py::none(), py::arg("noop") = py::none()); m.def("dequantize", &transformer_engine::pytorch::dequantize, "Dequantize", py::arg("input"), py::arg("otype")); - m.def("nvfp4_gemm_swizzled_scales_alignment", - &transformer_engine::pytorch::NVFP4Quantizer::gemm_swizzled_scales_alignment, - py::arg("optimize_for_gemm"), py::arg("with_rht"), py::arg("with_2d_quantization"), - py::arg("row_scaled_nvfp4"), py::arg("nvfp4_use_4over6"), py::arg("sm_arch"), - py::arg("disable_rht_cast_fusion")); m.def("create_empty_quantized_tensor", &transformer_engine::pytorch::create_empty_quantized_tensor, "Create an empty quantized tensor", py::arg("quantizer"), py::arg("shape"), From b2173dbdae9c57448d4d7cae9552214a3bdd0b55 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 2 Nov 2026 19:32:11 +0100 Subject: [PATCH 09/12] Use saved normalization inputs for backward shapes Signed-off-by: Pawel Gadzinski --- transformer_engine/pytorch/module/layernorm_linear.py | 6 ++---- transformer_engine/pytorch/module/layernorm_mlp.py | 11 +---------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/transformer_engine/pytorch/module/layernorm_linear.py b/transformer_engine/pytorch/module/layernorm_linear.py index 87812829208..87dbf533a31 100644 --- a/transformer_engine/pytorch/module/layernorm_linear.py +++ b/transformer_engine/pytorch/module/layernorm_linear.py @@ -238,7 +238,6 @@ class LayerNormLinearBwdArgs: requires_dgrad: bool = False requires_wgrad: bool = False ln_out_needs_gather: bool = False - inp_shape: Optional[torch.Size] = None # --- Normalization --- normalization: str = "LayerNorm" @@ -784,7 +783,6 @@ def _layernorm_linear_setup_ctx( bwd_args.requires_dgrad = fwd_args.input_requires_grad bwd_args.requires_wgrad = fwd_args.weight_requires_grad bwd_args.ln_out_needs_gather = ctx_attrs["ln_out_needs_gather"] - bwd_args.inp_shape = inp.shape # Normalization bwd_args.normalization = fwd_args.normalization @@ -974,7 +972,7 @@ def _layernorm_linear_backward_impl( ub_obj_wgrad = None ub_type_dgrad = None ub_type_wgrad = None - dgrad_shape = args.inp_shape + dgrad_shape = inputmat.shape if args.ub_overlap_ag: # Overlap grad_output all-gather with dgrad compute args.ub_obj_gradout = get_ub(args.ub_name + "_dgrad", args.fp8) @@ -1781,7 +1779,7 @@ def _layernorm_linear_backward_fake( dgrad = None if args.requires_dgrad: dgrad = TensorSpec( - shape=args.inp_shape, + shape=args.inputmat.shape, dtype=out_dtype, device=device, ) diff --git a/transformer_engine/pytorch/module/layernorm_mlp.py b/transformer_engine/pytorch/module/layernorm_mlp.py index d1cc8330729..bb06e691219 100644 --- a/transformer_engine/pytorch/module/layernorm_mlp.py +++ b/transformer_engine/pytorch/module/layernorm_mlp.py @@ -364,7 +364,6 @@ class LayerNormMLPBwdArgs: fc1_weight_requires_grad: bool = False fc1_bias_requires_grad: bool = False fc2_weight_requires_grad: bool = False - inp_shape: Optional[torch.Size] = None # --- Normalization --- normalization: str = "LayerNorm" @@ -1173,7 +1172,6 @@ def _layernorm_mlp_setup_ctx( bwd_args.fc1_weight_requires_grad = fc1_weight_requires_grad bwd_args.fc1_bias_requires_grad = fwd_args.fc1_bias_requires_grad bwd_args.fc2_weight_requires_grad = fc2_weight_requires_grad - bwd_args.inp_shape = inp.shape # Normalization bwd_args.normalization = fwd_args.normalization @@ -2432,15 +2430,8 @@ def _layernorm_mlp_backward_fake( dgrad = None if args.requires_dgrad: - inp_leading = args.grad_output.shape[0] - if args.sequence_parallel and not args.set_parallel_mode: - inp_leading = inp_leading // args.tp_size dgrad = TensorSpec( - shape=( - (in_features,) - if len(args.grad_output.shape) == 1 - else (inp_leading, *args.grad_output.shape[1:-1], in_features) - ), + shape=args.inputmat.shape, dtype=out_dtype, device=device, ) From 0e67f6bbfc88dba851b3bd4b6967bed79bc72519 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 2 Nov 2026 19:39:51 +0100 Subject: [PATCH 10/12] Preserve 2D MXFP8 normalization semantics Signed-off-by: Pawel Gadzinski --- .../pytorch/csrc/extensions/normalization.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index fae0315253d..811a32152c8 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -106,9 +106,12 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe if (quantizer.is_none() || IsFloat8Quantizers(quantizer.ptr())) { impl = Impl::FULLY_FUSED; } else if (IsMXFP8Quantizers(quantizer.ptr())) { - if (transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN") && outer_size % 128 == 0 && + auto mxfp8_quantizer_cpp = dynamic_cast(quantizer_cpp.get()); + NVTE_CHECK(mxfp8_quantizer_cpp != nullptr, "Could not cast to MXFP8 quantizer"); + // cuDNN quantizes 1D blocks and requires full 128x128 tiles. + if (!mxfp8_quantizer_cpp->with_2d_quantization && + transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN") && outer_size % 128 == 0 && inner_size % 128 == 0) { - // cuDNN MXFP8 kernel requires full 128x128 tiles impl = Impl::FULLY_FUSED; } } else if (detail::IsFloat8CurrentScalingQuantizers(quantizer.ptr()) && From b35179c4615998861d013fb70bff2651f9a39cdd Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Wed, 4 Nov 2026 15:11:26 +0100 Subject: [PATCH 11/12] Batch GEMM scale finalization when all quantizers request it Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_quantized_tensor.py | 111 ++++++++++++++++++ .../pytorch/csrc/extensions/cast.cpp | 67 ++++++++++- 2 files changed, 172 insertions(+), 6 deletions(-) diff --git a/tests/pytorch/test_quantized_tensor.py b/tests/pytorch/test_quantized_tensor.py index e6bcad04692..b1d88f3cc5f 100644 --- a/tests/pytorch/test_quantized_tensor.py +++ b/tests/pytorch/test_quantized_tensor.py @@ -1340,3 +1340,114 @@ def run(out=None): assert run(actual) is actual assert actual._with_gemm_swizzled_scales == optimize_for_gemm torch.testing.assert_close(actual.dequantize(), expected, atol=0, rtol=0) + + +@pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) +@pytest.mark.parametrize("operation", ["multi_tensor_quantize", "split_quantize"]) +@pytest.mark.parametrize( + "case", + [ + "all", + "compact", + "rowwise", + "mixed_usage", + "mixed_optimize", + "no_optimize", + "mixed_formats", + "already_swizzled", + "all_swizzled", + "empty", + ], +) +def test_multi_tensor_quantization_scale_layout(operation, case): + """Batched output finalization preserves each tensor's data, usage and layout.""" + shapes = [(64, 128)] * 3 + if case == "all_swizzled": + shapes = [(128, 128)] * 3 + if case == "empty": + shapes[1] = (0, 128) + quantizers = [] + for i in range(3): + if case == "mixed_formats" and i == 1: + quantizer = MXFP8Quantizer(te.DType.kFloat8E4M3) + else: + with_rht = case in ("all", "no_optimize", "all_swizzled") + with_rht |= case == "already_swizzled" and i != 1 + quantizer = NVFP4Quantizer( + with_rht=with_rht, + with_post_rht_amax=with_rht, + stochastic_rounding=False, + with_random_sign_mask=False, + ) + quantizer.optimize_for_gemm = case != "no_optimize" and not ( + case == "mixed_optimize" and i == 1 + ) + if case == "rowwise": + quantizer.set_usage(rowwise=True, columnwise=False) + elif case == "mixed_usage": + quantizer.set_usage(rowwise=i != 2, columnwise=i != 0) + quantizers.append(quantizer) + inputs = [torch.randn(shape, dtype=torch.bfloat16, device="cuda") for shape in shapes] + references = [quantizer(inp) for quantizer, inp in zip(quantizers, inputs)] + if operation == "multi_tensor_quantize": + outputs = tex.multi_tensor_quantize(inputs, quantizers) + else: + outputs = tex.split_quantize( + torch.cat(inputs), + [shape[0] for shape in shapes], + quantizers, + disable_bulk_allocation=case + in ( + "compact", + "rowwise", + "mixed_usage", + "mixed_optimize", + "mixed_formats", + "already_swizzled", + "empty", + ), + ) + for actual, expected, quantizer in zip(outputs, references, quantizers): + assert actual._with_gemm_swizzled_scales == quantizer.optimize_for_gemm + assert actual.shape == expected.shape + for usage in ("rowwise", "columnwise"): + actual_data = getattr(actual, f"_{usage}_data") + expected_data = getattr(expected, f"_{usage}_data") + actual_scales = getattr(actual, f"_{usage}_scale_inv") + expected_scales = getattr(expected, f"_{usage}_scale_inv") + assert (actual_data is None) == (expected_data is None) + assert (actual_scales is None) == (expected_scales is None) + if actual_data is not None: + torch.testing.assert_close(actual_data, expected_data, atol=0, rtol=0) + assert actual_scales.shape == expected_scales.shape + if actual._rowwise_data is not None: + torch.testing.assert_close( + actual.dequantize(dtype=torch.float32), + expected.dequantize(dtype=torch.float32), + atol=0, + rtol=0, + ) + if actual._columnwise_data is not None: + + def columnwise_view(tensor): + return NVFP4Tensor( + shape=(tensor.shape[-1], tensor.shape[0]), + dtype=torch.float32, + rowwise_data=tensor._columnwise_data, + rowwise_scale_inv=tensor._columnwise_scale_inv, + columnwise_data=None, + columnwise_scale_inv=None, + amax_rowwise=tensor._amax_columnwise, + amax_columnwise=None, + fp4_dtype=quantizer.dtype, + quantizer=quantizer, + with_gemm_swizzled_scales=tensor._with_gemm_swizzled_scales, + ) + + if not isinstance(quantizer, MXFP8Quantizer): + torch.testing.assert_close( + columnwise_view(actual).dequantize(dtype=torch.float32), + columnwise_view(expected).dequantize(dtype=torch.float32), + atol=0, + rtol=0, + ) diff --git a/transformer_engine/pytorch/csrc/extensions/cast.cpp b/transformer_engine/pytorch/csrc/extensions/cast.cpp index 226f7c92b04..4479a51e055 100644 --- a/transformer_engine/pytorch/csrc/extensions/cast.cpp +++ b/transformer_engine/pytorch/csrc/extensions/cast.cpp @@ -38,6 +38,65 @@ std::vector get_tensor_shape(const TensorWrapper &tensor) { return std::vector(shape.data, shape.data + shape.ndim); } +void optimize_tensors_for_gemm(const std::vector> &quantizers, + std::vector &tensors, + std::vector &tensors_py) { + if (tensors.empty()) { + return; + } + if (std::all_of(tensors.begin(), tensors.end(), + [](const auto &tensor) { return tensor.get_with_gemm_swizzled_scales(); })) { + return; + } + const auto scaling_mode = tensors.front().scaling_mode(); + const bool use_batch = + tensors.size() > 1 && + (scaling_mode == NVTE_MXFP8_1D_SCALING || scaling_mode == NVTE_NVFP4_1D_SCALING) && + std::all_of(quantizers.begin(), quantizers.end(), + [](const auto &quantizer) { return quantizer->optimize_for_gemm; }) && + std::all_of(tensors.begin(), tensors.end(), [scaling_mode](const auto &tensor) { + return tensor.scaling_mode() == scaling_mode; + }); + if (!use_batch) { + for (size_t i = 0; i < tensors.size(); ++i) { + quantizers[i]->optimize_tensor_for_gemm(tensors[i], tensors_py[i]); + } + return; + } + + for (bool rowwise : {true, false}) { + const bool has_scales = + std::any_of(tensors.begin(), tensors.end(), [rowwise](const auto &tensor) { + const auto scales = + rowwise ? tensor.get_rowwise_scale_inv() : tensor.get_columnwise_scale_inv(); + return !tensor.get_with_gemm_swizzled_scales() && scales.data_ptr != nullptr; + }); + if (has_scales) { + inplace_multi_tensor_swizzle_scales_for_gemm(tensors_py, rowwise, !rowwise); + } + } + for (size_t i = 0; i < tensors.size(); ++i) { + auto &tensor = tensors[i]; + if (tensor.get_with_gemm_swizzled_scales()) { + continue; + } + for (bool rowwise : {true, false}) { + const auto scales = + rowwise ? tensor.get_rowwise_scale_inv() : tensor.get_columnwise_scale_inv(); + if (scales.data_ptr != nullptr) { + const auto scales_py = tensors_py[i] + .attr(rowwise ? "_rowwise_scale_inv" : "_columnwise_scale_inv") + .cast(); + tensor.set_parameter(rowwise ? kNVTERowwiseScaleInv : kNVTEColumnwiseScaleInv, + scales_py.data_ptr(), static_cast(scales.dtype), scales.shape); + } + } + // Both orientations must be converted before publishing the shared layout flag. + tensors_py[i].attr("_with_gemm_swizzled_scales") = true; + tensor.set_with_gemm_swizzled_scales(true); + } +} + void allreduce_nvfp4_amax_tensors(NVFP4Quantizer *nvfp4_quantizer_cpp, std::vector &&amax_tensors) { if (!nvfp4_quantizer_cpp->with_amax_reduction || amax_tensors.empty()) { @@ -1159,9 +1218,7 @@ std::vector multi_tensor_quantize(const std::vector &ten // Perform multi-tensor quantization multi_tensor_quantize_impl(input_cpp_list, quantizer_list, quantizer_cpp_list, output_cpp_list); - for (size_t i = 0; i < output_cpp_list.size(); ++i) { - quantizer_cpp_list[i]->optimize_tensor_for_gemm(output_cpp_list[i], output_py_list[i]); - } + optimize_tensors_for_gemm(quantizer_cpp_list, output_cpp_list, output_py_list); return output_py_list; } @@ -2262,9 +2319,7 @@ std::vector split_quantize(const at::Tensor &tensor, multi_tensor_quantize_impl(input_list, quantizer_list, quantizer_cpp_list, output_cpp_list); } - for (size_t i = 0; i < output_cpp_list.size(); ++i) { - quantizer_cpp_list[i]->optimize_tensor_for_gemm(output_cpp_list[i], output_py_list[i]); - } + optimize_tensors_for_gemm(quantizer_cpp_list, output_cpp_list, output_py_list); return output_py_list; } From a285cec8d75c94af374c5963270c162edf5c7ebb Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 12 Nov 2026 16:11:05 +0100 Subject: [PATCH 12/12] Keep existing Linear behavior in LayerNorm compile support Restore Linear and fused-attention backward to main. Make normalization gradient setup optional and remove the unused SHAPE field kind and its test. Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 69 ------ .../dot_product_attention/backends.py | 6 +- .../pytorch/dynamo/custom_op.py | 43 ++-- transformer_engine/pytorch/module/linear.py | 200 ++++++++++++++---- 4 files changed, 178 insertions(+), 140 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 827566f7454..b21aee02a3a 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -1950,75 +1950,6 @@ def fn(x): assert out.shape == () -@pytest.mark.parametrize("with_optional_shape", [False, True]) -def test_custom_op_symbolic_shape(with_optional_shape): - """Shape arguments stay dynamic across the custom-op boundary.""" - from transformer_engine.pytorch.dynamo.custom_op import register_custom_op_with_autograd - - @dataclasses.dataclass - class ShapeArgs: - input_: torch.Tensor - shape: torch.Size - optional_shape: torch.Size | None - - @dataclasses.dataclass - class ShapeBwdArgs: - grad_output: torch.Tensor | None = None - shape: torch.Size | None = None - - def setup_grad_outputs(self, grads): - (self.grad_output,) = grads - - def impl(args): - assert isinstance(args.shape, torch.Size) - shape = args.shape if args.optional_shape is None else args.optional_shape - assert isinstance(shape, torch.Size) - return args.input_.reshape(shape).clone(), (), None - - def fake_impl(args): - shape = args.shape if args.optional_shape is None else args.optional_shape - return TensorSpec(shape=shape, dtype=args.input_.dtype, device=args.input_.device), (), None - - def setup_context(bwd_args, fwd_args, *_): - bwd_args.shape = fwd_args.input_.shape - return () - - def backward_impl(args): - return (args.grad_output.reshape(args.shape).clone(),) - - def backward_fake(args): - grad = args.grad_output - return (TensorSpec(shape=args.shape, dtype=grad.dtype, device=grad.device),) - - op = register_custom_op_with_autograd( - op_name=f"test_symbolic_shape_{with_optional_shape}", - input_tensors_for_grad=["input_"], - fwd_arg_type=ShapeArgs, - fwd_impl=impl, - fwd_fake_impl=fake_impl, - setup_context=setup_context, - bwd_arg_type=ShapeBwdArgs, - bwd_impl=backward_impl, - bwd_fake_impl=backward_fake, - ) - assert op is not None - - def fn(inp): - shape = torch.Size((inp.shape[0], 2, 4)) - return op(ShapeArgs(inp, shape, shape if with_optional_shape else None)) - - counter = CompileCounterWithBackend("aot_eager") - compiled = torch.compile(fn, fullgraph=True, backend=counter) - for batch in (2, 3, 5): - inp = torch.randn(batch, 8, requires_grad=True) - torch._dynamo.mark_dynamic(inp, 0) - out = compiled(inp) - torch.testing.assert_close(out, inp.reshape(batch, 2, 4)) - out.sum().backward() - torch.testing.assert_close(inp.grad, torch.ones_like(inp)) - assert counter.frame_count == 1 - - def test_to_tensor_spec_plain(): """``to_tensor_spec`` describes a plain tensor.""" t = torch.empty(2, 3, dtype=torch.float32) diff --git a/transformer_engine/pytorch/attention/dot_product_attention/backends.py b/transformer_engine/pytorch/attention/dot_product_attention/backends.py index 9817e78eaf8..c3503d0f1bf 100644 --- a/transformer_engine/pytorch/attention/dot_product_attention/backends.py +++ b/transformer_engine/pytorch/attention/dot_product_attention/backends.py @@ -11,7 +11,7 @@ import functools import inspect import os -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, Union import warnings import logging from packaging.version import Version as PkgVersion @@ -1668,10 +1668,6 @@ class FusedAttnBwdArgs: dO_quantizer: Optional[Any] = None dP_quantizer: Optional[Any] = None - def setup_grad_outputs(self, grads: Sequence[Any]) -> None: - """Assign the output gradient, ignoring auxiliary outputs.""" - self.grad_output = grads[0] - def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: """Pull saved tensors from ``ctx`` into the fields backward consumes.""" ( diff --git a/transformer_engine/pytorch/dynamo/custom_op.py b/transformer_engine/pytorch/dynamo/custom_op.py index 5a784abf332..1b8f3b4b1f1 100644 --- a/transformer_engine/pytorch/dynamo/custom_op.py +++ b/transformer_engine/pytorch/dynamo/custom_op.py @@ -18,7 +18,7 @@ keeps its saved-tensor and context-metadata contract. A ``torch.library`` custom op is narrower: it only accepts flat schema slots -(tensors, symbolic shapes and opaque objects) and returns a flat ``Tensor[]``. +(tensors plus opaque objects) and returns a flat ``Tensor[]``. Bridging the two takes three parts (below): a parsed per-op *arg plan* maps the args dataclass onto the op's input slots; a per-trace *output plan*, parsed from @@ -37,8 +37,7 @@ * ``TENSOR_OR_QUANTIZED`` -- a field that may be a plain tensor, a bare quantized storage, or ``None``: three slots (the tensor, its flat inner buffers, and a ``__kind__`` tag) so a quantized tensor crosses as its buffers. - * ``SHAPE`` -- a ``torch.Size`` / ``Optional[torch.Size]``: one ``SymInt[]`` slot. - * ``SIMPLE`` -- every remaining simple value (scalars, enums, + * ``SIMPLE`` -- every remaining simple value (scalars, enums, sizes, quantizers -- value-opaque constants baked into the graph -- and nested collections of them), gathered into one shared ``OpaqueValueBundle`` slot. * ``PROCESS_GROUP`` -- rides in the shared bundle too, as its c10d registry @@ -67,8 +66,8 @@ forward state + ``ctx_attrs`` (e.g. saved-tensor aliases) and return the tensors to persist; the plan's output ranges are stashed on ``ctx``; * on ``backward()`` the incoming flat grads are sliced per user output from the - stashed plan and passed in forward-output order to the backward - container's ``setup_grad_outputs`` method, + stashed plan (a ``grad_outputs`` field on the backward args receives the + whole tuple; otherwise ``grad_output`` receives the first output's grad), the container's optional ``setup_saved_tensors`` hook restores the saved tensors, then the *backward op* runs the real ``bwd_impl`` and returns the flat grads (``bwd_fake_impl`` is its data-free fake). @@ -398,7 +397,6 @@ class _FieldKind(Enum): TENSOR = "tensor" # one ``Tensor`` / ``Tensor?`` slot TENSOR_OR_QUANTIZED = "tensor_or_quantized" # 3 slots: tensor / inner / meta - SHAPE = "shape" # one ``SymInt[]`` / ``SymInt[]?`` slot PROCESS_GROUP = "process_group" # c10d group name inside the shared bundle SIMPLE = "simple" # value carried verbatim inside the shared bundle UNSUPPORTED = "unsupported" # no slots; only a trivial value may cross @@ -496,9 +494,6 @@ def _parse_field(name: str, annot: Any) -> _FieldPlan: if stripped is torch.Tensor: slot = _SlotSpec(name, "Tensor?" if is_optional else "Tensor") return _FieldPlan(name, _FieldKind.TENSOR, (slot,)) - if stripped is torch.Size: - slot = _SlotSpec(name, "SymInt[]?" if is_optional else "SymInt[]") - return _FieldPlan(name, _FieldKind.SHAPE, (slot,)) # A union mixing tensor types with anything else is a malformed signature # (e.g. a bare quantized-storage Optional, or Tensor | int): reject it at # registration instead of silently degrading to an unsupported field. @@ -639,7 +634,7 @@ def pack(self, obj: Any) -> Dict[str, Any]: for field in self.fields: value = getattr(obj, field.name, None) match field.kind: - case _FieldKind.TENSOR | _FieldKind.SHAPE: + case _FieldKind.TENSOR: slots[field.slots[0].name] = value case _FieldKind.TENSOR_OR_QUANTIZED: _pack_tensor_or_quantized(field, value, slots) @@ -675,9 +670,6 @@ def unpack(self, slots: Dict[str, Any]) -> Any: match field.kind: case _FieldKind.TENSOR: kwargs[field.name] = slots[field.slots[0].name] - case _FieldKind.SHAPE: - value = slots[field.slots[0].name] - kwargs[field.name] = None if value is None else torch.Size(value) case _FieldKind.TENSOR_OR_QUANTIZED: kwargs[field.name] = _unpack_tensor_or_quantized(field, slots) case _FieldKind.PROCESS_GROUP: @@ -1040,11 +1032,12 @@ def _register_autograd_for_op( the saved tuple + ``ctx_attrs`` to the module's ``setup_context`` and stashes the plan on ``ctx`` so backward can slice its grads per user output. """ - - tensor_list_positions = tuple(offset + 1 for offset in fwd_plan.tensor_or_quantized_offsets()) + bwd_takes_grad_tuple = any(f.name == "grad_outputs" for f in bwd_plan.fields) def _setup_context(ctx, inputs, output): - ctx.fwd_tensor_list_lengths = {i: len(inputs[i]) for i in tensor_list_positions} + ctx.fwd_tensor_list_lengths = { + i: len(value) for i, value in enumerate(inputs) if isinstance(value, list) + } fwd_obj = fwd_plan.unpack(dict(zip(fwd_plan.slot_names, inputs))) spec_obj = _spec_view(fwd_obj, fwd_plan.tensor_field_names()) @@ -1079,7 +1072,12 @@ def _autograd_backward(ctx, *grad_outputs): ctx.tensor_objects = None user_grads = _slice_user_grads(ctx.output_ranges, grad_outputs[0]) ctx.output_ranges = None - bwd_obj.setup_grad_outputs(user_grads) + if hasattr(bwd_obj, "setup_grad_outputs"): + bwd_obj.setup_grad_outputs(user_grads) + elif bwd_takes_grad_tuple: + bwd_obj.grad_outputs = tuple(user_grads) + else: + bwd_obj.grad_output = user_grads[0] kwargs = bwd_plan.pack(bwd_obj) bwd_args_flat = [kwargs[name] for name in bwd_plan.slot_names] grads = [_decode_none(g) for g in bwd_op(*bwd_args_flat)] @@ -1366,9 +1364,8 @@ def register_custom_op_with_autograd( non-differentiable input). * ``bwd_fake_impl(bwd_args)`` -- data-free twin of ``bwd_impl`` returning :class:`TensorSpec` grads. - * ``bwd_arg_type.setup_grad_outputs(grads)`` -- assign incoming gradients - to the container's fields. Receives one entry per forward user output, - in return order, including auxiliary outputs such as weight workspaces. + * ``bwd_arg_type.setup_grad_outputs(grads)`` -- optional hook overriding + the default output-gradient assignment. * ``bwd_arg_type.setup_saved_tensors(ctx)`` -- optional hook; skipped if absent. @@ -1377,8 +1374,10 @@ def register_custom_op_with_autograd( forward state and returns the tensors to persist; the framework saves them via ``ctx.save_for_backward``. Before ``bwd_impl`` runs, the framework restores them into the container's tensor fields through the - ``setup_saved_tensors`` hook and passes the incoming gradients to - ``setup_grad_outputs``, so ``bwd_impl`` receives a fully-populated + ``setup_saved_tensors`` hook and sets the incoming gradient directly -- + into a ``grad_outputs`` field (tuple, one grad per user output) if + ``bwd_arg_type`` declares one, else into ``grad_output`` (the first user + output's grad) -- so ``bwd_impl`` receives a fully-populated ``bwd_arg_type``. Registration touches experimental ``torch.library`` / opaque-object APIs diff --git a/transformer_engine/pytorch/module/linear.py b/transformer_engine/pytorch/module/linear.py index 2059d02ef41..0aeb91d57dd 100644 --- a/transformer_engine/pytorch/module/linear.py +++ b/transformer_engine/pytorch/module/linear.py @@ -5,7 +5,7 @@ """Linear API""" from dataclasses import dataclass, replace as dataclass_replace -from typing import Sequence, Any, Callable, Dict, Optional, Tuple, Union, List +from typing import Any, Callable, Dict, Optional, Tuple, Union, List import warnings import weakref @@ -31,11 +31,8 @@ _2X_ACC_WGRAD, ) from ._common import ( - compile_unsupported_quantizer_reason, can_reconstruct_wgrad_input_from_original, check_fp8_reduce_and_update, - fake_workspace_valid, - get_output_first_dim_size, noop_cat, set_quantizer_amax_reduction_group, set_quantizer_usage_for_wgrad_all_gather, @@ -91,6 +88,7 @@ TensorSpec, TensorOrQuantized, register_custom_op_with_autograd, + is_value_opaque_quantizer, ) from ..tensor.float8_tensor import Float8CurrentScalingQuantizer, Float8Quantizer from ..tensor.mxfp8_tensor import MXFP8Quantizer @@ -141,11 +139,8 @@ class LinearFwdArgs: fp8_output: bool save_original_input: bool backward_override: Optional[str] - fprop_use_split_accumulator: bool dgrad_use_split_accumulator: bool wgrad_use_split_accumulator: bool - recipe_custom: bool - fp8_recompute_forward: bool custom: bool debug: bool @@ -185,6 +180,54 @@ class LinearFwdArgs: cpu_offloading: bool is_grad_enabled: bool + def compile_unsupported_reason(self) -> Optional[str]: + """Reason this config can't use the torch.compile custom-op path (else None).""" + if self.debug: + return "debug instrumentation (nvidia-dlfw-inspect)" + if is_distributed_weight(self.weight): + return "a DistributedWeight (custom weight parallelism, e.g. GTP)" + if isinstance(self.inp, (QuantizedTensor, QuantizedTensorStorage)): + return "a quantized input tensor" + if self.fsdp_group is not None: + return "manual TE FSDP (fsdp_group); use FSDP2 or MCore FSDP" + if ( + self.fp8_output + and self.is_grad_enabled + and (self.input_requires_grad or self.weight_requires_grad or self.bias_requires_grad) + ): + return "differentiable fp8_output=True" + if self.cpu_offloading: + return "CPU activation offloading" + if self.wgrad_store is not None: + # Non-None only when delayed wgrad compute is on (see Linear.forward). + return "delayed wgrad compute (wgrad_store)" + if ( + self.grad_input_quantizer is not None + and self.is_grad_enabled + and self.input_requires_grad + and not (self.ub_overlap_rs_dgrad or self.ub_bulk_wgrad) + ): + # A quantized dgrad can't cross the op boundary: grads are packed + # one plain Tensor[] slot each (_pack_bwd_result). + return "a quantized input grad (fp8_grad=True)" + if self.fp8 and self.skip_fp8_weight_update is not None: + return "GPU-controlled FP8 weight cache updates" + if self.fuse_wgrad_accumulation: + return "fuse_wgrad_accumulation (main_grad)" + for quantizer in ( + self.input_quantizer, + self.weight_quantizer, + self.output_quantizer, + self.grad_input_quantizer, + self.grad_weight_quantizer, + self.grad_output_quantizer, + ): + # e.g. delayed-scaling Float8Quantizer and unregistered custom-recipe + # quantizers are not value-opaque and can't cross the custom-op boundary. + if quantizer is not None and not is_value_opaque_quantizer(quantizer): + return "a quantizer not registered as a torch.compile value-opaque type" + return None + @dataclass(slots=True) class LinearBwdArgs: @@ -259,10 +302,6 @@ class LinearBwdArgs: # --- Per-backward scratch state (populated inside _linear_backward_impl) --- ub_obj_gradout: Optional[Any] = None - def setup_grad_outputs(self, grads: Sequence[Any]) -> None: - """Unpack gradients in forward-output order, ignoring weight workspaces.""" - self.grad_output, _ = grads - def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: """Pull saved tensors from ``ctx`` into the fields backward consumes.""" ( @@ -275,6 +314,40 @@ def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: ) # pylint: disable=unbalanced-tuple-unpacking +def _out_leading_from_inp(leading: int, args: Union[LinearFwdArgs, LinearBwdArgs]) -> int: + """Output's leading (sequence) dim from the input's: sequence parallelism + gathers it (column-parallel) or scatters it (row-parallel).""" + if not args.sequence_parallel: + return leading + if args.parallel_mode == "column": + return leading * args.tp_size + if args.parallel_mode == "row": + return leading // args.tp_size + return leading + + +def _inp_leading_from_out(leading: int, args: Union[LinearFwdArgs, LinearBwdArgs]) -> int: + """Inverse of :func:`_out_leading_from_inp`.""" + if not args.sequence_parallel: + return leading + if args.parallel_mode == "column": + return leading // args.tp_size + if args.parallel_mode == "row": + return leading * args.tp_size + return leading + + +def _fake_workspace_valid(workspace: TensorSpec, quantizer: Optional[Quantizer]) -> bool: + """Spec-level mirror of ``_is_weight_workspace_valid``: the cached workspace + must already hold every inner buffer the quantizer's current usage needs.""" + if quantizer is None: + return True + required = TensorSpec( + shape=workspace.shape, dtype=workspace.dtype, quantizer=quantizer, device=workspace.device + ).inner_names() + return set(required) <= set(workspace.inner_names()) + + def _linear_forward_impl( args: LinearFwdArgs, ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple], Optional[Dict]]: @@ -325,7 +398,7 @@ def _linear_forward_impl( # to reuse the high-precision DPA output saved by attention backward. Validate # the resolved quantizer since this is not necessarily an informed user opt-in. if isinstance(input_quantizer, Float8Quantizer): - if args.recipe_custom: + if FP8GlobalStateManager.get_fp8_recipe().custom(): warnings.warn( "save_original_input is incompatible with delayed-scaling quantizers " "(Float8Quantizer). Disabling save_original_input for this module.", @@ -447,7 +520,7 @@ def _linear_forward_impl( inputmat = cast_if_needed(inp, activation_dtype) # Cast for AMP inputmat_total = inputmat - if cpu_offloading: + if is_cpu_offload_enabled(): start_offload(inputmat) nvtx_range_pop(f"{nvtx_label}.input_cast_comm") # ------------------------------------------------------ @@ -476,7 +549,10 @@ def _linear_forward_impl( if backward_override is not None: columnwise_usage = False if not columnwise_usage: - columnwise_usage = args.fp8_recompute_forward + columnwise_usage = ( + is_fp8_activation_recompute_enabled() + and not in_fp8_activation_recompute_phase() + ) weight_quantizer.set_usage(rowwise=True, columnwise=columnwise_usage) elif isinstance(weight, QuantizedTensor): weight_quantizer = weight._quantizer @@ -515,7 +591,11 @@ def _linear_forward_impl( weight_quantizer.calibrate(weight) # Choose whether to use GEMM kernel with split accumulator - use_split_accumulator = args.fprop_use_split_accumulator + use_split_accumulator = _2X_ACC_FPROP + if fp8: + recipe = FP8GlobalStateManager.get_fp8_recipe() + if hasattr(recipe, "fp8_gemm_fprop"): + use_split_accumulator = recipe.fp8_gemm_fprop.use_split_accumulator # Configure output quantizer if output_quantizer is not None: @@ -762,7 +842,10 @@ def _linear_forward_fake( if args.backward_override is not None: columnwise_usage = False if not columnwise_usage: - columnwise_usage = args.fp8_recompute_forward + columnwise_usage = ( + is_fp8_activation_recompute_enabled() + and not in_fp8_activation_recompute_phase() + ) weight_quantizer.set_usage(rowwise=True, columnwise=columnwise_usage) elif weight.is_quantized: weight_quantizer = weight.quantizer @@ -775,7 +858,7 @@ def _linear_forward_fake( else: weightmat_is_storage = True workspace = args.weight_workspace - if workspace is not None and not fake_workspace_valid(workspace, weight_quantizer): + if workspace is not None and not _fake_workspace_valid(workspace, weight_quantizer): # quantize_weight drops a stale workspace and builds a new one. workspace = None if workspace is not None: @@ -807,7 +890,7 @@ def _linear_forward_fake( # Output tensor: y = x @ w^T (quantized iff an output quantizer is set). # ------------------------------------------------------ inp_leading = inp.shape[0] if len(inp.shape) > 1 else 1 - out_leading = get_output_first_dim_size(inp_leading, args) + out_leading = _out_leading_from_inp(inp_leading, args) out = TensorSpec( shape=( (out_features,) @@ -934,7 +1017,10 @@ def _linear_setup_ctx( bwd_args.use_bias = bias is not None bwd_args.requires_dgrad = fwd_args.input_requires_grad bwd_args.requires_wgrad = fwd_args.weight_requires_grad - bwd_args.inp_shape = inp.shape + # Don't store inp_shape in the value bundle: under torch.compile(dynamic=True) + # inp.shape contains SymInt dims which are not hashable in OpaqueValueBundle. + # The backward reconstructs inp_shape from grad_output + weight + SP config. + bwd_args.inp_shape = None # Numerical / dtype config bwd_args.activation_dtype = fwd_args.activation_dtype @@ -1086,6 +1172,16 @@ def _linear_backward_impl(args: LinearBwdArgs) -> Tuple[Union[torch.Tensor, None ) nvtx_range_pop(f"{nvtx_label}.fsdp_gather") + # Reconstruct inp_shape when not stored (compiled mode with dynamic shapes). + if bwd_args.inp_shape is None: + in_features = saved_weight.shape[-1] + inp_leading = _inp_leading_from_out(grad_output.shape[0], bwd_args) + bwd_args.inp_shape = ( + torch.Size([in_features]) + if grad_output.ndim == 1 + else torch.Size([inp_leading, *grad_output.shape[1:-1], in_features]) + ) + # Configure Userbuffers communication (comm+GEMM overlap) bwd_args.ub_obj_gradout = None ub_obj_dgrad = None @@ -1669,13 +1765,20 @@ def _linear_backward_fake( dgrad = None if args.requires_dgrad: + # Input shape rederived from grad_output + SP config (inp_shape is not + # stored: torch.Size with SymInt cannot cross in OpaqueValueBundle). + dgrad_leading = _inp_leading_from_out(args.grad_output.shape[0], args) # Under UB reduce-scatter or bulk-wgrad overlap the returned dgrad is a # plain tensor; the quantizer only feeds the comm buffer. dgrad_quantizer = ( None if (args.ub_overlap_rs_dgrad or args.ub_bulk_wgrad) else args.grad_input_quantizer ) dgrad = TensorSpec( - shape=args.inp_shape, + shape=( + (in_features,) + if len(args.grad_output.shape) == 1 + else (dgrad_leading, *args.grad_output.shape[1:-1], in_features) + ), dtype=out_dtype, quantizer=dgrad_quantizer, device=args.grad_output.device, @@ -1783,7 +1886,7 @@ def backward( ) -> Tuple[Union[torch.Tensor, None], ...]: """Backward pass: compute gradients and reduce FP8 scaling factors.""" bwd_args: LinearBwdArgs = ctx.backward_objects - bwd_args.setup_grad_outputs((grad_output, _grad_weight_workspace)) + bwd_args.grad_output = grad_output bwd_args.setup_saved_tensors(ctx) nvtx_label = "transformer_engine._Linear.backward" if bwd_args.ub_name is not None: @@ -2303,6 +2406,16 @@ def forward( if get_ub_is_fp8(self.ub_name + "_dgrad", FP8GlobalStateManager.is_fp8_enabled()): fp8_grad = True + if torch.compiler.is_compiling() and _linear_op is not None: + reason = self._compile_eager_fallback_reason( + inp, skip_fp8_weight_update, fp8_output, fp8_grad, is_grad_enabled, debug + ) + if reason is not None: + # A break inside the try/finally below would skip the whole frame. + warn_compile_eager_fallback(reason) + torch._dynamo.graph_break(msg=f"te.Linear falling back to eager: {reason}") + return self._forward_eager_fallback(inp, is_first_microbatch, fp8_output, fp8_grad) + inp = self.prepare_forward(inp, allow_non_contiguous=isinstance(inp, QuantizedTensor)) try: weight_tensor, bias_tensor = self._get_weight_and_bias_tensors() @@ -2345,20 +2458,15 @@ def forward( if cache_name is not None and not (use_compiled_op and is_first_microbatch): weight_workspace = self._fp8_workspaces.get(cache_name) - fprop_use_split_accumulator = _2X_ACC_FPROP dgrad_use_split_accumulator = _2X_ACC_DGRAD wgrad_use_split_accumulator = _2X_ACC_WGRAD - recipe_custom = False if self.fp8: _recipe = FP8GlobalStateManager.get_fp8_recipe() backward_override = _recipe.backward_override - if hasattr(_recipe, "fp8_gemm_fprop"): - fprop_use_split_accumulator = _recipe.fp8_gemm_fprop.use_split_accumulator if hasattr(_recipe, "fp8_gemm_dgrad"): dgrad_use_split_accumulator = _recipe.fp8_gemm_dgrad.use_split_accumulator if hasattr(_recipe, "fp8_gemm_wgrad"): wgrad_use_split_accumulator = _recipe.fp8_gemm_wgrad.use_split_accumulator - recipe_custom = _recipe.custom() else: backward_override = None custom = is_custom(input_quantizer) or is_custom(weight_quantizer) @@ -2414,14 +2522,8 @@ def forward( fp8_output=fp8_output, save_original_input=self.save_original_input, backward_override=backward_override, - fprop_use_split_accumulator=fprop_use_split_accumulator, dgrad_use_split_accumulator=dgrad_use_split_accumulator, wgrad_use_split_accumulator=wgrad_use_split_accumulator, - recipe_custom=recipe_custom, - fp8_recompute_forward=( - is_fp8_activation_recompute_enabled() - and not in_fp8_activation_recompute_phase() - ), custom=custom, debug=debug, # weight-workspace caching @@ -2456,15 +2558,8 @@ def forward( ) if use_compiled_op: - fallback_reason = self._compile_eager_fallback_reason( - inp, - skip_fp8_weight_update, - fp8_output, - fp8_grad, - is_grad_enabled, - debug, - quantizers, - ) + # Safety net for quantizer-dependent conditions only. + fallback_reason = fwd_args.compile_unsupported_reason() if fallback_reason is not None: warn_compile_eager_fallback(fallback_reason) torch._dynamo.graph_break( @@ -2563,9 +2658,9 @@ def _compile_eager_fallback_reason( fp8_grad: bool, is_grad_enabled: bool, debug: bool, - quantizers: Sequence[Optional[Quantizer]], ) -> Optional[str]: - """Return a fallback reason after preparing the forward quantizers.""" + """Why this call can't use the compiled op (else None), decided before + prepare_forward. Quantizer checks stay in compile_unsupported_reason.""" if debug: return "debug instrumentation (nvidia-dlfw-inspect)" weight_tensor, bias_tensor = self._get_weight_and_bias_tensors() @@ -2599,7 +2694,24 @@ def _compile_eager_fallback_reason( return "a quantized input grad (fp8_grad=True)" if fp8 and skip_fp8_weight_update is not None: return "GPU-controlled FP8 weight cache updates" - return compile_unsupported_quantizer_reason(quantizers) + return None + + @torch._dynamo.disable + def _forward_eager_fallback( + self, + inp: torch.Tensor, + is_first_microbatch: Optional[bool], + fp8_output: bool, + fp8_grad: bool, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, ...]]: + """Re-run forward outside Dynamo (unsupported-config fallback).""" + return Linear.forward( + self, + inp, + is_first_microbatch=is_first_microbatch, + fp8_output=fp8_output, + fp8_grad=fp8_grad, + ) def _get_weight_and_bias_tensors(self) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: # Get concatenated weight and bias tensors