diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 6dea683618..b21aee02a3 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -22,6 +22,7 @@ except ImportError: # pragma: no cover counters = None from torch._subclasses.fake_tensor import FakeTensor, FakeTensorMode +from torch._dynamo.testing import CompileCounterWithBackend try: from torch._opaque_base import OpaqueBaseMeta @@ -45,6 +46,11 @@ from transformer_engine.common import recipe from transformer_engine.pytorch.constants import FP8FwdTensorIdx, FP8BwdTensorIdx from transformer_engine.pytorch.module.base import TransformerEngineBaseModule +from transformer_engine.pytorch.module import ( + linear as linear_module, + layernorm_linear as layernorm_linear_module, + layernorm_mlp as layernorm_mlp_module, +) from transformer_engine.pytorch.quantization import FP8GlobalStateManager, QuantizerRole from transformer_engine.pytorch.ops.basic.basic_linear import BasicLinear from transformer_engine.pytorch.ops.basic.bias import BiasBwdArgs @@ -129,7 +135,8 @@ def nvfp4_4over6(): if mxfp8_available: _all_recipes.append(recipe.MXFP8BlockScaling()) if nvfp4_available: - _all_recipes.append(recipe.NVFP4BlockScaling()) + # Disable stochastic rounding for exact eager/compiled comparisons. + _all_recipes.append(recipe.NVFP4BlockScaling(disable_stochastic_rounding=True)) _all_recipes.append(nvfp4_4over6()) _all_recipes.append(nvfp4_row_scaled()) @@ -175,30 +182,75 @@ def _assert_no_cudagraph_skips(enabled: bool): _EAGER_ATOL, _EAGER_RTOL = 0.0, 0.0 -def _assert_close_eager_compiled(fn, compiled, model, base): - """Run ``fn`` eagerly and ``compiled`` on identical inputs; assert the - forward output and the input / weight / bias gradients match.""" +def _flatten_outputs(out): + return list(out) if isinstance(out, (tuple, list)) else [out] + + +def _assert_close_eager_compiled(fn, compiled, model, base, *, dynamic_batch=False): + """Run ``fn`` eagerly and ``compiled`` on identical inputs; assert every + forward output and the input / parameter gradients match.""" inp_eager = base.detach().clone().requires_grad_(True) model.zero_grad(set_to_none=True) - out_eager = fn(inp_eager) - out_eager.sum().backward() - ref_out = out_eager.detach().clone() - ref_wgrad = model.weight.grad.detach().clone() + outs_eager = _flatten_outputs(fn(inp_eager)) + sum(o.sum() for o in outs_eager).backward() + ref_outs = [o.detach().clone() for o in outs_eager] ref_igrad = inp_eager.grad.detach().clone() - # bias=False keeps a 0-element ``bias`` around; grad is None then. - ref_bgrad = model.bias.grad.detach().clone() if model.bias.grad is not None else None + ref_pgrads = { + name: p.grad.detach().clone() for name, p in model.named_parameters() if p.grad is not None + } inp_compiled = base.detach().clone().requires_grad_(True) + if dynamic_batch: + torch._dynamo.mark_dynamic(inp_compiled, 0) model.zero_grad(set_to_none=True) # Clone before a later cuda-graph replay overwrites the static output buffer. - out_compiled = compiled(inp_compiled).clone() - out_compiled.sum().backward() + outs_compiled = [o.clone() for o in _flatten_outputs(compiled(inp_compiled))] + sum(o.sum() for o in outs_compiled).backward() - torch.testing.assert_close(out_compiled, ref_out, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) + assert len(outs_compiled) == len(ref_outs) + for out, ref in zip(outs_compiled, ref_outs): + torch.testing.assert_close(out, ref, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) torch.testing.assert_close(inp_compiled.grad, ref_igrad, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) - torch.testing.assert_close(model.weight.grad, ref_wgrad, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) - if ref_bgrad is not None: - torch.testing.assert_close(model.bias.grad, ref_bgrad, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) + pgrads = {name: p.grad for name, p in model.named_parameters() if p.grad is not None} + assert pgrads.keys() == ref_pgrads.keys() + for name, ref in ref_pgrads.items(): + torch.testing.assert_close( + pgrads[name], ref, atol=_EAGER_ATOL, rtol=_EAGER_RTOL, msg=f"grad mismatch: {name}" + ) + + +def _run_module_compile_test(model, fp8_recipe, compile_mode, in_features, dtype=torch.bfloat16): + device = "cuda" + + def fn(inp): + if fp8_recipe is None: + return model(inp) + with te.autocast(recipe=fp8_recipe): + return model(inp) + + torch._dynamo.reset() + if compile_mode == "reduce-overhead": + warm = torch.randn(32, in_features, dtype=dtype, device=device, requires_grad=True) + outs = _flatten_outputs(fn(warm)) + sum(o.sum() for o in outs).backward() + model.zero_grad(set_to_none=True) + compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) + + n_iters = 3 if compile_mode == "reduce-overhead" else 1 + with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): + for _ in range(n_iters): + base = torch.randn(32, in_features, dtype=dtype, device=device) + _assert_close_eager_compiled(fn, compiled, model, base) + + +_LINEAR_MODULES = ["Linear", "LayerNormLinear", "LayerNormMLP"] + + +def _make_module(module, *, dtype=torch.bfloat16, device="cuda", **kwargs): + """Small module with dimensions suitable for FP8 GEMMs.""" + return getattr(te, module)( + 64, 128 if module == "LayerNormMLP" else 32, params_dtype=dtype, device=device, **kwargs + ) # --------------------------------------------------------------------------- @@ -1909,11 +1961,19 @@ def test_to_tensor_spec_plain(): @pytest.mark.parametrize("factory, shape", _SPEC_QUANTIZERS) -def test_to_tensor_spec_quantized(factory, shape): +@pytest.mark.parametrize("swizzled", [False, True]) +def test_to_tensor_spec_quantized(factory, shape, swizzled): """``to_tensor_spec`` round-trips a quantized tensor back into a spec.""" q = factory() + has_scale_layout = isinstance(q, (MXFP8Quantizer, NVFP4Quantizer)) + if has_scale_layout: + q.optimize_for_gemm = not swizzled tensor = TensorSpec( - shape=shape, dtype=torch.bfloat16, quantizer=q, device=torch.device("cpu") + shape=shape, + dtype=torch.bfloat16, + quantizer=q, + device=torch.device("cpu"), + with_gemm_swizzled_scales=swizzled if has_scale_layout else None, ).create_tensor() spec = to_tensor_spec(tensor) @@ -1923,13 +1983,17 @@ def test_to_tensor_spec_quantized(factory, shape): # Same buffer layout as the original tensor. assert spec.inner_names() == tuple(q.inner_tensor_specs(shape)) # Rebuilding from the derived spec matches the original tensor's structure. - assert _signature(spec.create_tensor(), spec.inner_names()) == _signature( - tensor, spec.inner_names() - ) + rebuilt = spec.create_tensor() + assert _signature(rebuilt, spec.inner_names()) == _signature(tensor, spec.inner_names()) + if has_scale_layout: + assert tensor._with_gemm_swizzled_scales == swizzled + assert spec.with_gemm_swizzled_scales == swizzled + assert rebuilt._with_gemm_swizzled_scales == swizzled + assert spec.quantizer.optimize_for_gemm == q.optimize_for_gemm == (not swizzled) # --------------------------------------------------------------------------- -# te.Linear +# te.Linear / te.LayerNormLinear / te.LayerNormMLP # --------------------------------------------------------------------------- @@ -1937,20 +2001,62 @@ def test_to_tensor_spec_quantized(factory, shape): @pytest.mark.parametrize("compile_mode", _compile_modes) @pytest.mark.parametrize( "fp8_recipe", - [None, *_all_recipes], + [None, *_all_recipes] + + ( + [ + recipe.CustomRecipe( + qfactory=lambda _role: Float8CurrentScalingQuantizer( + tex.DType.kFloat8E4M3, device="cuda" + ) + ) + ] + if fp8_available + else [] + ), ids=lambda r: "bf16" if r is None else type(r).__name__, ) -def test_te_linear_compiles(fp8_recipe, compile_mode): - """ - torch.compile(fullgraph=True) of ``te.Linear`` under every built-in - recipe (plus the bf16-only baseline with no autocast), for both the default - backend and ``mode="reduce-overhead"`` (CUDA-graph trees). - """ - dtype = torch.bfloat16 - device = "cuda" +@pytest.mark.parametrize("module", _LINEAR_MODULES) +def test_te_modules_compile(module, fp8_recipe, compile_mode): + """Fullgraph forward/backward agrees with eager across modules, recipes and modes.""" + if module == "LayerNormMLP" and getattr(fp8_recipe, "backward_override", None) is not None: + pytest.skip("LayerNormMLP does not support backward_override, including in eager mode") + model = _make_module(module) + _run_module_compile_test(model, fp8_recipe, compile_mode, 64) + + +_non_2d_cases = [ + pytest.param((64,), None, id="rank1-bf16"), + pytest.param((2, 4, 4, 64), None, id="rank4-bf16"), + *( + pytest.param( + (2, 4, 16, 128) if isinstance(r, recipe.Float8BlockScaling) else (2, 4, 4, 64), + r, + id=f"rank4-{recipe_id(r)}", + ) + for r in _all_recipes + ), + *( + pytest.param((2, 4, 8, 64), r, id=f"rank4-aligned-{recipe_id(r)}") + for r in _all_recipes + if isinstance(r, recipe.NVFP4BlockScaling) and not r.disable_rht + ), +] - # FP8 GEMMs require leading dimensions divisible by 16. - model = te.Linear(64, 32, params_dtype=dtype, device=device) + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("compile_mode", _compile_modes) +@pytest.mark.parametrize("module", _LINEAR_MODULES) +@pytest.mark.parametrize("input_shape, fp8_recipe", _non_2d_cases) +def test_te_modules_compile_non_2d_input(input_shape, fp8_recipe, module, compile_mode): + """Compiled modules preserve non-matrix activation shapes and gradients.""" + if module == "LayerNormMLP" and getattr(fp8_recipe, "backward_override", None) is not None: + pytest.skip("LayerNormMLP does not support backward_override, including in eager mode") + dtype = torch.bfloat16 + if isinstance(fp8_recipe, recipe.Float8BlockScaling): + out_features = 256 if module == "LayerNormMLP" else 128 + model = getattr(te, module)(128, out_features, params_dtype=dtype, device="cuda") + else: + model = _make_module(module, dtype=dtype) def fn(inp): if fp8_recipe is None: @@ -1962,17 +2068,15 @@ def fn(inp): if compile_mode == "reduce-overhead": _cudagraph_warmup( fn, - torch.randn(32, 64, dtype=dtype, device=device, requires_grad=True), + torch.randn(input_shape, dtype=dtype, device="cuda", requires_grad=True), backward=True, ) model.zero_grad(set_to_none=True) compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) - # Iterate a few times so reduce-overhead actually replays a captured graph. - n_iters = 3 if compile_mode == "reduce-overhead" else 1 with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): - for _ in range(n_iters): - base = torch.randn(32, 64, dtype=dtype, device=device) + for _ in range(3 if compile_mode == "reduce-overhead" else 1): + base = torch.randn(input_shape, dtype=dtype, device="cuda") _assert_close_eager_compiled(fn, compiled, model, base) @@ -1987,19 +2091,22 @@ def fn(inp): + [ pytest.param(fp8_recipe, shape, id=f"{name}-{shape}") for name, fp8_recipe in ( + ("bf16", None), + ("mxfp8", recipe.MXFP8BlockScaling()), ("nvfp4", recipe.NVFP4BlockScaling()), ("nvfp4-no-rht", recipe.NVFP4BlockScaling(disable_rht=True)), ) - for shape in ((1, 128, 128), (2, 4, 16, 128)) + for shape in ((1, 128, 128), (128, 1, 128), (2, 4, 16, 128)) ], ) def test_te_linear_compiles_non_2d_input(fp8_recipe, input_shape, compile_mode): """Compiled Linear preserves non-matrix activation shapes in forward and backward.""" if isinstance(fp8_recipe, recipe.NVFP4BlockScaling) and not nvfp4_available: pytest.skip(reason_for_no_nvfp4) + if isinstance(fp8_recipe, recipe.MXFP8BlockScaling) and not mxfp8_available: + pytest.skip(reason_for_no_mxfp8) dtype = torch.bfloat16 - out_features = 32 if fp8_recipe is None else 128 - model = te.Linear(input_shape[-1], out_features, params_dtype=dtype, device="cuda") + model = te.Linear(input_shape[-1], 128, params_dtype=dtype, device="cuda") def fn(inp): if fp8_recipe is None: @@ -2026,37 +2133,14 @@ def fn(inp): @pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") @pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) @pytest.mark.parametrize("compile_mode", _compile_modes) -def test_te_linear_compile_with_quantized_fp8_weight(compile_mode): - """torch.compile of Linear with the weight initialized as an FP8 tensor - (exercises the wrapper op's ``register_torch_dispatch`` input flattening).""" - dtype = torch.bfloat16 - device = "cuda" +@pytest.mark.parametrize("module", ["Linear", "LayerNormLinear"]) +def test_te_modules_compile_with_quantized_fp8_weight(module, compile_mode): + """FP8 primary weights cross the wrapper-op boundary in both compile modes.""" fp8_recipe = recipe.Float8CurrentScaling() - with te.quantized_model_init(enabled=True, recipe=fp8_recipe): - model = te.Linear(64, 32, params_dtype=dtype, device=device) - + model = _make_module(module) assert isinstance(model.weight, te.Float8Tensor) - - def fn(inp): - with te.autocast(recipe=fp8_recipe): - return model(inp) - - torch._dynamo.reset() - if compile_mode == "reduce-overhead": - _cudagraph_warmup( - fn, - torch.randn(32, 64, dtype=dtype, device=device, requires_grad=True), - backward=True, - ) - model.zero_grad(set_to_none=True) - compiled = torch.compile(fn, fullgraph=True, mode=compile_mode) - - n_iters = 3 if compile_mode == "reduce-overhead" else 1 - with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): - for _ in range(n_iters): - base = torch.randn(32, 64, dtype=dtype, device=device) - _assert_close_eager_compiled(fn, compiled, model, base) + _run_module_compile_test(model, fp8_recipe, compile_mode, 64) @pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") @@ -2105,7 +2189,7 @@ def fn(inp): ) -# Configs rejected by LinearFwdArgs.compile_unsupported_reason() that a +# Configs rejected by the modules' early fallback checks that a # single-GPU unit test can construct. Distributed-only reasons (fsdp_group, # DistributedWeight) and CPU offloading need machinery this file doesn't have; # delayed scaling is a hard error (check_recipe_support), tested separately. @@ -2114,58 +2198,67 @@ def fn(inp): # output crossing the graph-break boundary gets a plain-tensor tangent); # "no_grad" = forward under no_grad. _FALLBACK_CASES = [ - "fp8_output_differentiable", - "fuse_wgrad_accumulation", - "delayed_wgrad", - "quantized_input", + (module, case) + for module in _LINEAR_MODULES + for case in ("fuse_wgrad_accumulation", "delayed_wgrad") +] + [ + ("Linear", "fp8_output_differentiable"), + ("Linear", "quantized_input"), + ("LayerNormMLP", "checkpoint"), ] -def _fallback_case(case, dtype, device): - """Build ``(model, fn, mode, post_backward, reason)`` for one case.""" +def _fallback_case(module, case, dtype, device): + """Build ``(model, fn, mode, post_backward, reason)`` for one fallback case.""" model_kwargs = {} if case == "fuse_wgrad_accumulation": model_kwargs["fuse_wgrad_accumulation"] = True elif case == "delayed_wgrad": model_kwargs["delay_wgrad_compute"] = True - model = te.Linear(64, 32, params_dtype=dtype, device=device, **model_kwargs) - - if case == "fp8_output_differentiable": - fp8_recipe = recipe.Float8CurrentScaling() + elif case == "checkpoint": + model_kwargs["checkpoint"] = True + model = _make_module(module, dtype=dtype, device=device, **model_kwargs) + fp8_recipe = recipe.Float8CurrentScaling() - def fn(inp): - with te.autocast(recipe=fp8_recipe): + def fn(inp): + # Preserve the original Linear BF16 accumulation/delayed-wgrad cases; + # the LayerNorm cases exercise those paths under FP8. + use_fp8 = module != "Linear" or case in ("fp8_output_differentiable", "quantized_input") + with te.autocast(enabled=use_fp8, recipe=fp8_recipe): + if case == "fp8_output_differentiable": return model(inp, fp8_output=True).dequantize() + return model(inp) + if case == "fp8_output_differentiable": return model, fn, "fwd_grad", None, "differentiable fp8_output=True" if case == "fuse_wgrad_accumulation": - model.weight.main_grad = torch.zeros_like(model.weight, dtype=torch.float32) - return model, model, "bwd", None, "fuse_wgrad_accumulation" + weights = ( + (model.fc1_weight, model.fc2_weight) if module == "LayerNormMLP" else (model.weight,) + ) + for weight in weights: + weight.main_grad = torch.zeros_like(weight, dtype=torch.float32) + return model, fn, "bwd", None, "fuse_wgrad_accumulation" if case == "delayed_wgrad": - return model, model, "bwd", model.backward_dw, "delayed wgrad compute" + return model, fn, "bwd", model.backward_dw, "delayed wgrad compute" if case == "quantized_input": - fp8_recipe = recipe.Float8CurrentScaling() - - def fn(inp): - with te.autocast(recipe=fp8_recipe): - return model(inp) - return model, fn, "no_grad", None, "a quantized input tensor" + if case == "checkpoint": + return model, fn, "bwd", None, "activation checkpointing" raise ValueError(case) @pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") @pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) -@pytest.mark.parametrize("case", _FALLBACK_CASES) -def test_te_linear_compile_eager_fallback(case): +@pytest.mark.parametrize("module,case", _FALLBACK_CASES) +def test_te_modules_compile_eager_fallback(module, case): """Configs unsupported on the compiled custom-op path must fall back to eager under ``torch.compile`` -- warning + numerics identical to eager -- and graph-break with the explicit reason under ``fullgraph=True``.""" dtype, device = torch.bfloat16, "cuda" torch.manual_seed(0) - model_ref, fn_ref, mode, post_bwd_ref, _ = _fallback_case(case, dtype, device) + model_ref, fn_ref, mode, post_bwd_ref, _ = _fallback_case(module, case, dtype, device) torch.manual_seed(0) - model, fn, _, post_bwd, reason = _fallback_case(case, dtype, device) + model, fn, _, post_bwd, reason = _fallback_case(module, case, dtype, device) def make_inp(): torch.manual_seed(1) @@ -2194,20 +2287,27 @@ def make_inp(): if post_bwd is not None: post_bwd() torch.testing.assert_close(inp.grad, inp_ref.grad, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) - # The fallback must also preserve the stateful side effects that define - # these cases (main_grad accumulation, delayed wgrad), not just inp.grad. - if getattr(model_ref.weight, "main_grad", None) is not None: - assert model.weight.grad is None - torch.testing.assert_close( - model.weight.main_grad, - model_ref.weight.main_grad, - atol=_EAGER_ATOL, - rtol=_EAGER_RTOL, - ) - else: - torch.testing.assert_close( - model.weight.grad, model_ref.weight.grad, atol=_EAGER_ATOL, rtol=_EAGER_RTOL - ) + # Check both gradients and stateful accumulation for every parameter. + ref_params = dict(model_ref.named_parameters()) + for name, param in model.named_parameters(): + ref_param = ref_params[name] + if getattr(ref_param, "main_grad", None) is not None: + assert param.grad is None + torch.testing.assert_close( + param.main_grad, + ref_param.main_grad, + atol=_EAGER_ATOL, + rtol=_EAGER_RTOL, + msg=f"main_grad mismatch: {name}", + ) + else: + torch.testing.assert_close( + param.grad, + ref_param.grad, + atol=_EAGER_ATOL, + rtol=_EAGER_RTOL, + msg=f"grad mismatch: {name}", + ) torch.testing.assert_close(out.detach(), out_ref.detach(), atol=_EAGER_ATOL, rtol=_EAGER_RTOL) torch._dynamo.reset() @@ -2264,14 +2364,17 @@ def fn(inp): ], ids=recipe_id, ) -def test_te_linear_compile_is_first_microbatch( - fp8_recipe, compile_mode, deferred_backward, monkeypatch +@pytest.mark.parametrize("module", _LINEAR_MODULES) +def test_te_modules_compile_is_first_microbatch( + module, fp8_recipe, compile_mode, deferred_backward, monkeypatch ): """Reuse cached weights across microbatches and refresh after optimizer updates.""" dtype, device = torch.bfloat16, "cuda" - model = te.Linear(128, 128, params_dtype=dtype, device=device) - ref_model = te.Linear(128, 128, params_dtype=dtype, device=device) + model = getattr(te, module)(128, 128, params_dtype=dtype, device=device) + ref_model = getattr(te, module)(128, 128, params_dtype=dtype, device=device) ref_model.load_state_dict(model.state_dict()) + cache_names = ("fc1_weight", "fc2_weight") if module == "LayerNormMLP" else ("weight",) + cached_workspaces = {} def fn(inp, is_first): with te.autocast(recipe=fp8_recipe): @@ -2315,17 +2418,20 @@ def replay(graph): out = compiled(inp, is_first).clone() torch.testing.assert_close(out, out_ref, atol=_EAGER_ATOL, rtol=_EAGER_RTOL) - workspace = model._fp8_workspaces["weight"] - if is_first: - cached_workspace = workspace - else: - assert workspace is cached_workspace - torch.testing.assert_close( - workspace.dequantize(), - ref_model._fp8_workspaces["weight"].dequantize(), - atol=_EAGER_ATOL, - rtol=_EAGER_RTOL, - ) + for name in cache_names: + workspace = model._fp8_workspaces[name] + if is_first: + if name in cached_workspaces: + assert workspace is not cached_workspaces[name] + cached_workspaces[name] = workspace + else: + assert workspace is cached_workspaces[name] + torch.testing.assert_close( + workspace.dequantize(), + ref_model._fp8_workspaces[name].dequantize(), + atol=_EAGER_ATOL, + rtol=_EAGER_RTOL, + ) pending.append((out, out_ref, inp, inp_ref)) if not deferred_backward or microbatch == 2: @@ -2346,7 +2452,7 @@ def replay(graph): for param, ref_param in zip(model.parameters(), ref_model.parameters()): param.add_(param.grad, alpha=-0.001) ref_param.add_(ref_param.grad, alpha=-0.001) - del cached_workspace, workspace, out, out_ref, result, ref_result + del workspace, out, out_ref, result, ref_result torch._dynamo.reset() @@ -2403,74 +2509,363 @@ def train_step(): @pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") -def test_te_linear_dynamic_shapes(): - """torch.compile of ``te.Linear`` with a ``mark_dynamic`` batch dimension: - one graph must serve all batch sizes -- no recompiles -- and match eager - numerically. - - Only the leading (batch/sequence) dims may be dynamic; the last dim is - fixed by the weight's ``in_features``. - """ - dtype = torch.bfloat16 - device = "cuda" - in_features, out_features = 64, 32 - model = te.Linear(in_features, out_features, params_dtype=dtype, device=device) +@pytest.mark.parametrize( + "module,bias_gelu_fusion", + [ + ("Linear", False), + ("LayerNormLinear", False), + ("LayerNormMLP", False), + ("LayerNormMLP", True), + ], +) +def test_te_modules_dynamic_shapes(module, bias_gelu_fusion): + """Batch sizes share the warmed-up outer graph and match eager outputs/gradients.""" + dtype, device = torch.bfloat16, "cuda" + model = _make_module(module, dtype=dtype, device=device) + if module == "LayerNormMLP": + model.bias_gelu_nvfusion = bias_gelu_fusion def fn(inp): return model(inp) torch._dynamo.reset() - compiled = torch.compile(fn, fullgraph=True) - - batch_sizes = [16, 32, 48] - - # Two warmup calls: the second absorbs the one-time recompile from module - # attributes lazily created during call one (e.g. the cached ``is_fsdp2``). + # Count the outer graph independently of lazy compilation of fused helpers. + compile_counter = CompileCounterWithBackend("inductor") + compiled = torch.compile(fn, fullgraph=True, backend=compile_counter) for _ in range(2): - warm = torch.randn(batch_sizes[0], in_features, dtype=dtype, device=device) + warm = torch.randn(16, 64, dtype=dtype, device=device) torch._dynamo.mark_dynamic(warm, 0) compiled(warm.requires_grad_(True)).sum().backward() model.zero_grad(set_to_none=True) - unique_graphs_baseline = _dynamo_counter("stats", "unique_graphs") - if not unique_graphs_baseline: - warnings.warn("unique_graphs counter is stale; skipping the recompile check") - - for batch in batch_sizes: - inp = torch.randn(batch, in_features, dtype=dtype, device=device, requires_grad=True) - # Mark batch dim as dynamic so Dynamo traces once and reuses across batch sizes. - torch._dynamo.mark_dynamic(inp, 0) - out = compiled(inp) - assert out.shape == (batch, out_features), f"wrong output shape for batch={batch}" - out.sum().backward() - assert inp.grad is not None, f"no input gradient for batch={batch}" - assert inp.grad.shape == inp.shape, f"wrong grad shape for batch={batch}" + frames_after_warmup = compile_counter.frame_count + assert frames_after_warmup > 0 - # Verify numerics against eager on each distinct batch size. - inp_eager = inp.detach().clone().requires_grad_(True) - model.zero_grad(set_to_none=True) - out_eager = model(inp_eager) - out_eager.sum().backward() - torch.testing.assert_close( - out.detach(), - out_eager.detach(), - atol=_EAGER_ATOL, - rtol=_EAGER_RTOL, - msg=f"forward mismatch at batch={batch}", - ) - torch.testing.assert_close( - inp.grad, - inp_eager.grad, - atol=_EAGER_ATOL, - rtol=_EAGER_RTOL, - msg=f"dgrad mismatch at batch={batch}", + for batch in (16, 32, 48): + base = torch.randn(batch, 64, dtype=dtype, device=device) + _assert_close_eager_compiled(fn, compiled, model, base, dynamic_batch=True) + + assert compile_counter.frame_count == frames_after_warmup + + +# --------------------------------------------------------------------------- +# Module fake/real metadata and LayerNorm variants +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("module", ["LayerNormLinear", "LayerNormMLP"]) +@pytest.mark.parametrize("loss_source", ["output", "normalization", "both"]) +@pytest.mark.parametrize("fp8_recipe", [None, recipe.Float8CurrentScaling()]) +def test_te_layernorm_output_gradients(module, loss_source, fp8_recipe): + """Route distinct gradients from each public output to the correct backward field.""" + if fp8_recipe is not None and not fp8_available: + pytest.skip(reason_for_no_fp8) + model = _make_module(module, return_layernorm_output=True) + + def fn(inp): + if fp8_recipe is None: + output, norm_output = model(inp) + else: + with te.autocast(recipe=fp8_recipe): + output, norm_output = model(inp) + if loss_source == "output": + return output + if loss_source == "normalization": + return norm_output + return output, 0.25 * norm_output + + torch._dynamo.reset() + compiled = torch.compile(fn, fullgraph=True) + base = torch.randn(32, 64, dtype=torch.bfloat16, device="cuda") + _assert_close_eager_compiled(fn, compiled, model, base) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize( + "case", ["linear", "layernorm_linear", "mlp_bias_gelu", "mlp_gemm_gelu", "mlp_swiglu"] +) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("bias", [False, True]) +@pytest.mark.parametrize("fp8_recipe", [None, recipe.Float8CurrentScaling()]) +def test_te_modules_fake_matches_real(monkeypatch, case, dtype, bias, fp8_recipe): + """Compare fake outputs, saved buffers and gradients against the eager kernels.""" + from transformer_engine.pytorch.dynamo.custom_op import _parse_arg_type, _spec_view + + if fp8_recipe is not None and not fp8_available: + pytest.skip(reason_for_no_fp8) + if fp8_recipe is not None and case == "mlp_gemm_gelu": + if dtype == torch.float32 or (dtype == torch.float16 and not bias): + pytest.skip( + "cuBLAS FP8 GELU requires matching output and auxiliary dtypes (BF16 without bias)" + ) + module_name = case if case in ("linear", "layernorm_linear") else "layernorm_mlp" + module = { + "linear": linear_module, + "layernorm_linear": layernorm_linear_module, + "layernorm_mlp": layernorm_mlp_module, + }[module_name] + checked = [] + + def assert_metadata(spec, real): + if isinstance(spec, TensorSpec): + assert real is not None + assert tuple(spec.shape) == tuple(real.shape) + assert spec.dtype == to_tensor_spec(real).dtype + assert spec.device == real.device + names = ( + tuple(real.__tensor_flatten__()[0]) + if hasattr(real, "__tensor_flatten__") + else ("data",) + ) + assert spec.inner_names() == names + for name, fake_inner in zip(names, spec.create_inner_tensors()): + real_inner = real if name == "data" else getattr(real, name) + assert fake_inner.shape == real_inner.shape + assert fake_inner.dtype == real_inner.dtype + assert fake_inner.stride() == real_inner.stride() + elif isinstance(spec, tuple): + assert isinstance(real, tuple) and len(spec) == len(real) + for fake_value, real_value in zip(spec, real): + assert_metadata(fake_value, real_value) + elif isinstance(spec, dict): + # No manual FSDP or checkpointing here; [] and None both denote no shards. + for key in spec.keys() | real.keys(): + if key in ("fsdp_shapes", "checkpoint"): + assert not spec.get(key) and not real.get(key) + else: + assert spec[key] == real[key] + else: + assert spec == real + + def checked_impl(direction): + real_impl = getattr(module, f"_{module_name}_{direction}_impl") + fake_impl = getattr(module, f"_{module_name}_{direction}_fake") + + def wrapper(args): + spec_args = _spec_view(args, _parse_arg_type(type(args)).tensor_field_names()) + # Fake execution may change quantizer usage; leave the real invocation untouched. + spec_args = dataclasses.replace( + spec_args, + **{ + field.name: value.copy() + for field in dataclasses.fields(spec_args) + if isinstance(value := getattr(spec_args, field.name), Quantizer) + }, + ) + expected = fake_impl(spec_args) + actual = real_impl(args) + assert_metadata(expected, actual) + checked.append(direction) + return actual + + return wrapper + + for direction in ("forward", "backward"): + monkeypatch.setattr(module, f"_{module_name}_{direction}_impl", checked_impl(direction)) + kwargs = dict(params_dtype=dtype, device="cuda", bias=bias) + if case != "linear": + kwargs["return_layernorm_output"] = True + if case == "linear": + model = te.Linear(64, 32, **kwargs) + elif case == "layernorm_linear": + model = te.LayerNormLinear(64, 32, **kwargs) + else: + model = te.LayerNormMLP( + 64, 128, activation="swiglu" if case == "mlp_swiglu" else "gelu", **kwargs ) + model.bias_gelu_nvfusion = case == "mlp_bias_gelu" + model.gemm_gelu_fusion = case == "mlp_gemm_gelu" + inp = torch.randn(2, 16, 64, dtype=dtype, device="cuda", requires_grad=True) + with te.autocast(enabled=fp8_recipe is not None, recipe=fp8_recipe): + outs = _flatten_outputs(model(inp)) + sum(out.sum() for out in outs).backward() + assert checked == ["forward", "backward"] + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) +@pytest.mark.parametrize( + "dtype,bias", [(torch.bfloat16, False), (torch.bfloat16, True), (torch.float16, True)] +) +@pytest.mark.parametrize("compile_mode", _compile_modes) +def test_te_layernorm_mlp_compile_fp8_gemm_gelu(dtype, bias, compile_mode): + """Fused FP8 GEMM-GELU must preserve the auxiliary tensor dtype through backward.""" + model = te.LayerNormMLP(64, 128, params_dtype=dtype, device="cuda", bias=bias) + model.gemm_gelu_fusion = True + _run_module_compile_test(model, recipe.Float8CurrentScaling(), compile_mode, 64, dtype=dtype) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize( + "quantization,disable_rht_fusion", + [("mxfp8", False), ("mxfp8_2d", False), ("nvfp4", False), ("nvfp4", True)], +) +@pytest.mark.parametrize("module", ["LayerNormLinear", "LayerNormMLP"]) +@pytest.mark.parametrize("normalization", ["LayerNorm", "RMSNorm"]) +@pytest.mark.parametrize( + "use_cudnn,input_shape,return_layernorm_output", + [ + (False, (128, 128), False), + (True, (128, 128), False), + (True, (64, 128), False), + (True, (2, 64, 128), False), + (True, (128, 128), True), + ], +) +def test_te_layernorm_compile_scale_layout( + monkeypatch, + module, + normalization, + use_cudnn, + input_shape, + return_layernorm_output, + quantization, + disable_rht_fusion, +): + """Saved normalization scales retain their layout through compiled backward.""" + if quantization.startswith("mxfp8") and not mxfp8_available: + pytest.skip(reason_for_no_mxfp8) + if quantization == "nvfp4" and not nvfp4_available: + pytest.skip(reason_for_no_nvfp4) + monkeypatch.setenv("NVTE_NORM_FWD_USE_CUDNN", str(int(use_cudnn))) + monkeypatch.setenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", str(int(disable_rht_fusion))) + model = getattr(te, module)( + 128, + 128, + params_dtype=torch.bfloat16, + device="cuda", + normalization=normalization, + return_layernorm_output=return_layernorm_output, + ) + if quantization == "mxfp8_2d": + + def qfactory(role): + return MXFP8Quantizer( + tex.DType.kFloat8E4M3, + with_2d_quantization=role is not None and role.tensor_type == "input", + ) + + fp8_recipe = recipe.CustomRecipe(qfactory=qfactory) + elif quantization == "mxfp8": + fp8_recipe = recipe.MXFP8BlockScaling() + else: + fp8_recipe = recipe.NVFP4BlockScaling(disable_stochastic_rounding=True) + + def fn(inp): + with te.autocast(recipe=fp8_recipe): + return model(inp) + + torch._dynamo.reset() + compiled = torch.compile(fn, fullgraph=True) + base = torch.randn(input_shape, dtype=torch.bfloat16, device="cuda") + _assert_close_eager_compiled(fn, compiled, model, base) - if unique_graphs_baseline: - unique_graphs_after = _dynamo_counter("stats", "unique_graphs") - assert unique_graphs_after == unique_graphs_baseline, ( - "Unexpected recompilation(s) across different batch sizes: " - f"{unique_graphs_after - unique_graphs_baseline} extra graph(s) compiled" + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("sequence_parallel", [False, True]) +@pytest.mark.parametrize("gathered", [False, True]) +@pytest.mark.parametrize("input_shape", [(128, 128), (2, 64, 128)]) +def test_te_layernorm_mlp_fake_gathered_output( + monkeypatch, sequence_parallel, gathered, input_shape +): + """The normalization gather is independent of set_parallel_mode.""" + from transformer_engine.pytorch.dynamo.custom_op import _parse_arg_type, _spec_view + + module = layernorm_mlp_module + real_impl = module._layernorm_mlp_forward_impl + checked = [] + + def checked_impl(args): + args = dataclasses.replace( + args, + sequence_parallel=sequence_parallel, + tp_size=2, + return_layernorm_output_gathered=gathered, ) + spec_args = _spec_view(args, _parse_arg_type(type(args)).tensor_field_names()) + expected = module._layernorm_mlp_forward_fake(spec_args) + actual = real_impl(args) + for spec, tensor in zip(expected[:2], actual[:2]): + assert tuple(spec.shape) == tuple(tensor.shape) + checked.append(True) + return actual + + # Emulate a two-rank gather while exercising the real normalization and GEMMs. + monkeypatch.setattr(module, "get_distributed_world_size", lambda group: 2) + monkeypatch.setattr( + module, + "gather_along_first_dim", + lambda tensor, group, quantizer=None: (torch.cat([tensor, tensor]), None), + ) + monkeypatch.setattr(module, "_layernorm_mlp_forward_impl", checked_impl) + model = te.LayerNormMLP( + 128, + 128, + params_dtype=torch.bfloat16, + device="cuda", + set_parallel_mode=False, + return_layernorm_output=True, + ) + with torch.no_grad(): + model(torch.randn(input_shape, dtype=torch.bfloat16, device="cuda")) + assert checked == [True] + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("normalization", ["LayerNorm", "RMSNorm"]) +@pytest.mark.parametrize("return_layernorm_output", [False, True]) +@pytest.mark.parametrize("zero_centered_gamma", [False, True]) +def test_te_layernorm_linear_compile_variants( + normalization, return_layernorm_output, zero_centered_gamma +): + """Norm-type / returned-norm-output / zero-centered-gamma variants of + ``te.LayerNormLinear`` under torch.compile, bf16 and FP8 current scaling.""" + model = te.LayerNormLinear( + 64, + 32, + params_dtype=torch.bfloat16, + device="cuda", + normalization=normalization, + return_layernorm_output=return_layernorm_output, + zero_centered_gamma=zero_centered_gamma, + ) + _run_module_compile_test(model, None, "default", 64) + if fp8_available: + _run_module_compile_test(model, recipe.Float8CurrentScaling(), "default", 64) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("activation", ["gelu", "swiglu", "relu", "qgeglu"]) +@pytest.mark.parametrize("normalization", ["LayerNorm", "RMSNorm"]) +@pytest.mark.parametrize("return_layernorm_output", [False, True]) +def test_te_layernorm_mlp_compile_variants(activation, normalization, return_layernorm_output): + """Activation / norm-type / returned-norm-output variants of + ``te.LayerNormMLP`` under torch.compile, bf16 and FP8 current scaling.""" + model = te.LayerNormMLP( + 64, + 128, + params_dtype=torch.bfloat16, + device="cuda", + activation=activation, + normalization=normalization, + return_layernorm_output=return_layernorm_output, + ) + _run_module_compile_test(model, None, "default", 64) + if fp8_available: + _run_module_compile_test(model, recipe.Float8CurrentScaling(), "default", 64) + + +@pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") +@pytest.mark.parametrize("bias", [True, False]) +def test_te_layernorm_mlp_compile_no_bias_or_frozen(bias): + """``te.LayerNormMLP`` without biases, and with frozen FC1 weight (exercises + the ``None`` saved-tensor / grad slots).""" + model = te.LayerNormMLP(64, 128, params_dtype=torch.bfloat16, device="cuda", bias=bias) + _run_module_compile_test(model, None, "default", 64) + model.fc1_weight.requires_grad_(False) + _run_module_compile_test(model, None, "default", 64) + if fp8_available: + _run_module_compile_test(model, recipe.Float8CurrentScaling(), "default", 64) # --------------------------------------------------------------------------- # @@ -2912,21 +3307,20 @@ def test_te_ops_bias_compile_with_fp8_input(backward): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize( - "dtype,quantization,shape,disable_rht_fusion,case,grads", + "dtype,quantization,shape,disable_rht_fusion", [ - (dtype, quantization, (2, 16, 32), False, case, grads) - for dtype in (torch.float16, torch.bfloat16) - for quantization in (None, "fp8") - for case in ("linear", "bias", "pair", "unfused") - for grads in ("all", "input", "weight", "bias", "none") - ] - + [ - (torch.bfloat16, "nvfp4", (2, 16, 128), False, "pair", "all"), - (torch.bfloat16, "nvfp4", (64, 128), False, "pair", "all"), - (torch.bfloat16, "nvfp4", (64, 128), True, "pair", "all"), - (torch.bfloat16, "nvfp4-no-rht", (2, 16, 128), False, "pair", "all"), + (torch.float16, None, (2, 16, 32), False), + (torch.bfloat16, None, (2, 16, 32), False), + (torch.float16, "fp8", (2, 16, 32), False), + (torch.bfloat16, "fp8", (2, 16, 32), False), + (torch.bfloat16, "nvfp4", (2, 16, 128), False), + (torch.bfloat16, "nvfp4", (64, 128), False), + (torch.bfloat16, "nvfp4", (64, 128), True), + (torch.bfloat16, "nvfp4-no-rht", (2, 16, 128), False), ], ) +@pytest.mark.parametrize("case", ["linear", "bias", "pair", "unfused"]) +@pytest.mark.parametrize("grads", ["all", "input", "weight", "bias", "none"]) def test_te_ops_linear_bias_compile( dtype, quantization, shape, disable_rht_fusion, case, grads, monkeypatch ): @@ -3063,13 +3457,29 @@ def unexpected_allocation(*args, **kwargs): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize("training", [False, True]) +@pytest.mark.parametrize("initial_device", ["cuda", "cpu", "meta"]) @pytest.mark.parametrize( - "initial_device,quantization", - [("cuda", None), ("cpu", None), ("meta", None), ("cuda", "nvfp4"), ("cuda", "nvfp4-no-rht")], + "quantization", + [ + None, + "fp8", + "fp8-block", + "mxfp8", + "nvfp4", + "nvfp4-no-rht", + "nvfp4-4over6", + "nvfp4-row-scaled", + ], ) def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device, quantization): """A fresh process prevents earlier GEMMs from hiding lazy workspace allocation.""" - if quantization is not None and not nvfp4_available: + if quantization == "fp8" and not fp8_available: + pytest.skip(reason_for_no_fp8) + if quantization == "fp8-block" and not fp8_block_scaling_available: + pytest.skip(reason_for_no_fp8_block_scaling) + if quantization == "mxfp8" and not mxfp8_available: + pytest.skip(reason_for_no_mxfp8) + if quantization is not None and quantization.startswith("nvfp4") and not nvfp4_available: pytest.skip(reason_for_no_nvfp4) script = textwrap.dedent( """ @@ -3084,17 +3494,27 @@ def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device, quantizati training, initial_device, quantization = sys.argv[1:] training = training == "True" - is_nvfp4 = quantization != "None" - quant_recipe = ( - recipe.NVFP4BlockScaling( - disable_rht=quantization == "nvfp4-no-rht", + is_nvfp4 = quantization.startswith("nvfp4") + if quantization == "None": + quant_recipe = None + elif is_nvfp4: + row_scaled = quantization == "nvfp4-row-scaled" + quant_recipe = recipe.NVFP4BlockScaling( + disable_rht=quantization != "nvfp4", disable_stochastic_rounding=True, + disable_2d_quantization=row_scaled, + row_scaled_activation=row_scaled, + nvfp4_4over6="all" if quantization == "nvfp4-4over6" else "none", + backward_override="dequantized" if row_scaled else None, ) - if is_nvfp4 - else None - ) - in_features, out_features = (128, 128) if is_nvfp4 else (32, 64) - batch_size = 32 + else: + quant_recipe = { + "fp8": recipe.Float8CurrentScaling, + "fp8-block": recipe.Float8BlockScaling, + "mxfp8": recipe.MXFP8BlockScaling, + }[quantization]() + in_features, out_features = (128, 128) if quant_recipe is not None else (32, 64) + batch_size = 128 if isinstance(quant_recipe, recipe.Float8BlockScaling) else 32 eager_tols = {"rtol": 0, "atol": 0} if is_nvfp4 else {} assert get_cublas_workspace.cache_info().currsize == 0 linear = te.ops.BasicLinear( diff --git a/transformer_engine/pytorch/dynamo/custom_op.py b/transformer_engine/pytorch/dynamo/custom_op.py index 1ed84017ec..1b8f3b4b1f 100644 --- a/transformer_engine/pytorch/dynamo/custom_op.py +++ b/transformer_engine/pytorch/dynamo/custom_op.py @@ -475,6 +475,9 @@ def _is_simple_annot(annot: Any) -> bool: if get_origin(annot) in (tuple, list): inner = [a for a in get_args(annot) if a is not Ellipsis] return bool(inner) and all(_is_simple_annot(a) for a in inner) + if get_origin(annot) is dict: + args = get_args(annot) + return len(args) == 2 and args[0] is str and (args[1] is Any or _is_simple_annot(args[1])) return False @@ -1069,7 +1072,9 @@ def _autograd_backward(ctx, *grad_outputs): ctx.tensor_objects = None user_grads = _slice_user_grads(ctx.output_ranges, grad_outputs[0]) ctx.output_ranges = None - if bwd_takes_grad_tuple: + if hasattr(bwd_obj, "setup_grad_outputs"): + bwd_obj.setup_grad_outputs(user_grads) + elif bwd_takes_grad_tuple: bwd_obj.grad_outputs = tuple(user_grads) else: bwd_obj.grad_output = user_grads[0] @@ -1359,6 +1364,8 @@ def register_custom_op_with_autograd( non-differentiable input). * ``bwd_fake_impl(bwd_args)`` -- data-free twin of ``bwd_impl`` returning :class:`TensorSpec` grads. + * ``bwd_arg_type.setup_grad_outputs(grads)`` -- optional hook overriding + the default output-gradient assignment. * ``bwd_arg_type.setup_saved_tensors(ctx)`` -- optional hook; skipped if absent. diff --git a/transformer_engine/pytorch/dynamo/fake.py b/transformer_engine/pytorch/dynamo/fake.py index f3621f9e12..5f2568e986 100644 --- a/transformer_engine/pytorch/dynamo/fake.py +++ b/transformer_engine/pytorch/dynamo/fake.py @@ -5,8 +5,11 @@ """Data-free descriptions of quantized operation outputs.""" from __future__ import annotations +import math from typing import Optional +import torch + from .tensor_spec import TensorSpec from ..quantized_tensor import Quantizer @@ -30,3 +33,40 @@ def fake_bgrad( ) ) return db, dx + + +def fake_activation( + input_: TensorSpec, + quantizer: Optional[Quantizer] = None, + *, + gated: bool = False, + dtype: Optional[torch.dtype] = None, +) -> TensorSpec: + """Describe an optionally quantized activation output.""" + shape = (*input_.shape[:-1], input_.shape[-1] // 2 if gated else input_.shape[-1]) + return TensorSpec( + shape=shape, dtype=dtype or input_.dtype, device=input_.device, quantizer=quantizer + ) + + +def fake_norm( + input_: TensorSpec, + quantizer: Optional[Quantizer] = None, + *, + normalization: str = "LayerNorm", + dtype: Optional[torch.dtype] = None, +) -> tuple[TensorSpec, Optional[TensorSpec], TensorSpec]: + """Describe norm output, mean and reciprocal standard deviation.""" + if normalization not in ("LayerNorm", "RMSNorm"): + raise ValueError(f"Unsupported normalization: {normalization}") + out = TensorSpec( + shape=input_.shape, dtype=dtype or input_.dtype, device=input_.device, quantizer=quantizer + ) + rows = math.prod(input_.shape[:-1]) + mean = ( + TensorSpec(shape=(rows,), dtype=torch.float32, device=input_.device) + if normalization == "LayerNorm" + else None + ) + rsigma = TensorSpec(shape=(rows,), dtype=torch.float32, device=input_.device) + return out, mean, rsigma diff --git a/transformer_engine/pytorch/module/_common.py b/transformer_engine/pytorch/module/_common.py index 3d04882698..fd1e9b067b 100644 --- a/transformer_engine/pytorch/module/_common.py +++ b/transformer_engine/pytorch/module/_common.py @@ -6,19 +6,33 @@ import dataclasses import queue -from typing import Any, Callable, List, Optional, Tuple, Union +from typing import Any, Callable, List, Optional, Sequence, Tuple, Union import torch from .. import cpp_extensions as tex from ..constants import TE_DType from ..distributed import in_fp8_activation_recompute_phase +from ..dynamo import TensorSpec, is_value_opaque_quantizer from ..export import is_in_onnx_export_mode from ..quantization import FP8GlobalStateManager +from ..quantized_tensor import Quantizer +from ..tensor.float8_blockwise_tensor import Float8BlockQuantizer from ..tensor.hybrid_tensor import HybridQuantizer from ..utils import get_default_init_method +def compile_unsupported_quantizer_reason( + quantizers: Sequence[Optional[Quantizer]], +) -> Optional[str]: + """Return a fallback reason if a quantizer cannot cross the custom-op boundary.""" + for quantizer in quantizers: + # Delayed-scaling and unregistered custom-recipe quantizers are not value-opaque. + if quantizer is not None and not is_value_opaque_quantizer(quantizer): + return "a quantizer not registered as a torch.compile value-opaque type" + return None + + def sum_bias_grad(tensor: torch.Tensor) -> torch.Tensor: """Reduce all token dimensions, preserving the final feature dimension.""" if tensor.ndim == 1: @@ -343,3 +357,70 @@ def check_fp8_reduce_and_update(restore_first_module: bool = False) -> bool: if restore_first_module or in_fp8_activation_recompute_phase(): qstate.is_first_fp8_module = first_fp8_module return result + + +def get_output_first_dim_size(input_first_dim_size: int, args: Any) -> int: + """Compute the output's first dimension size from the input's. + + Sequence parallelism gathers this dimension in column-parallel mode and + scatters it in row-parallel mode. ``args`` provides ``sequence_parallel``, + ``parallel_mode``, and ``tp_size``. + """ + if not args.sequence_parallel: + return input_first_dim_size + if args.parallel_mode == "column": + return input_first_dim_size * args.tp_size + if args.parallel_mode == "row": + return input_first_dim_size // args.tp_size + return input_first_dim_size + + +def fake_workspace_valid(workspace: TensorSpec, quantizer: Optional[Quantizer]) -> bool: + """Spec-level mirror of ``_is_weight_workspace_valid``.""" + if quantizer is None: + return True + if ( + isinstance(workspace.quantizer, Float8BlockQuantizer) + and workspace.quantizer.block_scaling_dim == 2 + and workspace.quantizer.rowwise_usage + ): + # 2D block scaling can reconstruct columnwise buffers from rowwise data. + return True + required = TensorSpec( + shape=workspace.shape, dtype=workspace.dtype, quantizer=quantizer, device=workspace.device + ).inner_names() + return set(required) <= set(workspace.inner_names()) + + +def fake_quantized_weight( + weight: TensorSpec, + quantizer: Optional[Quantizer], + workspace: Optional[TensorSpec], + *, + activation_dtype: torch.dtype, + cache_weight: bool, +) -> Tuple[TensorSpec, Optional[TensorSpec], Optional[str]]: + """Spec-level mirror of ``quantize_weight``: ``(weightmat, new_workspace, + save_alias)`` where ``save_alias`` names the forward input / output the + saved weight aliases (``None`` when it is a fresh tensor).""" + if weight.is_quantized: + return weight, None, "weight" + if workspace is not None and not fake_workspace_valid(workspace, quantizer): + workspace = None + if workspace is not None: + weightmat = dataclasses.replace(workspace) + weightmat.update_usage(rowwise_usage=True) + return weightmat, None, "weight_workspace" + weightmat = TensorSpec( + shape=tuple(weight.shape), + dtype=activation_dtype, + quantizer=quantizer, + device=weight.device, + ) + new_workspace = None + if cache_weight: + if weightmat.quantizer is not None: + weightmat.quantizer.internal = False + new_workspace = weightmat + weightmat.update_usage(rowwise_usage=True) + return weightmat, new_workspace, "new_weight_workspace" if cache_weight else None diff --git a/transformer_engine/pytorch/module/layernorm_linear.py b/transformer_engine/pytorch/module/layernorm_linear.py index cacbc6ffd5..cf49ff650c 100644 --- a/transformer_engine/pytorch/module/layernorm_linear.py +++ b/transformer_engine/pytorch/module/layernorm_linear.py @@ -6,8 +6,8 @@ import os import warnings import weakref -from dataclasses import dataclass -from typing import Any, Callable, Dict, Optional, Tuple, Union, List +from dataclasses import dataclass, replace as dataclass_replace +from typing import Sequence, Any, Callable, Dict, Optional, Tuple, Union, List import torch from torch.nn import init @@ -42,6 +42,9 @@ nvtx_range_push, needs_quantized_gemm, get_nvtx_range_context, + warn_compile_eager_fallback, + warn_if_compile_disabled, + check_gemm_dims, ) from ..distributed import ( set_tensor_model_parallel_attributes, @@ -63,9 +66,12 @@ from ..jit import no_torch_dynamo from ..graph import is_graph_capturing from ._common import ( + compile_unsupported_quantizer_reason, apply_normalization, check_fp8_reduce_and_update, + fake_workspace_valid, noop_cat, + get_output_first_dim_size, set_quantizer_amax_reduction_group, set_quantizer_usage_for_wgrad_all_gather, WeightGradStore, @@ -77,9 +83,15 @@ prepare_for_saving, restore_from_func_ctx, ) -from ..dynamo import TensorOrQuantized +from ..dynamo.fake import fake_norm +from ..dynamo import ( + TensorSpec, + TensorOrQuantized, + register_custom_op_with_autograd, +) from ...debug.pytorch.debug_state import TEDebugState from ..tensor.mxfp8_tensor import MXFP8Quantizer +from ..tensor.float8_blockwise_tensor import Float8BlockQuantizer from ..tensor.hybrid_tensor import HybridQuantizer from ..tensor.identity_tensor import IdentityQuantizer from ..cpu_offload import ( @@ -142,6 +154,7 @@ class LayerNormLinearFwdArgs: fp8: bool fp8_calibration: bool backward_override: Optional[str] + fprop_use_split_accumulator: bool dgrad_use_split_accumulator: bool wgrad_use_split_accumulator: bool debug: bool @@ -223,7 +236,6 @@ class LayerNormLinearBwdArgs: requires_dgrad: bool = False requires_wgrad: bool = False ln_out_needs_gather: bool = False - inp_shape: Optional[torch.Size] = None # --- Normalization --- normalization: str = "LayerNorm" @@ -277,6 +289,10 @@ class LayerNormLinearBwdArgs: # --- Per-backward scratch state (populated inside the backward impl) --- ub_obj_gradout: Optional[Any] = None + def setup_grad_outputs(self, grads: Sequence[Any]) -> None: + """Unpack gradients in forward-output order, ignoring weight workspaces.""" + self.grad_output, self.grad_ln_out, _ = grads + def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: """Pull saved tensors from ``ctx`` into the fields backward consumes.""" ( @@ -359,7 +375,7 @@ def _layernorm_linear_forward_impl( ln_bias = cast_if_needed(ln_bias, activation_dtype) nvtx_range_pop(f"{nvtx_label}.norm_input_cast") - if is_cpu_offload_enabled(): + if cpu_offloading: start_offload(inputmat) tp_world_size = get_distributed_world_size(tp_group) @@ -541,11 +557,7 @@ def _layernorm_linear_forward_impl( weight_quantizer.calibrate(weight) # Choose whether to use GEMM kernel with split accumulator - use_split_accumulator = _2X_ACC_FPROP - if fp8: - recipe = FP8GlobalStateManager.get_fp8_recipe() - if hasattr(recipe, "fp8_gemm_fprop"): - use_split_accumulator = recipe.fp8_gemm_fprop.use_split_accumulator + use_split_accumulator = args.fprop_use_split_accumulator # Configure output quantizer if output_quantizer is not None: @@ -767,7 +779,6 @@ def _layernorm_linear_setup_ctx( bwd_args.requires_dgrad = fwd_args.input_requires_grad bwd_args.requires_wgrad = fwd_args.weight_requires_grad bwd_args.ln_out_needs_gather = ctx_attrs["ln_out_needs_gather"] - bwd_args.inp_shape = inp.shape # Normalization bwd_args.normalization = fwd_args.normalization @@ -957,7 +968,7 @@ def _layernorm_linear_backward_impl( ub_obj_wgrad = None ub_type_dgrad = None ub_type_wgrad = None - dgrad_shape = args.inp_shape + dgrad_shape = inputmat.shape if args.ub_overlap_ag: # Overlap grad_output all-gather with dgrad compute args.ub_obj_gradout = get_ub(args.ub_name + "_dgrad", args.fp8) @@ -1463,6 +1474,353 @@ def wgrad_gemm( ) +def _layernorm_linear_forward_fake( + args: LayerNormLinearFwdArgs, +) -> Tuple[ + TensorSpec, + Optional[TensorSpec], + Optional[TensorSpec], + Optional[Tuple[Any, ...]], + Optional[Dict], +]: + """Shape/metadata-only twin of :func:`_layernorm_linear_forward_impl` for + torch.compile, returning ``TensorSpec`` descriptors for the outputs and + saved tensors instead of allocating real data.""" + if args.fsdp_group is not None and args.is_grad_enabled: + raise NotImplementedError( + "Compile-time LayerNormLinear forward does not support manual TE FSDP " + "(fsdp_group is not None); use FSDP2 or MCore FSDP." + ) + + weight_quantizer = args.weight_quantizer + fp8_or_debug = args.fp8 or args.debug + with_input_all_gather = args.parallel_mode == "column" and args.sequence_parallel + backward_needs_input = args.is_grad_enabled and args.weight_requires_grad + device = args.inp.device + + out_features, in_features = args.weight.shape + inp_leading = args.inp.shape[0] if len(args.inp.shape) > 1 else 1 + inputmat_aliases_inp = args.inp.dtype == args.activation_dtype + ln_weight_aliases = args.ln_weight.dtype == args.activation_dtype + + # Norm output quantizer usage -- mirrors the impl; the ln_out spec must be + # taken now, before the later rowwise-only usage for the all-gather. + if args.fp8: + if args.input_quantizer is None: + raise ValueError("Missing quantizer for input tensor") + args.input_quantizer.set_usage( + rowwise=True, + columnwise=backward_needs_input and args.backward_override is None, + ) + if with_input_all_gather and args.input_quantizer.supports_only_rowwise_all_gather(): + args.input_quantizer.set_usage(columnwise=False) + custom = is_custom(args.input_quantizer) + hybrid = isinstance(args.input_quantizer, HybridQuantizer) + identity = isinstance(args.input_quantizer, IdentityQuantizer) + with_quantized_norm = ( + args.fp8 + and not args.debug + and not args.return_layernorm_output + and not args.return_layernorm_output_gathered + and args.backward_override is None + and not custom + and not hybrid + and not identity + ) + # A custom quantizer's ln_out stays in high precision on the plain + # all-gather path (only the gathered copy is quantized). + ln_out_quantized = fp8_or_debug and not ( + with_input_all_gather and not args.return_layernorm_output_gathered and custom + ) + ln_out_spec, mu, rsigma = fake_norm( + args.inp, + args.input_quantizer if with_quantized_norm else None, + normalization=args.normalization, + dtype=args.activation_dtype, + ) + if ln_out_quantized and not with_quantized_norm: + ln_out_spec = TensorSpec( + shape=ln_out_spec.shape, + dtype=args.activation_dtype, + quantizer=args.input_quantizer, + device=device, + ) + if with_input_all_gather and fp8_or_debug: + args.input_quantizer.set_usage(rowwise=True, columnwise=False) + # ``ln_out_return`` is the high-precision norm output (or its all-gathered copy). + ln_out_return_is_total = with_input_all_gather and args.return_layernorm_output_gathered + # Whether the quantized ln_out is a different object from the returned one. + ln_out_rebound = ln_out_quantized and not with_quantized_norm + + # ------------------------------------------------------ + # Weight pipeline -- mirror ``quantize_weight`` / ``cast_if_needed``. + # ------------------------------------------------------ + new_weight_workspace = None + workspace = None # args.weight_workspace after validation + weightmat = None + weightmat_is_storage = False + weightmat_aliases_weight = False + is_weight_param_quantized = False + if fp8_or_debug: + is_weight_param_quantized = args.weight.is_quantized + if is_weight_param_quantized and not args.debug: + weight_quantizer = args.weight.quantizer + elif weight_quantizer is not None: + weight_quantizer.set_usage( + rowwise=True, + columnwise=args.is_grad_enabled + and not args.is_fsdp2 + and args.backward_override is None, + ) + + if args.weight.is_quantized: + # Primary-quantized args.weight: the impl reuses it as ``weightmat``. + weightmat = args.weight + weightmat_is_storage = True + weightmat_aliases_weight = True + else: + weightmat_is_storage = True + workspace = args.weight_workspace + if workspace is not None and not fake_workspace_valid(workspace, weight_quantizer): + # quantize_weight drops a stale workspace and builds a new one. + workspace = None + if workspace is not None: + # Copy, so the ``update_usage`` below stays off the input spec. + weightmat = dataclass_replace(workspace) + else: + weightmat = TensorSpec( + shape=tuple(args.weight.shape), + dtype=args.activation_dtype, + quantizer=weight_quantizer, + device=args.weight.device, + ) + if args.cache_weight: + # Persistent cache entries are wrappers, not bare storages. + if weightmat.quantizer is not None: + weightmat.quantizer.internal = False + new_weight_workspace = weightmat + weightmat.update_usage(rowwise_usage=True) + else: + weightmat_aliases_weight = args.weight.dtype == args.activation_dtype + weightmat = TensorSpec( + shape=tuple(args.weight.shape), dtype=args.activation_dtype, device=args.weight.device + ) + + # Bias cast: cuBLAS has no FP8 GEMM with FP32 args.bias. + bias_dtype = args.activation_dtype + if fp8_or_debug and args.activation_dtype == torch.float32: + bias_dtype = torch.bfloat16 + bias_aliases = args.bias is not None and args.bias.dtype == bias_dtype + + if args.output_quantizer is not None: + args.output_quantizer.set_usage(rowwise=True, columnwise=False) + + # ------------------------------------------------------ + # Outputs: y = norm(x) @ w^T and the optional norm output. + # ------------------------------------------------------ + requires_grad = args.is_grad_enabled and args.any_requires_grad() + out = TensorSpec( + shape=( + (out_features,) + if len(args.inp.shape) == 1 + else ( + get_output_first_dim_size(inp_leading, args), + *tuple(args.inp.shape[1:-1]), + out_features, + ) + ), + dtype=args.activation_dtype, + quantizer=args.output_quantizer, + requires_grad=requires_grad, + device=device, + ) + ln_out_for_return = None + if args.return_layernorm_output: + ln_leading = inp_leading + if args.return_layernorm_output_gathered and with_input_all_gather: + ln_leading = inp_leading * args.tp_size + ln_out_for_return = TensorSpec( + shape=( + (ln_leading, *tuple(args.inp.shape[1:])) + if len(args.inp.shape) > 1 + else (in_features,) + ), + dtype=args.activation_dtype, + requires_grad=requires_grad, + device=device, + ) + + # ------------------------------------------------------ + # Backward state -- saved-tensor layout + # (inputmat, wt_save, saved_weight, args.bias, args.ln_weight, ln_out, mu, rsigma). + # ------------------------------------------------------ + tensors_to_save_from_forward = None + ctx_attrs = None + if args.is_grad_enabled: + ln_out_needs_gather = args.weight_requires_grad and with_input_all_gather + + # Slot 5 -- ``ln_out_to_save``. + ln_out_alias = None + ln_out_to_save = None + if args.backward_override == "high_precision": + # ``ln_out_hp`` is taken before ln_out may be dropped, so it is always saved. + ln_out_to_save = TensorSpec( + shape=tuple(args.inp.shape), dtype=args.activation_dtype, device=device + ) + if args.return_layernorm_output and not ln_out_return_is_total: + ln_out_alias = "ln_out" + elif args.weight_requires_grad or args.return_layernorm_output: + ln_out_to_save = ln_out_spec + if ( + backward_needs_input + and args.backward_override is None + and ln_out_quantized + and ( + isinstance(args.input_quantizer, (MXFP8Quantizer, Float8BlockQuantizer)) + or not ln_out_needs_gather + ) + ): + ln_out_to_save.update_usage(rowwise_usage=False) + if args.return_layernorm_output and not ln_out_return_is_total and not ln_out_rebound: + ln_out_alias = "ln_out" + + # Slot 1 -- ``wt_save``, with the impl's alias dedup. + wt_alias = None + wt_save = None + if weightmat_aliases_weight: + wt_alias = "weight" + elif args.is_fsdp2: + pass # FSDP2 re-quantizes from the gathered args.weight in backward. + elif weightmat_is_storage and new_weight_workspace is not None: + wt_alias = "new_weight_workspace" + elif weightmat_is_storage and workspace is not None: + wt_alias = "weight_workspace" + elif weightmat_is_storage: + wt_save = weightmat + else: + wt_save = TensorSpec( + shape=tuple(args.weight.shape), + dtype=args.activation_dtype, + device=args.weight.device, + ) + + saved_tensor_aliases = ( + "inp" if inputmat_aliases_inp else None, + wt_alias, + "weight", + "bias" if bias_aliases else None, + "ln_weight" if ln_weight_aliases else None, + ln_out_alias, + None, + None, + ) + tensors_to_save_from_forward = ( + ( + None + if inputmat_aliases_inp + else TensorSpec( + shape=tuple(args.inp.shape), dtype=args.activation_dtype, device=device + ) + ), + wt_save, + None, + ( + None + if (args.bias is None or bias_aliases) + else TensorSpec(shape=tuple(args.bias.shape), dtype=bias_dtype, device=device) + ), + ( + None + if ln_weight_aliases + else TensorSpec( + shape=tuple(args.ln_weight.shape), dtype=args.activation_dtype, device=device + ) + ), + None if ln_out_alias is not None else ln_out_to_save, + mu, + rsigma, + ) + ctx_attrs = { + "fsdp_shapes": [], + "saved_tensor_aliases": saved_tensor_aliases, + "is_weight_param_quantized": is_weight_param_quantized, + "ln_out_needs_gather": ln_out_needs_gather, + } + + return out, ln_out_for_return, new_weight_workspace, tensors_to_save_from_forward, ctx_attrs + + +def _layernorm_linear_backward_fake( + args: LayerNormLinearBwdArgs, +) -> Tuple[Optional[TensorSpec], ...]: + """Allocation-free fake of :func:`_layernorm_linear_backward_impl` on + ``TensorSpec``. Returns ``(dgrad, dgamma, dbeta, wgrad, grad_bias)`` specs; + TP/SP gather/scatter happens inside the eager op, so the specs carry + rank-local shapes.""" + if args.fsdp_group is not None: + raise NotImplementedError( + "Fake LayerNormLinear backward does not support manual TE FSDP " + "(fsdp_group is not None); use FSDP2 or MCore FSDP." + ) + + weight = args.saved_weight + out_dtype = args.activation_dtype + out_features, in_features = weight.shape + device = args.grad_output.device + + # Mirrors the impl; affects dgrad's buffer layout. + if args.grad_input_quantizer is not None: + args.grad_input_quantizer.set_usage(rowwise=True, columnwise=False) + + dgrad = None + if args.requires_dgrad: + dgrad = TensorSpec( + shape=args.inputmat.shape, + dtype=out_dtype, + device=device, + ) + + # The norm backward always runs; its grads take the saved (cast) ln_weight dtype. + dgamma = TensorSpec(shape=(in_features,), dtype=args.ln_weight.dtype, device=device) + dbeta = None + if args.normalization == "LayerNorm": + dbeta = TensorSpec(shape=(in_features,), dtype=args.ln_weight.dtype, device=device) + + wgrad = None + # Under fuse_wgrad_accumulation the grad goes into main_grad in place. + if args.requires_wgrad and not args.fuse_wgrad_accumulation: + wgrad = TensorSpec( + shape=(out_features, in_features), + dtype=out_dtype, + quantizer=args.grad_weight_quantizer, + device=weight.device, + ) + + grad_bias = None + # FP8 backward computes bgrad in grad_output_preprocess whenever bias is + # used; in high precision it is fused into the wgrad GEMM, so it only + # exists when wgrad runs. + fp8_bwd = args.fp8 and args.backward_override is None + if args.use_bias and (args.requires_wgrad or fp8_bwd): + grad_bias = TensorSpec(shape=(out_features,), dtype=out_dtype, device=device) + + return dgrad, dgamma, dbeta, wgrad, grad_bias + + +# Custom op used under ``torch.compile``. +_layernorm_linear_op = register_custom_op_with_autograd( + op_name="layernorm_linear", + input_tensors_for_grad=["inp", "ln_weight", "ln_bias", "weight", "bias"], + fwd_arg_type=LayerNormLinearFwdArgs, + fwd_impl=_layernorm_linear_forward_impl, + fwd_fake_impl=_layernorm_linear_forward_fake, + setup_context=_layernorm_linear_setup_ctx, + bwd_arg_type=LayerNormLinearBwdArgs, + bwd_impl=_layernorm_linear_backward_impl, + bwd_fake_impl=_layernorm_linear_backward_fake, +) + + class _LayerNormLinear(torch.autograd.Function): """LayerNormLinear semi-top level module Calls custom cuda extensions. @@ -1527,8 +1885,7 @@ def backward( ) -> Tuple[Union[torch.Tensor, None], ...]: """Backward pass: compute gradients and reduce FP8 scaling factors.""" bwd_args: LayerNormLinearBwdArgs = ctx.backward_objects - bwd_args.grad_output = grad_output - bwd_args.grad_ln_out = grad_ln_out + bwd_args.setup_grad_outputs((grad_output, grad_ln_out, _grad_weight_workspace)) bwd_args.setup_saved_tensors(ctx) nvtx_label = "transformer_engine._LayerNormLinear.backward" if bwd_args.ub_name is not None: @@ -1553,6 +1910,22 @@ def backward( ) +@no_torch_dynamo() +def _layernorm_linear_eager( + inp: torch.Tensor, + ln_weight: torch.Tensor, + ln_bias: Optional[torch.Tensor], + weight: torch.Tensor, + bias: Optional[torch.Tensor], + fwd_args: LayerNormLinearFwdArgs, + is_grad_enabled: bool, +) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]: + """Run ``_LayerNormLinear`` eagerly, bypassing Dynamo.""" + if is_grad_enabled: + return _LayerNormLinear.apply(inp, ln_weight, ln_bias, weight, bias, fwd_args) + return _LayerNormLinear.forward(None, inp, ln_weight, ln_bias, weight, bias, fwd_args) + + class LayerNormLinear(TransformerEngineBaseModule): r""" Applies layer normalization followed by linear transformation to the incoming data. @@ -2012,7 +2385,6 @@ def reset_parameters(self, defer_init=False): elif self.parallel_mode == "column": set_tensor_model_parallel_attributes(getattr(self, bias), True, 0, 1) - @no_torch_dynamo() def forward( self, inp: torch.Tensor, @@ -2040,6 +2412,13 @@ def forward( * it also allows skipping gradient accumulation during the first microbatch (since it is the first gradient being produced) + + Under ``torch.compile``, the FP8 weight cache is replaced on + the first microbatch and reused on subsequent microbatches. + ``fuse_wgrad_accumulation=True`` remains unsupported. + With ``mode="reduce-overhead"``, call + ``torch.compiler.cudagraph_mark_step_begin()`` once before + each minibatch to keep the cache live across its microbatches. """ is_grad_enabled = torch.is_grad_enabled() @@ -2095,16 +2474,28 @@ def forward( weight_quantizer, weight_tensor ) + use_compiled_op = torch.compiler.is_compiling() and _layernorm_linear_op is not None + if _layernorm_linear_op is None and torch.compiler.is_compiling(): + warn_if_compile_disabled() + if use_compiled_op: + # Process groups cross the op boundary separately from quantizers. + for quantizer in (input_quantizer, grad_output_quantizer): + if getattr(quantizer, "amax_reduction_group", None) is not None: + set_quantizer_amax_reduction_group(quantizer, None) + cache_name = None if (is_first_microbatch is None or self.is_fsdp2) else "weight" - weight_workspace = ( - self._fp8_workspaces.get(cache_name) if cache_name is not None else None - ) + weight_workspace = None + if cache_name is not None and not (use_compiled_op and is_first_microbatch): + weight_workspace = self._fp8_workspaces.get(cache_name) + fprop_use_split_accumulator = _2X_ACC_FPROP dgrad_use_split_accumulator = _2X_ACC_DGRAD wgrad_use_split_accumulator = _2X_ACC_WGRAD if self.fp8: _recipe = FP8GlobalStateManager.get_fp8_recipe() backward_override = _recipe.backward_override + if hasattr(_recipe, "fp8_gemm_fprop"): + fprop_use_split_accumulator = _recipe.fp8_gemm_fprop.use_split_accumulator if hasattr(_recipe, "fp8_gemm_dgrad"): dgrad_use_split_accumulator = _recipe.fp8_gemm_dgrad.use_split_accumulator if hasattr(_recipe, "fp8_gemm_wgrad"): @@ -2130,6 +2521,25 @@ def forward( linear_bias_tensor = ( bias_tensor if (self.apply_bias and not self.gemm_bias_unfused_add) else None ) + if use_compiled_op: + fallback_reason = self._compile_eager_fallback_reason( + inp, + skip_fp8_weight_update, + fp8_output, + fp8_grad, + is_grad_enabled, + debug, + quantizers, + ) + if fallback_reason is not None: + warn_compile_eager_fallback(fallback_reason) + torch._dynamo.graph_break( + msg=f"te.LayerNormLinear falling back to eager: {fallback_reason}" + ) + use_compiled_op = False + + wgrad_store = None if use_compiled_op else self.wgrad_store + fwd_args = LayerNormLinearFwdArgs( # tensors inp=inp, @@ -2172,6 +2582,7 @@ def forward( fp8=self.fp8, fp8_calibration=self.fp8_calibration, backward_override=backward_override, + fprop_use_split_accumulator=fprop_use_split_accumulator, dgrad_use_split_accumulator=dgrad_use_split_accumulator, wgrad_use_split_accumulator=wgrad_use_split_accumulator, debug=debug, @@ -2199,30 +2610,24 @@ def forward( is_fsdp2=self.is_fsdp2, # weight-grad scheduling fuse_wgrad_accumulation=self.fuse_wgrad_accumulation, - wgrad_store=self.wgrad_store, + wgrad_store=wgrad_store, # misc cpu_offloading=is_cpu_offload_enabled(), is_grad_enabled=is_grad_enabled, ) - if is_grad_enabled: - out, ln_out, new_weight_workspace = _LayerNormLinear.apply( - inp, - self.layer_norm_weight, - self.layer_norm_bias, - weight_tensor, - linear_bias_tensor, - fwd_args, - ) + if use_compiled_op: + check_gemm_dims(inp, weight_tensor) + out, ln_out, new_weight_workspace = _layernorm_linear_op(fwd_args) else: - out, ln_out, new_weight_workspace = _LayerNormLinear.forward( - None, + out, ln_out, new_weight_workspace = _layernorm_linear_eager( inp, self.layer_norm_weight, self.layer_norm_bias, weight_tensor, linear_bias_tensor, fwd_args, + is_grad_enabled, ) if new_weight_workspace is not None and cache_name is not None: @@ -2289,6 +2694,57 @@ def _get_debug_quantizers(self, fp8_output, fp8_grad, is_grad_enabled): for name, q in zip(names, original_quantizers) ) + def _compile_eager_fallback_reason( + self, + inp: torch.Tensor, + skip_fp8_weight_update: Optional[torch.Tensor], + fp8_output: bool, + fp8_grad: bool, + is_grad_enabled: bool, + debug: bool, + quantizers: Sequence[Optional[Quantizer]], + ) -> Optional[str]: + """Return a fallback reason after preparing the forward quantizers.""" + if debug: + return "debug instrumentation (nvidia-dlfw-inspect)" + weight_tensor, bias_tensor = self._get_weight_and_bias_tensors() + if is_distributed_weight(weight_tensor): + return "a DistributedWeight (custom weight parallelism, e.g. GTP)" + if isinstance(inp, (QuantizedTensor, QuantizedTensorStorage)): + return "a quantized input tensor" + if self.fsdp_group is not None: + return "manual TE FSDP (fsdp_group); use FSDP2 or MCore FSDP" + any_requires_grad = ( + inp.requires_grad + or self.layer_norm_weight.requires_grad + or (self.layer_norm_bias is not None and self.layer_norm_bias.requires_grad) + or weight_tensor.requires_grad + or (bias_tensor is not None and bias_tensor.requires_grad) + ) + if fp8_output and is_grad_enabled and any_requires_grad: + return "differentiable fp8_output=True" + if is_cpu_offload_enabled(): + return "CPU activation offloading" + # A queued store can be enabled between forward and backward. + if self.wgrad_store is not None and ( + self.wgrad_store.context is not None or self.wgrad_store.delay_wgrad_compute() + ): + return "delayed wgrad compute (wgrad_store)" + if self.fuse_wgrad_accumulation: + return "fuse_wgrad_accumulation (main_grad)" + fp8 = FP8GlobalStateManager.is_fp8_enabled() + needs_dgrad = is_grad_enabled and inp.requires_grad + if ( + fp8 + and fp8_grad + and needs_dgrad + and not (self.ub_overlap_rs_dgrad or self.ub_bulk_wgrad) + ): + return "a quantized input grad (fp8_grad=True)" + if fp8 and skip_fp8_weight_update is not None: + return "GPU-controlled FP8 weight cache updates" + return compile_unsupported_quantizer_reason(quantizers) + def _get_weight_and_bias_tensors(self): # Get concatenated weight and bias tensors unfused_weights = self._get_weight_tensors() diff --git a/transformer_engine/pytorch/module/layernorm_mlp.py b/transformer_engine/pytorch/module/layernorm_mlp.py index fb7c211b6b..842f0a6181 100644 --- a/transformer_engine/pytorch/module/layernorm_mlp.py +++ b/transformer_engine/pytorch/module/layernorm_mlp.py @@ -47,6 +47,8 @@ clear_tensor_data, needs_quantized_gemm, get_nvtx_range_context, + warn_compile_eager_fallback, + warn_if_compile_disabled, ) from ..distributed import ( set_tensor_model_parallel_attributes, @@ -70,9 +72,11 @@ from ..tensor.hybrid_tensor import HybridQuantizer from ..tensor.identity_tensor import IdentityQuantizer from ._common import ( + compile_unsupported_quantizer_reason, sum_bias_grad, apply_normalization, check_fp8_reduce_and_update, + fake_quantized_weight, set_quantizer_amax_reduction_group, set_quantizer_usage_for_wgrad_all_gather, WeightGradStore, @@ -84,12 +88,18 @@ mark_activation_offload, ) from ..quantized_tensor import ( + QuantizedTensor, QuantizedTensorStorage, Quantizer, prepare_for_saving, restore_from_func_ctx, ) -from ..dynamo import TensorOrQuantized +from ..dynamo.fake import fake_activation, fake_norm +from ..dynamo import ( + TensorSpec, + TensorOrQuantized, + register_custom_op_with_autograd, +) from ..cpp_extensions import ( general_gemm, ) @@ -99,8 +109,10 @@ __all__ = ["LayerNormMLP"] -def _get_act_func_supported_list(recipe: Optional[Recipe] = None): - if recipe is None: +def _get_act_func_supported_list( + recipe: Optional[Recipe] = None, dbias_fusion: Optional[bool] = None +): + if dbias_fusion is False or (dbias_fusion is None and recipe is None): # bf16 (recipe is None): return { "gelu": (tex.gelu, tex.dgelu, None), @@ -116,7 +128,7 @@ def _get_act_func_supported_list(recipe: Optional[Recipe] = None): "swiglu": (tex.swiglu, tex.dswiglu, None), "clamped_swiglu": (tex.clamped_swiglu, tex.clamped_dswiglu, None), } - if recipe.delayed() or recipe.mxfp8(): + if dbias_fusion is True or recipe.delayed() or recipe.mxfp8(): # Delayed scaling, fusion supported list: [tex.dbias_dgelu, tex.dbias_drelu, tex.dbias_dqgelu, tex.dbias_dsrelu] # MXFP8: [tex.dbias_dgelu, tex.dbias_drelu, tex.dbias_dqgelu, tex.dbias_dsrelu] return { @@ -159,18 +171,26 @@ def _get_act_func_supported_list(recipe: Optional[Recipe] = None): raise NotImplementedError(f"Unhandled recipe type {recipe}") -def _act_func(activation: str, recipe: Optional[Recipe] = None): +def _act_func( + activation: str, recipe: Optional[Recipe] = None, dbias_fusion: Optional[bool] = None +): # based on each quantization mode, we have different kernel fusion supported: # bf16 (recipe is None): [tex.dbias_dgelu, tex.dbias_drelu, tex.dbias_dqgelu, tex.dbias_dsrelu] # Delayed scaling, fusion supported list: [tex.dbias_dgelu, tex.dbias_drelu, tex.dbias_dqgelu, tex.dbias_dsrelu] # MXFP8: [tex.dbias_dgelu, tex.dbias_drelu, tex.dbias_dqgelu, tex.dbias_dsrelu] # Per-tensor current scaling or fp8 blockwise scaling: [] - funcs = _get_act_func_supported_list(recipe) + funcs = _get_act_func_supported_list(recipe, dbias_fusion) if activation not in funcs: raise NotImplementedError("Activation type " + activation + " is not supported!") return funcs[activation] +# Activations whose output halves the last dim (gated linear units). +_GATED_ACTIVATIONS = frozenset( + {"geglu", "glu", "qgeglu", "reglu", "sreglu", "swiglu", "clamped_swiglu"} +) + + @dataclass(slots=True) class LayerNormMLPFwdArgs: """Single-argument bag for the forward path of :class:`_LayerNormMLP`.""" @@ -231,8 +251,13 @@ class LayerNormMLPFwdArgs: fp8: bool fp8_calibration: bool backward_override: Optional[str] + fprop_use_split_accumulator: bool dgrad_use_split_accumulator: bool wgrad_use_split_accumulator: bool + # The recipe itself can't cross the op boundary. + recipe_float8_block_scaling: bool + recipe_custom: bool + recipe_dbias_dact_fusion: bool debug: bool # --- Weight-workspace caching --- @@ -336,7 +361,6 @@ class LayerNormMLPBwdArgs: fc1_weight_requires_grad: bool = False fc1_bias_requires_grad: bool = False fc2_weight_requires_grad: bool = False - inp_shape: Optional[torch.Size] = None # --- Normalization --- normalization: str = "LayerNorm" @@ -353,7 +377,9 @@ class LayerNormMLPBwdArgs: # --- Numerical / dtype config --- activation_dtype: Optional[torch.dtype] = None fp8: bool = False - fp8_recipe: Optional[Any] = None + recipe_float8_block_scaling: bool = False + recipe_custom: bool = False + recipe_dbias_dact_fusion: bool = False dgrad_use_split_accumulator: bool = _2X_ACC_DGRAD wgrad_use_split_accumulator: bool = _2X_ACC_WGRAD backward_override: Optional[str] = None @@ -397,6 +423,10 @@ class LayerNormMLPBwdArgs: # --- Per-backward scratch state (populated inside the backward impl) --- ub_obj_gradout: Optional[Any] = None + def setup_grad_outputs(self, grads: Sequence[Any]) -> None: + """Unpack gradients in forward-output order, ignoring weight workspaces.""" + self.grad_output, self.grad_ln_out, _, _ = grads + def setup_saved_tensors(self, ctx: torch.autograd.function.FunctionCtx) -> None: """Pull saved tensors from ``ctx`` into the fields backward consumes.""" self.set_saved_tensors(restore_from_func_ctx(ctx)) @@ -534,16 +564,14 @@ def _layernorm_mlp_forward_impl( assert inp_shape[-1] == in_features, "GEMM not possible" inputmat = inp - activation_func = _act_func( - activation, FP8GlobalStateManager.get_fp8_recipe() if fp8 else None - )[0] + activation_func = _act_func(activation, dbias_fusion=fp8 and args.recipe_dbias_dact_fusion)[0] # Cast for native AMP inputmat = cast_if_needed(inputmat, activation_dtype) ln_weight_cast = cast_if_needed(ln_weight, activation_dtype) if ln_bias is not None: ln_bias = cast_if_needed(ln_bias, activation_dtype) - if is_cpu_offload_enabled(): + if cpu_offloading: start_offload(inputmat) tp_world_size = get_distributed_world_size(tp_group) @@ -562,11 +590,7 @@ def _layernorm_mlp_forward_impl( ub_overlap_rs = ub_overlap_rs and is_grad_enabled # Choose whether to use GEMM kernel with split accumulator - use_split_accumulator = _2X_ACC_FPROP - if fp8: - recipe = FP8GlobalStateManager.get_fp8_recipe() - if hasattr(recipe, "fp8_gemm_fprop"): - use_split_accumulator = recipe.fp8_gemm_fprop.use_split_accumulator + use_split_accumulator = args.fprop_use_split_accumulator # Configure quantizer for norm output if fp8: @@ -812,12 +836,11 @@ def _layernorm_mlp_forward_impl( else: fc1_out, *_ = fc1_outputs if fp8: - recipe = FP8GlobalStateManager.get_fp8_recipe() - if recipe.float8_block_scaling(): + if args.recipe_float8_block_scaling: # tex.quantize does not support GELU fusion for blockwise act_out = activation_func(fc1_out, None, **act_params) act_out = tex.quantize(act_out, fc2_input_quantizer) - elif recipe.custom(): + elif args.recipe_custom: # tex.quantize does not support custom quantizers act_out = activation_func(fc1_out, None, **act_params) act_out = fc2_input_quantizer(act_out) @@ -1144,7 +1167,6 @@ def _layernorm_mlp_setup_ctx( bwd_args.fc1_weight_requires_grad = fc1_weight_requires_grad bwd_args.fc1_bias_requires_grad = fwd_args.fc1_bias_requires_grad bwd_args.fc2_weight_requires_grad = fc2_weight_requires_grad - bwd_args.inp_shape = inp.shape # Normalization bwd_args.normalization = fwd_args.normalization @@ -1163,7 +1185,9 @@ def _layernorm_mlp_setup_ctx( # Numerical / dtype config bwd_args.activation_dtype = fwd_args.activation_dtype bwd_args.fp8 = fp8 - bwd_args.fp8_recipe = FP8GlobalStateManager.get_fp8_recipe() if fp8 else None + bwd_args.recipe_float8_block_scaling = fwd_args.recipe_float8_block_scaling + bwd_args.recipe_custom = fwd_args.recipe_custom + bwd_args.recipe_dbias_dact_fusion = fwd_args.recipe_dbias_dact_fusion bwd_args.dgrad_use_split_accumulator = fwd_args.dgrad_use_split_accumulator bwd_args.wgrad_use_split_accumulator = fwd_args.wgrad_use_split_accumulator bwd_args.backward_override = fwd_args.backward_override @@ -1625,7 +1649,7 @@ def _layernorm_mlp_backward_impl( # Whether to set grad arg in general_gemm grad_arg = True - if args.fp8 and args.fp8_recipe.float8_block_scaling(): + if args.fp8 and args.recipe_float8_block_scaling: grad_arg = False # Arguments to include in wgrad GEMM closure @@ -1671,7 +1695,7 @@ def fc2_wgrad_gemm( # Update grad bias if needed if fc2_bias_grad is None: - if args.fp8 and args.fp8_recipe.float8_block_scaling() and fc2_bias is not None: + if args.fp8 and args.recipe_float8_block_scaling and fc2_bias is not None: # BGRAD not fused with GEMM for float8 blockwise gemm. fc2_bias_grad_ = sum_bias_grad(act_out) fc2_bias_grad = fc2_bias_grad_ @@ -1704,12 +1728,13 @@ def fc2_wgrad_gemm( fc1_bias_grad = sum_bias_grad(dact) dact = args.fc1_grad_output_quantizer(dact) elif ( - _act_func(args.activation, args.fp8_recipe if args.fp8 else None)[2] is not None + _act_func(args.activation, dbias_fusion=args.fp8 and args.recipe_dbias_dact_fusion)[2] + is not None and args.fp8 ): # Fusion: gemm, bias + gelu + quantize dbias_dact_quantize_func = _act_func( - args.activation, args.fp8_recipe if args.fp8 else None + args.activation, dbias_fusion=args.fp8 and args.recipe_dbias_dact_fusion )[2] fc1_bias_grad, dact = dbias_dact_quantize_func( fc2_dgrad, @@ -1721,7 +1746,7 @@ def fc2_wgrad_gemm( # Fusion: gemm + gelu, if not fc2_dgrad_gemm_gelu_fusion: activation_func_bwd = _act_func( - args.activation, args.fp8_recipe if args.fp8 else None + args.activation, dbias_fusion=args.fp8 and args.recipe_dbias_dact_fusion )[1] dact = activation_func_bwd( fc2_dgrad, fc1_out.to(args.activation_dtype), None, **act_params @@ -1734,7 +1759,7 @@ def fc2_wgrad_gemm( args.fc1_grad_output_quantizer, (Float8BlockQuantizer, IdentityQuantizer), ) - or args.fp8_recipe.custom() + or args.recipe_custom ): fc1_bias_grad = sum_bias_grad(dact) dact = args.fc1_grad_output_quantizer(dact) @@ -2066,6 +2091,406 @@ def fc1_wgrad_gemm( ) +def _layernorm_mlp_forward_fake( + args: LayerNormMLPFwdArgs, +) -> Tuple[ + TensorSpec, + Optional[TensorSpec], + Optional[TensorSpec], + Optional[TensorSpec], + Optional[Tuple[Any, ...]], + Optional[Dict], +]: + """Shape/metadata-only twin of :func:`_layernorm_mlp_forward_impl` for + torch.compile (no args.activation checkpointing: that path falls back to eager).""" + if args.fsdp_group is not None and args.is_grad_enabled: + raise NotImplementedError( + "Compile-time LayerNormMLP forward does not support manual TE FSDP " + "(fsdp_group is not None); use FSDP2 or MCore FSDP." + ) + if args.checkpoint and args.is_grad_enabled: + raise NotImplementedError( + "Compile-time LayerNormMLP forward does not support args.activation checkpointing" + ) + + fc1_weight_quantizer = args.fc1_weight_quantizer + fc2_weight_quantizer = args.fc2_weight_quantizer + fp8_or_debug = args.fp8 or args.debug + bias_gelu_fusion = args.bias_gelu_fusion + gemm_gelu_fusion = args.gemm_gelu_fusion + device = args.inp.device + + in_features = args.ln_weight.shape[0] + fc1_out_features = args.fc1_weight.shape[0] + fc2_out_features = args.fc2_weight.shape[0] + inp_leading = args.inp.shape[0] if len(args.inp.shape) > 1 else 1 + gemm_leading_shape = ( + (inp_leading * args.tp_size, *args.inp.shape[1:-1]) + if args.sequence_parallel + else tuple(args.inp.shape[:-1]) + ) + inputmat_aliases_inp = args.inp.dtype == args.activation_dtype + ln_weight_aliases = args.ln_weight.dtype == args.activation_dtype + + backwards_needs_fc1_input = args.fc1_weight_requires_grad and args.is_grad_enabled + + # Norm output quantizer usage -- mirrors the impl; the ln_out spec must be + # taken now, before the later rowwise-only usage for the all-gather. + if args.fp8: + if args.fc1_input_quantizer is None: + raise ValueError("Missing quantizer for FC1 input tensor") + args.fc1_input_quantizer.set_usage(rowwise=True, columnwise=backwards_needs_fc1_input) + if args.sequence_parallel and args.fc1_input_quantizer.supports_only_rowwise_all_gather(): + args.fc1_input_quantizer.set_usage(columnwise=False) + custom = is_custom(args.fc1_input_quantizer) + hybrid = isinstance(args.fc1_input_quantizer, HybridQuantizer) + identity = isinstance(args.fc1_input_quantizer, IdentityQuantizer) + with_quantized_norm = ( + args.fp8 + and not args.debug + and not args.return_layernorm_output + and not args.return_layernorm_output_gathered + and not custom + and not hybrid + and not identity + ) + # A custom quantizer's ln_out stays in high precision on the plain + # all-gather path (only the gathered copy is quantized). + ln_out_quantized = fp8_or_debug and not ( + args.sequence_parallel and not args.return_layernorm_output_gathered and custom + ) + ln_out, mu, rsigma = fake_norm( + args.inp, + args.fc1_input_quantizer if with_quantized_norm else None, + normalization=args.normalization, + dtype=args.activation_dtype, + ) + if ln_out_quantized and not with_quantized_norm: + ln_out = TensorSpec( + shape=ln_out.shape, + dtype=args.activation_dtype, + quantizer=args.fc1_input_quantizer, + device=device, + ) + if args.sequence_parallel and fp8_or_debug: + args.fc1_input_quantizer.set_usage(rowwise=True, columnwise=False) + ln_out_return_is_total = args.sequence_parallel and args.return_layernorm_output_gathered + ln_out_rebound = ln_out_quantized and not with_quantized_norm + + # ------------------------------------------------------ + # Weights -- mirror ``quantize_weight`` / ``cast_if_needed``. + # ------------------------------------------------------ + new_fc1_weight_workspace = None + new_fc2_weight_workspace = None + if fp8_or_debug: + if args.fc1_weight.is_quantized and not args.debug: + fc1_weight_quantizer = args.fc1_weight.quantizer + elif fc1_weight_quantizer is not None: + fc1_weight_quantizer.set_usage( + rowwise=True, columnwise=args.is_grad_enabled and not args.is_fsdp2 + ) + if args.fc2_weight.is_quantized and not args.debug: + fc2_weight_quantizer = args.fc2_weight.quantizer + elif fc2_weight_quantizer is not None: + fc2_weight_quantizer.set_usage( + rowwise=True, columnwise=args.is_grad_enabled and not args.is_fsdp2 + ) + fc1_weight_final, new_fc1_weight_workspace, fc1_wt_alias = fake_quantized_weight( + args.fc1_weight, + fc1_weight_quantizer, + args.fc1_weight_workspace, + activation_dtype=args.activation_dtype, + cache_weight=args.cache_weight, + ) + fc2_weight_final, new_fc2_weight_workspace, fc2_wt_alias = fake_quantized_weight( + args.fc2_weight, + fc2_weight_quantizer, + args.fc2_weight_workspace, + activation_dtype=args.activation_dtype, + cache_weight=args.cache_weight, + ) + fc1_wt_alias = ( + None if fc1_wt_alias is None else fc1_wt_alias.replace("weight", "fc1_weight") + ) + fc2_wt_alias = ( + None if fc2_wt_alias is None else fc2_wt_alias.replace("weight", "fc2_weight") + ) + else: + fc1_weight_final = TensorSpec( + shape=tuple(args.fc1_weight.shape), + dtype=args.activation_dtype, + device=args.fc1_weight.device, + ) + fc2_weight_final = TensorSpec( + shape=tuple(args.fc2_weight.shape), + dtype=args.activation_dtype, + device=args.fc2_weight.device, + ) + fc1_wt_alias = "fc1_weight" if args.fc1_weight.dtype == args.activation_dtype else None + fc2_wt_alias = "fc2_weight" if args.fc2_weight.dtype == args.activation_dtype else None + + # Bias cast: cuBLAS has no FP8 GEMM with FP32 bias. + bias_dtype = args.activation_dtype + if fp8_or_debug and args.activation_dtype == torch.float32: + bias_dtype = torch.bfloat16 + fc1_bias_aliases = args.fc1_bias is not None and args.fc1_bias.dtype == bias_dtype + fc2_bias_aliases = args.fc2_bias is not None and args.fc2_bias.dtype == bias_dtype + + # ------------------------------------------------------ + # FC1 GEMM + args.activation (see the impl for the fusion rules). + # ------------------------------------------------------ + if args.activation != "gelu": + gemm_gelu_fusion = bias_gelu_fusion = False + else: + if not args.fp8: + gemm_gelu_fusion = True + if gemm_gelu_fusion and bias_gelu_fusion: + gemm_gelu_fusion = False + if args.debug: + gemm_gelu_fusion = False + fc1_out = None + fc1_out_without_bias = None + if bias_gelu_fusion: + fc1_out_without_bias = TensorSpec( + shape=(*gemm_leading_shape, fc1_out_features), + dtype=args.activation_dtype, + device=device, + ) + else: + # generic_gemm stores the pre-GELU auxiliary output in the bias dtype + # for low-precision inputs, defaulting to BF16 when there is no bias. + fc1_out_dtype = args.activation_dtype + if gemm_gelu_fusion and args.fp8: + fc1_out_dtype = bias_dtype if args.fc1_bias is not None else torch.bfloat16 + fc1_out = TensorSpec( + shape=(*gemm_leading_shape, fc1_out_features), + dtype=fc1_out_dtype, + device=device, + ) + act_out = fake_activation( + fc1_out_without_bias if bias_gelu_fusion else fc1_out, + args.fc2_input_quantizer if fp8_or_debug else None, + gated=args.activation in _GATED_ACTIVATIONS, + dtype=args.activation_dtype, + ) + if args.fc2_output_quantizer is not None: + args.fc2_output_quantizer.set_usage(rowwise=True, columnwise=False) + + # ------------------------------------------------------ + # Outputs. + # ------------------------------------------------------ + requires_grad = args.is_grad_enabled and args.any_requires_grad() + out_leading = inp_leading + if args.sequence_parallel and not args.set_parallel_mode: + out_leading = inp_leading * args.tp_size + fc2_out = TensorSpec( + shape=( + (fc2_out_features,) + if len(args.inp.shape) == 1 + else (out_leading, *tuple(args.inp.shape[1:-1]), fc2_out_features) + ), + dtype=args.activation_dtype, + quantizer=args.fc2_output_quantizer, + requires_grad=requires_grad, + device=device, + ) + ln_out_for_return = None + if args.return_layernorm_output: + ln_leading = inp_leading + if ln_out_return_is_total: + ln_leading = inp_leading * args.tp_size + ln_out_for_return = TensorSpec( + shape=( + (ln_leading, *tuple(args.inp.shape[1:])) + if len(args.inp.shape) > 1 + else (in_features,) + ), + dtype=args.activation_dtype, + requires_grad=requires_grad, + device=device, + ) + + # ------------------------------------------------------ + # Backward state -- saved-tensor layout (see the impl). + # ------------------------------------------------------ + tensors_to_save_from_forward = None + ctx_attrs = None + if args.is_grad_enabled: + if not args.fc1_weight_requires_grad: + ln_out = None + if not args.fc2_weight_requires_grad: + act_out = None + fc1_wt_save = fc1_weight_final + fc2_wt_save = fc2_weight_final + if args.is_fsdp2: + if fc1_wt_alias != "fc1_weight": + fc1_wt_save, fc1_wt_alias = None, None + if fc2_wt_alias != "fc2_weight": + fc2_wt_save, fc2_wt_alias = None, None + ln_out_alias = ( + "ln_out" + if args.return_layernorm_output + and ln_out is not None + and not ln_out_return_is_total + and not ln_out_rebound + else None + ) + saved_tensor_aliases = ( + "inp" if inputmat_aliases_inp else None, + "ln_weight" if ln_weight_aliases else None, + ln_out_alias, + fc1_wt_alias, + "fc1_weight", + "fc1_bias" if fc1_bias_aliases else None, + None, + None, + None, + fc2_wt_alias, + "fc2_weight", + "fc2_bias" if fc2_bias_aliases else None, + None, + None, + ) + saved = ( + TensorSpec(shape=tuple(args.inp.shape), dtype=args.activation_dtype, device=device), + TensorSpec( + shape=tuple(args.ln_weight.shape), dtype=args.activation_dtype, device=device + ), + ln_out, + fc1_wt_save, + None, + ( + None + if args.fc1_bias is None + else TensorSpec(shape=tuple(args.fc1_bias.shape), dtype=bias_dtype, device=device) + ), + fc1_out, + fc1_out_without_bias, + act_out, + fc2_wt_save, + None, + ( + None + if args.fc2_bias is None + else TensorSpec(shape=tuple(args.fc2_bias.shape), dtype=bias_dtype, device=device) + ), + mu, + rsigma, + ) + tensors_to_save_from_forward = tuple( + None if alias is not None else spec for alias, spec in zip(saved_tensor_aliases, saved) + ) + ctx_attrs = { + "saved_tensor_aliases": saved_tensor_aliases, + "fsdp_shapes": None, + "is_recomputation": False, + } + + return ( + fc2_out, + ln_out_for_return, + new_fc1_weight_workspace, + new_fc2_weight_workspace, + tensors_to_save_from_forward, + ctx_attrs, + ) + + +def _layernorm_mlp_backward_fake( + args: LayerNormMLPBwdArgs, +) -> Tuple[Optional[TensorSpec], ...]: + """Allocation-free fake of :func:`_layernorm_mlp_backward_impl` on + ``TensorSpec``. Returns ``(dgrad, dgamma, dbeta, fc1_wgrad, fc1_bias_grad, + fc2_wgrad, fc2_bias_grad)`` specs with rank-local shapes.""" + if args.fsdp_group is not None: + raise NotImplementedError( + "Fake LayerNormMLP backward does not support manual TE FSDP " + "(fsdp_group is not None); use FSDP2 or MCore FSDP." + ) + out_dtype = args.activation_dtype + device = args.grad_output.device + in_features = args.ln_weight.shape[-1] + fc1_out_features = args.fc1_weight.shape[0] + fc2_out_features, act_features = args.fc2_weight.shape + + if args.fc1_grad_output_quantizer is not None: + args.fc1_grad_output_quantizer.set_usage(rowwise=True, columnwise=True) + + dgrad = None + if args.requires_dgrad: + dgrad = TensorSpec( + shape=args.inputmat.shape, + dtype=out_dtype, + device=device, + ) + + # The norm backward always runs; its grads take the saved (cast) ln_weight dtype. + dgamma = TensorSpec(shape=(in_features,), dtype=args.ln_weight.dtype, device=device) + dbeta = None + if args.normalization == "LayerNorm": + dbeta = TensorSpec(shape=(in_features,), dtype=args.ln_weight.dtype, device=device) + + fc1_wgrad = None + if args.fc1_weight_requires_grad and not args.fuse_wgrad_accumulation: + fc1_wgrad = TensorSpec( + shape=(fc1_out_features, in_features), + dtype=out_dtype, + quantizer=args.fc1_grad_weight_quantizer, + device=device, + ) + fc1_bias_grad = None + # FP8, debug and the fused bias-gelu path always produce it; in high + # precision it is fused into the FC1 wgrad GEMM or summed when only the + # bias needs a grad. + if args.fc1_bias is not None and any( + ( + args.fp8, + args.debug, + args.bias_gelu_fusion, + args.fc1_weight_requires_grad, + args.fc1_bias_requires_grad, + ) + ): + fc1_bias_grad = TensorSpec(shape=(fc1_out_features,), dtype=out_dtype, device=device) + + fc2_wgrad = None + if args.fc2_weight_requires_grad and not args.fuse_wgrad_accumulation: + fc2_wgrad = TensorSpec( + shape=(fc2_out_features, act_features), + dtype=out_dtype, + quantizer=args.fc2_grad_weight_quantizer, + device=device, + ) + fc2_bias_grad = None + fp8_bwd = args.fp8 and args.backward_override is None + if args.use_bias and (args.fc2_weight_requires_grad or fp8_bwd): + fc2_bias_grad = TensorSpec(shape=(fc2_out_features,), dtype=out_dtype, device=device) + + return dgrad, dgamma, dbeta, fc1_wgrad, fc1_bias_grad, fc2_wgrad, fc2_bias_grad + + +# Custom op used under ``torch.compile``. +_layernorm_mlp_op = register_custom_op_with_autograd( + op_name="layernorm_mlp", + input_tensors_for_grad=[ + "inp", + "ln_weight", + "ln_bias", + "fc1_weight", + "fc1_bias", + "fc2_weight", + "fc2_bias", + ], + fwd_arg_type=LayerNormMLPFwdArgs, + fwd_impl=_layernorm_mlp_forward_impl, + fwd_fake_impl=_layernorm_mlp_forward_fake, + setup_context=_layernorm_mlp_setup_ctx, + bwd_arg_type=LayerNormMLPBwdArgs, + bwd_impl=_layernorm_mlp_backward_impl, + bwd_fake_impl=_layernorm_mlp_backward_fake, +) + + class _LayerNormMLP(torch.autograd.Function): """LayerNormMLP semi-top level module Calls custom cuda extensions. @@ -2136,8 +2561,9 @@ def backward( ) -> Tuple[Union[torch.Tensor, None], ...]: """Backward pass: compute gradients and reduce FP8 scaling factors.""" bwd_args: LayerNormMLPBwdArgs = ctx.backward_objects - bwd_args.grad_output = grad_output - bwd_args.grad_ln_out = grad_ln_out + bwd_args.setup_grad_outputs( + (grad_output, grad_ln_out, _grad_fc1_weight_workspace, _grad_fc2_weight_workspace) + ) with get_nvtx_range_context("_LayerNormMLP_backward"): _layernorm_mlp_recompute(bwd_args, ctx) ( @@ -2168,6 +2594,18 @@ def backward( ) +@no_torch_dynamo() +def _layernorm_mlp_eager( + tensors: Tuple[Optional[torch.Tensor], ...], + fwd_args: LayerNormMLPFwdArgs, + is_grad_enabled: bool, +) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor]]: + """Run ``_LayerNormMLP`` eagerly, bypassing Dynamo.""" + if is_grad_enabled: + return _LayerNormMLP.apply(*tensors, fwd_args) + return _LayerNormMLP.forward(None, *tensors, fwd_args) + + class LayerNormMLP(TransformerEngineBaseModule): r""" Applies layer normalization on the input followed by the MLP module, consisting of @@ -2586,7 +3024,6 @@ def reset_parameters(self, defer_init=False): if self.set_parallel_mode: setattr(self.fc2_bias, "sequence_parallel", self.sequence_parallel) - @no_torch_dynamo() def forward( self, inp: torch.Tensor, @@ -2612,6 +3049,13 @@ def forward( * it also allows skipping gradient accumulation during the first microbatch (since it is the first gradient being produced) + + Under ``torch.compile``, the FP8 weight cache is replaced on + the first microbatch and reused on subsequent microbatches. + ``fuse_wgrad_accumulation=True`` remains unsupported. + With ``mode="reduce-overhead"``, call + ``torch.compiler.cudagraph_mark_step_begin()`` once before + each minibatch to keep the cache live across its microbatches. """ is_grad_enabled = torch.is_grad_enabled() @@ -2686,28 +3130,45 @@ def forward( if self.bias_gelu_nvfusion and not use_reentrant_activation_recompute(): self.fast_setattr("bias_gelu_nvfusion", False) + use_compiled_op = torch.compiler.is_compiling() and _layernorm_mlp_op is not None + if _layernorm_mlp_op is None and torch.compiler.is_compiling(): + warn_if_compile_disabled() + if use_compiled_op: + # Process groups cross the op boundary separately from quantizers. + for quantizer in (fc1_input_quantizer, fc2_grad_output_quantizer): + if getattr(quantizer, "amax_reduction_group", None) is not None: + set_quantizer_amax_reduction_group(quantizer, None) + cache_name_fc1 = ( None if (is_first_microbatch is None or self.is_fsdp2) else "fc1_weight" ) cache_name_fc2 = ( None if (is_first_microbatch is None or self.is_fsdp2) else "fc2_weight" ) - fc1_weight_workspace = ( - self._fp8_workspaces.get(cache_name_fc1) if cache_name_fc1 is not None else None - ) - fc2_weight_workspace = ( - self._fp8_workspaces.get(cache_name_fc2) if cache_name_fc2 is not None else None - ) + fc1_weight_workspace = None + fc2_weight_workspace = None + if cache_name_fc1 is not None and not (use_compiled_op and is_first_microbatch): + fc1_weight_workspace = self._fp8_workspaces.get(cache_name_fc1) + fc2_weight_workspace = self._fp8_workspaces.get(cache_name_fc2) + fprop_use_split_accumulator = _2X_ACC_FPROP dgrad_use_split_accumulator = _2X_ACC_DGRAD wgrad_use_split_accumulator = _2X_ACC_WGRAD + recipe_float8_block_scaling = False + recipe_custom = False + recipe_dbias_dact_fusion = False if self.fp8: _recipe = FP8GlobalStateManager.get_fp8_recipe() backward_override = _recipe.backward_override + if hasattr(_recipe, "fp8_gemm_fprop"): + fprop_use_split_accumulator = _recipe.fp8_gemm_fprop.use_split_accumulator if hasattr(_recipe, "fp8_gemm_dgrad"): dgrad_use_split_accumulator = _recipe.fp8_gemm_dgrad.use_split_accumulator if hasattr(_recipe, "fp8_gemm_wgrad"): wgrad_use_split_accumulator = _recipe.fp8_gemm_wgrad.use_split_accumulator + recipe_float8_block_scaling = _recipe.float8_block_scaling() + recipe_custom = _recipe.custom() + recipe_dbias_dact_fusion = _recipe.delayed() or _recipe.mxfp8() else: backward_override = None @@ -2727,6 +3188,24 @@ def forward( fc2_bias_tensor = ( fc2_bias if self.apply_bias and not self.gemm_bias_unfused_add else None ) + if use_compiled_op: + fallback_reason = self._compile_eager_fallback_reason( + inp, + skip_fp8_weight_update, + fp8_output, + is_grad_enabled, + debug, + quantizers, + ) + if fallback_reason is not None: + warn_compile_eager_fallback(fallback_reason) + torch._dynamo.graph_break( + msg=f"te.LayerNormMLP falling back to eager: {fallback_reason}" + ) + use_compiled_op = False + + wgrad_store = None if use_compiled_op else self.wgrad_store + fwd_args = LayerNormMLPFwdArgs( # tensors inp=inp, @@ -2785,8 +3264,12 @@ def forward( fp8=self.fp8, fp8_calibration=self.fp8_calibration, backward_override=backward_override, + fprop_use_split_accumulator=fprop_use_split_accumulator, dgrad_use_split_accumulator=dgrad_use_split_accumulator, wgrad_use_split_accumulator=wgrad_use_split_accumulator, + recipe_float8_block_scaling=recipe_float8_block_scaling, + recipe_custom=recipe_custom, + recipe_dbias_dact_fusion=recipe_dbias_dact_fusion, debug=debug, # weight-workspace caching is_first_microbatch=is_first_microbatch, @@ -2810,7 +3293,7 @@ def forward( is_fsdp2=self.is_fsdp2, # weight-grad scheduling fuse_wgrad_accumulation=self.fuse_wgrad_accumulation, - wgrad_store=self.wgrad_store, + wgrad_store=wgrad_store, # activation checkpointing checkpoint=self.checkpoint, fp8_meta=self.fp8_meta if self.checkpoint else None, @@ -2820,28 +3303,21 @@ def forward( is_grad_enabled=is_grad_enabled, ) - if is_grad_enabled: - out, ln_out, new_fc1_ws, new_fc2_ws = _LayerNormMLP.apply( - inp, - self.layer_norm_weight, - self.layer_norm_bias, - fc1_weight, - fc1_bias, - fc2_weight, - fc2_bias_tensor, - fwd_args, - ) + if use_compiled_op: + out, ln_out, new_fc1_ws, new_fc2_ws = _layernorm_mlp_op(fwd_args) else: - out, ln_out, new_fc1_ws, new_fc2_ws = _LayerNormMLP.forward( - None, - inp, - self.layer_norm_weight, - self.layer_norm_bias, - fc1_weight, - fc1_bias, - fc2_weight, - fc2_bias_tensor, + out, ln_out, new_fc1_ws, new_fc2_ws = _layernorm_mlp_eager( + ( + inp, + self.layer_norm_weight, + self.layer_norm_bias, + fc1_weight, + fc1_bias, + fc2_weight, + fc2_bias_tensor, + ), fwd_args, + is_grad_enabled, ) if new_fc1_ws is not None and cache_name_fc1 is not None: @@ -2867,6 +3343,44 @@ def forward( return out, ln_out return out + def _compile_eager_fallback_reason( + self, + inp: torch.Tensor, + skip_fp8_weight_update: Optional[torch.Tensor], + fp8_output: bool, + is_grad_enabled: bool, + debug: bool, + quantizers: Sequence[Optional[Quantizer]], + ) -> Optional[str]: + """Return a fallback reason after preparing the forward quantizers.""" + if debug: + return "debug instrumentation (nvidia-dlfw-inspect)" + if self.checkpoint and is_grad_enabled: + return "activation checkpointing (checkpoint=True)" + if isinstance(inp, (QuantizedTensor, QuantizedTensorStorage)): + return "a quantized input tensor" + if self.fsdp_group is not None: + return "manual TE FSDP (fsdp_group); use FSDP2 or MCore FSDP" + if ( + fp8_output + and is_grad_enabled + and (inp.requires_grad or any(p.requires_grad for p in self.parameters())) + ): + return "differentiable fp8_output=True" + if is_cpu_offload_enabled(): + return "CPU activation offloading" + # A queued store can be enabled between forward and backward. + if self.wgrad_store is not None and ( + self.wgrad_store.context is not None or self.wgrad_store.delay_wgrad_compute() + ): + return "delayed wgrad compute (wgrad_store)" + if self.fuse_wgrad_accumulation: + return "fuse_wgrad_accumulation (main_grad)" + fp8 = FP8GlobalStateManager.is_fp8_enabled() + if fp8 and skip_fp8_weight_update is not None: + return "GPU-controlled FP8 weight cache updates" + return compile_unsupported_quantizer_reason(quantizers) + def _get_quantizers(self, fp8_output, is_grad_enabled): if self.fp8: self._warn_missing_output_quantizer_role(fp8_output, False)