diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 472917d56f0..7140f9f8573 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -42,6 +42,7 @@ from transformer_engine.pytorch.module.base import TransformerEngineBaseModule from transformer_engine.pytorch.quantization import FP8GlobalStateManager, QuantizerRole from transformer_engine.pytorch.ops.basic.basic_linear import BasicLinear +from transformer_engine.pytorch.ops.basic.bias import BiasBwdArgs from transformer_engine.pytorch.ops.fuser import OperationFuser from transformer_engine.pytorch.ops.op import BasicOperation, OperationContext from transformer_engine.pytorch.tensor.float8_tensor import Float8CurrentScalingQuantizer @@ -54,6 +55,7 @@ Quantizer, ) from transformer_engine.pytorch.dynamo import TensorSpec, to_tensor_spec +from transformer_engine.pytorch.dynamo.fake import fake_bgrad from transformer_engine.pytorch import ( is_fp8_available, is_mxfp8_available, @@ -1972,17 +1974,33 @@ def fn(inp): @pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") @pytest.mark.parametrize("compile_mode", _compile_modes) @pytest.mark.parametrize( - "input_shape", - [(64,), (2, 4, 4, 64)], - ids=["rank1", "rank4"], + "fp8_recipe,input_shape", + [ + pytest.param(None, (64,), id="bf16-rank1"), + pytest.param(None, (2, 4, 4, 64), id="bf16-rank4"), + ] + + [ + pytest.param(fp8_recipe, shape, id=f"{name}-{shape}") + for name, fp8_recipe in ( + ("nvfp4", recipe.NVFP4BlockScaling()), + ("nvfp4-no-rht", recipe.NVFP4BlockScaling(disable_rht=True)), + ) + for shape in ((1, 128, 128), (2, 4, 16, 128)) + ], ) -def test_te_linear_compiles_non_2d_input(input_shape, compile_mode): +def test_te_linear_compiles_non_2d_input(fp8_recipe, input_shape, compile_mode): """Compiled Linear preserves non-matrix activation shapes in forward and backward.""" + if isinstance(fp8_recipe, recipe.NVFP4BlockScaling) and not nvfp4_available: + pytest.skip(reason_for_no_nvfp4) dtype = torch.bfloat16 - model = te.Linear(64, 32, params_dtype=dtype, device="cuda") + out_features = 32 if fp8_recipe is None else 128 + model = te.Linear(input_shape[-1], out_features, params_dtype=dtype, device="cuda") def fn(inp): - return model(inp) + if fp8_recipe is None: + return model(inp) + with te.autocast(recipe=fp8_recipe): + return model(inp) torch._dynamo.reset() if compile_mode == "reduce-overhead": @@ -2852,12 +2870,83 @@ def test_te_ops_bias_compile_with_fp8_input(backward): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -@pytest.mark.parametrize("quantization", [None, "fp8"]) -@pytest.mark.parametrize("case", ["linear", "bias", "pair", "unfused"]) -@pytest.mark.parametrize("grads", ["all", "input", "weight", "bias", "none"]) -def test_te_ops_linear_bias_compile(dtype, quantization, case, grads, monkeypatch): - _check_linear_bias_compile(dtype, quantization, case, grads, monkeypatch) +@pytest.mark.parametrize( + "dtype,quantization,shape,disable_rht_fusion,case,grads", + [ + (dtype, quantization, (2, 16, 32), False, case, grads) + for dtype in (torch.float16, torch.bfloat16) + for quantization in (None, "fp8") + for case in ("linear", "bias", "pair", "unfused") + for grads in ("all", "input", "weight", "bias", "none") + ] + + [ + (torch.bfloat16, "nvfp4", (2, 16, 128), False, "pair", "all"), + (torch.bfloat16, "nvfp4", (64, 128), False, "pair", "all"), + (torch.bfloat16, "nvfp4", (64, 128), True, "pair", "all"), + (torch.bfloat16, "nvfp4-no-rht", (2, 16, 128), False, "pair", "all"), + ], +) +def test_te_ops_linear_bias_compile( + dtype, quantization, shape, disable_rht_fusion, case, grads, monkeypatch +): + monkeypatch.setenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", str(int(disable_rht_fusion))) + _check_linear_bias_compile(dtype, quantization, case, grads, monkeypatch, input_shape=shape) + + +@pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) +@pytest.mark.parametrize("mode", ["rht", "2d", "row-scaled"]) +def test_te_ops_bias_nvfp4_layout(mode, monkeypatch): + quantizer = NVFP4Quantizer( + columnwise=True, + with_rht=mode == "rht", + with_post_rht_amax=mode == "rht", + with_2d_quantization=mode == "2d", + row_scaled_nvfp4=mode == "row-scaled", + with_random_sign_mask=False, + ) + + def bgrad(inp): + dx, ((db,),), _ = te.ops.Bias.compile_ops[1](BiasBwdArgs(inp, quantizer)) + return dx, db + + torch._dynamo.reset() + compiled, graphs = _compile_with_graphs(bgrad) + # Pairs preserve alignment, including distinct nonzero remainders. + shapes = ((128, 128), (256, 256), (96, 160), (160, 224)) + if mode == "row-scaled": + shapes = shapes[:1] + cases = [(False, True, shape) for shape in shapes] + if mode == "rht": + cases += [(True, True, (128, 128)), (False, True, (128, 128))] + if mode != "row-scaled": + cases.append((False, False, (128, 128))) + + for i, (disable_fusion, optimize_for_gemm, shape) in enumerate(cases): + monkeypatch.setenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", str(int(disable_fusion))) + quantizer.optimize_for_gemm = optimize_for_gemm + inp = torch.randn(shape, device="cuda", dtype=torch.bfloat16) + torch._dynamo.mark_dynamic(inp, (0, 1)) + previous_graph_count = len(graphs) + dx, db = compiled(inp) + expected_db, expected_dx = tex.bgrad_quantize(inp, quantizer) + assert dx._with_gemm_swizzled_scales == expected_dx._with_gemm_swizzled_scales + torch.testing.assert_close(db, expected_db, atol=0, rtol=0) + torch.testing.assert_close(dx.dequantize(), expected_dx.dequantize(), atol=0, rtol=0) + torch.testing.assert_close( + dx._columnwise_data, expected_dx._columnwise_data, atol=0, rtol=0 + ) + if i < len(shapes) and i % 2 == 1: + assert len(graphs) == previous_graph_count + + shape = (64, 1, 128) + inp = torch.randn(shape, device="cuda", dtype=torch.bfloat16) + _, expected_dx = tex.bgrad_quantize(inp, quantizer) + _, dx_spec = fake_bgrad(to_tensor_spec(inp), quantizer) + assert dx_spec.with_gemm_swizzled_scales == expected_dx._with_gemm_swizzled_scales + for name, (inner_shape, inner_dtype) in dx_spec.quantizer.inner_tensor_specs(shape).items(): + inner = getattr(expected_dx, name) + assert inner.shape == inner_shape + assert inner.dtype == inner_dtype @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @@ -2928,22 +3017,43 @@ def unexpected_allocation(*args, **kwargs): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize("training", [False, True]) -@pytest.mark.parametrize("initial_device", ["cuda", "cpu", "meta"]) -def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device): +@pytest.mark.parametrize( + "initial_device,quantization", + [("cuda", None), ("cpu", None), ("meta", None), ("cuda", "nvfp4"), ("cuda", "nvfp4-no-rht")], +) +def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device, quantization): """A fresh process prevents earlier GEMMs from hiding lazy workspace allocation.""" + if quantization is not None and not nvfp4_available: + pytest.skip(reason_for_no_nvfp4) script = textwrap.dedent( """ import contextlib + import copy import sys import torch import transformer_engine.pytorch as te + from transformer_engine.common import recipe from transformer_engine.pytorch.cpp_extensions.gemm import get_cublas_workspace from torch._dynamo.utils import counters - training, initial_device = sys.argv[1:] + training, initial_device, quantization = sys.argv[1:] training = training == "True" + is_nvfp4 = quantization != "None" + quant_recipe = ( + recipe.NVFP4BlockScaling( + disable_rht=quantization == "nvfp4-no-rht", + disable_stochastic_rounding=True, + ) + if is_nvfp4 + else None + ) + in_features, out_features = (128, 128) if is_nvfp4 else (32, 64) + batch_size = 32 + eager_tols = {"rtol": 0, "atol": 0} if is_nvfp4 else {} assert get_cublas_workspace.cache_info().currsize == 0 - linear = te.ops.BasicLinear(32, 64, device=initial_device, dtype=torch.bfloat16) + linear = te.ops.BasicLinear( + in_features, out_features, device=initial_device, dtype=torch.bfloat16 + ) if initial_device == "cpu": assert get_cublas_workspace.cache_info().currsize == 0 linear.cuda() @@ -2952,28 +3062,35 @@ def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device): linear.to_empty(device="cuda") linear.reset_parameters() assert get_cublas_workspace.cache_info().currsize > 0 - model = te.ops.Sequential(linear, te.ops.Bias(64, dtype=torch.bfloat16)) - x = torch.randn(32, 32, device="cuda", dtype=torch.bfloat16, requires_grad=training) + model = te.ops.Sequential(linear, te.ops.Bias(out_features, dtype=torch.bfloat16)) + reference_model = copy.deepcopy(model) + x = torch.randn( + batch_size, in_features, device="cuda", dtype=torch.bfloat16, requires_grad=training + ) targets = (x, *model.parameters()) - dy = torch.randn(32, 64, device="cuda", dtype=torch.bfloat16) + reference_targets = (x, *reference_model.parameters()) + dy = torch.randn(batch_size, out_features, device="cuda", dtype=torch.bfloat16) - def forward(x): - return model(x) + def forward(x, module): + if quant_recipe is None: + return module(x) + with te.autocast(recipe=quant_recipe): + return module(x) compiled = torch.compile(forward, fullgraph=True, mode="reduce-overhead") counters.clear() with contextlib.nullcontext() if training else torch.no_grad(): for _ in range(5): torch.compiler.cudagraph_mark_step_begin() - actual = compiled(x) + actual = compiled(x, model) if training: grads = torch.autograd.grad(actual, targets, dy) # Eager reference runs only after the first compiled invocation. - expected = forward(x) - torch.testing.assert_close(actual, expected) + expected = forward(x, reference_model) + torch.testing.assert_close(actual, expected, **eager_tols) if training: - expected_grads = torch.autograd.grad(expected, targets, dy) - torch.testing.assert_close(grads, expected_grads) + expected_grads = torch.autograd.grad(expected, reference_targets, dy) + torch.testing.assert_close(grads, expected_grads, **eager_tols) del grads, expected_grads del actual, expected torch.cuda.synchronize() @@ -2984,7 +3101,7 @@ def forward(x): """ ) result = subprocess.run( - [sys.executable, "-c", script, str(training), initial_device], + [sys.executable, "-c", script, str(training), initial_device, str(quantization)], capture_output=True, text=True, check=False, @@ -2994,29 +3111,44 @@ def forward(x): def _check_linear_bias_compile( - dtype, quantization, case, grads, monkeypatch, *, backward_override=None + dtype, + quantization, + case, + grads, + monkeypatch, + *, + backward_override=None, + input_shape=(2, 16, 32), ): + is_nvfp4 = quantization in ("nvfp4", "nvfp4-no-rht") if quantization == "fp8" and not fp8_available: pytest.skip(reason_for_no_fp8) + if is_nvfp4 and not nvfp4_available: + pytest.skip(reason_for_no_nvfp4) if (case == "linear" and grads == "bias") or (case == "bias" and grads == "weight"): pytest.skip("Requested parameter is absent") torch._dynamo.reset() if case == "unfused": monkeypatch.setattr(OperationFuser, "forward_fusion_functions", []) + in_features = input_shape[-1] + out_features = 128 if is_nvfp4 else 64 ops = [] if case != "bias": - ops.append(te.ops.BasicLinear(32, 64, dtype=dtype)) + ops.append(te.ops.BasicLinear(in_features, out_features, dtype=dtype)) ops[-1].weight.requires_grad_(grads in ("all", "weight")) if case != "linear": - ops.append(te.ops.Bias(32 if case == "bias" else 64, dtype=dtype)) + ops.append(te.ops.Bias(in_features if case == "bias" else out_features, dtype=dtype)) ops[-1].bias.requires_grad_(grads in ("all", "bias")) model = te.ops.Sequential(*ops) eager_model = copy.deepcopy(model) - quant_recipe = ( - recipe.Float8CurrentScaling(backward_override=backward_override) - if quantization == "fp8" - else None - ) + quant_recipe = None + if quantization == "fp8": + quant_recipe = recipe.Float8CurrentScaling(backward_override=backward_override) + elif is_nvfp4: + quant_recipe = recipe.NVFP4BlockScaling( + disable_rht=quantization == "nvfp4-no-rht", disable_stochastic_rounding=True + ) + eager_tols = {"rtol": 0, "atol": 0} if is_nvfp4 else {} def run(x, module=model): if quant_recipe is None: @@ -3025,16 +3157,17 @@ def run(x, module=model): return module(x) compiled, graphs = _compile_with_graphs(run) - x = torch.randn(2, 16, 32, device="cuda", dtype=dtype) + x = torch.randn(input_shape, device="cuda", dtype=dtype) x.requires_grad_(grads in ("all", "input")) targets = tuple(t for t in (x, *model.parameters()) if t.requires_grad) eager_targets = tuple(t for t in (x, *eager_model.parameters()) if t.requires_grad) graph_count = None with torch.no_grad() if grads == "none" else contextlib.nullcontext(): for iteration in range(3): + check_reference = iteration == 0 and not is_nvfp4 with torch.no_grad(): for tensor in (x, *model.parameters()): - if iteration == 0: + if check_reference: # These binary fractions survive FP8 quantization exactly, # allowing a strict comparison with native PyTorch. tensor.copy_(torch.randint(-2, 3, tensor.shape, device=tensor.device) / 16) @@ -3044,8 +3177,8 @@ def run(x, module=model): eager_param.copy_(param) actual = compiled(x) expected = run(x, eager_model) - torch.testing.assert_close(actual, expected) - if iteration == 0: + torch.testing.assert_close(actual, expected, **eager_tols) + if check_reference: reference = x.double() for op in ops: reference = ( @@ -3059,14 +3192,14 @@ def run(x, module=model): if targets: dy = ( torch.randint_like(actual, -2, 3) / 16 - if iteration == 0 + if check_reference else torch.randn_like(actual) ) actual_grads = torch.autograd.grad(actual, targets, dy) torch.testing.assert_close( - actual_grads, torch.autograd.grad(expected, eager_targets, dy) + actual_grads, torch.autograd.grad(expected, eager_targets, dy), **eager_tols ) - if iteration == 0: + if check_reference: torch.testing.assert_close( actual_grads, torch.autograd.grad(reference, targets, dy.double()), diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index 1103225b8a0..9a3c6314054 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -341,6 +341,8 @@ class MXFP8Quantizer : public Quantizer { class NVFP4Quantizer : public Quantizer { public: + using ShapeAlignment = std::optional>; + // amax reduction for low precision FP4 AG bool with_amax_reduction; c10::intrusive_ptr amax_reduction_group; @@ -422,7 +424,21 @@ class NVFP4Quantizer : public Quantizer { */ static bool is_eligible_for_2d_swizzle_fusion(const std::vector& shape); + // Flattened row/column alignment for directly emitted GEMM-swizzled scales, or no such path. + static ShapeAlignment gemm_swizzled_scales_alignment(bool optimize_for_gemm, bool with_rht, + bool with_2d_quantization, + bool row_scaled_nvfp4, bool nvfp4_use_4over6, + int sm_arch, + std::optional disable_rht_cast_fusion); + private: + static ShapeAlignment rht_cast_fusion_alignment(int sm_arch, + std::optional disable_rht_cast_fusion, + bool for_grouped_kernel = false); + static ShapeAlignment get_2d_swizzle_fusion_alignment(int sm_arch); + static bool meets_alignment(const std::vector& shape, const ShapeAlignment& alignment); + bool emits_gemm_swizzled_scales(const std::vector& shape) const; + void quantize_with_rht_unfused_helper(const TensorWrapper& input, TensorWrapper& out, TensorWrapper& rht_output_t_cpp, QuantizationConfigWrapper& quant_config, diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index 2046408eba8..bbb91a11d35 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -210,6 +210,11 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { py::arg("output") = py::none(), py::arg("noop") = py::none()); m.def("dequantize", &transformer_engine::pytorch::dequantize, "Dequantize", py::arg("input"), py::arg("otype")); + m.def("nvfp4_gemm_swizzled_scales_alignment", + &transformer_engine::pytorch::NVFP4Quantizer::gemm_swizzled_scales_alignment, + py::arg("optimize_for_gemm"), py::arg("with_rht"), py::arg("with_2d_quantization"), + py::arg("row_scaled_nvfp4"), py::arg("nvfp4_use_4over6"), py::arg("sm_arch"), + py::arg("disable_rht_cast_fusion")); m.def("create_empty_quantized_tensor", &transformer_engine::pytorch::create_empty_quantized_tensor, "Create an empty quantized tensor", py::arg("quantizer"), py::arg("shape"), diff --git a/transformer_engine/pytorch/csrc/quantizer.cpp b/transformer_engine/pytorch/csrc/quantizer.cpp index fba268d2723..ba0410bcacb 100644 --- a/transformer_engine/pytorch/csrc/quantizer.cpp +++ b/transformer_engine/pytorch/csrc/quantizer.cpp @@ -1934,50 +1934,72 @@ void NVFP4Quantizer::set_quantization_params(TensorWrapper* tensor) const { columnwise_data.shape); } -bool NVFP4Quantizer::is_eligible_for_rht_cast_fusion(const std::vector& shape, - bool for_grouped_kernel) { - if (transformer_engine::getenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION")) { - return false; - } - const auto [rows, cols] = get_2d_dims(shape); - const size_t row_align = for_grouped_kernel ? 128 : 64; - const int sm_arch = transformer_engine::cuda::sm_arch(); +NVFP4Quantizer::ShapeAlignment NVFP4Quantizer::rht_cast_fusion_alignment( + int sm_arch, std::optional disable_rht_cast_fusion, bool for_grouped_kernel) { + const bool disabled = + disable_rht_cast_fusion.has_value() + ? *disable_rht_cast_fusion + : transformer_engine::getenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION"); const bool supported_arch = (sm_arch >= 100 && sm_arch <= 110) || (!for_grouped_kernel && (sm_arch == 120 || sm_arch == 121)); - return rows % row_align == 0 && cols % 128 == 0 && supported_arch; + if (disabled || !supported_arch) { + return std::nullopt; + } + return std::pair{for_grouped_kernel ? 128 : 64, 128}; } -bool NVFP4Quantizer::is_eligible_for_2d_swizzle_fusion(const std::vector& shape) { - const auto [rows, cols] = get_2d_dims(shape); - return rows % 128 == 0 && cols % 128 == 0 && transformer_engine::cuda::sm_arch() >= 100 && - transformer_engine::cuda::sm_arch() <= 110; +NVFP4Quantizer::ShapeAlignment NVFP4Quantizer::get_2d_swizzle_fusion_alignment(int sm_arch) { + if (sm_arch < 100 || sm_arch > 110) { + return std::nullopt; + } + return std::pair{128, 128}; } -namespace { - -// Whether this quantizer should emit GEMM-swizzled scale factors directly from the -// quantize kernel (RHT cast-fusion path or the plain 2D quantize path), avoiding the -// standalone post-quantize swizzle pass. -bool nvfp4_emits_gemm_swizzled_scales(const NVFP4Quantizer& q, const std::vector& shape) { - if (!q.optimize_for_gemm) { +bool NVFP4Quantizer::meets_alignment(const std::vector& shape, + const ShapeAlignment& alignment) { + if (!alignment) { return false; } - if (q.with_rht) { - const int sm_arch = transformer_engine::cuda::sm_arch(); + const auto [rows, cols] = get_2d_dims(shape); + return rows % alignment->first == 0 && cols % alignment->second == 0; +} + +NVFP4Quantizer::ShapeAlignment NVFP4Quantizer::gemm_swizzled_scales_alignment( + bool optimize_for_gemm, bool with_rht, bool with_2d_quantization, bool row_scaled_nvfp4, + bool nvfp4_use_4over6, int sm_arch, std::optional disable_rht_cast_fusion) { + if (!optimize_for_gemm) { + return std::nullopt; + } + if (with_rht) { // The SM12x no-TMEM RHT path reuses the compact-scale 1D TMA quantizer. - // Keep the existing post-quantize swizzle until that path gains an in-kernel swizzle epilogue. if (sm_arch == 120 || sm_arch == 121) { - return false; + return std::nullopt; } - return NVFP4Quantizer::is_eligible_for_rht_cast_fusion(shape); + return rht_cast_fusion_alignment(sm_arch, disable_rht_cast_fusion); } - // Plain (non-RHT) 2D quantize kernel can bake the swizzled layout for aligned shapes. - return q.with_2d_quantization && !q.row_scaled_nvfp4 && - q.nvfp4_4over6_mode == kNVTENVFP44Over6Disabled && - NVFP4Quantizer::is_eligible_for_2d_swizzle_fusion(shape); + if (with_2d_quantization && !row_scaled_nvfp4 && !nvfp4_use_4over6) { + return get_2d_swizzle_fusion_alignment(sm_arch); + } + return std::nullopt; } -} // namespace +bool NVFP4Quantizer::is_eligible_for_rht_cast_fusion(const std::vector& shape, + bool for_grouped_kernel) { + return meets_alignment(shape, rht_cast_fusion_alignment(transformer_engine::cuda::sm_arch(), + std::nullopt, for_grouped_kernel)); +} + +bool NVFP4Quantizer::is_eligible_for_2d_swizzle_fusion(const std::vector& shape) { + return meets_alignment(shape, + get_2d_swizzle_fusion_alignment(transformer_engine::cuda::sm_arch())); +} + +bool NVFP4Quantizer::emits_gemm_swizzled_scales(const std::vector& shape) const { + return meets_alignment(shape, gemm_swizzled_scales_alignment( + optimize_for_gemm, with_rht, with_2d_quantization, + row_scaled_nvfp4, nvfp4_4over6_mode != kNVTENVFP44Over6Disabled, + transformer_engine::cuda::sm_arch(), std::nullopt)); +} std::pair NVFP4Quantizer::create_tensor( const std::vector& shape, DType dtype, std::optional device_opt, @@ -1992,7 +2014,7 @@ std::pair NVFP4Quantizer::create_tensor( // Swizzled SF is emitted directly by the RHT cast-fusion kernel or by the plain // 2D quantize kernel (for aligned shapes); other quantize paths reject it and go // through the standalone post-quantize swizzle pass instead. - const bool with_gemm_swizzled_scales = nvfp4_emits_gemm_swizzled_scales(*this, shape); + const bool with_gemm_swizzled_scales = emits_gemm_swizzled_scales(shape); NVTE_CHECK(flat_first_dim % NVFP4_BLOCK_SIZE == 0, "First dim for NVFP4 must be divisible by ", NVFP4_BLOCK_SIZE, " (got shape=", shape, ")"); NVTE_CHECK(flat_last_dim % NVFP4_BLOCK_SIZE == 0, @@ -2344,7 +2366,7 @@ std::pair NVFP4Quantizer::convert_and_update_tensor( // Swizzled SF is emitted directly by the RHT cast-fusion kernel or by the plain // 2D quantize kernel (for aligned shapes); other quantize paths reject it and go // through the standalone post-quantize swizzle pass instead. - const bool with_gemm_swizzled_scales = nvfp4_emits_gemm_swizzled_scales(*this, shape); + const bool with_gemm_swizzled_scales = emits_gemm_swizzled_scales(shape); const bool row_scaled_nvfp4 = this->row_scaled_nvfp4; const bool disable_second_level_scale = this->disable_second_level_scale; diff --git a/transformer_engine/pytorch/dynamo/fake.py b/transformer_engine/pytorch/dynamo/fake.py new file mode 100644 index 00000000000..5f856d62214 --- /dev/null +++ b/transformer_engine/pytorch/dynamo/fake.py @@ -0,0 +1,94 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Data-free descriptions of quantized operation outputs.""" + +from __future__ import annotations +import math +import os +from typing import Optional + +import torch +import transformer_engine_torch as tex + +from .tensor_spec import TensorSpec +from ..quantized_tensor import Quantizer +from ..tensor.nvfp4_tensor import NVFP4Quantizer +from ..utils import get_device_compute_capability + + +@torch.compiler.assume_constant_result +def _nvfp4_gemm_swizzled_scales_alignment( + optimize_for_gemm: bool, + with_rht: bool, + with_2d_quantization: bool, + row_scaled_nvfp4: bool, + nvfp4_use_4over6: bool, + sm_arch: int, + disable_rht_cast_fusion: bool, +) -> Optional[tuple[int, int]]: + return tex.nvfp4_gemm_swizzled_scales_alignment( + optimize_for_gemm, + with_rht, + with_2d_quantization, + row_scaled_nvfp4, + nvfp4_use_4over6, + sm_arch, + disable_rht_cast_fusion, + ) + + +def _make_fused_quantization_output_spec( + shape: tuple[int, ...], + dtype: torch.dtype, + device: Optional[torch.device], + quantizer: Optional[Quantizer], +) -> TensorSpec: + """Describe a native output without a separate post-quantize swizzle.""" + spec = TensorSpec(shape=tuple(shape), dtype=dtype, device=device, quantizer=quantizer) + quantizer = spec.quantizer + if not isinstance(quantizer, NVFP4Quantizer): + return spec + major, minor = get_device_compute_capability(device) + # Direct environ access lets Dynamo guard changes to the flag. + disable_rht_cast_fusion = bool( + int(os.environ.get("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", "0") or "0") + ) + alignment = _nvfp4_gemm_swizzled_scales_alignment( + quantizer.optimize_for_gemm, + quantizer.with_rht, + quantizer.with_2d_quantization, + quantizer.row_scaled_nvfp4, + quantizer.nvfp4_use_4over6, + major * 10 + minor, + disable_rht_cast_fusion, + ) + # Keep divisibility checks visible to Dynamo without specializing the dimensions. + rows, cols = math.prod(shape[:-1]), shape[-1] + if alignment is None: + spec.with_gemm_swizzled_scales = False + else: + row_alignment, col_alignment = alignment + spec.with_gemm_swizzled_scales = bool( + rows % row_alignment == 0 and cols % col_alignment == 0 + ) + return spec + + +def fake_bgrad( + grad_output: TensorSpec, + quantizer: Optional[Quantizer] = None, +) -> tuple[TensorSpec, TensorSpec]: + """Describe bias reduction and optional gradient quantization, in native order.""" + db = TensorSpec( + shape=(grad_output.shape[-1],), dtype=grad_output.dtype, device=grad_output.device + ) + dx = ( + grad_output + if quantizer is None + else _make_fused_quantization_output_spec( + grad_output.shape, grad_output.dtype, grad_output.device, quantizer + ) + ) + return db, dx diff --git a/transformer_engine/pytorch/dynamo/tensor_spec.py b/transformer_engine/pytorch/dynamo/tensor_spec.py index 4cfe225952a..d96c49ca7a5 100644 --- a/transformer_engine/pytorch/dynamo/tensor_spec.py +++ b/transformer_engine/pytorch/dynamo/tensor_spec.py @@ -29,6 +29,7 @@ class TensorSpec: quantizer: Optional[Any] = None requires_grad: bool = False device: Optional[torch.device] = field(default=None) + with_gemm_swizzled_scales: Optional[bool] = None def __post_init__(self) -> None: # Own a private copy of the quantizer so usage changes (update_usage) @@ -94,9 +95,12 @@ def create_metadata(self) -> Dict[str, Any]: "dtype": self.dtype, "requires_grad": self.requires_grad, } - return self.quantizer.create_metadata( + meta = self.quantizer.create_metadata( tuple(self.shape), dtype=self.dtype, requires_grad=self.requires_grad ) + if self.with_gemm_swizzled_scales is not None: + meta["nontensor_kwargs"]["with_gemm_swizzled_scales"] = self.with_gemm_swizzled_scales + return meta def create_inner_tensors(self) -> List[torch.Tensor]: """Materialize the flat inner tensors (in :meth:`inner_names` order). @@ -162,4 +166,5 @@ def to_tensor_spec(tensor: Any) -> TensorSpec: quantizer=getattr(tensor, "_quantizer", None), requires_grad=requires_grad, device=tensor.device, + with_gemm_swizzled_scales=getattr(tensor, "_with_gemm_swizzled_scales", None), ) diff --git a/transformer_engine/pytorch/ops/basic/bias.py b/transformer_engine/pytorch/ops/basic/bias.py index 70b34da1851..b9fbd1fad08 100644 --- a/transformer_engine/pytorch/ops/basic/bias.py +++ b/transformer_engine/pytorch/ops/basic/bias.py @@ -12,6 +12,7 @@ import transformer_engine_torch as tex from ...dynamo import TensorOrQuantized, TensorSpec +from ...dynamo.fake import fake_bgrad from ...quantization import FP8GlobalStateManager from ..op import BasicOperation from ...utils import canonicalize_device, canonicalize_dtype @@ -194,13 +195,8 @@ def backward_compute( @classmethod def backward_compute_fake(cls, args: BiasBwdArgs): dy = args.grad_output - dx = None - if len(dy.shape) > 1 and args.grad_input_quantizer is not None: - dx = TensorSpec( - shape=dy.shape, - dtype=dy.dtype, - device=dy.device, - quantizer=args.grad_input_quantizer, - ) - db = TensorSpec(shape=(dy.shape[-1],), dtype=dy.dtype, device=dy.device) + quantizer = args.grad_input_quantizer if len(dy.shape) > 1 else None + db, dx = fake_bgrad(dy, quantizer) + if quantizer is None: + dx = None return dx, [(db,)], [()] diff --git a/transformer_engine/pytorch/tensor/nvfp4_tensor.py b/transformer_engine/pytorch/tensor/nvfp4_tensor.py index 3b4c7fd2d89..938d9a77d4c 100644 --- a/transformer_engine/pytorch/tensor/nvfp4_tensor.py +++ b/transformer_engine/pytorch/tensor/nvfp4_tensor.py @@ -330,29 +330,11 @@ def get_scale_shape(self, shape: Iterable[int], columnwise: bool) -> Tuple[int, @staticmethod def get_columnwise_shape(shape: Iterable[int]) -> Tuple[int, ...]: - """Calculate the shape of a tensor after columnwise quantization. - - For NVFP4 columnwise quantization, it's performing 16x1 quantization block scaling. - - Parameters - ---------- - shape : Iterable[int] - Original shape of the tensor - - Returns - ------- - Tuple[int, ...] - New shape with dimensions rearranged for columnwise layout. - For a shape (d1, d2, ..., dn), returns (dn, d1, d2, ..., dn-1). - Returns empty tuple for empty input shape. - """ - if len(shape) == 0: - return tuple() - # and then after AG, a reorganize kernel will be called to restore the shape - colwise_shape = [shape[-1]] - for i in range(len(shape) - 1): - colwise_shape.append(shape[i]) - return tuple(colwise_shape) + """Return the flattened columnwise shape before FP4 packing.""" + shape = tuple(shape) + if not shape: + return () + return (shape[-1], math.prod(shape[:-1])) @staticmethod def convert_shape_for_fp4(shape: Iterable[int]) -> Tuple[int, ...]: @@ -418,7 +400,8 @@ def inner_tensor_specs( amax_rowwise_shape = (math.prod(shape[:-1]),) if self.row_scaled_nvfp4 else (1,) specs["_amax_rowwise"] = (amax_rowwise_shape, torch.float32) if self.columnwise_usage: - specs["_amax_columnwise"] = ((1,), torch.float32) + amax_columnwise_shape = (shape[-1],) if self.row_scaled_nvfp4 else (1,) + specs["_amax_columnwise"] = (amax_columnwise_shape, torch.float32) return specs diff --git a/transformer_engine/pytorch/tensor/storage/nvfp4_tensor_storage.py b/transformer_engine/pytorch/tensor/storage/nvfp4_tensor_storage.py index cd1c15b9438..97b66a8341f 100644 --- a/transformer_engine/pytorch/tensor/storage/nvfp4_tensor_storage.py +++ b/transformer_engine/pytorch/tensor/storage/nvfp4_tensor_storage.py @@ -258,7 +258,9 @@ def size(self, dim: Optional[int] = None) -> Union[torch.Size, int]: byte_shape = list(self._rowwise_data.size()) shape = byte_shape[:-1] + [byte_shape[-1] * 2] elif self._columnwise_data is not None: - warnings.warn("Attempting to get shape of NVFP4 tensor with only column-wise data.") + # Dynamo queries columnwise-only fake shapes; warning emission breaks fullgraph tracing. + if not torch.compiler.is_compiling(): + warnings.warn("Attempting to get shape of NVFP4 tensor with only column-wise data.") byte_shape = list(self._columnwise_data.size()) shape = byte_shape[1:-1] + [byte_shape[-1] * 2, byte_shape[0]] if shape is None: