From 62f954d8517db9902139790cf62b54998291cd05 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 5 Oct 2026 16:28:23 +0200 Subject: [PATCH 01/15] Fix NVFP4 compile metadata for non-matrix inputs Describe columnwise FP4 buffers with the flattened 2D shape used by native quantization. Cover 3D and 4D inputs, singleton dimensions, RHT variants, buffer metadata, and compiled forward/backward with CUDA graphs. Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 48 ++++++++++++++++--- .../pytorch/tensor/nvfp4_tensor.py | 2 +- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index eae6f0a8a24..bc3c225b0b9 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -1661,6 +1661,26 @@ def test_storage_flatten_unflatten_roundtrip(factory, shape): ] +@pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) +@pytest.mark.parametrize("shape", [(64, 128), (1, 64, 128), (64, 1, 128), (2, 4, 8, 128)]) +@pytest.mark.parametrize("with_rht", [False, True]) +@pytest.mark.parametrize("optimize_for_gemm", [False, True]) +@pytest.mark.parametrize("rowwise, columnwise", _USAGE_COMBOS) +def test_nvfp4_inner_tensor_specs_match_quantize( + shape, with_rht, optimize_for_gemm, rowwise, columnwise +): + """NVFP4 fake buffers match native quantization for every input rank.""" + quantizer = _nvfp4(with_rht=with_rht) + quantizer.set_usage(rowwise=rowwise, columnwise=columnwise) + quantizer.optimize_for_gemm = optimize_for_gemm + tensor = quantizer(torch.randn(shape, dtype=torch.bfloat16, device="cuda")) + for name, (expected_shape, expected_dtype) in quantizer.inner_tensor_specs(shape).items(): + inner = getattr(tensor, name) + assert inner.shape == expected_shape, name + assert inner.dtype == expected_dtype, name + assert inner.is_contiguous(), name + + @pytest.mark.parametrize("factory, shape", _SPEC_QUANTIZERS) @pytest.mark.parametrize("rowwise, columnwise", _USAGE_COMBOS) @pytest.mark.parametrize("internal", [False, True], ids=["wrapper", "internal"]) @@ -1938,17 +1958,33 @@ def fn(inp): @pytest.mark.skipif(not _opaque_available, reason="torch opaque object API not available") @pytest.mark.parametrize("compile_mode", _compile_modes) @pytest.mark.parametrize( - "input_shape", - [(64,), (2, 4, 4, 64)], - ids=["rank1", "rank4"], + "fp8_recipe,input_shape", + [pytest.param(None, (64,), id="bf16-rank1")] + + [ + pytest.param(fp8_recipe, shape, id=f"{name}-{shape}") + for name, fp8_recipe in ( + ("bf16", None), + ("mxfp8", recipe.MXFP8BlockScaling()), + ("nvfp4", recipe.NVFP4BlockScaling()), + ("nvfp4-no-rht", recipe.NVFP4BlockScaling(disable_rht=True)), + ) + for shape in ((1, 128, 128), (128, 1, 128), (2, 4, 16, 128)) + ], ) -def test_te_linear_compiles_non_2d_input(input_shape, compile_mode): +def test_te_linear_compiles_non_2d_input(fp8_recipe, input_shape, compile_mode): """Compiled Linear preserves non-matrix activation shapes in forward and backward.""" + if isinstance(fp8_recipe, recipe.NVFP4BlockScaling) and not nvfp4_available: + pytest.skip(reason_for_no_nvfp4) + if isinstance(fp8_recipe, recipe.MXFP8BlockScaling) and not mxfp8_available: + pytest.skip(reason_for_no_mxfp8) dtype = torch.bfloat16 - model = te.Linear(64, 32, params_dtype=dtype, device="cuda") + model = te.Linear(input_shape[-1], 128, params_dtype=dtype, device="cuda") def fn(inp): - return model(inp) + if fp8_recipe is None: + return model(inp) + with te.autocast(recipe=fp8_recipe): + return model(inp) torch._dynamo.reset() if compile_mode == "reduce-overhead": diff --git a/transformer_engine/pytorch/tensor/nvfp4_tensor.py b/transformer_engine/pytorch/tensor/nvfp4_tensor.py index 3b4c7fd2d89..c49e1036260 100644 --- a/transformer_engine/pytorch/tensor/nvfp4_tensor.py +++ b/transformer_engine/pytorch/tensor/nvfp4_tensor.py @@ -407,7 +407,7 @@ def inner_tensor_specs( ) if self.columnwise_usage: specs["_columnwise_data"] = ( - type(self).convert_shape_for_fp4(type(self).get_columnwise_shape(shape)), + (shape[-1], math.prod(shape[:-1]) // 2), torch.uint8, ) specs["_columnwise_scale_inv"] = ( From ae93a9f2448ca8322dbb90e7e4d725b43db2e17f Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 12 Oct 2026 16:11:44 +0200 Subject: [PATCH 02/15] [PyTorch] Fix NVFP4 metadata in compiled Linear operations Preserve the actual scale layout across custom-op boundaries and describe Bias backward output with the native fused-quantization layout. Keep the columnwise storage warning outside tracing. Cover compiled ops.Linear gradients with compact and swizzled scales, including disabled RHT fusion and CUDA Graphs. Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 43 ++++++++++++ transformer_engine/pytorch/dynamo/fake.py | 67 +++++++++++++++++++ .../pytorch/dynamo/tensor_spec.py | 11 ++- transformer_engine/pytorch/ops/basic/bias.py | 14 ++-- .../tensor/storage/nvfp4_tensor_storage.py | 3 +- 5 files changed, 127 insertions(+), 11 deletions(-) create mode 100644 transformer_engine/pytorch/dynamo/fake.py diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 8162189b94d..e6c894cf242 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -2896,6 +2896,49 @@ def test_te_ops_linear_bias_compile(dtype, quantization, case, grads, monkeypatc _check_linear_bias_compile(dtype, quantization, case, grads, monkeypatch) +@pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) +@pytest.mark.parametrize("compile_mode", _compile_modes) +@pytest.mark.parametrize("bias", [False, True]) +@pytest.mark.parametrize( + "shape,with_rht,disable_rht_fusion", + [ + ((2, 16, 128), False, False), + ((2, 16, 128), True, False), + ((64, 128), True, False), + ((64, 128), True, True), + ], +) +def test_te_ops_linear_nvfp4_compile( + shape, with_rht, disable_rht_fusion, bias, compile_mode, monkeypatch +): + monkeypatch.setenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", str(int(disable_rht_fusion))) + torch._dynamo.reset() + model = te.ops.Sequential(te.ops.Linear(128, 128, bias=bias, dtype=torch.bfloat16)) + eager_model = copy.deepcopy(model) + quant_recipe = recipe.NVFP4BlockScaling( + disable_rht=not with_rht, disable_stochastic_rounding=True + ) + + def run(x, module=model): + with te.autocast(recipe=quant_recipe): + return module(x) + + compiled = torch.compile(run, fullgraph=True, mode=compile_mode) + with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): + for _ in range(3): + torch.compiler.cudagraph_mark_step_begin() + x = torch.randn(shape, device="cuda", dtype=torch.bfloat16, requires_grad=True) + eager_x = x.detach().clone().requires_grad_() + actual = compiled(x) + expected = run(eager_x, eager_model) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + dy = torch.randn_like(actual) + grads = torch.autograd.grad(actual, (x, *model.parameters()), dy) + eager_grads = torch.autograd.grad(expected, (eager_x, *eager_model.parameters()), dy) + torch.testing.assert_close(grads, eager_grads, rtol=0, atol=0) + del actual, expected, grads, eager_grads, x, eager_x, dy + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) @pytest.mark.parametrize("backward_override", ["high_precision", "dequantized"]) diff --git a/transformer_engine/pytorch/dynamo/fake.py b/transformer_engine/pytorch/dynamo/fake.py new file mode 100644 index 00000000000..b200624cad9 --- /dev/null +++ b/transformer_engine/pytorch/dynamo/fake.py @@ -0,0 +1,67 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Data-free descriptions of quantized operation outputs.""" + +from __future__ import annotations +import math +import os +from typing import Optional + +import torch + +from .tensor_spec import TensorSpec +from ..quantized_tensor import Quantizer +from ..tensor.nvfp4_tensor import NVFP4Quantizer +from ..utils import get_device_compute_capability + + +def _make_fused_quantization_output_spec( + shape: tuple[int, ...], + dtype: torch.dtype, + device: Optional[torch.device], + quantizer: Optional[Quantizer], +) -> TensorSpec: + """Describe a native output without a separate post-quantize swizzle.""" + spec = TensorSpec(shape=tuple(shape), dtype=dtype, device=device, quantizer=quantizer) + quantizer = spec.quantizer + if not isinstance(quantizer, NVFP4Quantizer) or not quantizer.optimize_for_gemm: + return spec + rows, cols = math.prod(shape[:-1]), shape[-1] + if not (10, 0) <= get_device_compute_capability(device) <= (11, 0): + spec.with_gemm_swizzled_scales = False + elif quantizer.with_rht: + spec.with_gemm_swizzled_scales = bool( + not int(os.getenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", "0") or "0") + and rows % 64 == 0 + and cols % 128 == 0 + ) + else: + spec.with_gemm_swizzled_scales = bool( + quantizer.with_2d_quantization + and not quantizer.row_scaled_nvfp4 + and not quantizer.nvfp4_use_4over6 + and rows % 128 == 0 + and cols % 128 == 0 + ) + return spec + + +def fake_bgrad( + grad_output: TensorSpec, + quantizer: Optional[Quantizer] = None, + *, + dtype: Optional[torch.dtype] = None, +) -> tuple[TensorSpec, TensorSpec]: + """Describe bias reduction and optional gradient quantization, in native order.""" + dtype = dtype or grad_output.dtype + db = TensorSpec(shape=(grad_output.shape[-1],), dtype=dtype, device=grad_output.device) + dx = ( + grad_output + if quantizer is None and dtype == grad_output.dtype + else _make_fused_quantization_output_spec( + grad_output.shape, dtype, grad_output.device, quantizer + ) + ) + return db, dx diff --git a/transformer_engine/pytorch/dynamo/tensor_spec.py b/transformer_engine/pytorch/dynamo/tensor_spec.py index 4cfe225952a..4a0886b23b9 100644 --- a/transformer_engine/pytorch/dynamo/tensor_spec.py +++ b/transformer_engine/pytorch/dynamo/tensor_spec.py @@ -29,6 +29,7 @@ class TensorSpec: quantizer: Optional[Any] = None requires_grad: bool = False device: Optional[torch.device] = field(default=None) + with_gemm_swizzled_scales: Optional[bool] = None def __post_init__(self) -> None: # Own a private copy of the quantizer so usage changes (update_usage) @@ -37,6 +38,10 @@ def __post_init__(self) -> None: if self.quantizer is not None: q = self.quantizer self.quantizer = q.copy() if hasattr(q, "copy") else _copy.copy(q) + if self.with_gemm_swizzled_scales is None: + self.with_gemm_swizzled_scales = self.quantizer.storage_metadata(self.dtype)[ + "nontensor_kwargs" + ].get("with_gemm_swizzled_scales") @property def is_quantized(self) -> bool: @@ -94,9 +99,12 @@ def create_metadata(self) -> Dict[str, Any]: "dtype": self.dtype, "requires_grad": self.requires_grad, } - return self.quantizer.create_metadata( + meta = self.quantizer.create_metadata( tuple(self.shape), dtype=self.dtype, requires_grad=self.requires_grad ) + if self.with_gemm_swizzled_scales is not None: + meta["nontensor_kwargs"]["with_gemm_swizzled_scales"] = self.with_gemm_swizzled_scales + return meta def create_inner_tensors(self) -> List[torch.Tensor]: """Materialize the flat inner tensors (in :meth:`inner_names` order). @@ -162,4 +170,5 @@ def to_tensor_spec(tensor: Any) -> TensorSpec: quantizer=getattr(tensor, "_quantizer", None), requires_grad=requires_grad, device=tensor.device, + with_gemm_swizzled_scales=getattr(tensor, "_with_gemm_swizzled_scales", None), ) diff --git a/transformer_engine/pytorch/ops/basic/bias.py b/transformer_engine/pytorch/ops/basic/bias.py index 70b34da1851..b9fbd1fad08 100644 --- a/transformer_engine/pytorch/ops/basic/bias.py +++ b/transformer_engine/pytorch/ops/basic/bias.py @@ -12,6 +12,7 @@ import transformer_engine_torch as tex from ...dynamo import TensorOrQuantized, TensorSpec +from ...dynamo.fake import fake_bgrad from ...quantization import FP8GlobalStateManager from ..op import BasicOperation from ...utils import canonicalize_device, canonicalize_dtype @@ -194,13 +195,8 @@ def backward_compute( @classmethod def backward_compute_fake(cls, args: BiasBwdArgs): dy = args.grad_output - dx = None - if len(dy.shape) > 1 and args.grad_input_quantizer is not None: - dx = TensorSpec( - shape=dy.shape, - dtype=dy.dtype, - device=dy.device, - quantizer=args.grad_input_quantizer, - ) - db = TensorSpec(shape=(dy.shape[-1],), dtype=dy.dtype, device=dy.device) + quantizer = args.grad_input_quantizer if len(dy.shape) > 1 else None + db, dx = fake_bgrad(dy, quantizer) + if quantizer is None: + dx = None return dx, [(db,)], [()] diff --git a/transformer_engine/pytorch/tensor/storage/nvfp4_tensor_storage.py b/transformer_engine/pytorch/tensor/storage/nvfp4_tensor_storage.py index cd1c15b9438..e4796214ac3 100644 --- a/transformer_engine/pytorch/tensor/storage/nvfp4_tensor_storage.py +++ b/transformer_engine/pytorch/tensor/storage/nvfp4_tensor_storage.py @@ -258,7 +258,8 @@ def size(self, dim: Optional[int] = None) -> Union[torch.Size, int]: byte_shape = list(self._rowwise_data.size()) shape = byte_shape[:-1] + [byte_shape[-1] * 2] elif self._columnwise_data is not None: - warnings.warn("Attempting to get shape of NVFP4 tensor with only column-wise data.") + if not torch.compiler.is_compiling(): + warnings.warn("Attempting to get shape of NVFP4 tensor with only column-wise data.") byte_shape = list(self._columnwise_data.size()) shape = byte_shape[1:-1] + [byte_shape[-1] * 2, byte_shape[0]] if shape is None: From 06edd0b6c2fe40f4efed9df7511e9094a49ef158 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 12 Oct 2026 16:58:43 +0200 Subject: [PATCH 03/15] [PyTorch] Consolidate NVFP4 Linear compile tests Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 158 ++++++++++++---------------- 1 file changed, 70 insertions(+), 88 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index e6c894cf242..1b06bc25083 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -1695,26 +1695,6 @@ def test_storage_flatten_unflatten_roundtrip(factory, shape): ] -@pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) -@pytest.mark.parametrize("shape", [(64, 128), (1, 64, 128), (64, 1, 128), (2, 4, 8, 128)]) -@pytest.mark.parametrize("with_rht", [False, True]) -@pytest.mark.parametrize("optimize_for_gemm", [False, True]) -@pytest.mark.parametrize("rowwise, columnwise", _USAGE_COMBOS) -def test_nvfp4_inner_tensor_specs_match_quantize( - shape, with_rht, optimize_for_gemm, rowwise, columnwise -): - """NVFP4 fake buffers match native quantization for every input rank.""" - quantizer = _nvfp4(with_rht=with_rht) - quantizer.set_usage(rowwise=rowwise, columnwise=columnwise) - quantizer.optimize_for_gemm = optimize_for_gemm - tensor = quantizer(torch.randn(shape, dtype=torch.bfloat16, device="cuda")) - for name, (expected_shape, expected_dtype) in quantizer.inner_tensor_specs(shape).items(): - inner = getattr(tensor, name) - assert inner.shape == expected_shape, name - assert inner.dtype == expected_dtype, name - assert inner.is_contiguous(), name - - @pytest.mark.parametrize("factory, shape", _SPEC_QUANTIZERS) @pytest.mark.parametrize("rowwise, columnwise", _USAGE_COMBOS) @pytest.mark.parametrize("internal", [False, True], ids=["wrapper", "internal"]) @@ -2888,55 +2868,26 @@ def test_te_ops_bias_compile_with_fp8_input(backward): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -@pytest.mark.parametrize("quantization", [None, "fp8"]) -@pytest.mark.parametrize("case", ["linear", "bias", "pair", "unfused"]) -@pytest.mark.parametrize("grads", ["all", "input", "weight", "bias", "none"]) -def test_te_ops_linear_bias_compile(dtype, quantization, case, grads, monkeypatch): - _check_linear_bias_compile(dtype, quantization, case, grads, monkeypatch) - - -@pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) -@pytest.mark.parametrize("compile_mode", _compile_modes) -@pytest.mark.parametrize("bias", [False, True]) @pytest.mark.parametrize( - "shape,with_rht,disable_rht_fusion", + "dtype,quantization,shape,disable_rht_fusion", [ - ((2, 16, 128), False, False), - ((2, 16, 128), True, False), - ((64, 128), True, False), - ((64, 128), True, True), + (torch.float16, None, (2, 16, 32), False), + (torch.bfloat16, None, (2, 16, 32), False), + (torch.float16, "fp8", (2, 16, 32), False), + (torch.bfloat16, "fp8", (2, 16, 32), False), + (torch.bfloat16, "nvfp4", (2, 16, 128), False), + (torch.bfloat16, "nvfp4", (64, 128), False), + (torch.bfloat16, "nvfp4", (64, 128), True), + (torch.bfloat16, "nvfp4-no-rht", (2, 16, 128), False), ], ) -def test_te_ops_linear_nvfp4_compile( - shape, with_rht, disable_rht_fusion, bias, compile_mode, monkeypatch +@pytest.mark.parametrize("case", ["linear", "bias", "pair", "unfused"]) +@pytest.mark.parametrize("grads", ["all", "input", "weight", "bias", "none"]) +def test_te_ops_linear_bias_compile( + dtype, quantization, shape, disable_rht_fusion, case, grads, monkeypatch ): monkeypatch.setenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", str(int(disable_rht_fusion))) - torch._dynamo.reset() - model = te.ops.Sequential(te.ops.Linear(128, 128, bias=bias, dtype=torch.bfloat16)) - eager_model = copy.deepcopy(model) - quant_recipe = recipe.NVFP4BlockScaling( - disable_rht=not with_rht, disable_stochastic_rounding=True - ) - - def run(x, module=model): - with te.autocast(recipe=quant_recipe): - return module(x) - - compiled = torch.compile(run, fullgraph=True, mode=compile_mode) - with _assert_no_cudagraph_skips(compile_mode == "reduce-overhead"): - for _ in range(3): - torch.compiler.cudagraph_mark_step_begin() - x = torch.randn(shape, device="cuda", dtype=torch.bfloat16, requires_grad=True) - eager_x = x.detach().clone().requires_grad_() - actual = compiled(x) - expected = run(eager_x, eager_model) - torch.testing.assert_close(actual, expected, rtol=0, atol=0) - dy = torch.randn_like(actual) - grads = torch.autograd.grad(actual, (x, *model.parameters()), dy) - eager_grads = torch.autograd.grad(expected, (eager_x, *eager_model.parameters()), dy) - torch.testing.assert_close(grads, eager_grads, rtol=0, atol=0) - del actual, expected, grads, eager_grads, x, eager_x, dy + _check_linear_bias_compile(dtype, quantization, case, grads, monkeypatch, input_shape=shape) @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @@ -3008,21 +2959,35 @@ def unexpected_allocation(*args, **kwargs): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize("training", [False, True]) @pytest.mark.parametrize("initial_device", ["cuda", "cpu", "meta"]) -def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device): +@pytest.mark.parametrize("quantization", [None, "nvfp4", "nvfp4-no-rht"]) +def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device, quantization): """A fresh process prevents earlier GEMMs from hiding lazy workspace allocation.""" + if quantization is not None and not nvfp4_available: + pytest.skip(reason_for_no_nvfp4) script = textwrap.dedent( """ import contextlib import sys import torch import transformer_engine.pytorch as te + from transformer_engine.common import recipe from transformer_engine.pytorch.cpp_extensions.gemm import get_cublas_workspace from torch._dynamo.utils import counters - training, initial_device = sys.argv[1:] + training, initial_device, quantization = sys.argv[1:] training = training == "True" + quant_recipe = ( + recipe.NVFP4BlockScaling( + disable_rht=quantization == "nvfp4-no-rht", disable_stochastic_rounding=True + ) + if quantization != "None" else None + ) + in_features, out_features = (128, 128) if quant_recipe is not None else (32, 64) + eager_tols = {"rtol": 0, "atol": 0} if quant_recipe is not None else {} assert get_cublas_workspace.cache_info().currsize == 0 - linear = te.ops.BasicLinear(32, 64, device=initial_device, dtype=torch.bfloat16) + linear = te.ops.BasicLinear( + in_features, out_features, device=initial_device, dtype=torch.bfloat16 + ) if initial_device == "cpu": assert get_cublas_workspace.cache_info().currsize == 0 linear.cuda() @@ -3031,13 +2996,14 @@ def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device): linear.to_empty(device="cuda") linear.reset_parameters() assert get_cublas_workspace.cache_info().currsize > 0 - model = te.ops.Sequential(linear, te.ops.Bias(64, dtype=torch.bfloat16)) - x = torch.randn(32, 32, device="cuda", dtype=torch.bfloat16, requires_grad=training) + model = te.ops.Sequential(linear, te.ops.Bias(out_features, dtype=torch.bfloat16)) + x = torch.randn(32, in_features, device="cuda", dtype=torch.bfloat16, requires_grad=training) targets = (x, *model.parameters()) - dy = torch.randn(32, 64, device="cuda", dtype=torch.bfloat16) + dy = torch.randn(32, out_features, device="cuda", dtype=torch.bfloat16) def forward(x): - return model(x) + with te.autocast(enabled=quant_recipe is not None, recipe=quant_recipe): + return model(x) compiled = torch.compile(forward, fullgraph=True, mode="reduce-overhead") counters.clear() @@ -3049,10 +3015,10 @@ def forward(x): grads = torch.autograd.grad(actual, targets, dy) # Eager reference runs only after the first compiled invocation. expected = forward(x) - torch.testing.assert_close(actual, expected) + torch.testing.assert_close(actual, expected, **eager_tols) if training: expected_grads = torch.autograd.grad(expected, targets, dy) - torch.testing.assert_close(grads, expected_grads) + torch.testing.assert_close(grads, expected_grads, **eager_tols) del grads, expected_grads del actual, expected torch.cuda.synchronize() @@ -3063,7 +3029,7 @@ def forward(x): """ ) result = subprocess.run( - [sys.executable, "-c", script, str(training), initial_device], + [sys.executable, "-c", script, str(training), initial_device, str(quantization)], capture_output=True, text=True, check=False, @@ -3073,29 +3039,44 @@ def forward(x): def _check_linear_bias_compile( - dtype, quantization, case, grads, monkeypatch, *, backward_override=None + dtype, + quantization, + case, + grads, + monkeypatch, + *, + backward_override=None, + input_shape=(2, 16, 32), ): + is_nvfp4 = quantization in ("nvfp4", "nvfp4-no-rht") if quantization == "fp8" and not fp8_available: pytest.skip(reason_for_no_fp8) + if is_nvfp4 and not nvfp4_available: + pytest.skip(reason_for_no_nvfp4) if (case == "linear" and grads == "bias") or (case == "bias" and grads == "weight"): pytest.skip("Requested parameter is absent") torch._dynamo.reset() if case == "unfused": monkeypatch.setattr(OperationFuser, "forward_fusion_functions", []) + in_features = input_shape[-1] + out_features = 128 if is_nvfp4 else 64 ops = [] if case != "bias": - ops.append(te.ops.BasicLinear(32, 64, dtype=dtype)) + ops.append(te.ops.BasicLinear(in_features, out_features, dtype=dtype)) ops[-1].weight.requires_grad_(grads in ("all", "weight")) if case != "linear": - ops.append(te.ops.Bias(32 if case == "bias" else 64, dtype=dtype)) + ops.append(te.ops.Bias(in_features if case == "bias" else out_features, dtype=dtype)) ops[-1].bias.requires_grad_(grads in ("all", "bias")) model = te.ops.Sequential(*ops) eager_model = copy.deepcopy(model) - quant_recipe = ( - recipe.Float8CurrentScaling(backward_override=backward_override) - if quantization == "fp8" - else None - ) + quant_recipe = None + if quantization == "fp8": + quant_recipe = recipe.Float8CurrentScaling(backward_override=backward_override) + elif is_nvfp4: + quant_recipe = recipe.NVFP4BlockScaling( + disable_rht=quantization == "nvfp4-no-rht", disable_stochastic_rounding=True + ) + eager_tols = {"rtol": 0, "atol": 0} if is_nvfp4 else {} def run(x, module=model): if quant_recipe is None: @@ -3104,16 +3085,17 @@ def run(x, module=model): return module(x) compiled, graphs = _compile_with_graphs(run) - x = torch.randn(2, 16, 32, device="cuda", dtype=dtype) + x = torch.randn(input_shape, device="cuda", dtype=dtype) x.requires_grad_(grads in ("all", "input")) targets = tuple(t for t in (x, *model.parameters()) if t.requires_grad) eager_targets = tuple(t for t in (x, *eager_model.parameters()) if t.requires_grad) graph_count = None with torch.no_grad() if grads == "none" else contextlib.nullcontext(): for iteration in range(3): + check_reference = iteration == 0 and not is_nvfp4 with torch.no_grad(): for tensor in (x, *model.parameters()): - if iteration == 0: + if check_reference: # These binary fractions survive FP8 quantization exactly, # allowing a strict comparison with native PyTorch. tensor.copy_(torch.randint(-2, 3, tensor.shape, device=tensor.device) / 16) @@ -3123,8 +3105,8 @@ def run(x, module=model): eager_param.copy_(param) actual = compiled(x) expected = run(x, eager_model) - torch.testing.assert_close(actual, expected) - if iteration == 0: + torch.testing.assert_close(actual, expected, **eager_tols) + if check_reference: reference = x.double() for op in ops: reference = ( @@ -3138,14 +3120,14 @@ def run(x, module=model): if targets: dy = ( torch.randint_like(actual, -2, 3) / 16 - if iteration == 0 + if check_reference else torch.randn_like(actual) ) actual_grads = torch.autograd.grad(actual, targets, dy) torch.testing.assert_close( - actual_grads, torch.autograd.grad(expected, eager_targets, dy) + actual_grads, torch.autograd.grad(expected, eager_targets, dy), **eager_tols ) - if iteration == 0: + if check_reference: torch.testing.assert_close( actual_grads, torch.autograd.grad(reference, targets, dy.double()), From 47453031227b9bb1037b1e0352783f31078984a0 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 12 Oct 2026 17:22:03 +0200 Subject: [PATCH 04/15] [PyTorch] Cover supported recipes in cold CUDA Graph tests Exercise current and block FP8, MXFP8, and NVFP4 variants in the existing fresh-process test. Keep scale-layout defaults in create_metadata instead of duplicating them in TensorSpec initialization. Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 54 +++++++++++++++---- .../pytorch/dynamo/tensor_spec.py | 4 -- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 1b06bc25083..119fc7c2969 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -2959,10 +2959,28 @@ def unexpected_allocation(*args, **kwargs): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize("training", [False, True]) @pytest.mark.parametrize("initial_device", ["cuda", "cpu", "meta"]) -@pytest.mark.parametrize("quantization", [None, "nvfp4", "nvfp4-no-rht"]) +@pytest.mark.parametrize( + "quantization", + [ + None, + "fp8", + "fp8-block", + "mxfp8", + "nvfp4", + "nvfp4-no-rht", + "nvfp4-4over6", + "nvfp4-row-scaled", + ], +) def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device, quantization): """A fresh process prevents earlier GEMMs from hiding lazy workspace allocation.""" - if quantization is not None and not nvfp4_available: + if quantization == "fp8" and not fp8_available: + pytest.skip(reason_for_no_fp8) + if quantization == "fp8-block" and not fp8_block_scaling_available: + pytest.skip(reason_for_no_fp8_block_scaling) + if quantization == "mxfp8" and not mxfp8_available: + pytest.skip(reason_for_no_mxfp8) + if quantization is not None and quantization.startswith("nvfp4") and not nvfp4_available: pytest.skip(reason_for_no_nvfp4) script = textwrap.dedent( """ @@ -2976,14 +2994,28 @@ def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device, quantizati training, initial_device, quantization = sys.argv[1:] training = training == "True" - quant_recipe = ( - recipe.NVFP4BlockScaling( - disable_rht=quantization == "nvfp4-no-rht", disable_stochastic_rounding=True + is_nvfp4 = quantization.startswith("nvfp4") + if quantization == "None": + quant_recipe = None + elif is_nvfp4: + row_scaled = quantization == "nvfp4-row-scaled" + quant_recipe = recipe.NVFP4BlockScaling( + disable_rht=quantization != "nvfp4", + disable_stochastic_rounding=True, + disable_2d_quantization=row_scaled, + row_scaled_activation=row_scaled, + nvfp4_4over6="all" if quantization == "nvfp4-4over6" else "none", + backward_override="dequantized" if row_scaled else None, ) - if quantization != "None" else None - ) + else: + quant_recipe = { + "fp8": recipe.Float8CurrentScaling, + "fp8-block": recipe.Float8BlockScaling, + "mxfp8": recipe.MXFP8BlockScaling, + }[quantization]() in_features, out_features = (128, 128) if quant_recipe is not None else (32, 64) - eager_tols = {"rtol": 0, "atol": 0} if quant_recipe is not None else {} + batch_size = 128 if isinstance(quant_recipe, recipe.Float8BlockScaling) else 32 + eager_tols = {"rtol": 0, "atol": 0} if is_nvfp4 else {} assert get_cublas_workspace.cache_info().currsize == 0 linear = te.ops.BasicLinear( in_features, out_features, device=initial_device, dtype=torch.bfloat16 @@ -2997,9 +3029,11 @@ def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device, quantizati linear.reset_parameters() assert get_cublas_workspace.cache_info().currsize > 0 model = te.ops.Sequential(linear, te.ops.Bias(out_features, dtype=torch.bfloat16)) - x = torch.randn(32, in_features, device="cuda", dtype=torch.bfloat16, requires_grad=training) + x = torch.randn( + batch_size, in_features, device="cuda", dtype=torch.bfloat16, requires_grad=training + ) targets = (x, *model.parameters()) - dy = torch.randn(32, out_features, device="cuda", dtype=torch.bfloat16) + dy = torch.randn(batch_size, out_features, device="cuda", dtype=torch.bfloat16) def forward(x): with te.autocast(enabled=quant_recipe is not None, recipe=quant_recipe): diff --git a/transformer_engine/pytorch/dynamo/tensor_spec.py b/transformer_engine/pytorch/dynamo/tensor_spec.py index 4a0886b23b9..d96c49ca7a5 100644 --- a/transformer_engine/pytorch/dynamo/tensor_spec.py +++ b/transformer_engine/pytorch/dynamo/tensor_spec.py @@ -38,10 +38,6 @@ def __post_init__(self) -> None: if self.quantizer is not None: q = self.quantizer self.quantizer = q.copy() if hasattr(q, "copy") else _copy.copy(q) - if self.with_gemm_swizzled_scales is None: - self.with_gemm_swizzled_scales = self.quantizer.storage_metadata(self.dtype)[ - "nontensor_kwargs" - ].get("with_gemm_swizzled_scales") @property def is_quantized(self) -> bool: From 7788fdea091b51e87a685f00398364f424759764 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 12 Oct 2026 18:54:14 +0200 Subject: [PATCH 05/15] [PyTorch] Isolate eager references in cold CUDA Graph tests Skip TE autocast for unquantized inputs and give the eager reference its own model so quantizer usage changes do not invalidate compiled graphs. Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 119fc7c2969..627b445252b 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -2985,6 +2985,7 @@ def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device, quantizati script = textwrap.dedent( """ import contextlib + import copy import sys import torch import transformer_engine.pytorch as te @@ -3029,29 +3030,33 @@ def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device, quantizati linear.reset_parameters() assert get_cublas_workspace.cache_info().currsize > 0 model = te.ops.Sequential(linear, te.ops.Bias(out_features, dtype=torch.bfloat16)) + reference_model = copy.deepcopy(model) x = torch.randn( batch_size, in_features, device="cuda", dtype=torch.bfloat16, requires_grad=training ) targets = (x, *model.parameters()) + reference_targets = (x, *reference_model.parameters()) dy = torch.randn(batch_size, out_features, device="cuda", dtype=torch.bfloat16) - def forward(x): - with te.autocast(enabled=quant_recipe is not None, recipe=quant_recipe): - return model(x) + def forward(x, module): + if quant_recipe is None: + return module(x) + with te.autocast(recipe=quant_recipe): + return module(x) compiled = torch.compile(forward, fullgraph=True, mode="reduce-overhead") counters.clear() with contextlib.nullcontext() if training else torch.no_grad(): for _ in range(5): torch.compiler.cudagraph_mark_step_begin() - actual = compiled(x) + actual = compiled(x, model) if training: grads = torch.autograd.grad(actual, targets, dy) # Eager reference runs only after the first compiled invocation. - expected = forward(x) + expected = forward(x, reference_model) torch.testing.assert_close(actual, expected, **eager_tols) if training: - expected_grads = torch.autograd.grad(expected, targets, dy) + expected_grads = torch.autograd.grad(expected, reference_targets, dy) torch.testing.assert_close(grads, expected_grads, **eager_tols) del grads, expected_grads del actual, expected From 040ed019d385c8a998f84080e6fecba811f4f632 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Tue, 13 Oct 2026 13:16:08 +0200 Subject: [PATCH 06/15] Share NVFP4 layout eligibility between native and fake execution Expose native alignment requirements through a metadata-only binding. Keep symbolic shape checks and environment reads visible to Dynamo, and cover dynamic batches and configuration changes through the Bias custom op. Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 55 ++++++++++++ transformer_engine/pytorch/csrc/common.h | 5 ++ .../pytorch/csrc/extensions/pybind.cpp | 5 ++ transformer_engine/pytorch/csrc/quantizer.cpp | 88 +++++++++++++------ transformer_engine/pytorch/dynamo/fake.py | 58 ++++++++---- 5 files changed, 166 insertions(+), 45 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 627b445252b..caacf9c5b89 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -42,6 +42,7 @@ from transformer_engine.pytorch.module.base import TransformerEngineBaseModule from transformer_engine.pytorch.quantization import FP8GlobalStateManager, QuantizerRole from transformer_engine.pytorch.ops.basic.basic_linear import BasicLinear +from transformer_engine.pytorch.ops.basic.bias import BiasBwdArgs from transformer_engine.pytorch.ops.fuser import OperationFuser from transformer_engine.pytorch.ops.op import BasicOperation, OperationContext from transformer_engine.pytorch.tensor.float8_tensor import Float8CurrentScalingQuantizer @@ -54,6 +55,7 @@ Quantizer, ) from transformer_engine.pytorch.dynamo import TensorSpec, to_tensor_spec +from transformer_engine.pytorch.dynamo.fake import fake_bgrad from transformer_engine.pytorch import ( is_fp8_available, is_mxfp8_available, @@ -2890,6 +2892,59 @@ def test_te_ops_linear_bias_compile( _check_linear_bias_compile(dtype, quantization, case, grads, monkeypatch, input_shape=shape) +@pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) +@pytest.mark.parametrize("mode", ["rht", "1d", "2d", "row-scaled", "4over6"]) +@pytest.mark.parametrize("optimize_for_gemm", [False, True]) +def test_te_ops_bias_nvfp4_layout(mode, optimize_for_gemm, monkeypatch): + quantizer = NVFP4Quantizer( + columnwise=mode != "row-scaled", + with_rht=mode == "rht", + with_post_rht_amax=mode == "rht", + with_2d_quantization=mode in ("2d", "4over6"), + row_scaled_nvfp4=mode == "row-scaled", + nvfp4_use_4over6=mode == "4over6", + nvfp4_e4m3_max=448 if mode == "4over6" else 0, + with_random_sign_mask=False, + ) + quantizer.optimize_for_gemm = optimize_for_gemm + + def bgrad(inp): + dx, ((db,),), _ = te.ops.Bias.compile_ops[1](BiasBwdArgs(inp, quantizer)) + return dx, db + + torch._dynamo.reset() + compiled, graphs = _compile_with_graphs(bgrad) + # Each pair introduces different batch sizes with the same alignment. + for disable_fusion in (False, True, False): + monkeypatch.setenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", str(int(disable_fusion))) + for i, batch in enumerate((64, 192, 96, 160, 128, 256)): + inp = torch.randn(batch, 128, device="cuda", dtype=torch.bfloat16) + torch._dynamo.mark_dynamic(inp, 0) + previous_graph_count = len(graphs) + dx, db = compiled(inp) + expected_db, expected_dx = tex.bgrad_quantize(inp, quantizer) + layout = expected_dx._with_gemm_swizzled_scales + assert dx._with_gemm_swizzled_scales == layout, (mode, disable_fusion, batch) + torch.testing.assert_close(db, expected_db, atol=0, rtol=0) + torch.testing.assert_close(dx.dequantize(), expected_dx.dequantize(), atol=0, rtol=0) + if i % 2 == 1: + assert len(graphs) == previous_graph_count + + quantizer.optimize_for_gemm = not optimize_for_gemm + inp = torch.randn(128, 128, device="cuda", dtype=torch.bfloat16) + torch._dynamo.mark_dynamic(inp, 0) + dx, _ = compiled(inp) + _, expected_dx = tex.bgrad_quantize(inp, quantizer) + assert dx._with_gemm_swizzled_scales == expected_dx._with_gemm_swizzled_scales + torch.testing.assert_close(dx.dequantize(), expected_dx.dequantize(), atol=0, rtol=0) + + for shape in ((2, 32, 128), (2, 64, 128), (128, 64)): + inp = torch.randn(shape, device="cuda", dtype=torch.bfloat16) + _, expected_dx = tex.bgrad_quantize(inp, quantizer) + _, dx_spec = fake_bgrad(to_tensor_spec(inp), quantizer) + assert dx_spec.with_gemm_swizzled_scales == expected_dx._with_gemm_swizzled_scales + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) @pytest.mark.parametrize("backward_override", ["high_precision", "dequantized"]) diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index 1103225b8a0..896d291f664 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -339,6 +339,11 @@ class MXFP8Quantizer : public Quantizer { std::vector get_scale_shape(const std::vector& shape, bool columnwise) const; }; +// Flattened row/column alignment for directly emitted GEMM-swizzled scales, or no such path. +std::optional> nvfp4_gemm_swizzled_scales_alignment( + bool optimize_for_gemm, bool with_rht, bool with_2d_quantization, bool row_scaled_nvfp4, + bool nvfp4_use_4over6, int sm_arch, bool disable_rht_cast_fusion); + class NVFP4Quantizer : public Quantizer { public: // amax reduction for low precision FP4 AG diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index 2046408eba8..d63efd9cddb 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -210,6 +210,11 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { py::arg("output") = py::none(), py::arg("noop") = py::none()); m.def("dequantize", &transformer_engine::pytorch::dequantize, "Dequantize", py::arg("input"), py::arg("otype")); + m.def("nvfp4_gemm_swizzled_scales_alignment", + &transformer_engine::pytorch::nvfp4_gemm_swizzled_scales_alignment, + py::arg("optimize_for_gemm"), py::arg("with_rht"), py::arg("with_2d_quantization"), + py::arg("row_scaled_nvfp4"), py::arg("nvfp4_use_4over6"), py::arg("sm_arch"), + py::arg("disable_rht_cast_fusion")); m.def("create_empty_quantized_tensor", &transformer_engine::pytorch::create_empty_quantized_tensor, "Create an empty quantized tensor", py::arg("quantizer"), py::arg("shape"), diff --git a/transformer_engine/pytorch/csrc/quantizer.cpp b/transformer_engine/pytorch/csrc/quantizer.cpp index fba268d2723..ffe05197f3b 100644 --- a/transformer_engine/pytorch/csrc/quantizer.cpp +++ b/transformer_engine/pytorch/csrc/quantizer.cpp @@ -1934,47 +1934,79 @@ void NVFP4Quantizer::set_quantization_params(TensorWrapper* tensor) const { columnwise_data.shape); } -bool NVFP4Quantizer::is_eligible_for_rht_cast_fusion(const std::vector& shape, - bool for_grouped_kernel) { - if (transformer_engine::getenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION")) { - return false; - } - const auto [rows, cols] = get_2d_dims(shape); - const size_t row_align = for_grouped_kernel ? 128 : 64; - const int sm_arch = transformer_engine::cuda::sm_arch(); +namespace { + +using ShapeAlignment = std::optional>; + +ShapeAlignment nvfp4_rht_cast_fusion_alignment(int sm_arch, bool disable_rht_cast_fusion, + bool for_grouped_kernel = false) { const bool supported_arch = (sm_arch >= 100 && sm_arch <= 110) || (!for_grouped_kernel && (sm_arch == 120 || sm_arch == 121)); - return rows % row_align == 0 && cols % 128 == 0 && supported_arch; + if (disable_rht_cast_fusion || !supported_arch) { + return std::nullopt; + } + return std::pair{for_grouped_kernel ? 128 : 64, 128}; } -bool NVFP4Quantizer::is_eligible_for_2d_swizzle_fusion(const std::vector& shape) { +ShapeAlignment nvfp4_2d_swizzle_fusion_alignment(int sm_arch) { + if (sm_arch < 100 || sm_arch > 110) { + return std::nullopt; + } + return std::pair{128, 128}; +} + +bool meets_alignment(const std::vector& shape, const ShapeAlignment& alignment) { + if (!alignment) { + return false; + } const auto [rows, cols] = get_2d_dims(shape); - return rows % 128 == 0 && cols % 128 == 0 && transformer_engine::cuda::sm_arch() >= 100 && - transformer_engine::cuda::sm_arch() <= 110; + return rows % alignment->first == 0 && cols % alignment->second == 0; } -namespace { +} // namespace -// Whether this quantizer should emit GEMM-swizzled scale factors directly from the -// quantize kernel (RHT cast-fusion path or the plain 2D quantize path), avoiding the -// standalone post-quantize swizzle pass. -bool nvfp4_emits_gemm_swizzled_scales(const NVFP4Quantizer& q, const std::vector& shape) { - if (!q.optimize_for_gemm) { - return false; +std::optional> nvfp4_gemm_swizzled_scales_alignment( + bool optimize_for_gemm, bool with_rht, bool with_2d_quantization, bool row_scaled_nvfp4, + bool nvfp4_use_4over6, int sm_arch, bool disable_rht_cast_fusion) { + if (!optimize_for_gemm) { + return std::nullopt; } - if (q.with_rht) { - const int sm_arch = transformer_engine::cuda::sm_arch(); + if (with_rht) { // The SM12x no-TMEM RHT path reuses the compact-scale 1D TMA quantizer. - // Keep the existing post-quantize swizzle until that path gains an in-kernel swizzle epilogue. if (sm_arch == 120 || sm_arch == 121) { - return false; + return std::nullopt; } - return NVFP4Quantizer::is_eligible_for_rht_cast_fusion(shape); + return nvfp4_rht_cast_fusion_alignment(sm_arch, disable_rht_cast_fusion); } - // Plain (non-RHT) 2D quantize kernel can bake the swizzled layout for aligned shapes. - return q.with_2d_quantization && !q.row_scaled_nvfp4 && - q.nvfp4_4over6_mode == kNVTENVFP44Over6Disabled && - NVFP4Quantizer::is_eligible_for_2d_swizzle_fusion(shape); + if (with_2d_quantization && !row_scaled_nvfp4 && !nvfp4_use_4over6) { + return nvfp4_2d_swizzle_fusion_alignment(sm_arch); + } + return std::nullopt; +} + +bool NVFP4Quantizer::is_eligible_for_rht_cast_fusion(const std::vector& shape, + bool for_grouped_kernel) { + return meets_alignment(shape, + nvfp4_rht_cast_fusion_alignment( + transformer_engine::cuda::sm_arch(), + transformer_engine::getenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION"), + for_grouped_kernel)); +} + +bool NVFP4Quantizer::is_eligible_for_2d_swizzle_fusion(const std::vector& shape) { + return meets_alignment(shape, + nvfp4_2d_swizzle_fusion_alignment(transformer_engine::cuda::sm_arch())); +} + +namespace { + +bool nvfp4_emits_gemm_swizzled_scales(const NVFP4Quantizer& q, const std::vector& shape) { + return meets_alignment( + shape, + nvfp4_gemm_swizzled_scales_alignment( + q.optimize_for_gemm, q.with_rht, q.with_2d_quantization, q.row_scaled_nvfp4, + q.nvfp4_4over6_mode != kNVTENVFP44Over6Disabled, transformer_engine::cuda::sm_arch(), + transformer_engine::getenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION"))); } } // namespace diff --git a/transformer_engine/pytorch/dynamo/fake.py b/transformer_engine/pytorch/dynamo/fake.py index b200624cad9..5652a8952e7 100644 --- a/transformer_engine/pytorch/dynamo/fake.py +++ b/transformer_engine/pytorch/dynamo/fake.py @@ -10,6 +10,7 @@ from typing import Optional import torch +import transformer_engine_torch as tex from .tensor_spec import TensorSpec from ..quantized_tensor import Quantizer @@ -17,6 +18,27 @@ from ..utils import get_device_compute_capability +@torch.compiler.assume_constant_result +def _nvfp4_gemm_swizzled_scales_alignment( + optimize_for_gemm: bool, + with_rht: bool, + with_2d_quantization: bool, + row_scaled_nvfp4: bool, + nvfp4_use_4over6: bool, + sm_arch: int, + disable_rht_cast_fusion: bool, +) -> Optional[tuple[int, int]]: + return tex.nvfp4_gemm_swizzled_scales_alignment( + optimize_for_gemm, + with_rht, + with_2d_quantization, + row_scaled_nvfp4, + nvfp4_use_4over6, + sm_arch, + disable_rht_cast_fusion, + ) + + def _make_fused_quantization_output_spec( shape: tuple[int, ...], dtype: torch.dtype, @@ -26,25 +48,27 @@ def _make_fused_quantization_output_spec( """Describe a native output without a separate post-quantize swizzle.""" spec = TensorSpec(shape=tuple(shape), dtype=dtype, device=device, quantizer=quantizer) quantizer = spec.quantizer - if not isinstance(quantizer, NVFP4Quantizer) or not quantizer.optimize_for_gemm: + if not isinstance(quantizer, NVFP4Quantizer): return spec + major, minor = get_device_compute_capability(device) + # Direct environ access lets Dynamo guard changes to the flag. + disable_rht_cast_fusion = bool( + int(os.environ.get("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", "0") or "0") + ) + alignment = _nvfp4_gemm_swizzled_scales_alignment( + quantizer.optimize_for_gemm, + quantizer.with_rht, + quantizer.with_2d_quantization, + quantizer.row_scaled_nvfp4, + quantizer.nvfp4_use_4over6, + major * 10 + minor, + disable_rht_cast_fusion, + ) + # Keep shape predicates visible to Dynamo so only alignment, not batch size, is guarded. rows, cols = math.prod(shape[:-1]), shape[-1] - if not (10, 0) <= get_device_compute_capability(device) <= (11, 0): - spec.with_gemm_swizzled_scales = False - elif quantizer.with_rht: - spec.with_gemm_swizzled_scales = bool( - not int(os.getenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", "0") or "0") - and rows % 64 == 0 - and cols % 128 == 0 - ) - else: - spec.with_gemm_swizzled_scales = bool( - quantizer.with_2d_quantization - and not quantizer.row_scaled_nvfp4 - and not quantizer.nvfp4_use_4over6 - and rows % 128 == 0 - and cols % 128 == 0 - ) + spec.with_gemm_swizzled_scales = alignment is not None and bool( + rows % alignment[0] == 0 and cols % alignment[1] == 0 + ) return spec From daa4b353fa5df82097070bbc899bc527f88e8c81 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Tue, 13 Oct 2026 13:33:06 +0200 Subject: [PATCH 07/15] Fix row-scaled NVFP4 columnwise amax metadata Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 21 +++++++++++++++++-- .../pytorch/tensor/nvfp4_tensor.py | 3 ++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index caacf9c5b89..ddb8d138e48 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -2895,9 +2895,10 @@ def test_te_ops_linear_bias_compile( @pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) @pytest.mark.parametrize("mode", ["rht", "1d", "2d", "row-scaled", "4over6"]) @pytest.mark.parametrize("optimize_for_gemm", [False, True]) -def test_te_ops_bias_nvfp4_layout(mode, optimize_for_gemm, monkeypatch): +@pytest.mark.parametrize("columnwise", [False, True]) +def test_te_ops_bias_nvfp4_layout(mode, optimize_for_gemm, columnwise, monkeypatch): quantizer = NVFP4Quantizer( - columnwise=mode != "row-scaled", + columnwise=columnwise, with_rht=mode == "rht", with_post_rht_amax=mode == "rht", with_2d_quantization=mode in ("2d", "4over6"), @@ -2927,6 +2928,18 @@ def bgrad(inp): assert dx._with_gemm_swizzled_scales == layout, (mode, disable_fusion, batch) torch.testing.assert_close(db, expected_db, atol=0, rtol=0) torch.testing.assert_close(dx.dequantize(), expected_dx.dequantize(), atol=0, rtol=0) + if columnwise: + torch.testing.assert_close( + dx._columnwise_data, expected_dx._columnwise_data, atol=0, rtol=0 + ) + if mode == "row-scaled": + torch.testing.assert_close( + dx._amax_rowwise, inp.abs().float().amax(1), atol=0, rtol=0 + ) + if columnwise: + torch.testing.assert_close( + dx._amax_columnwise, inp.abs().float().amax(0), atol=0, rtol=0 + ) if i % 2 == 1: assert len(graphs) == previous_graph_count @@ -2943,6 +2956,10 @@ def bgrad(inp): _, expected_dx = tex.bgrad_quantize(inp, quantizer) _, dx_spec = fake_bgrad(to_tensor_spec(inp), quantizer) assert dx_spec.with_gemm_swizzled_scales == expected_dx._with_gemm_swizzled_scales + for name, (inner_shape, inner_dtype) in dx_spec.quantizer.inner_tensor_specs(shape).items(): + inner = getattr(expected_dx, name) + assert inner.shape == inner_shape + assert inner.dtype == inner_dtype @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") diff --git a/transformer_engine/pytorch/tensor/nvfp4_tensor.py b/transformer_engine/pytorch/tensor/nvfp4_tensor.py index c49e1036260..97185d3b7ea 100644 --- a/transformer_engine/pytorch/tensor/nvfp4_tensor.py +++ b/transformer_engine/pytorch/tensor/nvfp4_tensor.py @@ -418,7 +418,8 @@ def inner_tensor_specs( amax_rowwise_shape = (math.prod(shape[:-1]),) if self.row_scaled_nvfp4 else (1,) specs["_amax_rowwise"] = (amax_rowwise_shape, torch.float32) if self.columnwise_usage: - specs["_amax_columnwise"] = ((1,), torch.float32) + amax_columnwise_shape = (shape[-1],) if self.row_scaled_nvfp4 else (1,) + specs["_amax_columnwise"] = (amax_columnwise_shape, torch.float32) return specs From 6e2cba75f7133814c9aece03b316fc219f274e9d Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Tue, 13 Oct 2026 14:51:27 +0200 Subject: [PATCH 08/15] Share alignment checks with fake execution and verify native metadata Signed-off-by: Pawel Gadzinski --- qa/L0_pytorch_unittest/test.sh | 1 + tests/pytorch/test_dynamo_fakes.py | 146 ++++++++++++++++++ transformer_engine/pytorch/csrc/common.h | 3 + .../pytorch/csrc/extensions/pybind.cpp | 2 + transformer_engine/pytorch/csrc/quantizer.cpp | 10 +- transformer_engine/pytorch/dynamo/fake.py | 14 +- 6 files changed, 169 insertions(+), 7 deletions(-) create mode 100644 tests/pytorch/test_dynamo_fakes.py diff --git a/qa/L0_pytorch_unittest/test.sh b/qa/L0_pytorch_unittest/test.sh index 2a486450a20..5d34d6c14d6 100644 --- a/qa/L0_pytorch_unittest/test.sh +++ b/qa/L0_pytorch_unittest/test.sh @@ -43,6 +43,7 @@ python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_mxfp8.xml $TE_PA python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_weight_swizzle_in_layers.xml $TE_PATH/tests/pytorch/test_weight_swizzle_in_layers.py || test_fail "test_weight_swizzle_in_layers.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_quantized_tensor.xml $TE_PATH/tests/pytorch/test_quantized_tensor.py || test_fail "test_quantized_tensor.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_torch_compile.xml $TE_PATH/tests/pytorch/test_torch_compile.py || test_fail "test_torch_compile.py" +python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_dynamo_fakes.xml $TE_PATH/tests/pytorch/test_dynamo_fakes.py || test_fail "test_dynamo_fakes.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_float8blockwisetensor.xml $TE_PATH/tests/pytorch/test_float8blockwisetensor.py || test_fail "test_float8blockwisetensor.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_float8_blockwise_scaling_exact.xml $TE_PATH/tests/pytorch/test_float8_blockwise_scaling_exact.py || test_fail "test_float8_blockwise_scaling_exact.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_float8_blockwise_gemm_exact.xml $TE_PATH/tests/pytorch/test_float8_blockwise_gemm_exact.py || test_fail "test_float8_blockwise_gemm_exact.py" diff --git a/tests/pytorch/test_dynamo_fakes.py b/tests/pytorch/test_dynamo_fakes.py new file mode 100644 index 00000000000..721986049a0 --- /dev/null +++ b/tests/pytorch/test_dynamo_fakes.py @@ -0,0 +1,146 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +"""Check fake output contracts against native operations.""" + +import pytest +import torch +from torch._subclasses.fake_tensor import FakeTensor, FakeTensorMode + +import transformer_engine.pytorch as te +import transformer_engine_torch as tex +from transformer_engine.pytorch.dynamo import to_tensor_spec +from transformer_engine.pytorch.dynamo.fake import fake_bgrad +from transformer_engine.pytorch.tensor.nvfp4_tensor import NVFP4Quantizer + + +nvfp4_available, reason_for_no_nvfp4 = te.is_nvfp4_available(return_reason=True) + + +def _assert_matches_native(spec, native): + if spec.is_quantized: + names, native_metadata = native.__tensor_flatten__() + assert spec.inner_names() == tuple(names) + fake_metadata = spec.create_metadata() + for metadata in (fake_metadata, native_metadata): + metadata["nontensor_kwargs"].pop("quantizer") + assert fake_metadata == native_metadata + native_inner = [getattr(native, name) for name in names] + else: + native_inner = [native] + assert spec.shape == native.shape + with FakeTensorMode(): + fake_inner = spec.create_inner_tensors() + assert len(fake_inner) == len(native_inner) + for fake, actual in zip(fake_inner, native_inner): + assert isinstance(fake, FakeTensor) + assert fake.shape == actual.shape + assert fake.stride() == actual.stride() + assert fake.dtype == actual.dtype + assert fake.device == actual.device + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("shape", [(32,), (64, 128), (2, 32, 128), (0, 128)]) +def test_fake_bgrad_unquantized_matches_native(dtype, shape): + inp = torch.randn(shape, dtype=dtype, device="cuda") + native_outputs = tex.bgrad_quantize(inp, None) + with FakeTensorMode() as mode: + specs = fake_bgrad(to_tensor_spec(mode.from_tensor(inp))) + for spec, native in zip(specs, native_outputs): + _assert_matches_native(spec, native) + + +@pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) +@pytest.mark.parametrize("mode", ["rht", "1d", "2d", "row-scaled", "4over6"]) +@pytest.mark.parametrize("optimize_for_gemm", [False, True]) +@pytest.mark.parametrize("columnwise", [False, True]) +@pytest.mark.parametrize("internal", [False, True]) +@pytest.mark.parametrize("disable_rht_fusion", [False, True]) +@pytest.mark.parametrize( + "shape", + [ + (32, 32), + (64, 128), + (96, 128), + (128, 64), + (128, 128), + (128, 160), + (2, 32, 128), + (2, 64, 128), + (2, 1, 32, 128), + (0, 128), + ], +) +def test_fake_bgrad_nvfp4_matches_native( + mode, optimize_for_gemm, columnwise, internal, disable_rht_fusion, shape, monkeypatch +): + monkeypatch.setenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", str(int(disable_rht_fusion))) + quantizer = NVFP4Quantizer( + columnwise=columnwise, + with_rht=mode == "rht", + with_post_rht_amax=mode == "rht", + with_2d_quantization=mode in ("2d", "4over6"), + row_scaled_nvfp4=mode == "row-scaled", + nvfp4_use_4over6=mode == "4over6", + nvfp4_e4m3_max=448 if mode == "4over6" else 0, + with_random_sign_mask=False, + ) + quantizer.optimize_for_gemm = optimize_for_gemm + quantizer.internal = internal + inp = torch.randn(shape, dtype=torch.bfloat16, device="cuda") + native_outputs = tex.bgrad_quantize(inp, quantizer) + with FakeTensorMode() as fake_mode: + specs = fake_bgrad(to_tensor_spec(fake_mode.from_tensor(inp)), quantizer) + for spec, native in zip(specs, native_outputs): + _assert_matches_native(spec, native) + + +@pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) +@pytest.mark.parametrize("with_rht", [False, True]) +def test_fake_bgrad_dynamic_alignment(with_rht, monkeypatch): + monkeypatch.setenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", "0") + quantizer = NVFP4Quantizer( + with_rht=with_rht, + with_post_rht_amax=with_rht, + with_2d_quantization=not with_rht, + with_random_sign_mask=False, + ) + quantizer.optimize_for_gemm = True + graphs = [] + + def backend(graph, _inputs): + graphs.append(graph) + return graph.forward + + def describe(inp): + _, spec = fake_bgrad(to_tensor_spec(inp), quantizer) + return inp + spec.with_gemm_swizzled_scales + + torch._dynamo.reset() + compiled = torch.compile(describe, backend=backend, fullgraph=True) + # Pairs preserve alignment, including distinct nonzero remainders. + for i, shape in enumerate( + ( + (64, 128), + (192, 384), + (96, 160), + (160, 224), + (160, 160), + (288, 288), + (128, 64), + (256, 192), + (128, 128), + (256, 256), + ) + ): + inp = torch.zeros(shape, device="cuda", dtype=torch.bfloat16) + torch._dynamo.mark_dynamic(inp, (0, 1)) + previous_graph_count = len(graphs) + actual = compiled(inp) + _, native = tex.bgrad_quantize(inp, quantizer) + torch.testing.assert_close(actual, inp + native._with_gemm_swizzled_scales, atol=0, rtol=0) + if i % 2 == 1: + assert len(graphs) == previous_graph_count diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index 896d291f664..c0e22a53181 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -344,6 +344,9 @@ std::optional> nvfp4_gemm_swizzled_scales_alignment( bool optimize_for_gemm, bool with_rht, bool with_2d_quantization, bool row_scaled_nvfp4, bool nvfp4_use_4over6, int sm_arch, bool disable_rht_cast_fusion); +bool meets_quantization_alignment(size_t rows, size_t cols, + const std::optional>& alignment); + class NVFP4Quantizer : public Quantizer { public: // amax reduction for low precision FP4 AG diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index d63efd9cddb..3c432b3d573 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -215,6 +215,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { py::arg("optimize_for_gemm"), py::arg("with_rht"), py::arg("with_2d_quantization"), py::arg("row_scaled_nvfp4"), py::arg("nvfp4_use_4over6"), py::arg("sm_arch"), py::arg("disable_rht_cast_fusion")); + m.def("meets_quantization_alignment", &transformer_engine::pytorch::meets_quantization_alignment, + py::arg("rows"), py::arg("cols"), py::arg("alignment")); m.def("create_empty_quantized_tensor", &transformer_engine::pytorch::create_empty_quantized_tensor, "Create an empty quantized tensor", py::arg("quantizer"), py::arg("shape"), diff --git a/transformer_engine/pytorch/csrc/quantizer.cpp b/transformer_engine/pytorch/csrc/quantizer.cpp index ffe05197f3b..587143904f1 100644 --- a/transformer_engine/pytorch/csrc/quantizer.cpp +++ b/transformer_engine/pytorch/csrc/quantizer.cpp @@ -1956,15 +1956,17 @@ ShapeAlignment nvfp4_2d_swizzle_fusion_alignment(int sm_arch) { } bool meets_alignment(const std::vector& shape, const ShapeAlignment& alignment) { - if (!alignment) { - return false; - } const auto [rows, cols] = get_2d_dims(shape); - return rows % alignment->first == 0 && cols % alignment->second == 0; + return meets_quantization_alignment(rows, cols, alignment); } } // namespace +bool meets_quantization_alignment(size_t rows, size_t cols, + const std::optional>& alignment) { + return alignment && rows % alignment->first == 0 && cols % alignment->second == 0; +} + std::optional> nvfp4_gemm_swizzled_scales_alignment( bool optimize_for_gemm, bool with_rht, bool with_2d_quantization, bool row_scaled_nvfp4, bool nvfp4_use_4over6, int sm_arch, bool disable_rht_cast_fusion) { diff --git a/transformer_engine/pytorch/dynamo/fake.py b/transformer_engine/pytorch/dynamo/fake.py index 5652a8952e7..8b2a78f0c50 100644 --- a/transformer_engine/pytorch/dynamo/fake.py +++ b/transformer_engine/pytorch/dynamo/fake.py @@ -39,6 +39,13 @@ def _nvfp4_gemm_swizzled_scales_alignment( ) +@torch.compiler.assume_constant_result +def _meets_quantization_alignment( + rows: int, cols: int, alignment: Optional[tuple[int, int]] +) -> bool: + return tex.meets_quantization_alignment(rows, cols, alignment) + + def _make_fused_quantization_output_spec( shape: tuple[int, ...], dtype: torch.dtype, @@ -64,10 +71,11 @@ def _make_fused_quantization_output_spec( major * 10 + minor, disable_rht_cast_fusion, ) - # Keep shape predicates visible to Dynamo so only alignment, not batch size, is guarded. + # Guard only zero/nonzero remainders, keeping other batch sizes in the same graph. rows, cols = math.prod(shape[:-1]), shape[-1] - spec.with_gemm_swizzled_scales = alignment is not None and bool( - rows % alignment[0] == 0 and cols % alignment[1] == 0 + row_alignment, col_alignment = alignment or (1, 1) + spec.with_gemm_swizzled_scales = _meets_quantization_alignment( + int(bool(rows % row_alignment)), int(bool(cols % col_alignment)), alignment ) return spec From c91597f1ae43e4fb2052782adb48a1359ed1df25 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Tue, 13 Oct 2026 18:41:49 +0200 Subject: [PATCH 09/15] Simplify fake NVFP4 alignment checks Signed-off-by: Pawel Gadzinski --- transformer_engine/pytorch/csrc/common.h | 3 --- .../pytorch/csrc/extensions/pybind.cpp | 2 -- transformer_engine/pytorch/csrc/quantizer.cpp | 10 ++++------ transformer_engine/pytorch/dynamo/fake.py | 20 ++++++++----------- 4 files changed, 12 insertions(+), 23 deletions(-) diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index c0e22a53181..896d291f664 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -344,9 +344,6 @@ std::optional> nvfp4_gemm_swizzled_scales_alignment( bool optimize_for_gemm, bool with_rht, bool with_2d_quantization, bool row_scaled_nvfp4, bool nvfp4_use_4over6, int sm_arch, bool disable_rht_cast_fusion); -bool meets_quantization_alignment(size_t rows, size_t cols, - const std::optional>& alignment); - class NVFP4Quantizer : public Quantizer { public: // amax reduction for low precision FP4 AG diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index 3c432b3d573..d63efd9cddb 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -215,8 +215,6 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { py::arg("optimize_for_gemm"), py::arg("with_rht"), py::arg("with_2d_quantization"), py::arg("row_scaled_nvfp4"), py::arg("nvfp4_use_4over6"), py::arg("sm_arch"), py::arg("disable_rht_cast_fusion")); - m.def("meets_quantization_alignment", &transformer_engine::pytorch::meets_quantization_alignment, - py::arg("rows"), py::arg("cols"), py::arg("alignment")); m.def("create_empty_quantized_tensor", &transformer_engine::pytorch::create_empty_quantized_tensor, "Create an empty quantized tensor", py::arg("quantizer"), py::arg("shape"), diff --git a/transformer_engine/pytorch/csrc/quantizer.cpp b/transformer_engine/pytorch/csrc/quantizer.cpp index 587143904f1..ffe05197f3b 100644 --- a/transformer_engine/pytorch/csrc/quantizer.cpp +++ b/transformer_engine/pytorch/csrc/quantizer.cpp @@ -1956,17 +1956,15 @@ ShapeAlignment nvfp4_2d_swizzle_fusion_alignment(int sm_arch) { } bool meets_alignment(const std::vector& shape, const ShapeAlignment& alignment) { + if (!alignment) { + return false; + } const auto [rows, cols] = get_2d_dims(shape); - return meets_quantization_alignment(rows, cols, alignment); + return rows % alignment->first == 0 && cols % alignment->second == 0; } } // namespace -bool meets_quantization_alignment(size_t rows, size_t cols, - const std::optional>& alignment) { - return alignment && rows % alignment->first == 0 && cols % alignment->second == 0; -} - std::optional> nvfp4_gemm_swizzled_scales_alignment( bool optimize_for_gemm, bool with_rht, bool with_2d_quantization, bool row_scaled_nvfp4, bool nvfp4_use_4over6, int sm_arch, bool disable_rht_cast_fusion) { diff --git a/transformer_engine/pytorch/dynamo/fake.py b/transformer_engine/pytorch/dynamo/fake.py index 8b2a78f0c50..2b13a06ca2d 100644 --- a/transformer_engine/pytorch/dynamo/fake.py +++ b/transformer_engine/pytorch/dynamo/fake.py @@ -39,13 +39,6 @@ def _nvfp4_gemm_swizzled_scales_alignment( ) -@torch.compiler.assume_constant_result -def _meets_quantization_alignment( - rows: int, cols: int, alignment: Optional[tuple[int, int]] -) -> bool: - return tex.meets_quantization_alignment(rows, cols, alignment) - - def _make_fused_quantization_output_spec( shape: tuple[int, ...], dtype: torch.dtype, @@ -71,12 +64,15 @@ def _make_fused_quantization_output_spec( major * 10 + minor, disable_rht_cast_fusion, ) - # Guard only zero/nonzero remainders, keeping other batch sizes in the same graph. + # Keep divisibility checks visible to Dynamo without specializing the dimensions. rows, cols = math.prod(shape[:-1]), shape[-1] - row_alignment, col_alignment = alignment or (1, 1) - spec.with_gemm_swizzled_scales = _meets_quantization_alignment( - int(bool(rows % row_alignment)), int(bool(cols % col_alignment)), alignment - ) + if alignment is None: + spec.with_gemm_swizzled_scales = False + else: + row_alignment, col_alignment = alignment + spec.with_gemm_swizzled_scales = bool( + rows % row_alignment == 0 and cols % col_alignment == 0 + ) return spec From e6ad24370c72e86f159fc3d767b04da59fa95235 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Wed, 14 Oct 2026 11:14:11 +0200 Subject: [PATCH 10/15] Consolidate NVFP4 compile regression tests Signed-off-by: Pawel Gadzinski --- qa/L0_pytorch_unittest/test.sh | 1 - tests/pytorch/test_dynamo_fakes.py | 146 ---------------------------- tests/pytorch/test_torch_compile.py | 112 ++++++++++----------- 3 files changed, 50 insertions(+), 209 deletions(-) delete mode 100644 tests/pytorch/test_dynamo_fakes.py diff --git a/qa/L0_pytorch_unittest/test.sh b/qa/L0_pytorch_unittest/test.sh index 5d34d6c14d6..2a486450a20 100644 --- a/qa/L0_pytorch_unittest/test.sh +++ b/qa/L0_pytorch_unittest/test.sh @@ -43,7 +43,6 @@ python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_mxfp8.xml $TE_PA python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_weight_swizzle_in_layers.xml $TE_PATH/tests/pytorch/test_weight_swizzle_in_layers.py || test_fail "test_weight_swizzle_in_layers.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_quantized_tensor.xml $TE_PATH/tests/pytorch/test_quantized_tensor.py || test_fail "test_quantized_tensor.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_torch_compile.xml $TE_PATH/tests/pytorch/test_torch_compile.py || test_fail "test_torch_compile.py" -python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_dynamo_fakes.xml $TE_PATH/tests/pytorch/test_dynamo_fakes.py || test_fail "test_dynamo_fakes.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_float8blockwisetensor.xml $TE_PATH/tests/pytorch/test_float8blockwisetensor.py || test_fail "test_float8blockwisetensor.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_float8_blockwise_scaling_exact.xml $TE_PATH/tests/pytorch/test_float8_blockwise_scaling_exact.py || test_fail "test_float8_blockwise_scaling_exact.py" python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_float8_blockwise_gemm_exact.xml $TE_PATH/tests/pytorch/test_float8_blockwise_gemm_exact.py || test_fail "test_float8_blockwise_gemm_exact.py" diff --git a/tests/pytorch/test_dynamo_fakes.py b/tests/pytorch/test_dynamo_fakes.py deleted file mode 100644 index 721986049a0..00000000000 --- a/tests/pytorch/test_dynamo_fakes.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# See LICENSE for license information. - -"""Check fake output contracts against native operations.""" - -import pytest -import torch -from torch._subclasses.fake_tensor import FakeTensor, FakeTensorMode - -import transformer_engine.pytorch as te -import transformer_engine_torch as tex -from transformer_engine.pytorch.dynamo import to_tensor_spec -from transformer_engine.pytorch.dynamo.fake import fake_bgrad -from transformer_engine.pytorch.tensor.nvfp4_tensor import NVFP4Quantizer - - -nvfp4_available, reason_for_no_nvfp4 = te.is_nvfp4_available(return_reason=True) - - -def _assert_matches_native(spec, native): - if spec.is_quantized: - names, native_metadata = native.__tensor_flatten__() - assert spec.inner_names() == tuple(names) - fake_metadata = spec.create_metadata() - for metadata in (fake_metadata, native_metadata): - metadata["nontensor_kwargs"].pop("quantizer") - assert fake_metadata == native_metadata - native_inner = [getattr(native, name) for name in names] - else: - native_inner = [native] - assert spec.shape == native.shape - with FakeTensorMode(): - fake_inner = spec.create_inner_tensors() - assert len(fake_inner) == len(native_inner) - for fake, actual in zip(fake_inner, native_inner): - assert isinstance(fake, FakeTensor) - assert fake.shape == actual.shape - assert fake.stride() == actual.stride() - assert fake.dtype == actual.dtype - assert fake.device == actual.device - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") -@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) -@pytest.mark.parametrize("shape", [(32,), (64, 128), (2, 32, 128), (0, 128)]) -def test_fake_bgrad_unquantized_matches_native(dtype, shape): - inp = torch.randn(shape, dtype=dtype, device="cuda") - native_outputs = tex.bgrad_quantize(inp, None) - with FakeTensorMode() as mode: - specs = fake_bgrad(to_tensor_spec(mode.from_tensor(inp))) - for spec, native in zip(specs, native_outputs): - _assert_matches_native(spec, native) - - -@pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) -@pytest.mark.parametrize("mode", ["rht", "1d", "2d", "row-scaled", "4over6"]) -@pytest.mark.parametrize("optimize_for_gemm", [False, True]) -@pytest.mark.parametrize("columnwise", [False, True]) -@pytest.mark.parametrize("internal", [False, True]) -@pytest.mark.parametrize("disable_rht_fusion", [False, True]) -@pytest.mark.parametrize( - "shape", - [ - (32, 32), - (64, 128), - (96, 128), - (128, 64), - (128, 128), - (128, 160), - (2, 32, 128), - (2, 64, 128), - (2, 1, 32, 128), - (0, 128), - ], -) -def test_fake_bgrad_nvfp4_matches_native( - mode, optimize_for_gemm, columnwise, internal, disable_rht_fusion, shape, monkeypatch -): - monkeypatch.setenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", str(int(disable_rht_fusion))) - quantizer = NVFP4Quantizer( - columnwise=columnwise, - with_rht=mode == "rht", - with_post_rht_amax=mode == "rht", - with_2d_quantization=mode in ("2d", "4over6"), - row_scaled_nvfp4=mode == "row-scaled", - nvfp4_use_4over6=mode == "4over6", - nvfp4_e4m3_max=448 if mode == "4over6" else 0, - with_random_sign_mask=False, - ) - quantizer.optimize_for_gemm = optimize_for_gemm - quantizer.internal = internal - inp = torch.randn(shape, dtype=torch.bfloat16, device="cuda") - native_outputs = tex.bgrad_quantize(inp, quantizer) - with FakeTensorMode() as fake_mode: - specs = fake_bgrad(to_tensor_spec(fake_mode.from_tensor(inp)), quantizer) - for spec, native in zip(specs, native_outputs): - _assert_matches_native(spec, native) - - -@pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) -@pytest.mark.parametrize("with_rht", [False, True]) -def test_fake_bgrad_dynamic_alignment(with_rht, monkeypatch): - monkeypatch.setenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", "0") - quantizer = NVFP4Quantizer( - with_rht=with_rht, - with_post_rht_amax=with_rht, - with_2d_quantization=not with_rht, - with_random_sign_mask=False, - ) - quantizer.optimize_for_gemm = True - graphs = [] - - def backend(graph, _inputs): - graphs.append(graph) - return graph.forward - - def describe(inp): - _, spec = fake_bgrad(to_tensor_spec(inp), quantizer) - return inp + spec.with_gemm_swizzled_scales - - torch._dynamo.reset() - compiled = torch.compile(describe, backend=backend, fullgraph=True) - # Pairs preserve alignment, including distinct nonzero remainders. - for i, shape in enumerate( - ( - (64, 128), - (192, 384), - (96, 160), - (160, 224), - (160, 160), - (288, 288), - (128, 64), - (256, 192), - (128, 128), - (256, 256), - ) - ): - inp = torch.zeros(shape, device="cuda", dtype=torch.bfloat16) - torch._dynamo.mark_dynamic(inp, (0, 1)) - previous_graph_count = len(graphs) - actual = compiled(inp) - _, native = tex.bgrad_quantize(inp, quantizer) - torch.testing.assert_close(actual, inp + native._with_gemm_swizzled_scales, atol=0, rtol=0) - if i % 2 == 1: - assert len(graphs) == previous_graph_count diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index ddb8d138e48..f292b749b94 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -1975,26 +1975,26 @@ def fn(inp): @pytest.mark.parametrize("compile_mode", _compile_modes) @pytest.mark.parametrize( "fp8_recipe,input_shape", - [pytest.param(None, (64,), id="bf16-rank1")] + [ + pytest.param(None, (64,), id="bf16-rank1"), + pytest.param(None, (2, 4, 4, 64), id="bf16-rank4"), + ] + [ pytest.param(fp8_recipe, shape, id=f"{name}-{shape}") for name, fp8_recipe in ( - ("bf16", None), - ("mxfp8", recipe.MXFP8BlockScaling()), ("nvfp4", recipe.NVFP4BlockScaling()), ("nvfp4-no-rht", recipe.NVFP4BlockScaling(disable_rht=True)), ) - for shape in ((1, 128, 128), (128, 1, 128), (2, 4, 16, 128)) + for shape in ((1, 128, 128), (2, 4, 16, 128)) ], ) def test_te_linear_compiles_non_2d_input(fp8_recipe, input_shape, compile_mode): """Compiled Linear preserves non-matrix activation shapes in forward and backward.""" if isinstance(fp8_recipe, recipe.NVFP4BlockScaling) and not nvfp4_available: pytest.skip(reason_for_no_nvfp4) - if isinstance(fp8_recipe, recipe.MXFP8BlockScaling) and not mxfp8_available: - pytest.skip(reason_for_no_mxfp8) dtype = torch.bfloat16 - model = te.Linear(input_shape[-1], 128, params_dtype=dtype, device="cuda") + out_features = 32 if fp8_recipe is None else 128 + model = te.Linear(input_shape[-1], out_features, params_dtype=dtype, device="cuda") def fn(inp): if fp8_recipe is None: @@ -2871,20 +2871,21 @@ def test_te_ops_bias_compile_with_fp8_input(backward): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize( - "dtype,quantization,shape,disable_rht_fusion", + "dtype,quantization,shape,disable_rht_fusion,case,grads", [ - (torch.float16, None, (2, 16, 32), False), - (torch.bfloat16, None, (2, 16, 32), False), - (torch.float16, "fp8", (2, 16, 32), False), - (torch.bfloat16, "fp8", (2, 16, 32), False), - (torch.bfloat16, "nvfp4", (2, 16, 128), False), - (torch.bfloat16, "nvfp4", (64, 128), False), - (torch.bfloat16, "nvfp4", (64, 128), True), - (torch.bfloat16, "nvfp4-no-rht", (2, 16, 128), False), + (dtype, quantization, (2, 16, 32), False, case, grads) + for dtype in (torch.float16, torch.bfloat16) + for quantization in (None, "fp8") + for case in ("linear", "bias", "pair", "unfused") + for grads in ("all", "input", "weight", "bias", "none") + ] + + [ + (torch.bfloat16, "nvfp4", (2, 16, 128), False, "pair", "all"), + (torch.bfloat16, "nvfp4", (64, 128), False, "pair", "all"), + (torch.bfloat16, "nvfp4", (64, 128), True, "pair", "all"), + (torch.bfloat16, "nvfp4-no-rht", (2, 16, 128), False, "pair", "all"), ], ) -@pytest.mark.parametrize("case", ["linear", "bias", "pair", "unfused"]) -@pytest.mark.parametrize("grads", ["all", "input", "weight", "bias", "none"]) def test_te_ops_linear_bias_compile( dtype, quantization, shape, disable_rht_fusion, case, grads, monkeypatch ): @@ -2915,17 +2916,30 @@ def bgrad(inp): torch._dynamo.reset() compiled, graphs = _compile_with_graphs(bgrad) - # Each pair introduces different batch sizes with the same alignment. + # Pairs preserve alignment, including distinct nonzero remainders. for disable_fusion in (False, True, False): monkeypatch.setenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", str(int(disable_fusion))) - for i, batch in enumerate((64, 192, 96, 160, 128, 256)): - inp = torch.randn(batch, 128, device="cuda", dtype=torch.bfloat16) - torch._dynamo.mark_dynamic(inp, 0) + for i, shape in enumerate( + ( + (64, 128), + (192, 384), + (96, 160), + (160, 224), + (160, 160), + (288, 288), + (128, 64), + (256, 192), + (128, 128), + (256, 256), + ) + ): + inp = torch.randn(shape, device="cuda", dtype=torch.bfloat16) + torch._dynamo.mark_dynamic(inp, (0, 1)) previous_graph_count = len(graphs) dx, db = compiled(inp) expected_db, expected_dx = tex.bgrad_quantize(inp, quantizer) layout = expected_dx._with_gemm_swizzled_scales - assert dx._with_gemm_swizzled_scales == layout, (mode, disable_fusion, batch) + assert dx._with_gemm_swizzled_scales == layout, (mode, disable_fusion, shape) torch.testing.assert_close(db, expected_db, atol=0, rtol=0) torch.testing.assert_close(dx.dequantize(), expected_dx.dequantize(), atol=0, rtol=0) if columnwise: @@ -2945,7 +2959,7 @@ def bgrad(inp): quantizer.optimize_for_gemm = not optimize_for_gemm inp = torch.randn(128, 128, device="cuda", dtype=torch.bfloat16) - torch._dynamo.mark_dynamic(inp, 0) + torch._dynamo.mark_dynamic(inp, (0, 1)) dx, _ = compiled(inp) _, expected_dx = tex.bgrad_quantize(inp, quantizer) assert dx._with_gemm_swizzled_scales == expected_dx._with_gemm_swizzled_scales @@ -3030,29 +3044,13 @@ def unexpected_allocation(*args, **kwargs): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize("training", [False, True]) -@pytest.mark.parametrize("initial_device", ["cuda", "cpu", "meta"]) @pytest.mark.parametrize( - "quantization", - [ - None, - "fp8", - "fp8-block", - "mxfp8", - "nvfp4", - "nvfp4-no-rht", - "nvfp4-4over6", - "nvfp4-row-scaled", - ], + "initial_device,quantization", + [("cuda", None), ("cpu", None), ("meta", None), ("cuda", "nvfp4"), ("cuda", "nvfp4-no-rht")], ) def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device, quantization): """A fresh process prevents earlier GEMMs from hiding lazy workspace allocation.""" - if quantization == "fp8" and not fp8_available: - pytest.skip(reason_for_no_fp8) - if quantization == "fp8-block" and not fp8_block_scaling_available: - pytest.skip(reason_for_no_fp8_block_scaling) - if quantization == "mxfp8" and not mxfp8_available: - pytest.skip(reason_for_no_mxfp8) - if quantization is not None and quantization.startswith("nvfp4") and not nvfp4_available: + if quantization is not None and not nvfp4_available: pytest.skip(reason_for_no_nvfp4) script = textwrap.dedent( """ @@ -3067,27 +3065,17 @@ def test_te_ops_linear_bias_cold_cudagraphs(training, initial_device, quantizati training, initial_device, quantization = sys.argv[1:] training = training == "True" - is_nvfp4 = quantization.startswith("nvfp4") - if quantization == "None": - quant_recipe = None - elif is_nvfp4: - row_scaled = quantization == "nvfp4-row-scaled" - quant_recipe = recipe.NVFP4BlockScaling( - disable_rht=quantization != "nvfp4", + is_nvfp4 = quantization != "None" + quant_recipe = ( + recipe.NVFP4BlockScaling( + disable_rht=quantization == "nvfp4-no-rht", disable_stochastic_rounding=True, - disable_2d_quantization=row_scaled, - row_scaled_activation=row_scaled, - nvfp4_4over6="all" if quantization == "nvfp4-4over6" else "none", - backward_override="dequantized" if row_scaled else None, ) - else: - quant_recipe = { - "fp8": recipe.Float8CurrentScaling, - "fp8-block": recipe.Float8BlockScaling, - "mxfp8": recipe.MXFP8BlockScaling, - }[quantization]() - in_features, out_features = (128, 128) if quant_recipe is not None else (32, 64) - batch_size = 128 if isinstance(quant_recipe, recipe.Float8BlockScaling) else 32 + if is_nvfp4 + else None + ) + in_features, out_features = (128, 128) if is_nvfp4 else (32, 64) + batch_size = 32 eager_tols = {"rtol": 0, "atol": 0} if is_nvfp4 else {} assert get_cublas_workspace.cache_info().currsize == 0 linear = te.ops.BasicLinear( From 2566a942521531d7e101ec72756064e807f76a6c Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Wed, 14 Oct 2026 11:46:39 +0200 Subject: [PATCH 11/15] Move NVFP4 alignment helpers into the quantizer Signed-off-by: Pawel Gadzinski --- transformer_engine/pytorch/csrc/common.h | 19 +++++-- .../pytorch/csrc/extensions/pybind.cpp | 2 +- transformer_engine/pytorch/csrc/quantizer.cpp | 51 ++++++++----------- 3 files changed, 35 insertions(+), 37 deletions(-) diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index 896d291f664..563a9c6ac02 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -339,13 +339,10 @@ class MXFP8Quantizer : public Quantizer { std::vector get_scale_shape(const std::vector& shape, bool columnwise) const; }; -// Flattened row/column alignment for directly emitted GEMM-swizzled scales, or no such path. -std::optional> nvfp4_gemm_swizzled_scales_alignment( - bool optimize_for_gemm, bool with_rht, bool with_2d_quantization, bool row_scaled_nvfp4, - bool nvfp4_use_4over6, int sm_arch, bool disable_rht_cast_fusion); - class NVFP4Quantizer : public Quantizer { public: + using ShapeAlignment = std::optional>; + // amax reduction for low precision FP4 AG bool with_amax_reduction; c10::intrusive_ptr amax_reduction_group; @@ -427,7 +424,19 @@ class NVFP4Quantizer : public Quantizer { */ static bool is_eligible_for_2d_swizzle_fusion(const std::vector& shape); + // Flattened row/column alignment for directly emitted GEMM-swizzled scales, or no such path. + static ShapeAlignment gemm_swizzled_scales_alignment(bool optimize_for_gemm, bool with_rht, + bool with_2d_quantization, + bool row_scaled_nvfp4, bool nvfp4_use_4over6, + int sm_arch, bool disable_rht_cast_fusion); + private: + static ShapeAlignment rht_cast_fusion_alignment(int sm_arch, bool disable_rht_cast_fusion, + bool for_grouped_kernel = false); + static ShapeAlignment get_2d_swizzle_fusion_alignment(int sm_arch); + static bool meets_alignment(const std::vector& shape, const ShapeAlignment& alignment); + bool emits_gemm_swizzled_scales(const std::vector& shape) const; + void quantize_with_rht_unfused_helper(const TensorWrapper& input, TensorWrapper& out, TensorWrapper& rht_output_t_cpp, QuantizationConfigWrapper& quant_config, diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index d63efd9cddb..bbb91a11d35 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -211,7 +211,7 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.def("dequantize", &transformer_engine::pytorch::dequantize, "Dequantize", py::arg("input"), py::arg("otype")); m.def("nvfp4_gemm_swizzled_scales_alignment", - &transformer_engine::pytorch::nvfp4_gemm_swizzled_scales_alignment, + &transformer_engine::pytorch::NVFP4Quantizer::gemm_swizzled_scales_alignment, py::arg("optimize_for_gemm"), py::arg("with_rht"), py::arg("with_2d_quantization"), py::arg("row_scaled_nvfp4"), py::arg("nvfp4_use_4over6"), py::arg("sm_arch"), py::arg("disable_rht_cast_fusion")); diff --git a/transformer_engine/pytorch/csrc/quantizer.cpp b/transformer_engine/pytorch/csrc/quantizer.cpp index ffe05197f3b..35bf392b9dc 100644 --- a/transformer_engine/pytorch/csrc/quantizer.cpp +++ b/transformer_engine/pytorch/csrc/quantizer.cpp @@ -1934,12 +1934,8 @@ void NVFP4Quantizer::set_quantization_params(TensorWrapper* tensor) const { columnwise_data.shape); } -namespace { - -using ShapeAlignment = std::optional>; - -ShapeAlignment nvfp4_rht_cast_fusion_alignment(int sm_arch, bool disable_rht_cast_fusion, - bool for_grouped_kernel = false) { +NVFP4Quantizer::ShapeAlignment NVFP4Quantizer::rht_cast_fusion_alignment( + int sm_arch, bool disable_rht_cast_fusion, bool for_grouped_kernel) { const bool supported_arch = (sm_arch >= 100 && sm_arch <= 110) || (!for_grouped_kernel && (sm_arch == 120 || sm_arch == 121)); if (disable_rht_cast_fusion || !supported_arch) { @@ -1948,14 +1944,15 @@ ShapeAlignment nvfp4_rht_cast_fusion_alignment(int sm_arch, bool disable_rht_cas return std::pair{for_grouped_kernel ? 128 : 64, 128}; } -ShapeAlignment nvfp4_2d_swizzle_fusion_alignment(int sm_arch) { +NVFP4Quantizer::ShapeAlignment NVFP4Quantizer::get_2d_swizzle_fusion_alignment(int sm_arch) { if (sm_arch < 100 || sm_arch > 110) { return std::nullopt; } return std::pair{128, 128}; } -bool meets_alignment(const std::vector& shape, const ShapeAlignment& alignment) { +bool NVFP4Quantizer::meets_alignment(const std::vector& shape, + const ShapeAlignment& alignment) { if (!alignment) { return false; } @@ -1963,9 +1960,7 @@ bool meets_alignment(const std::vector& shape, const ShapeAlignment& ali return rows % alignment->first == 0 && cols % alignment->second == 0; } -} // namespace - -std::optional> nvfp4_gemm_swizzled_scales_alignment( +NVFP4Quantizer::ShapeAlignment NVFP4Quantizer::gemm_swizzled_scales_alignment( bool optimize_for_gemm, bool with_rht, bool with_2d_quantization, bool row_scaled_nvfp4, bool nvfp4_use_4over6, int sm_arch, bool disable_rht_cast_fusion) { if (!optimize_for_gemm) { @@ -1976,41 +1971,35 @@ std::optional> nvfp4_gemm_swizzled_scales_alignment( if (sm_arch == 120 || sm_arch == 121) { return std::nullopt; } - return nvfp4_rht_cast_fusion_alignment(sm_arch, disable_rht_cast_fusion); + return rht_cast_fusion_alignment(sm_arch, disable_rht_cast_fusion); } if (with_2d_quantization && !row_scaled_nvfp4 && !nvfp4_use_4over6) { - return nvfp4_2d_swizzle_fusion_alignment(sm_arch); + return get_2d_swizzle_fusion_alignment(sm_arch); } return std::nullopt; } bool NVFP4Quantizer::is_eligible_for_rht_cast_fusion(const std::vector& shape, bool for_grouped_kernel) { - return meets_alignment(shape, - nvfp4_rht_cast_fusion_alignment( - transformer_engine::cuda::sm_arch(), - transformer_engine::getenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION"), - for_grouped_kernel)); + return meets_alignment(shape, rht_cast_fusion_alignment(transformer_engine::cuda::sm_arch(), + transformer_engine::getenv( + "NVTE_NVFP4_DISABLE_RHT_CAST_FUSION"), + for_grouped_kernel)); } bool NVFP4Quantizer::is_eligible_for_2d_swizzle_fusion(const std::vector& shape) { return meets_alignment(shape, - nvfp4_2d_swizzle_fusion_alignment(transformer_engine::cuda::sm_arch())); + get_2d_swizzle_fusion_alignment(transformer_engine::cuda::sm_arch())); } -namespace { - -bool nvfp4_emits_gemm_swizzled_scales(const NVFP4Quantizer& q, const std::vector& shape) { +bool NVFP4Quantizer::emits_gemm_swizzled_scales(const std::vector& shape) const { return meets_alignment( - shape, - nvfp4_gemm_swizzled_scales_alignment( - q.optimize_for_gemm, q.with_rht, q.with_2d_quantization, q.row_scaled_nvfp4, - q.nvfp4_4over6_mode != kNVTENVFP44Over6Disabled, transformer_engine::cuda::sm_arch(), - transformer_engine::getenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION"))); + shape, gemm_swizzled_scales_alignment( + optimize_for_gemm, with_rht, with_2d_quantization, row_scaled_nvfp4, + nvfp4_4over6_mode != kNVTENVFP44Over6Disabled, transformer_engine::cuda::sm_arch(), + transformer_engine::getenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION"))); } -} // namespace - std::pair NVFP4Quantizer::create_tensor( const std::vector& shape, DType dtype, std::optional device_opt, bool pin_memory) const { @@ -2024,7 +2013,7 @@ std::pair NVFP4Quantizer::create_tensor( // Swizzled SF is emitted directly by the RHT cast-fusion kernel or by the plain // 2D quantize kernel (for aligned shapes); other quantize paths reject it and go // through the standalone post-quantize swizzle pass instead. - const bool with_gemm_swizzled_scales = nvfp4_emits_gemm_swizzled_scales(*this, shape); + const bool with_gemm_swizzled_scales = emits_gemm_swizzled_scales(shape); NVTE_CHECK(flat_first_dim % NVFP4_BLOCK_SIZE == 0, "First dim for NVFP4 must be divisible by ", NVFP4_BLOCK_SIZE, " (got shape=", shape, ")"); NVTE_CHECK(flat_last_dim % NVFP4_BLOCK_SIZE == 0, @@ -2376,7 +2365,7 @@ std::pair NVFP4Quantizer::convert_and_update_tensor( // Swizzled SF is emitted directly by the RHT cast-fusion kernel or by the plain // 2D quantize kernel (for aligned shapes); other quantize paths reject it and go // through the standalone post-quantize swizzle pass instead. - const bool with_gemm_swizzled_scales = nvfp4_emits_gemm_swizzled_scales(*this, shape); + const bool with_gemm_swizzled_scales = emits_gemm_swizzled_scales(shape); const bool row_scaled_nvfp4 = this->row_scaled_nvfp4; const bool disable_second_level_scale = this->disable_second_level_scale; From abc7e7bdf15b43ae55d2d342428d7fb10384bfb5 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Wed, 14 Oct 2026 12:47:15 +0200 Subject: [PATCH 12/15] Refine NVFP4 metadata helpers and empty fake outputs Signed-off-by: Pawel Gadzinski --- transformer_engine/pytorch/csrc/common.h | 6 ++- transformer_engine/pytorch/csrc/quantizer.cpp | 23 ++++++----- transformer_engine/pytorch/dynamo/fake.py | 11 +++-- transformer_engine/pytorch/permutation.py | 41 +++++++++++-------- .../tensor/storage/nvfp4_tensor_storage.py | 1 + 5 files changed, 47 insertions(+), 35 deletions(-) diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index 563a9c6ac02..9a3c6314054 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -428,10 +428,12 @@ class NVFP4Quantizer : public Quantizer { static ShapeAlignment gemm_swizzled_scales_alignment(bool optimize_for_gemm, bool with_rht, bool with_2d_quantization, bool row_scaled_nvfp4, bool nvfp4_use_4over6, - int sm_arch, bool disable_rht_cast_fusion); + int sm_arch, + std::optional disable_rht_cast_fusion); private: - static ShapeAlignment rht_cast_fusion_alignment(int sm_arch, bool disable_rht_cast_fusion, + static ShapeAlignment rht_cast_fusion_alignment(int sm_arch, + std::optional disable_rht_cast_fusion, bool for_grouped_kernel = false); static ShapeAlignment get_2d_swizzle_fusion_alignment(int sm_arch); static bool meets_alignment(const std::vector& shape, const ShapeAlignment& alignment); diff --git a/transformer_engine/pytorch/csrc/quantizer.cpp b/transformer_engine/pytorch/csrc/quantizer.cpp index 35bf392b9dc..ba0410bcacb 100644 --- a/transformer_engine/pytorch/csrc/quantizer.cpp +++ b/transformer_engine/pytorch/csrc/quantizer.cpp @@ -1935,10 +1935,14 @@ void NVFP4Quantizer::set_quantization_params(TensorWrapper* tensor) const { } NVFP4Quantizer::ShapeAlignment NVFP4Quantizer::rht_cast_fusion_alignment( - int sm_arch, bool disable_rht_cast_fusion, bool for_grouped_kernel) { + int sm_arch, std::optional disable_rht_cast_fusion, bool for_grouped_kernel) { + const bool disabled = + disable_rht_cast_fusion.has_value() + ? *disable_rht_cast_fusion + : transformer_engine::getenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION"); const bool supported_arch = (sm_arch >= 100 && sm_arch <= 110) || (!for_grouped_kernel && (sm_arch == 120 || sm_arch == 121)); - if (disable_rht_cast_fusion || !supported_arch) { + if (disabled || !supported_arch) { return std::nullopt; } return std::pair{for_grouped_kernel ? 128 : 64, 128}; @@ -1962,7 +1966,7 @@ bool NVFP4Quantizer::meets_alignment(const std::vector& shape, NVFP4Quantizer::ShapeAlignment NVFP4Quantizer::gemm_swizzled_scales_alignment( bool optimize_for_gemm, bool with_rht, bool with_2d_quantization, bool row_scaled_nvfp4, - bool nvfp4_use_4over6, int sm_arch, bool disable_rht_cast_fusion) { + bool nvfp4_use_4over6, int sm_arch, std::optional disable_rht_cast_fusion) { if (!optimize_for_gemm) { return std::nullopt; } @@ -1982,9 +1986,7 @@ NVFP4Quantizer::ShapeAlignment NVFP4Quantizer::gemm_swizzled_scales_alignment( bool NVFP4Quantizer::is_eligible_for_rht_cast_fusion(const std::vector& shape, bool for_grouped_kernel) { return meets_alignment(shape, rht_cast_fusion_alignment(transformer_engine::cuda::sm_arch(), - transformer_engine::getenv( - "NVTE_NVFP4_DISABLE_RHT_CAST_FUSION"), - for_grouped_kernel)); + std::nullopt, for_grouped_kernel)); } bool NVFP4Quantizer::is_eligible_for_2d_swizzle_fusion(const std::vector& shape) { @@ -1993,11 +1995,10 @@ bool NVFP4Quantizer::is_eligible_for_2d_swizzle_fusion(const std::vector } bool NVFP4Quantizer::emits_gemm_swizzled_scales(const std::vector& shape) const { - return meets_alignment( - shape, gemm_swizzled_scales_alignment( - optimize_for_gemm, with_rht, with_2d_quantization, row_scaled_nvfp4, - nvfp4_4over6_mode != kNVTENVFP44Over6Disabled, transformer_engine::cuda::sm_arch(), - transformer_engine::getenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION"))); + return meets_alignment(shape, gemm_swizzled_scales_alignment( + optimize_for_gemm, with_rht, with_2d_quantization, + row_scaled_nvfp4, nvfp4_4over6_mode != kNVTENVFP44Over6Disabled, + transformer_engine::cuda::sm_arch(), std::nullopt)); } std::pair NVFP4Quantizer::create_tensor( diff --git a/transformer_engine/pytorch/dynamo/fake.py b/transformer_engine/pytorch/dynamo/fake.py index 2b13a06ca2d..5f856d62214 100644 --- a/transformer_engine/pytorch/dynamo/fake.py +++ b/transformer_engine/pytorch/dynamo/fake.py @@ -79,17 +79,16 @@ def _make_fused_quantization_output_spec( def fake_bgrad( grad_output: TensorSpec, quantizer: Optional[Quantizer] = None, - *, - dtype: Optional[torch.dtype] = None, ) -> tuple[TensorSpec, TensorSpec]: """Describe bias reduction and optional gradient quantization, in native order.""" - dtype = dtype or grad_output.dtype - db = TensorSpec(shape=(grad_output.shape[-1],), dtype=dtype, device=grad_output.device) + db = TensorSpec( + shape=(grad_output.shape[-1],), dtype=grad_output.dtype, device=grad_output.device + ) dx = ( grad_output - if quantizer is None and dtype == grad_output.dtype + if quantizer is None else _make_fused_quantization_output_spec( - grad_output.shape, dtype, grad_output.device, quantizer + grad_output.shape, grad_output.dtype, grad_output.device, quantizer ) ) return db, dx diff --git a/transformer_engine/pytorch/permutation.py b/transformer_engine/pytorch/permutation.py index 5e6bdb4dfcd..f34a6a37346 100644 --- a/transformer_engine/pytorch/permutation.py +++ b/transformer_engine/pytorch/permutation.py @@ -92,16 +92,14 @@ def _moe_permute_index_map_fake( # pylint: disable=unused-argument max_token_num: int, ) -> Tuple[torch.Tensor, torch.Tensor]: """Fake implementation for shape inference.""" + if not inp.numel(): + return inp.clone(), torch.empty(0, dtype=torch.int32, device=inp.device) num_tokens = inp.shape[0] topK = index.shape[1] - if num_tokens > 0: - assert ( - num_out_tokens >= 0 - ), f"moe_permute (index map) requires num_out_tokens >= 0, got {num_out_tokens}." - output_tokens = num_out_tokens if num_out_tokens > 0 else num_tokens * topK - else: - # Match `moe_permute_index_map_forward` empty-input fast path (ignores num_out_tokens). - output_tokens = 0 + assert ( + num_out_tokens >= 0 + ), f"moe_permute (index map) requires num_out_tokens >= 0, got {num_out_tokens}." + output_tokens = num_out_tokens if num_out_tokens > 0 else num_tokens * topK # row_id_map is 1D with size = num_tokens * topK fake_output = torch.empty((output_tokens, inp.shape[1]), dtype=inp.dtype, device=inp.device) @@ -201,6 +199,8 @@ def _moe_unpermute_index_map_forward_fake( # pylint: disable=unused-argument topK: int, ) -> torch.Tensor: """Fake implementation for shape inference.""" + if not inp.numel(): + return inp.clone() # Output shape: (num_tokens, hidden_size) return torch.empty((num_tokens, inp.shape[1]), dtype=inp.dtype, device=inp.device) @@ -442,15 +442,19 @@ def _moe_permute_mask_map_forward_fake( # pylint: disable=unused-argument num_tokens = inp.shape[0] hidden_size = inp.shape[1] num_experts = routing_map.shape[1] - if num_tokens > 0: - assert num_out_tokens > 0, ( - f"moe_permute (mask map) requires num_out_tokens > 0, got {num_out_tokens}. " - "Use int(routing_map.sum()) or num_tokens * top_k." + if not inp.numel(): + return ( + inp.clone(), + torch.empty((0, num_experts * 2 + 1), dtype=torch.int32, device=inp.device), + torch.empty( + 0, dtype=probs.dtype if probs is not None else torch.float32, device=inp.device + ), ) - out_rows = num_out_tokens - else: - # Match `moe_permute_mask_map_forward` empty-input fast path (ignores num_out_tokens). - out_rows = 0 + assert num_out_tokens > 0, ( + f"moe_permute (mask map) requires num_out_tokens > 0, got {num_out_tokens}. " + "Use int(routing_map.sum()) or num_tokens * top_k." + ) + out_rows = num_out_tokens # row_id_map: (num_tokens, num_experts * 2 + 1) fake_output = torch.empty((out_rows, hidden_size), dtype=inp.dtype, device=inp.device) fake_row_id_map = torch.empty( @@ -615,6 +619,8 @@ def _moe_unpermute_mask_map_forward_fake( # pylint: disable=unused-argument pad_offsets: Optional[torch.Tensor], ) -> torch.Tensor: """Fake implementation for shape inference.""" + if not inp.numel(): + return inp.clone() return torch.empty((num_tokens, hidden_size), dtype=inp.dtype, device=inp.device) @@ -1168,6 +1174,9 @@ def _moe_chunk_sort_forward_fake( # pylint: disable=unused-argument probs: Optional[torch.Tensor], ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """Fake for shape inference.""" + if not inp.numel(): + probs_out = probs.clone() if probs is not None else torch.empty(0, device=inp.device) + return inp.clone(), probs_out, torch.empty(0, device=inp.device, dtype=torch.int32) num_tokens = inp.shape[0] hidden_size = inp.shape[1] fake_output = torch.empty((num_tokens, hidden_size), dtype=inp.dtype, device=inp.device) diff --git a/transformer_engine/pytorch/tensor/storage/nvfp4_tensor_storage.py b/transformer_engine/pytorch/tensor/storage/nvfp4_tensor_storage.py index e4796214ac3..97b66a8341f 100644 --- a/transformer_engine/pytorch/tensor/storage/nvfp4_tensor_storage.py +++ b/transformer_engine/pytorch/tensor/storage/nvfp4_tensor_storage.py @@ -258,6 +258,7 @@ def size(self, dim: Optional[int] = None) -> Union[torch.Size, int]: byte_shape = list(self._rowwise_data.size()) shape = byte_shape[:-1] + [byte_shape[-1] * 2] elif self._columnwise_data is not None: + # Dynamo queries columnwise-only fake shapes; warning emission breaks fullgraph tracing. if not torch.compiler.is_compiling(): warnings.warn("Attempting to get shape of NVFP4 tensor with only column-wise data.") byte_shape = list(self._columnwise_data.size()) From 805b51b9591bc03b6bd2b48b90cf7c62fd5160c9 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Wed, 14 Oct 2026 13:06:34 +0200 Subject: [PATCH 13/15] Keep MoE fake changes out of the NVFP4 fix Signed-off-by: Pawel Gadzinski --- transformer_engine/pytorch/permutation.py | 41 +++++++++-------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/transformer_engine/pytorch/permutation.py b/transformer_engine/pytorch/permutation.py index f34a6a37346..5e6bdb4dfcd 100644 --- a/transformer_engine/pytorch/permutation.py +++ b/transformer_engine/pytorch/permutation.py @@ -92,14 +92,16 @@ def _moe_permute_index_map_fake( # pylint: disable=unused-argument max_token_num: int, ) -> Tuple[torch.Tensor, torch.Tensor]: """Fake implementation for shape inference.""" - if not inp.numel(): - return inp.clone(), torch.empty(0, dtype=torch.int32, device=inp.device) num_tokens = inp.shape[0] topK = index.shape[1] - assert ( - num_out_tokens >= 0 - ), f"moe_permute (index map) requires num_out_tokens >= 0, got {num_out_tokens}." - output_tokens = num_out_tokens if num_out_tokens > 0 else num_tokens * topK + if num_tokens > 0: + assert ( + num_out_tokens >= 0 + ), f"moe_permute (index map) requires num_out_tokens >= 0, got {num_out_tokens}." + output_tokens = num_out_tokens if num_out_tokens > 0 else num_tokens * topK + else: + # Match `moe_permute_index_map_forward` empty-input fast path (ignores num_out_tokens). + output_tokens = 0 # row_id_map is 1D with size = num_tokens * topK fake_output = torch.empty((output_tokens, inp.shape[1]), dtype=inp.dtype, device=inp.device) @@ -199,8 +201,6 @@ def _moe_unpermute_index_map_forward_fake( # pylint: disable=unused-argument topK: int, ) -> torch.Tensor: """Fake implementation for shape inference.""" - if not inp.numel(): - return inp.clone() # Output shape: (num_tokens, hidden_size) return torch.empty((num_tokens, inp.shape[1]), dtype=inp.dtype, device=inp.device) @@ -442,19 +442,15 @@ def _moe_permute_mask_map_forward_fake( # pylint: disable=unused-argument num_tokens = inp.shape[0] hidden_size = inp.shape[1] num_experts = routing_map.shape[1] - if not inp.numel(): - return ( - inp.clone(), - torch.empty((0, num_experts * 2 + 1), dtype=torch.int32, device=inp.device), - torch.empty( - 0, dtype=probs.dtype if probs is not None else torch.float32, device=inp.device - ), + if num_tokens > 0: + assert num_out_tokens > 0, ( + f"moe_permute (mask map) requires num_out_tokens > 0, got {num_out_tokens}. " + "Use int(routing_map.sum()) or num_tokens * top_k." ) - assert num_out_tokens > 0, ( - f"moe_permute (mask map) requires num_out_tokens > 0, got {num_out_tokens}. " - "Use int(routing_map.sum()) or num_tokens * top_k." - ) - out_rows = num_out_tokens + out_rows = num_out_tokens + else: + # Match `moe_permute_mask_map_forward` empty-input fast path (ignores num_out_tokens). + out_rows = 0 # row_id_map: (num_tokens, num_experts * 2 + 1) fake_output = torch.empty((out_rows, hidden_size), dtype=inp.dtype, device=inp.device) fake_row_id_map = torch.empty( @@ -619,8 +615,6 @@ def _moe_unpermute_mask_map_forward_fake( # pylint: disable=unused-argument pad_offsets: Optional[torch.Tensor], ) -> torch.Tensor: """Fake implementation for shape inference.""" - if not inp.numel(): - return inp.clone() return torch.empty((num_tokens, hidden_size), dtype=inp.dtype, device=inp.device) @@ -1174,9 +1168,6 @@ def _moe_chunk_sort_forward_fake( # pylint: disable=unused-argument probs: Optional[torch.Tensor], ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """Fake for shape inference.""" - if not inp.numel(): - probs_out = probs.clone() if probs is not None else torch.empty(0, device=inp.device) - return inp.clone(), probs_out, torch.empty(0, device=inp.device, dtype=torch.int32) num_tokens = inp.shape[0] hidden_size = inp.shape[1] fake_output = torch.empty((num_tokens, hidden_size), dtype=inp.dtype, device=inp.device) From 5bc7286eaadb66bbecfd93f53b6f694fbce7712d Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Wed, 14 Oct 2026 14:20:44 +0200 Subject: [PATCH 14/15] Reduce NVFP4 Bias layout test combinations Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 99 +++++++++++------------------ 1 file changed, 36 insertions(+), 63 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index f292b749b94..3c1621db197 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -2894,21 +2894,16 @@ def test_te_ops_linear_bias_compile( @pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) -@pytest.mark.parametrize("mode", ["rht", "1d", "2d", "row-scaled", "4over6"]) -@pytest.mark.parametrize("optimize_for_gemm", [False, True]) -@pytest.mark.parametrize("columnwise", [False, True]) -def test_te_ops_bias_nvfp4_layout(mode, optimize_for_gemm, columnwise, monkeypatch): +@pytest.mark.parametrize("mode", ["rht", "2d", "row-scaled"]) +def test_te_ops_bias_nvfp4_layout(mode, monkeypatch): quantizer = NVFP4Quantizer( - columnwise=columnwise, + columnwise=True, with_rht=mode == "rht", with_post_rht_amax=mode == "rht", - with_2d_quantization=mode in ("2d", "4over6"), + with_2d_quantization=mode == "2d", row_scaled_nvfp4=mode == "row-scaled", - nvfp4_use_4over6=mode == "4over6", - nvfp4_e4m3_max=448 if mode == "4over6" else 0, with_random_sign_mask=False, ) - quantizer.optimize_for_gemm = optimize_for_gemm def bgrad(inp): dx, ((db,),), _ = te.ops.Bias.compile_ops[1](BiasBwdArgs(inp, quantizer)) @@ -2917,63 +2912,41 @@ def bgrad(inp): torch._dynamo.reset() compiled, graphs = _compile_with_graphs(bgrad) # Pairs preserve alignment, including distinct nonzero remainders. - for disable_fusion in (False, True, False): + shapes = ((128, 128), (256, 256), (96, 160), (160, 224)) + if mode == "row-scaled": + shapes = shapes[:1] + cases = [(False, True, shape) for shape in shapes] + if mode == "rht": + cases += [(True, True, (128, 128)), (False, True, (128, 128))] + if mode != "row-scaled": + cases.append((False, False, (128, 128))) + + for i, (disable_fusion, optimize_for_gemm, shape) in enumerate(cases): monkeypatch.setenv("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", str(int(disable_fusion))) - for i, shape in enumerate( - ( - (64, 128), - (192, 384), - (96, 160), - (160, 224), - (160, 160), - (288, 288), - (128, 64), - (256, 192), - (128, 128), - (256, 256), - ) - ): - inp = torch.randn(shape, device="cuda", dtype=torch.bfloat16) - torch._dynamo.mark_dynamic(inp, (0, 1)) - previous_graph_count = len(graphs) - dx, db = compiled(inp) - expected_db, expected_dx = tex.bgrad_quantize(inp, quantizer) - layout = expected_dx._with_gemm_swizzled_scales - assert dx._with_gemm_swizzled_scales == layout, (mode, disable_fusion, shape) - torch.testing.assert_close(db, expected_db, atol=0, rtol=0) - torch.testing.assert_close(dx.dequantize(), expected_dx.dequantize(), atol=0, rtol=0) - if columnwise: - torch.testing.assert_close( - dx._columnwise_data, expected_dx._columnwise_data, atol=0, rtol=0 - ) - if mode == "row-scaled": - torch.testing.assert_close( - dx._amax_rowwise, inp.abs().float().amax(1), atol=0, rtol=0 - ) - if columnwise: - torch.testing.assert_close( - dx._amax_columnwise, inp.abs().float().amax(0), atol=0, rtol=0 - ) - if i % 2 == 1: - assert len(graphs) == previous_graph_count + quantizer.optimize_for_gemm = optimize_for_gemm + inp = torch.randn(shape, device="cuda", dtype=torch.bfloat16) + torch._dynamo.mark_dynamic(inp, (0, 1)) + previous_graph_count = len(graphs) + dx, db = compiled(inp) + expected_db, expected_dx = tex.bgrad_quantize(inp, quantizer) + assert dx._with_gemm_swizzled_scales == expected_dx._with_gemm_swizzled_scales + torch.testing.assert_close(db, expected_db, atol=0, rtol=0) + torch.testing.assert_close(dx.dequantize(), expected_dx.dequantize(), atol=0, rtol=0) + torch.testing.assert_close( + dx._columnwise_data, expected_dx._columnwise_data, atol=0, rtol=0 + ) + if i < len(shapes) and i % 2 == 1: + assert len(graphs) == previous_graph_count - quantizer.optimize_for_gemm = not optimize_for_gemm - inp = torch.randn(128, 128, device="cuda", dtype=torch.bfloat16) - torch._dynamo.mark_dynamic(inp, (0, 1)) - dx, _ = compiled(inp) + shape = (2, 32, 128) + inp = torch.randn(shape, device="cuda", dtype=torch.bfloat16) _, expected_dx = tex.bgrad_quantize(inp, quantizer) - assert dx._with_gemm_swizzled_scales == expected_dx._with_gemm_swizzled_scales - torch.testing.assert_close(dx.dequantize(), expected_dx.dequantize(), atol=0, rtol=0) - - for shape in ((2, 32, 128), (2, 64, 128), (128, 64)): - inp = torch.randn(shape, device="cuda", dtype=torch.bfloat16) - _, expected_dx = tex.bgrad_quantize(inp, quantizer) - _, dx_spec = fake_bgrad(to_tensor_spec(inp), quantizer) - assert dx_spec.with_gemm_swizzled_scales == expected_dx._with_gemm_swizzled_scales - for name, (inner_shape, inner_dtype) in dx_spec.quantizer.inner_tensor_specs(shape).items(): - inner = getattr(expected_dx, name) - assert inner.shape == inner_shape - assert inner.dtype == inner_dtype + _, dx_spec = fake_bgrad(to_tensor_spec(inp), quantizer) + assert dx_spec.with_gemm_swizzled_scales == expected_dx._with_gemm_swizzled_scales + for name, (inner_shape, inner_dtype) in dx_spec.quantizer.inner_tensor_specs(shape).items(): + inner = getattr(expected_dx, name) + assert inner.shape == inner_shape + assert inner.dtype == inner_dtype @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") From dc6098e2e88560e747baaaf916aeede2ca922061 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 19 Oct 2026 15:19:43 -0700 Subject: [PATCH 15/15] Fix NVFP4 columnwise shapes in the shared helper Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 2 +- .../pytorch/tensor/nvfp4_tensor.py | 30 ++++--------------- 2 files changed, 7 insertions(+), 25 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 3c1621db197..7140f9f8573 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -2938,7 +2938,7 @@ def bgrad(inp): if i < len(shapes) and i % 2 == 1: assert len(graphs) == previous_graph_count - shape = (2, 32, 128) + shape = (64, 1, 128) inp = torch.randn(shape, device="cuda", dtype=torch.bfloat16) _, expected_dx = tex.bgrad_quantize(inp, quantizer) _, dx_spec = fake_bgrad(to_tensor_spec(inp), quantizer) diff --git a/transformer_engine/pytorch/tensor/nvfp4_tensor.py b/transformer_engine/pytorch/tensor/nvfp4_tensor.py index 97185d3b7ea..938d9a77d4c 100644 --- a/transformer_engine/pytorch/tensor/nvfp4_tensor.py +++ b/transformer_engine/pytorch/tensor/nvfp4_tensor.py @@ -330,29 +330,11 @@ def get_scale_shape(self, shape: Iterable[int], columnwise: bool) -> Tuple[int, @staticmethod def get_columnwise_shape(shape: Iterable[int]) -> Tuple[int, ...]: - """Calculate the shape of a tensor after columnwise quantization. - - For NVFP4 columnwise quantization, it's performing 16x1 quantization block scaling. - - Parameters - ---------- - shape : Iterable[int] - Original shape of the tensor - - Returns - ------- - Tuple[int, ...] - New shape with dimensions rearranged for columnwise layout. - For a shape (d1, d2, ..., dn), returns (dn, d1, d2, ..., dn-1). - Returns empty tuple for empty input shape. - """ - if len(shape) == 0: - return tuple() - # and then after AG, a reorganize kernel will be called to restore the shape - colwise_shape = [shape[-1]] - for i in range(len(shape) - 1): - colwise_shape.append(shape[i]) - return tuple(colwise_shape) + """Return the flattened columnwise shape before FP4 packing.""" + shape = tuple(shape) + if not shape: + return () + return (shape[-1], math.prod(shape[:-1])) @staticmethod def convert_shape_for_fp4(shape: Iterable[int]) -> Tuple[int, ...]: @@ -407,7 +389,7 @@ def inner_tensor_specs( ) if self.columnwise_usage: specs["_columnwise_data"] = ( - (shape[-1], math.prod(shape[:-1]) // 2), + type(self).convert_shape_for_fp4(type(self).get_columnwise_shape(shape)), torch.uint8, ) specs["_columnwise_scale_inv"] = (