From 3a60ea418a48296aaf19cb89dab84c078b38fcc7 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Tue, 20 Oct 2026 09:09:20 -0700 Subject: [PATCH 01/31] [PyTorch] Compile activation operations and fused bias gradients Add activation forward/backward and fused bias-gradient custom ops with shared fake output descriptions. Cover quantized saved inputs, gated and dynamic activations, AMP, and MLP training with CUDA Graphs. Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 241 +++++++++++++++++ transformer_engine/pytorch/dynamo/fake.py | 39 ++- .../pytorch/ops/basic/activation.py | 246 ++++++++++++------ .../pytorch/ops/basic/swiglu.py | 199 +++++++++----- .../ops/fused/backward_activation_bias.py | 74 +++--- 5 files changed, 619 insertions(+), 180 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index d63ffd495df..37d3db981dc 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -3232,3 +3232,244 @@ def run(x, module=model): forward=case in ("bias", "unfused"), backward=case != "linear" and bool(targets), ) + + +def _ops_activation_reference(name, x): + gate = None + if name in ("GLU", "GEGLU", "QGEGLU", "ReGLU", "SReGLU", "SwiGLU"): + x, gate = x.chunk(2, dim=-1) + if name in ("GELU", "GEGLU"): + x = torch.nn.functional.gelu(x, approximate="tanh") + elif name in ("QGELU", "QGEGLU"): + x = x * torch.sigmoid(1.702 * x) + elif name in ("ReLU", "ReGLU"): + x = x.relu() + elif name in ("SReLU", "SReGLU"): + x = x.relu().square() + elif name in ("SiLU", "SwiGLU"): + x = torch.nn.functional.silu(x) + elif name == "GLU": + x = x.sigmoid() + else: + raise ValueError(name) + return x if gate is None else x * gate + + +def _check_te_ops_activation_compile( + activation, + *, + dtype=torch.float32, + layout="contiguous", + cache_input=False, + autocast=False, + training=True, + interleave_size=None, + dynamic=False, +): + if (layout == "fp8" or cache_input) and not fp8_available: + pytest.skip(reason_for_no_fp8) + torch._dynamo.reset() + kwargs = {"cache_quantized_input": cache_input} + if interleave_size is not None: + kwargs["glu_interleave_size"] = interleave_size + model = te.ops.Sequential(getattr(te.ops, activation)(**kwargs)) + + def run(x): + with torch.autocast("cuda", dtype=torch.bfloat16, enabled=autocast): + return model(x) + + compiled, graphs = _compile_with_graphs(run) + eager_reference = autocast or dynamic + tols = {} if eager_reference else dtype_tols(dtype) + for step, rows in enumerate((16, 16, 32, 48) if dynamic else (32,)): + shape = (rows, 64) if dynamic else (2, 16, 64) + x = torch.randn(shape, device="cuda", dtype=dtype) + if layout == "fp8" or (cache_input and not eager_reference): + quantizer = Float8CurrentScalingQuantizer(tex.DType.kFloat8E4M3, device="cuda") + x = quantizer(x) + if layout != "fp8": + x = x.dequantize() + if layout == "strided": + x = x.transpose(0, 1) + x = x.detach().requires_grad_(training) + if dynamic: + torch._dynamo.mark_dynamic(x, 0) + ref_x = ( + x.detach().clone() + if eager_reference + else (x.dequantize() if layout == "fp8" else x).detach().double() + ).requires_grad_(training) + with torch.set_grad_enabled(training): + actual = compiled(x) + expected = ( + run(ref_x) if eager_reference else _ops_activation_reference(activation, ref_x) + ) + torch.testing.assert_close(actual.to(expected.dtype), expected, **tols) + if training: + dy = torch.randn_like(actual) + got = torch.autograd.grad(actual, x, dy)[0] + want = torch.autograd.grad(expected, ref_x, dy.to(expected.dtype))[0] + torch.testing.assert_close(got.to(want.dtype), want, **tols) + if autocast: + assert actual.dtype == torch.bfloat16 + torch.testing.assert_close(x, ref_x) + if dynamic: + if step == 1: + graph_count = len(graphs) + elif step > 1: + assert len(graphs) == graph_count + _assert_custom_ops(graphs, activation.lower(), backward=training) + + +@pytest.mark.parametrize( + "activation", + [ + "GELU", + "GEGLU", + "GLU", + "QGELU", + "QGEGLU", + "ReLU", + "ReGLU", + "SReLU", + "SReGLU", + "SiLU", + "SwiGLU", + ], +) +@pytest.mark.parametrize( + "dtype,layout,cache_input", + [ + (torch.float32, "contiguous", False), + (torch.float16, "contiguous", False), + (torch.bfloat16, "contiguous", False), + (torch.bfloat16, "strided", False), + (torch.bfloat16, "fp8", False), + (torch.bfloat16, "contiguous", True), + (torch.bfloat16, "fp8", True), + ], +) +def test_te_ops_activation_compile(activation, dtype, layout, cache_input): + _check_te_ops_activation_compile( + activation, dtype=dtype, layout=layout, cache_input=cache_input + ) + + +@pytest.mark.parametrize("activation", ["GELU", "SwiGLU"]) +@pytest.mark.parametrize("cache_input", [False, True]) +@pytest.mark.parametrize("training", [False, True]) +def test_te_ops_activation_autocast_compile(activation, cache_input, training): + _check_te_ops_activation_compile( + activation, cache_input=cache_input, autocast=True, training=training + ) + + +@pytest.mark.parametrize("cache_input", [False, True]) +@pytest.mark.parametrize("interleave_size", [None, 4]) +def test_te_ops_swiglu_dynamic_compile(cache_input, interleave_size): + _check_te_ops_activation_compile( + "SwiGLU", cache_input=cache_input, interleave_size=interleave_size, dynamic=True + ) + + +@pytest.mark.parametrize("activation", ["GELU", "ReLU", "GEGLU", "SwiGLU"]) +@pytest.mark.parametrize("quantization", [None, "fp8", "nvfp4", "nvfp4_no_rht"]) +@pytest.mark.parametrize( + "training,mode", [(False, "default"), (True, "default"), (True, "reduce-overhead")] +) +def test_te_ops_activation_mlp_compile(activation, quantization, training, mode, monkeypatch): + nvfp4 = quantization in ("nvfp4", "nvfp4_no_rht") + if nvfp4 and not nvfp4_available: + pytest.skip(reason_for_no_nvfp4) + if quantization == "fp8" and not fp8_available: + pytest.skip(reason_for_no_fp8) + dtype = torch.float16 if quantization is None else torch.bfloat16 + tols = {"rtol": 0, "atol": 0} if nvfp4 or not training else {} + torch._dynamo.reset() + counters.clear() + monkeypatch.setattr(torch.backends.cuda.matmul, "allow_tf32", False) + in_features, hidden = (128, 128) if nvfp4 else (32, 64) + gated = activation in ("GEGLU", "SwiGLU") + model = ( + te.ops.Sequential( + te.ops.Linear(in_features, hidden * (2 if gated else 1), device="cuda", dtype=dtype), + getattr(te.ops, activation)(), + te.ops.Linear(hidden, in_features, device="cuda", dtype=dtype), + ) + .train(training) + .requires_grad_(training) + ) + if training: + with torch.no_grad(): + for param in model.parameters(): + param.copy_(torch.randint(0, 3, param.shape, device=param.device) / 32) + eager_model = copy.deepcopy(model) + if nvfp4: + quant_recipe = recipe.NVFP4BlockScaling( + disable_rht=quantization == "nvfp4_no_rht", disable_stochastic_rounding=True + ) + else: + quant_recipe = recipe.Float8CurrentScaling(backward_override=None) if quantization else None + + def run(x, module=model): + if quant_recipe is None: + return module(x) + with te.autocast(recipe=quant_recipe): + return module(x) + + if mode == "default": + compiled, graphs = _compile_with_graphs(run) + else: + compiled, graphs = torch.compile(run, fullgraph=True, mode=mode), [] + optimizer = torch.optim.SGD(model.parameters(), lr=0.01) + eager_optimizer = torch.optim.SGD(eager_model.parameters(), lr=0.01) + for step in range(4 if training else 1): + torch.compiler.cudagraph_mark_step_begin() + shape = (2, 16, in_features) if training else (32, in_features) + x = torch.randn(shape, device="cuda", dtype=dtype, requires_grad=training) + if training and step == 0: + # Keep the independent reference reductions well-conditioned. + with torch.no_grad(): + x.copy_(torch.randint(0, 9, x.shape, device=x.device) / 8) + eager_x = x.detach().clone().requires_grad_(training) + with torch.set_grad_enabled(training): + actual = compiled(x) + expected = run(eager_x, eager_model) + torch.testing.assert_close(actual, expected, **tols) + if not training: + continue + dy = torch.randn_like(actual) + if step == 0: + dy.copy_(torch.randint(0, 9, dy.shape, device=dy.device) / 8) + actual.backward(dy) + expected.backward(dy) + torch.testing.assert_close(x.grad, eager_x.grad, **tols) + for param, reference in zip(model.parameters(), eager_model.parameters()): + torch.testing.assert_close(param.grad, reference.grad, **tols) + if quantization is None and step == 0: + ref_x = x.detach().clone().requires_grad_() + params = [p.detach().clone().requires_grad_() for p in model.parameters()] + ref = torch.nn.functional.linear(ref_x.float(), *[p.float() for p in params[:2]]) + ref = _ops_activation_reference(activation, ref.to(dtype).float()).to(dtype) + ref = torch.nn.functional.linear(ref.float(), *[p.float() for p in params[2:]]) + ref = ref.to(dtype) + grads = torch.autograd.grad(ref, (ref_x, *params), dy) + torch.testing.assert_close(actual, ref, **dtype_tols(dtype)) + for got, want in zip((x.grad, *(p.grad for p in model.parameters())), grads): + torch.testing.assert_close(got, want, **dtype_tols(dtype)) + optimizer.step() + eager_optimizer.step() + optimizer.zero_grad(set_to_none=True) + eager_optimizer.zero_grad(set_to_none=True) + if step == 1: + graph_count = len(graphs) + elif step > 1: + assert len(graphs) == graph_count + del actual, expected, x, eager_x + if mode == "default": + fused = training and quantization is not None and activation in ("GELU", "ReLU") + _assert_custom_ops(graphs, activation.lower(), backward=training and not fused) + _assert_custom_ops(graphs, "backwardactivationbias", forward=False, backward=fused) + else: + assert not counters["inductor"]["cudagraph_skips"], counters["inductor"] + assert counters["inductor"]["cudagraph_recorded_non_static_inputs"] > 0 diff --git a/transformer_engine/pytorch/dynamo/fake.py b/transformer_engine/pytorch/dynamo/fake.py index 5f856d62214..e14c9c4b6b5 100644 --- a/transformer_engine/pytorch/dynamo/fake.py +++ b/transformer_engine/pytorch/dynamo/fake.py @@ -76,19 +76,48 @@ def _make_fused_quantization_output_spec( return spec +def fake_activation( + input_: TensorSpec, + quantizer: Optional[Quantizer] = None, + *, + dtype: Optional[torch.dtype] = None, + gated: bool = False, +) -> TensorSpec: + """Describe an activation, including optional output quantization.""" + shape = input_.shape + if gated: + shape = (*shape[:-1], shape[-1] // 2) + return _make_fused_quantization_output_spec( + shape, dtype or input_.dtype, input_.device, quantizer + ) + + +def fake_activation_backward( + input_: TensorSpec, + quantizer: Optional[Quantizer] = None, + *, + dtype: Optional[torch.dtype] = None, +) -> TensorSpec: + """Describe the gradient of the original, pre-activation input.""" + return _make_fused_quantization_output_spec( + input_.shape, dtype or input_.dtype, input_.device, quantizer + ) + + def fake_bgrad( grad_output: TensorSpec, quantizer: Optional[Quantizer] = None, + *, + dtype: Optional[torch.dtype] = None, ) -> tuple[TensorSpec, TensorSpec]: """Describe bias reduction and optional gradient quantization, in native order.""" - db = TensorSpec( - shape=(grad_output.shape[-1],), dtype=grad_output.dtype, device=grad_output.device - ) + dtype = dtype or grad_output.dtype + db = TensorSpec(shape=(grad_output.shape[-1],), dtype=dtype, device=grad_output.device) dx = ( grad_output - if quantizer is None + if quantizer is None and dtype == grad_output.dtype else _make_fused_quantization_output_spec( - grad_output.shape, grad_output.dtype, grad_output.device, quantizer + grad_output.shape, dtype, grad_output.device, quantizer ) ) return db, dx diff --git a/transformer_engine/pytorch/ops/basic/activation.py b/transformer_engine/pytorch/ops/basic/activation.py index ffcb72a9554..dbaaaa106bb 100644 --- a/transformer_engine/pytorch/ops/basic/activation.py +++ b/transformer_engine/pytorch/ops/basic/activation.py @@ -8,12 +8,15 @@ import abc import math from collections.abc import Iterable, Sequence +from dataclasses import dataclass from typing import Any, Optional import torch import transformer_engine_torch as tex from ...constants import DType +from ...dynamo import TensorSpec, TensorOrQuantized +from ...dynamo.fake import fake_activation, fake_activation_backward from ...cpu_offload import is_cpu_offload_enabled, mark_activation_offload from ...tensor.float8_tensor import Float8CurrentScalingQuantizer, Quantizer from ...utils import _compile_safe_warn, clear_tensor_data @@ -36,6 +39,28 @@ ] +@dataclass(slots=True) +class ActivationFwdArgs: + """Activation inputs and quantization configuration.""" + + input_: TensorOrQuantized + dtype: torch.dtype + output_quantizer: Optional[Quantizer] + input_quantizer: Optional[Quantizer] + grad_input_quantizer: Optional[Quantizer] + requires_grad: bool + + +@dataclass(slots=True) +class ActivationBwdArgs: + """Activation gradient and saved input.""" + + grad_output: TensorOrQuantized + input_: TensorOrQuantized + dtype: torch.dtype + grad_input_quantizer: Optional[Quantizer] + + class _ActivationOperation(BasicOperation, metaclass=abc.ABCMeta): r"""Apply activation function @@ -69,83 +94,124 @@ def __init__(self, *, cache_quantized_input: bool = False): super().__init__() self.cache_quantized_input: bool = cache_quantized_input + @staticmethod @abc.abstractmethod - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: """Forward implementation Implementation from transformer_engine.pytorch.cpp_extensions. """ + @staticmethod @abc.abstractmethod - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: """Backward implementation Implementation from transformer_engine_torch. """ - def op_forward( - self, - ctx: OperationContext, - input_: torch.Tensor, - prev_op_grad_output_quantizer: Optional[Quantizer], - next_op_input_quantizer: Optional[Quantizer], - ) -> torch.Tensor: + fwd_args_type = ActivationFwdArgs + bwd_args_type = ActivationBwdArgs + _output_halves_last_dim: bool = False - # Compute dtype - dtype: torch.dtype - if torch.is_autocast_enabled(): - dtype = torch.get_autocast_dtype("cuda") - else: - dtype = input_.dtype + def compile_unsupported_reason(self, mode: str) -> Optional[str]: + if is_cpu_offload_enabled(): + return "CPU offloading" + return super().compile_unsupported_reason(mode) + + def pack_forward_args( + self, + basic_op_ctxs, + input_, + *, + prev_op_grad_output_quantizer, + next_op_input_quantizer, + basic_op_kwargs, + **unused, # pylint: disable=unused-argument + ) -> ActivationFwdArgs: + if basic_op_kwargs[0]: + raise ValueError("Activation forward does not expect keyword arguments") + dtype = torch.get_autocast_dtype("cuda") if torch.is_autocast_enabled() else input_.dtype if dtype not in (torch.float32, torch.float16, torch.bfloat16): raise RuntimeError(f"Unsupported dtype ({dtype})") - - # Check input tensor - x = maybe_dequantize(input_.contiguous(), dtype) - - # Launch kernel - y = self._activation_forward_impl(x, next_op_input_quantizer) - - # Quantize input to FP8 before caching if needed + input_quantizer = None if self.cache_quantized_input: - input_quantizer = Float8CurrentScalingQuantizer(DType.kFloat8E4M3, x.device) + input_quantizer = Float8CurrentScalingQuantizer(DType.kFloat8E4M3, input_.device) input_quantizer.set_usage(rowwise=True, columnwise=False) - x = input_quantizer(x) - - # Save state for backward pass - if ctx.requires_grad: - if is_cpu_offload_enabled(): - mark_activation_offload(x) - ctx.save_for_backward(x) - ctx.dtype = dtype - ctx.prev_op_grad_output_quantizer = prev_op_grad_output_quantizer - - return y - - def op_backward( - self, - ctx: OperationContext, - grad_output: torch.Tensor, - ) -> tuple[torch.Tensor, tuple[()]]: - - # Saved tensors from forward pass - (x,) = ctx.saved_tensors - - # Check input tensor - x = maybe_dequantize(x.contiguous(), ctx.dtype) - - # Check grad output tensor - dy = maybe_dequantize(grad_output.contiguous(), x.dtype) + input_quantizer.internal = True + return ActivationFwdArgs( + input_, + dtype, + next_op_input_quantizer, + input_quantizer, + prev_op_grad_output_quantizer, + basic_op_ctxs[0].requires_grad, + ) - # Launch kernel - dx = self._activation_backward_impl(dy, x, ctx.prev_op_grad_output_quantizer) + @classmethod + def forward_compute(cls, args: ActivationFwdArgs, *, in_custom_op: bool = False): + x = maybe_dequantize(args.input_, args.dtype).contiguous() + y = cls._activation_forward_impl(x, args.output_quantizer) + if args.input_quantizer is not None: + x = args.input_quantizer(x) + # Custom ops return only fresh saved tensors; setup retains the original input. + save_input = args.requires_grad and (not in_custom_op or args.input_quantizer is not None) + return y, [()], (x,) if save_input else () + + @classmethod + def forward_compute_fake(cls, args: ActivationFwdArgs): + x = args.input_ + y = fake_activation( + x, + args.output_quantizer, + dtype=args.dtype, + gated=cls._output_halves_last_dim, + ) + saved = () + if args.requires_grad and args.input_quantizer is not None: + saved = ( + TensorSpec( + shape=x.shape, dtype=args.dtype, device=x.device, quantizer=args.input_quantizer + ), + ) + return y, [()], saved - # Clear input tensor if possible - clear_tensor_data(x) + def forward_setup_context(self, basic_op_ctxs, args, aux) -> None: + ctx = basic_op_ctxs[0] + x = aux[0] if aux else args.input_ + if is_cpu_offload_enabled(): + mark_activation_offload(x) + ctx.save_for_backward(x) + ctx.dtype = args.dtype + ctx.prev_op_grad_output_quantizer = args.grad_input_quantizer + + def pack_backward_args( + self, basic_op_ctxs, grad_output, **unused # pylint: disable=unused-argument + ) -> ActivationBwdArgs: + ctx = basic_op_ctxs[0] + return ActivationBwdArgs( + grad_output, ctx.saved_tensors[0], ctx.dtype, ctx.prev_op_grad_output_quantizer + ) - return dx, () + @classmethod + def backward_compute(cls, args: ActivationBwdArgs, *, in_custom_op: bool = False): + x = maybe_dequantize(args.input_, args.dtype).contiguous() + dy = maybe_dequantize(args.grad_output, args.dtype).contiguous() + dx = cls._activation_backward_impl(dy, x, args.grad_input_quantizer) + if not in_custom_op: + clear_tensor_data(x) + return dx, [()], [()] + + @classmethod + def backward_compute_fake(cls, args: ActivationBwdArgs): + dx = fake_activation_backward( + args.input_, + args.grad_input_quantizer, + dtype=args.dtype, + ) + return dx, [()], [()] class GELU(_ActivationOperation): @@ -161,10 +227,12 @@ class GELU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.gelu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dgelu(*args, **kwargs) @@ -193,10 +261,14 @@ class GLU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + _output_halves_last_dim = True + + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.glu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dglu(*args, **kwargs) @@ -228,10 +300,14 @@ class GEGLU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + _output_halves_last_dim = True + + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.geglu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dgeglu(*args, **kwargs) @@ -247,10 +323,12 @@ class QGELU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.qgelu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dqgelu(*args, **kwargs) @@ -280,10 +358,14 @@ class QGEGLU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + _output_halves_last_dim = True + + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.qgeglu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dqgeglu(*args, **kwargs) @@ -296,10 +378,12 @@ class ReLU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.relu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.drelu(*args, **kwargs) @@ -325,10 +409,14 @@ class ReGLU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + _output_halves_last_dim = True + + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.reglu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dreglu(*args, **kwargs) @@ -343,10 +431,12 @@ class SReLU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.srelu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dsrelu(*args, **kwargs) @@ -600,10 +690,14 @@ class SReGLU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + _output_halves_last_dim = True + + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.sreglu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dsreglu(*args, **kwargs) @@ -616,8 +710,10 @@ class SiLU(_ActivationOperation): """ - def _activation_forward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: return tex.silu(*args, **kwargs) - def _activation_backward_impl(self, *args, **kwargs) -> torch.Tensor: + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: return tex.dsilu(*args, **kwargs) diff --git a/transformer_engine/pytorch/ops/basic/swiglu.py b/transformer_engine/pytorch/ops/basic/swiglu.py index 8009de9b4ae..b9faa675681 100644 --- a/transformer_engine/pytorch/ops/basic/swiglu.py +++ b/transformer_engine/pytorch/ops/basic/swiglu.py @@ -7,6 +7,7 @@ from __future__ import annotations from collections.abc import Iterable, Sequence import math +from dataclasses import dataclass, fields from typing import Any, Optional import torch @@ -18,6 +19,7 @@ from ...utils import clear_tensor_data from ..op import BasicOperation, OperationContext from .._common import maybe_dequantize +from .activation import _ActivationOperation, ActivationFwdArgs, ActivationBwdArgs __all__ = [ "SwiGLU", @@ -29,7 +31,28 @@ ] -class SwiGLU(BasicOperation): +@dataclass(slots=True) +class SwiGLUFwdArgs(ActivationFwdArgs): + """Activation configuration with the GLU input layout.""" + + glu_interleave_size: Optional[int] + + +@dataclass(slots=True) +class SwiGLUBwdArgs(ActivationBwdArgs): + """Activation gradient with the GLU input layout.""" + + glu_interleave_size: Optional[int] + + +def _interleave_glu(tensor: torch.Tensor, size: int, *, reverse: bool = False) -> torch.Tensor: + shape = tensor.shape + blocks = shape[-1] // (2 * size) + middle = (blocks, 2) if reverse else (2, blocks) + return tensor.reshape(-1, *middle, size).transpose(1, 2).contiguous().view(shape) + + +class SwiGLU(_ActivationOperation): r"""Swish gated linear unit The input tensor is split into chunks :math:``a`` and :math:``b`` @@ -88,13 +111,120 @@ def __init__( self.cache_quantized_input: bool = cache_quantized_input self.glu_interleave_size: Optional[int] = glu_interleave_size + fwd_args_type = SwiGLUFwdArgs + bwd_args_type = SwiGLUBwdArgs + _output_halves_last_dim = True + + @staticmethod + def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: + return tex.swiglu(*args, **kwargs) + + @staticmethod + def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: + return tex.dswiglu(*args, **kwargs) + + def pack_forward_args(self, *args, **kwargs) -> SwiGLUFwdArgs: + args = super().pack_forward_args(*args, **kwargs) + return SwiGLUFwdArgs( + **{field.name: getattr(args, field.name) for field in fields(args)}, + glu_interleave_size=self.glu_interleave_size, + ) + + @classmethod + def forward_compute(cls, args: SwiGLUFwdArgs, *, in_custom_op: bool = False): + x = maybe_dequantize(args.input_, args.dtype).contiguous() + swiglu_in = x + if args.glu_interleave_size is not None: + swiglu_in = _interleave_glu(x, args.glu_interleave_size, reverse=True) + out = cls._activation_forward_impl(swiglu_in, args.output_quantizer) + if args.input_quantizer is not None: + x = args.input_quantizer(x) + save_input = args.requires_grad and (not in_custom_op or args.input_quantizer is not None) + return out, [()], (x,) if save_input else () + + def pack_backward_args(self, *args, **kwargs) -> SwiGLUBwdArgs: + args = super().pack_backward_args(*args, **kwargs) + return SwiGLUBwdArgs( + **{field.name: getattr(args, field.name) for field in fields(args)}, + glu_interleave_size=self.glu_interleave_size, + ) + + @classmethod + def backward_compute(cls, args: SwiGLUBwdArgs, *, in_custom_op: bool = False): + x = maybe_dequantize(args.input_, args.dtype).contiguous() + dy = maybe_dequantize(args.grad_output, args.dtype).contiguous() + quantizer = args.grad_input_quantizer + if args.glu_interleave_size is not None: + x = _interleave_glu(x, args.glu_interleave_size, reverse=True) + quantizer = None + dx = cls._activation_backward_impl(dy, x, quantizer) + if args.glu_interleave_size is not None: + dx = _interleave_glu(dx, args.glu_interleave_size) + if not in_custom_op: + clear_tensor_data(args.input_) + return dx, [()], [()] + + @classmethod + def backward_compute_fake(cls, args: SwiGLUBwdArgs): + dx, params, extras = super().backward_compute_fake(args) + if args.glu_interleave_size is not None: + dx.quantizer = None + return dx, params, extras + + +class SiTUGLU(SwiGLU): + r"""Soft-capped SiLU gated linear unit used by Kimi K3. + + See the `Kimi K3 technical report + `__. + + The input is split into gate and up-projection halves and computes + + .. math:: + + \beta_1 \tanh(a / \beta_1) \sigma(a) + \; \beta_2 \tanh(b / \beta_2). + + Parameters + ---------- + beta1 : float, default = 4.0 + Positive gate soft-cap parameter. + beta2 : float, default = 25.0 + Positive up-branch soft-cap parameter. + cache_quantized_input : bool, default = False + Quantize the saved input for backward, as in :class:`SwiGLU`. + glu_interleave_size : int, optional + Block-interleaved GLU layout, as in :class:`SwiGLU`. + """ + + fwd_args_type = None + bwd_args_type = None + + def __init__( + self, + *, + beta1: float = 4.0, + beta2: float = 25.0, + cache_quantized_input: bool = False, + glu_interleave_size: Optional[int] = None, + ) -> None: + super().__init__( + cache_quantized_input=cache_quantized_input, + glu_interleave_size=glu_interleave_size, + ) + self.beta1 = float(beta1) + self.beta2 = float(beta2) + if not math.isfinite(self.beta1) or self.beta1 <= 0.0: + raise ValueError(f"beta1 must be finite and positive, got {self.beta1}") + if not math.isfinite(self.beta2) or self.beta2 <= 0.0: + raise ValueError(f"beta2 must be finite and positive, got {self.beta2}") + def _tex_swiglu_forward( self, input_: torch.Tensor, quantizer: Optional[Quantizer], ) -> torch.Tensor: - """Call the Transformer Engine SwiGLU forward kernel.""" - return tex.swiglu(input_, quantizer) + return tex.situglu(input_, quantizer, self.beta1, self.beta2) def _tex_swiglu_backward( self, @@ -102,8 +232,7 @@ def _tex_swiglu_backward( input_: torch.Tensor, quantizer: Optional[Quantizer], ) -> torch.Tensor: - """Call the Transformer Engine SwiGLU backward kernel.""" - return tex.dswiglu(grad_output, input_, quantizer) + return tex.dsituglu(grad_output, input_, quantizer, self.beta1, self.beta2) def op_forward( self, @@ -213,66 +342,6 @@ def op_backward( return dx, () -class SiTUGLU(SwiGLU): - r"""Soft-capped SiLU gated linear unit used by Kimi K3. - - See the `Kimi K3 technical report - `__. - - The input is split into gate and up-projection halves and computes - - .. math:: - - \beta_1 \tanh(a / \beta_1) \sigma(a) - \; \beta_2 \tanh(b / \beta_2). - - Parameters - ---------- - beta1 : float, default = 4.0 - Positive gate soft-cap parameter. - beta2 : float, default = 25.0 - Positive up-branch soft-cap parameter. - cache_quantized_input : bool, default = False - Quantize the saved input for backward, as in :class:`SwiGLU`. - glu_interleave_size : int, optional - Block-interleaved GLU layout, as in :class:`SwiGLU`. - """ - - def __init__( - self, - *, - beta1: float = 4.0, - beta2: float = 25.0, - cache_quantized_input: bool = False, - glu_interleave_size: Optional[int] = None, - ) -> None: - super().__init__( - cache_quantized_input=cache_quantized_input, - glu_interleave_size=glu_interleave_size, - ) - self.beta1 = float(beta1) - self.beta2 = float(beta2) - if not math.isfinite(self.beta1) or self.beta1 <= 0.0: - raise ValueError(f"beta1 must be finite and positive, got {self.beta1}") - if not math.isfinite(self.beta2) or self.beta2 <= 0.0: - raise ValueError(f"beta2 must be finite and positive, got {self.beta2}") - - def _tex_swiglu_forward( - self, - input_: torch.Tensor, - quantizer: Optional[Quantizer], - ) -> torch.Tensor: - return tex.situglu(input_, quantizer, self.beta1, self.beta2) - - def _tex_swiglu_backward( - self, - grad_output: torch.Tensor, - input_: torch.Tensor, - quantizer: Optional[Quantizer], - ) -> torch.Tensor: - return tex.dsituglu(grad_output, input_, quantizer, self.beta1, self.beta2) - - class ClampedSwiGLU(BasicOperation): r"""GPT-OSS Implementation based on `GPT-OSS `__. diff --git a/transformer_engine/pytorch/ops/fused/backward_activation_bias.py b/transformer_engine/pytorch/ops/fused/backward_activation_bias.py index 3950316a3c2..dedb6c45e19 100644 --- a/transformer_engine/pytorch/ops/fused/backward_activation_bias.py +++ b/transformer_engine/pytorch/ops/fused/backward_activation_bias.py @@ -6,29 +6,36 @@ from __future__ import annotations from typing import Optional - -import torch +from dataclasses import dataclass import transformer_engine_torch as tex from transformer_engine.pytorch.quantization import Recipe from transformer_engine.pytorch.ops.basic import Bias from transformer_engine.pytorch.ops.basic.activation import ( _ActivationOperation, + ActivationBwdArgs, GELU, ReLU, ) from transformer_engine.pytorch.ops.op import ( FusedOperation, FusibleOperation, - OperationContext, ) from ...utils import clear_tensor_data +from ...dynamo.fake import fake_bgrad from .._common import maybe_dequantize _fused_activations = {GELU: tex.dbias_dgelu, ReLU: tex.dbias_drelu} _fusible_activations = tuple(_fused_activations.keys()) +@dataclass(slots=True) +class BackwardActivationBiasArgs(ActivationBwdArgs): + """Activation gradient with a fused bias reduction.""" + + activation: str + + class BackwardActivationBias(FusedOperation): """Fused backward dact + dbias + quantize @@ -38,47 +45,44 @@ class BackwardActivationBias(FusedOperation): def __init__(self, *, bias: Bias, activation: _ActivationOperation): super().__init__((bias, activation)) - self._fused_function = _fused_activations[type(activation)] - - def fuser_backward( - self, - basic_op_ctxs: list[OperationContext], - grad_output: torch.Tensor, - *, - basic_op_grad_extra_outputs: list[tuple[torch.Tensor, ...]], - ) -> tuple[ - torch.Tensor, - list[tuple[Optional[torch.Tensor], ...]], - list[tuple[()]], - ]: - # Get basic operation contexts - bias_op_ctx = basic_op_ctxs[0] - activation_op_ctx = basic_op_ctxs[1] + bwd_args_type = BackwardActivationBiasArgs - # Saved tensors from forward pass - (act_input,) = activation_op_ctx.saved_tensors + def compile_unsupported_reason(self, mode: str) -> Optional[str]: + reason = super().compile_unsupported_reason(mode) + return reason or self.basic_ops[1].compile_unsupported_reason(mode) - # Check activation input tensor - act_input = maybe_dequantize(act_input.contiguous(), activation_op_ctx.dtype) - - # Check grad output tensor - dy = maybe_dequantize(grad_output.contiguous(), act_input.dtype) - - # Get previous op quantizer - quantizer = bias_op_ctx.grad_input_quantizer + def pack_backward_args( + self, basic_op_ctxs, grad_output, **unused # pylint: disable=unused-argument + ) -> BackwardActivationBiasArgs: + bias_ctx, act_ctx = basic_op_ctxs + quantizer = bias_ctx.grad_input_quantizer if quantizer is None: raise RuntimeError( "BackwardActivationBias requires previous op's grad output quantizer, " "but Bias context has no quantizer" ) + return BackwardActivationBiasArgs( + grad_output, + act_ctx.saved_tensors[0], + act_ctx.dtype, + quantizer, + type(self.basic_ops[1]).__name__.lower(), + ) + + @classmethod + def backward_compute(cls, args: BackwardActivationBiasArgs, *, in_custom_op: bool = False): + x = maybe_dequantize(args.input_, args.dtype).contiguous() + dy = maybe_dequantize(args.grad_output, args.dtype).contiguous() + compute = {"gelu": tex.dbias_dgelu, "relu": tex.dbias_drelu}[args.activation] + db, dx = compute(dy, x, args.grad_input_quantizer) + if not in_custom_op: + clear_tensor_data(x) + return dx, [(db,), ()], [(), ()] - # Launch kernel - db, dx = self._fused_function(dy, act_input, quantizer) - - # Clear activation input tensor - clear_tensor_data(act_input) - + @classmethod + def backward_compute_fake(cls, args: BackwardActivationBiasArgs): + db, dx = fake_bgrad(args.grad_output, args.grad_input_quantizer, dtype=args.dtype) return dx, [(db,), ()], [(), ()] @staticmethod From 267639c90f6bc492a0b65bd31781ab5b61cfef02 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 2 Nov 2026 19:20:53 +0100 Subject: [PATCH 02/31] [PyTorch] Honor optimize_for_gemm in quantized outputs Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_quantized_tensor.py | 102 ++++++++++++++++++ tests/pytorch/test_torch_compile.py | 3 +- transformer_engine/pytorch/csrc/common.h | 3 + .../pytorch/csrc/extensions/activation.cpp | 3 + .../pytorch/csrc/extensions/bias.cpp | 5 + .../pytorch/csrc/extensions/cast.cpp | 16 +-- .../pytorch/csrc/extensions/gemm.cpp | 1 + .../pytorch/csrc/extensions/normalization.cpp | 23 ++-- .../pytorch/csrc/extensions/swizzle.cpp | 18 +++- transformer_engine/pytorch/dynamo/fake.py | 72 +------------ .../pytorch/quantized_tensor.py | 3 + 11 files changed, 160 insertions(+), 89 deletions(-) diff --git a/tests/pytorch/test_quantized_tensor.py b/tests/pytorch/test_quantized_tensor.py index 6cd1bd403b1..b4280df0333 100644 --- a/tests/pytorch/test_quantized_tensor.py +++ b/tests/pytorch/test_quantized_tensor.py @@ -8,6 +8,7 @@ import pytest import torch +import transformer_engine_torch as tex import transformer_engine.common.recipe import transformer_engine.pytorch as te @@ -1233,3 +1234,104 @@ def test_restore_real_quantized_storage(monkeypatch, compiled_metadata): assert restored[0] is x and restored[2] is None assert isinstance(tail, tuple) and tail[0] is x torch.testing.assert_close(restored[1].dequantize(), expected, rtol=0, atol=0) + + +@pytest.mark.parametrize( + "quantization,shape", + [ + pytest.param( + "mxfp8", + (128, 128), + marks=pytest.mark.skipif(not mxfp8_available, reason=reason_for_no_mxfp8), + ), + *[ + pytest.param( + name, + shape, + marks=pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4), + ) + for name, shape in [ + ("nvfp4", (32, 128)), + ("nvfp4-rht", (32, 128)), + ("nvfp4-rht", (128, 128)), + ("nvfp4-2d", (128, 128)), + ("nvfp4_row_scaled", (128, 128)), + ("nvfp4_4over6", (32, 128)), + ] + ], + ], +) +@pytest.mark.parametrize("optimize_for_gemm", [False, True]) +@pytest.mark.parametrize( + "operation", + [ + "quantize", + "multi_tensor_quantize", + "split_quantize", + "gelu", + "dgelu", + "scaled_srelu", + "dbias_dgelu", + "bgrad", + "layernorm", + "rmsnorm", + ], +) +def test_native_quantization_scale_layout( + quantization, shape, optimize_for_gemm, operation, monkeypatch +): + """Every producer honors the quantizer's requested scale layout.""" + monkeypatch.setenv("NVTE_NORM_FWD_USE_CUDNN", "1") + if quantization in ("nvfp4-rht", "nvfp4-2d"): + quantizer = NVFP4Quantizer( + with_rht=quantization == "nvfp4-rht", + with_post_rht_amax=quantization == "nvfp4-rht", + with_2d_quantization=quantization == "nvfp4-2d", + stochastic_rounding=False, + with_random_sign_mask=False, + ) + else: + quantizer = make_quantizer(quantization) + inp = torch.randn(shape, dtype=torch.bfloat16, device="cuda") + grad = torch.randn_like(inp) + weight = torch.randn(shape[-1], dtype=inp.dtype, device=inp.device) + bias = torch.randn_like(weight) + scales = torch.rand(shape[0], dtype=torch.float32, device=inp.device) + + def run(out=None): + if operation == "quantize": + return quantizer(inp) + if operation == "multi_tensor_quantize": + return tex.multi_tensor_quantize([inp], [quantizer])[0] + if operation == "split_quantize": + return tex.split_quantize(inp, [shape[0]], [quantizer])[0] + if operation == "gelu": + return tex.gelu(inp, quantizer) + if operation == "dgelu": + return tex.dgelu(grad, inp, quantizer) + if operation == "scaled_srelu": + return tex.scaled_srelu(inp, scales, quantizer) + if operation == "dbias_dgelu": + return tex.dbias_dgelu(grad, inp, quantizer)[1] + if operation == "bgrad": + return tex.bgrad_quantize(inp, quantizer)[1] + if operation == "layernorm": + return tex.layernorm_fwd( + inp, weight, bias, 1e-5, out, quantizer, te.DType.kBFloat16, 0, False + )[0] + if operation == "rmsnorm": + return tex.rmsnorm_fwd(inp, weight, 1e-5, out, quantizer, te.DType.kBFloat16, 0, False)[ + 0 + ] + raise AssertionError(operation) + + quantizer.optimize_for_gemm = False + expected = run().dequantize() + quantizer.optimize_for_gemm = optimize_for_gemm + actual = run() + assert actual._with_gemm_swizzled_scales == optimize_for_gemm + torch.testing.assert_close(actual.dequantize(), expected, atol=0, rtol=0) + if operation in ("layernorm", "rmsnorm"): + assert run(actual) is actual + assert actual._with_gemm_swizzled_scales == optimize_for_gemm + torch.testing.assert_close(actual.dequantize(), expected, atol=0, rtol=0) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 6868de7ef18..e9ab4df8221 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -2970,7 +2970,8 @@ def bgrad(inp): previous_graph_count = len(graphs) dx, db = compiled(inp) expected_db, expected_dx = tex.bgrad_quantize(inp, quantizer) - assert dx._with_gemm_swizzled_scales == expected_dx._with_gemm_swizzled_scales + assert dx._with_gemm_swizzled_scales == optimize_for_gemm + assert expected_dx._with_gemm_swizzled_scales == optimize_for_gemm torch.testing.assert_close(db, expected_db, atol=0, rtol=0) torch.testing.assert_close(dx.dequantize(), expected_dx.dequantize(), atol=0, rtol=0) torch.testing.assert_close( diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index 9a3c6314054..c3ecc35eb1e 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -135,6 +135,9 @@ class Quantizer { virtual void quantize(const TensorWrapper& input, TensorWrapper& out, const std::optional& noop_flag = std::nullopt) = 0; + /*! @brief Apply the requested GEMM scale layout to a computed tensor. */ + void optimize_tensor_for_gemm(TensorWrapper& tensor, py::handle tensor_py) const; + virtual ~Quantizer() = default; DType dtype = DType::kNumTypes; diff --git a/transformer_engine/pytorch/csrc/extensions/activation.cpp b/transformer_engine/pytorch/csrc/extensions/activation.cpp index 86419f0d3ed..cdd56e2e4a7 100644 --- a/transformer_engine/pytorch/csrc/extensions/activation.cpp +++ b/transformer_engine/pytorch/csrc/extensions/activation.cpp @@ -125,6 +125,7 @@ py::object activation_helper(const at::Tensor& input, py::handle quantizer, int NVTE_ERROR("Invalid activation implementation (", static_cast(impl), ")"); } + quantizer_cpp->optimize_tensor_for_gemm(out_nvte, out_py); return out_py; } @@ -241,6 +242,7 @@ py::object dactivation_helper(const at::Tensor& grad_output, const at::Tensor& i NVTE_ERROR("Invalid activation implementation (", static_cast(impl), ")"); } + quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); return grad_input_py; } } // namespace @@ -438,6 +440,7 @@ py::object maybe_quantize(const at::Tensor& tensor, py::handle quantizer) { auto fake_dtype = GetTransformerEngineDType(tensor.scalar_type()); auto [out_nvte, out_py] = quantizer_cpp->create_tensor(shape, fake_dtype); quantizer_cpp->quantize(tensor_nvte, out_nvte); + quantizer_cpp->optimize_tensor_for_gemm(out_nvte, out_py); return out_py; } diff --git a/transformer_engine/pytorch/csrc/extensions/bias.cpp b/transformer_engine/pytorch/csrc/extensions/bias.cpp index 2dbeb965a1a..3631b1641d2 100644 --- a/transformer_engine/pytorch/csrc/extensions/bias.cpp +++ b/transformer_engine/pytorch/csrc/extensions/bias.cpp @@ -67,6 +67,7 @@ std::vector bgrad_quantize(const at::Tensor &grad_output, py::handle // Trivial impl if tensors are empty if (product(shape) == 0) { grad_bias_torch.zero_(); + quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } @@ -91,6 +92,7 @@ std::vector bgrad_quantize(const at::Tensor &grad_output, py::handle if (!with_fused_kernel) { sum_bias_grad(grad_bias_torch, grad_output_torch); quantizer_cpp->quantize(grad_output_nvte, grad_input_nvte); + quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } @@ -116,6 +118,7 @@ std::vector bgrad_quantize(const at::Tensor &grad_output, py::handle workspace_nvte.data(), stream); }); + quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } @@ -149,6 +152,7 @@ std::vector dact_dbias( // Return immediately if tensors are empty if (product(output_shape) == 0) { grad_bias_torch.zero_(); + quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } @@ -259,6 +263,7 @@ std::vector dact_dbias( NVTE_ERROR("Invalid implementation"); } + quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } diff --git a/transformer_engine/pytorch/csrc/extensions/cast.cpp b/transformer_engine/pytorch/csrc/extensions/cast.cpp index 3d61dc525f8..226f7c92b04 100644 --- a/transformer_engine/pytorch/csrc/extensions/cast.cpp +++ b/transformer_engine/pytorch/csrc/extensions/cast.cpp @@ -81,11 +81,7 @@ py::object quantize(const at::Tensor &tensor, py::handle quantizer, const py::ob // Perform quantization quantizer_cpp->quantize(input_cpp, output_cpp, noop_flag_cpp); - // Post-quantize swizzle for quantizers whose kernel does not bake - // the GEMM-swizzled scale layout in directly - if (quantizer_cpp->optimize_for_gemm && !output_cpp.get_with_gemm_swizzled_scales()) { - inplace_swizzle_scale_for_gemm(output_py); - } + quantizer_cpp->optimize_tensor_for_gemm(output_cpp, output_py); return output_py; } @@ -128,9 +124,7 @@ py::object nvfp4_quantize_with_amax(const at::Tensor &tensor, py::handle quantiz } nvfp4_quantizer_cpp->quantize_impl(input_cpp, output_cpp, std::nullopt, false); - if (quantizer_cpp->optimize_for_gemm && !output_cpp.get_with_gemm_swizzled_scales()) { - inplace_swizzle_scale_for_gemm(output_py); - } + quantizer_cpp->optimize_tensor_for_gemm(output_cpp, output_py); return output_py; } @@ -1165,6 +1159,9 @@ std::vector multi_tensor_quantize(const std::vector &ten // Perform multi-tensor quantization multi_tensor_quantize_impl(input_cpp_list, quantizer_list, quantizer_cpp_list, output_cpp_list); + for (size_t i = 0; i < output_cpp_list.size(); ++i) { + quantizer_cpp_list[i]->optimize_tensor_for_gemm(output_cpp_list[i], output_py_list[i]); + } return output_py_list; } @@ -2265,6 +2262,9 @@ std::vector split_quantize(const at::Tensor &tensor, multi_tensor_quantize_impl(input_list, quantizer_list, quantizer_cpp_list, output_cpp_list); } + for (size_t i = 0; i < output_cpp_list.size(); ++i) { + quantizer_cpp_list[i]->optimize_tensor_for_gemm(output_cpp_list[i], output_py_list[i]); + } return output_py_list; } diff --git a/transformer_engine/pytorch/csrc/extensions/gemm.cpp b/transformer_engine/pytorch/csrc/extensions/gemm.cpp index c3484802246..124230b02bf 100644 --- a/transformer_engine/pytorch/csrc/extensions/gemm.cpp +++ b/transformer_engine/pytorch/csrc/extensions/gemm.cpp @@ -424,6 +424,7 @@ std::vector gemm(py::handle A, bool transa, py::handle B, bool trans // Quantize the output std::unique_ptr my_quantizer = convert_quantizer(quantizer); my_quantizer->quantize(unquantized_D_tensor, D_tensor); + my_quantizer->optimize_tensor_for_gemm(D_tensor, D); } // Pack outputs std::vector out; diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index e93c01026a9..fae0315253d 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -135,14 +135,14 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe // Output tensor TensorWrapper out_nvte; if (out.is_none()) { - if (impl == Impl::FULLY_FUSED) { - // FP8 has no special logic to optimize for GEMM, MXFP8 cuDNN - // kernel does not support GEMM swizzled scales - quantizer_cpp->optimize_for_gemm = false; - } std::tie(out_nvte, out) = quantizer_cpp->create_tensor(shape, out_dtype); } else { - out_nvte = makeTransformerEngineTensor(out, quantizer); + std::tie(out_nvte, out) = quantizer_cpp->convert_and_update_tensor(out); + } + if (impl == Impl::FULLY_FUSED && IsMXFP8Quantizers(quantizer.ptr())) { + // cuDNN writes compact scales, including into preallocated outputs. + out_nvte.set_with_gemm_swizzled_scales(false); + out.attr("_with_gemm_swizzled_scales") = false; } // Construct unquantized output tensor if needed @@ -213,6 +213,7 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe } } + quantizer_cpp->optimize_tensor_for_gemm(out_nvte, out); return {out, py::cast(mu_py), py::cast(rsigma_py)}; } @@ -384,12 +385,13 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w // Output tensor TensorWrapper out_nvte; if (out.is_none()) { - if (impl == Impl::FUSED_NORM_QUANT_UNSWIZZLED) { - quantizer_cpp->optimize_for_gemm = false; - } std::tie(out_nvte, out) = quantizer_cpp->create_tensor(shape, out_dtype); } else { - out_nvte = makeTransformerEngineTensor(out, quantizer); + std::tie(out_nvte, out) = quantizer_cpp->convert_and_update_tensor(out); + } + if (impl == Impl::FUSED_NORM_QUANT_UNSWIZZLED) { + out_nvte.set_with_gemm_swizzled_scales(false); + out.attr("_with_gemm_swizzled_scales") = false; } // Construct unquantized output tensor if needed @@ -458,6 +460,7 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w } } + quantizer_cpp->optimize_tensor_for_gemm(out_nvte, out); return {out, py::none(), py::cast(rsigma_py)}; } diff --git a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp index 03eec7d80fd..052c06a8080 100644 --- a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp +++ b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp @@ -727,10 +727,9 @@ void inplace_multi_tensor_swizzle_scales_for_gemm_unchecked(std::vector Optional[tuple[int, int]]: - return tex.nvfp4_gemm_swizzled_scales_alignment( - optimize_for_gemm, - with_rht, - with_2d_quantization, - row_scaled_nvfp4, - nvfp4_use_4over6, - sm_arch, - disable_rht_cast_fusion, - ) - - -def _make_fused_quantization_output_spec( - shape: tuple[int, ...], - dtype: torch.dtype, - device: Optional[torch.device], - quantizer: Optional[Quantizer], -) -> TensorSpec: - """Describe a native output without a separate post-quantize swizzle.""" - spec = TensorSpec(shape=tuple(shape), dtype=dtype, device=device, quantizer=quantizer) - quantizer = spec.quantizer - if not isinstance(quantizer, NVFP4Quantizer): - return spec - major, minor = get_device_compute_capability(device) - # Direct environ access lets Dynamo guard changes to the flag. - disable_rht_cast_fusion = bool( - int(os.environ.get("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", "0") or "0") - ) - alignment = _nvfp4_gemm_swizzled_scales_alignment( - quantizer.optimize_for_gemm, - quantizer.with_rht, - quantizer.with_2d_quantization, - quantizer.row_scaled_nvfp4, - quantizer.nvfp4_use_4over6, - major * 10 + minor, - disable_rht_cast_fusion, - ) - # Keep divisibility checks visible to Dynamo without specializing the dimensions. - rows, cols = math.prod(shape[:-1]), shape[-1] - if alignment is None: - spec.with_gemm_swizzled_scales = False - else: - row_alignment, col_alignment = alignment - spec.with_gemm_swizzled_scales = bool( - rows % row_alignment == 0 and cols % col_alignment == 0 - ) - return spec def fake_bgrad( @@ -87,8 +22,11 @@ def fake_bgrad( dx = ( grad_output if quantizer is None - else _make_fused_quantization_output_spec( - grad_output.shape, grad_output.dtype, grad_output.device, quantizer + else TensorSpec( + shape=grad_output.shape, + dtype=grad_output.dtype, + device=grad_output.device, + quantizer=quantizer, ) ) return db, dx diff --git a/transformer_engine/pytorch/quantized_tensor.py b/transformer_engine/pytorch/quantized_tensor.py index 204e703e1d9..c486bfc7586 100644 --- a/transformer_engine/pytorch/quantized_tensor.py +++ b/transformer_engine/pytorch/quantized_tensor.py @@ -450,6 +450,9 @@ class Quantizer(abc.ABC): """Whether to solely optimize for matrix multiplication + MXFP8 and NVFP4 outputs use GEMM-swizzled scales even when a separate + swizzle is needed after quantization. + The resulting quantized tensors are not guaranteed to support any operation other than matrix multiplication. Use with care since this is likely to break communication, checkpointing, and many From 466ef8903dc99978bfa214f802a9d571f728f778 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 2 Nov 2026 19:32:11 +0100 Subject: [PATCH 03/31] Remove the unused scale-layout binding and refine regression coverage Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_quantized_tensor.py | 9 +++++++-- tests/pytorch/test_torch_compile.py | 6 +++++- transformer_engine/pytorch/csrc/extensions/pybind.cpp | 5 ----- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/pytorch/test_quantized_tensor.py b/tests/pytorch/test_quantized_tensor.py index b4280df0333..e6bcad04692 100644 --- a/tests/pytorch/test_quantized_tensor.py +++ b/tests/pytorch/test_quantized_tensor.py @@ -8,10 +8,10 @@ import pytest import torch -import transformer_engine_torch as tex import transformer_engine.common.recipe import transformer_engine.pytorch as te +import transformer_engine_torch as tex from transformer_engine.pytorch import ( Float8Quantizer, Float8CurrentScalingQuantizer, @@ -1304,7 +1304,12 @@ def run(out=None): if operation == "multi_tensor_quantize": return tex.multi_tensor_quantize([inp], [quantizer])[0] if operation == "split_quantize": - return tex.split_quantize(inp, [shape[0]], [quantizer])[0] + return tex.split_quantize( + inp, + [shape[0]], + [quantizer], + disable_bulk_allocation=shape[0] % 64 != 0 or quantization == "nvfp4-2d", + )[0] if operation == "gelu": return tex.gelu(inp, quantizer) if operation == "dgelu": diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index e9ab4df8221..6dea6836187 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -2984,7 +2984,11 @@ def bgrad(inp): inp = torch.randn(shape, device="cuda", dtype=torch.bfloat16) _, expected_dx = tex.bgrad_quantize(inp, quantizer) _, dx_spec = fake_bgrad(to_tensor_spec(inp), quantizer) - assert dx_spec.with_gemm_swizzled_scales == expected_dx._with_gemm_swizzled_scales + metadata = dx_spec.create_metadata() + assert ( + metadata["nontensor_kwargs"]["with_gemm_swizzled_scales"] + == expected_dx._with_gemm_swizzled_scales + ) for name, (inner_shape, inner_dtype) in dx_spec.quantizer.inner_tensor_specs(shape).items(): inner = getattr(expected_dx, name) assert inner.shape == inner_shape diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index c852d4bac1e..895b9a96e87 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -213,11 +213,6 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { py::arg("output") = py::none(), py::arg("noop") = py::none()); m.def("dequantize", &transformer_engine::pytorch::dequantize, "Dequantize", py::arg("input"), py::arg("otype")); - m.def("nvfp4_gemm_swizzled_scales_alignment", - &transformer_engine::pytorch::NVFP4Quantizer::gemm_swizzled_scales_alignment, - py::arg("optimize_for_gemm"), py::arg("with_rht"), py::arg("with_2d_quantization"), - py::arg("row_scaled_nvfp4"), py::arg("nvfp4_use_4over6"), py::arg("sm_arch"), - py::arg("disable_rht_cast_fusion")); m.def("create_empty_quantized_tensor", &transformer_engine::pytorch::create_empty_quantized_tensor, "Create an empty quantized tensor", py::arg("quantizer"), py::arg("shape"), From 0e67f6bbfc88dba851b3bd4b6967bed79bc72519 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Mon, 2 Nov 2026 19:39:51 +0100 Subject: [PATCH 04/31] Preserve 2D MXFP8 normalization semantics Signed-off-by: Pawel Gadzinski --- .../pytorch/csrc/extensions/normalization.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index fae0315253d..811a32152c8 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -106,9 +106,12 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe if (quantizer.is_none() || IsFloat8Quantizers(quantizer.ptr())) { impl = Impl::FULLY_FUSED; } else if (IsMXFP8Quantizers(quantizer.ptr())) { - if (transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN") && outer_size % 128 == 0 && + auto mxfp8_quantizer_cpp = dynamic_cast(quantizer_cpp.get()); + NVTE_CHECK(mxfp8_quantizer_cpp != nullptr, "Could not cast to MXFP8 quantizer"); + // cuDNN quantizes 1D blocks and requires full 128x128 tiles. + if (!mxfp8_quantizer_cpp->with_2d_quantization && + transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN") && outer_size % 128 == 0 && inner_size % 128 == 0) { - // cuDNN MXFP8 kernel requires full 128x128 tiles impl = Impl::FULLY_FUSED; } } else if (detail::IsFloat8CurrentScalingQuantizers(quantizer.ptr()) && From b35179c4615998861d013fb70bff2651f9a39cdd Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Wed, 4 Nov 2026 15:11:26 +0100 Subject: [PATCH 05/31] Batch GEMM scale finalization when all quantizers request it Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_quantized_tensor.py | 111 ++++++++++++++++++ .../pytorch/csrc/extensions/cast.cpp | 67 ++++++++++- 2 files changed, 172 insertions(+), 6 deletions(-) diff --git a/tests/pytorch/test_quantized_tensor.py b/tests/pytorch/test_quantized_tensor.py index e6bcad04692..b1d88f3cc5f 100644 --- a/tests/pytorch/test_quantized_tensor.py +++ b/tests/pytorch/test_quantized_tensor.py @@ -1340,3 +1340,114 @@ def run(out=None): assert run(actual) is actual assert actual._with_gemm_swizzled_scales == optimize_for_gemm torch.testing.assert_close(actual.dequantize(), expected, atol=0, rtol=0) + + +@pytest.mark.skipif(not nvfp4_available, reason=reason_for_no_nvfp4) +@pytest.mark.parametrize("operation", ["multi_tensor_quantize", "split_quantize"]) +@pytest.mark.parametrize( + "case", + [ + "all", + "compact", + "rowwise", + "mixed_usage", + "mixed_optimize", + "no_optimize", + "mixed_formats", + "already_swizzled", + "all_swizzled", + "empty", + ], +) +def test_multi_tensor_quantization_scale_layout(operation, case): + """Batched output finalization preserves each tensor's data, usage and layout.""" + shapes = [(64, 128)] * 3 + if case == "all_swizzled": + shapes = [(128, 128)] * 3 + if case == "empty": + shapes[1] = (0, 128) + quantizers = [] + for i in range(3): + if case == "mixed_formats" and i == 1: + quantizer = MXFP8Quantizer(te.DType.kFloat8E4M3) + else: + with_rht = case in ("all", "no_optimize", "all_swizzled") + with_rht |= case == "already_swizzled" and i != 1 + quantizer = NVFP4Quantizer( + with_rht=with_rht, + with_post_rht_amax=with_rht, + stochastic_rounding=False, + with_random_sign_mask=False, + ) + quantizer.optimize_for_gemm = case != "no_optimize" and not ( + case == "mixed_optimize" and i == 1 + ) + if case == "rowwise": + quantizer.set_usage(rowwise=True, columnwise=False) + elif case == "mixed_usage": + quantizer.set_usage(rowwise=i != 2, columnwise=i != 0) + quantizers.append(quantizer) + inputs = [torch.randn(shape, dtype=torch.bfloat16, device="cuda") for shape in shapes] + references = [quantizer(inp) for quantizer, inp in zip(quantizers, inputs)] + if operation == "multi_tensor_quantize": + outputs = tex.multi_tensor_quantize(inputs, quantizers) + else: + outputs = tex.split_quantize( + torch.cat(inputs), + [shape[0] for shape in shapes], + quantizers, + disable_bulk_allocation=case + in ( + "compact", + "rowwise", + "mixed_usage", + "mixed_optimize", + "mixed_formats", + "already_swizzled", + "empty", + ), + ) + for actual, expected, quantizer in zip(outputs, references, quantizers): + assert actual._with_gemm_swizzled_scales == quantizer.optimize_for_gemm + assert actual.shape == expected.shape + for usage in ("rowwise", "columnwise"): + actual_data = getattr(actual, f"_{usage}_data") + expected_data = getattr(expected, f"_{usage}_data") + actual_scales = getattr(actual, f"_{usage}_scale_inv") + expected_scales = getattr(expected, f"_{usage}_scale_inv") + assert (actual_data is None) == (expected_data is None) + assert (actual_scales is None) == (expected_scales is None) + if actual_data is not None: + torch.testing.assert_close(actual_data, expected_data, atol=0, rtol=0) + assert actual_scales.shape == expected_scales.shape + if actual._rowwise_data is not None: + torch.testing.assert_close( + actual.dequantize(dtype=torch.float32), + expected.dequantize(dtype=torch.float32), + atol=0, + rtol=0, + ) + if actual._columnwise_data is not None: + + def columnwise_view(tensor): + return NVFP4Tensor( + shape=(tensor.shape[-1], tensor.shape[0]), + dtype=torch.float32, + rowwise_data=tensor._columnwise_data, + rowwise_scale_inv=tensor._columnwise_scale_inv, + columnwise_data=None, + columnwise_scale_inv=None, + amax_rowwise=tensor._amax_columnwise, + amax_columnwise=None, + fp4_dtype=quantizer.dtype, + quantizer=quantizer, + with_gemm_swizzled_scales=tensor._with_gemm_swizzled_scales, + ) + + if not isinstance(quantizer, MXFP8Quantizer): + torch.testing.assert_close( + columnwise_view(actual).dequantize(dtype=torch.float32), + columnwise_view(expected).dequantize(dtype=torch.float32), + atol=0, + rtol=0, + ) diff --git a/transformer_engine/pytorch/csrc/extensions/cast.cpp b/transformer_engine/pytorch/csrc/extensions/cast.cpp index 226f7c92b04..4479a51e055 100644 --- a/transformer_engine/pytorch/csrc/extensions/cast.cpp +++ b/transformer_engine/pytorch/csrc/extensions/cast.cpp @@ -38,6 +38,65 @@ std::vector get_tensor_shape(const TensorWrapper &tensor) { return std::vector(shape.data, shape.data + shape.ndim); } +void optimize_tensors_for_gemm(const std::vector> &quantizers, + std::vector &tensors, + std::vector &tensors_py) { + if (tensors.empty()) { + return; + } + if (std::all_of(tensors.begin(), tensors.end(), + [](const auto &tensor) { return tensor.get_with_gemm_swizzled_scales(); })) { + return; + } + const auto scaling_mode = tensors.front().scaling_mode(); + const bool use_batch = + tensors.size() > 1 && + (scaling_mode == NVTE_MXFP8_1D_SCALING || scaling_mode == NVTE_NVFP4_1D_SCALING) && + std::all_of(quantizers.begin(), quantizers.end(), + [](const auto &quantizer) { return quantizer->optimize_for_gemm; }) && + std::all_of(tensors.begin(), tensors.end(), [scaling_mode](const auto &tensor) { + return tensor.scaling_mode() == scaling_mode; + }); + if (!use_batch) { + for (size_t i = 0; i < tensors.size(); ++i) { + quantizers[i]->optimize_tensor_for_gemm(tensors[i], tensors_py[i]); + } + return; + } + + for (bool rowwise : {true, false}) { + const bool has_scales = + std::any_of(tensors.begin(), tensors.end(), [rowwise](const auto &tensor) { + const auto scales = + rowwise ? tensor.get_rowwise_scale_inv() : tensor.get_columnwise_scale_inv(); + return !tensor.get_with_gemm_swizzled_scales() && scales.data_ptr != nullptr; + }); + if (has_scales) { + inplace_multi_tensor_swizzle_scales_for_gemm(tensors_py, rowwise, !rowwise); + } + } + for (size_t i = 0; i < tensors.size(); ++i) { + auto &tensor = tensors[i]; + if (tensor.get_with_gemm_swizzled_scales()) { + continue; + } + for (bool rowwise : {true, false}) { + const auto scales = + rowwise ? tensor.get_rowwise_scale_inv() : tensor.get_columnwise_scale_inv(); + if (scales.data_ptr != nullptr) { + const auto scales_py = tensors_py[i] + .attr(rowwise ? "_rowwise_scale_inv" : "_columnwise_scale_inv") + .cast(); + tensor.set_parameter(rowwise ? kNVTERowwiseScaleInv : kNVTEColumnwiseScaleInv, + scales_py.data_ptr(), static_cast(scales.dtype), scales.shape); + } + } + // Both orientations must be converted before publishing the shared layout flag. + tensors_py[i].attr("_with_gemm_swizzled_scales") = true; + tensor.set_with_gemm_swizzled_scales(true); + } +} + void allreduce_nvfp4_amax_tensors(NVFP4Quantizer *nvfp4_quantizer_cpp, std::vector &&amax_tensors) { if (!nvfp4_quantizer_cpp->with_amax_reduction || amax_tensors.empty()) { @@ -1159,9 +1218,7 @@ std::vector multi_tensor_quantize(const std::vector &ten // Perform multi-tensor quantization multi_tensor_quantize_impl(input_cpp_list, quantizer_list, quantizer_cpp_list, output_cpp_list); - for (size_t i = 0; i < output_cpp_list.size(); ++i) { - quantizer_cpp_list[i]->optimize_tensor_for_gemm(output_cpp_list[i], output_py_list[i]); - } + optimize_tensors_for_gemm(quantizer_cpp_list, output_cpp_list, output_py_list); return output_py_list; } @@ -2262,9 +2319,7 @@ std::vector split_quantize(const at::Tensor &tensor, multi_tensor_quantize_impl(input_list, quantizer_list, quantizer_cpp_list, output_cpp_list); } - for (size_t i = 0; i < output_cpp_list.size(); ++i) { - quantizer_cpp_list[i]->optimize_tensor_for_gemm(output_cpp_list[i], output_py_list[i]); - } + optimize_tensors_for_gemm(quantizer_cpp_list, output_cpp_list, output_py_list); return output_py_list; } From d64234e6c6c17ef0da10826e81ec892f1ae30d6d Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Wed, 4 Nov 2026 19:36:34 +0100 Subject: [PATCH 06/31] Prepare reused GEMM outputs before quantization Signed-off-by: Pawel Gadzinski --- tests/pytorch/nvfp4/test_nvfp4_gemm_exact.py | 27 +++++++++++++++++++ .../pytorch/csrc/extensions/gemm.cpp | 4 +++ 2 files changed, 31 insertions(+) diff --git a/tests/pytorch/nvfp4/test_nvfp4_gemm_exact.py b/tests/pytorch/nvfp4/test_nvfp4_gemm_exact.py index 5062cf6b42f..c72c441e43a 100644 --- a/tests/pytorch/nvfp4/test_nvfp4_gemm_exact.py +++ b/tests/pytorch/nvfp4/test_nvfp4_gemm_exact.py @@ -66,6 +66,33 @@ def gemm(w_q, x_q): torch.testing.assert_close(gemm(w_test, x_test), gemm(w_ref, x_ref), atol=0, rtol=0) +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +@pytest.mark.parametrize("shape", [(32, 128), (128, 128)]) +@pytest.mark.parametrize("with_rht", [False, True]) +@pytest.mark.parametrize("optimize_for_gemm", [False, True]) +def test_gemm_reuse_quantized_output(shape, with_rht, optimize_for_gemm): + """Requantizing a GEMM output resets its previous scale layout.""" + torch.manual_seed(42) + weight = torch.randn(shape[-1], shape[-1], device="cuda", dtype=torch.bfloat16) + inp = torch.randn(shape, device="cuda", dtype=torch.bfloat16) + next_inp = torch.randn_like(inp) + quantizer = NVFP4Quantizer( + with_rht=with_rht, + with_post_rht_amax=with_rht, + stochastic_rounding=False, + with_random_sign_mask=False, + ) + quantizer.optimize_for_gemm = optimize_for_gemm + out = general_gemm(weight, inp, quantization_params=quantizer)[0] + expected = quantizer(general_gemm(weight, next_inp)[0]) + + actual = general_gemm(weight, next_inp, quantization_params=quantizer, out=out)[0] + + assert actual is out + assert actual._with_gemm_swizzled_scales == optimize_for_gemm + torch.testing.assert_close(actual.dequantize(), expected.dequantize(), atol=0, rtol=0) + + def check_nvfp4_gemm_versus_reference( x_dtype: torch.dtype, w_dtype: torch.dtype, diff --git a/transformer_engine/pytorch/csrc/extensions/gemm.cpp b/transformer_engine/pytorch/csrc/extensions/gemm.cpp index 124230b02bf..2c27705062f 100644 --- a/transformer_engine/pytorch/csrc/extensions/gemm.cpp +++ b/transformer_engine/pytorch/csrc/extensions/gemm.cpp @@ -227,6 +227,7 @@ std::vector gemm(py::handle A, bool transa, py::handle B, bool trans DType output_dtype = out_dtype ? *out_dtype : A_tensor.dtype(); // Output tensor + const bool reuse_output = !D.is_none(); TensorWrapper D_tensor; if (D.is_none()) { std::tie(D_tensor, D) = createOutputTensor(D_shape, output_dtype, quantizer); @@ -423,6 +424,9 @@ std::vector gemm(py::handle A, bool transa, py::handle B, bool trans if (unfused_quantization_needed) { // Quantize the output std::unique_ptr my_quantizer = convert_quantizer(quantizer); + if (reuse_output) { + std::tie(D_tensor, D) = my_quantizer->convert_and_update_tensor(D); + } my_quantizer->quantize(unquantized_D_tensor, D_tensor); my_quantizer->optimize_tensor_for_gemm(D_tensor, D); } From 19713927538e9030501812731cfffe133c2f7667 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Wed, 4 Nov 2026 20:06:53 +0100 Subject: [PATCH 07/31] Separate compact NVFP4 scales when post-swizzling Signed-off-by: Pawel Gadzinski --- .../pytorch/csrc/extensions/cast.cpp | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/transformer_engine/pytorch/csrc/extensions/cast.cpp b/transformer_engine/pytorch/csrc/extensions/cast.cpp index 4479a51e055..f685f6ac727 100644 --- a/transformer_engine/pytorch/csrc/extensions/cast.cpp +++ b/transformer_engine/pytorch/csrc/extensions/cast.cpp @@ -1441,8 +1441,7 @@ std::tuple, std::vector> bulk_allocate_mx return retval; } -// allocate fp4 data, fp8 scalings, and amax values -// layout: [fp4_data0, ..., fp4_dataN, fp8_scaling0, ..., fp8_scalingN, amax0, ..., amaxN] +// Bulk-allocate FP4 data, scales, and amax values. // amax buffer will be zeroed out by later amax kernels, so we can use empty to allocate std::tuple, std::vector, bool> bulk_allocate_nvfp4_tensors( std::vector> &shape_list, std::vector &quantizer_py_list, @@ -1510,6 +1509,25 @@ std::tuple, std::vector, bool> bulk_alloc const bool with_gemm_swizzled_scales = group_optimize_for_gemm && group_with_rht && all_tensors_rht_cast_fusion_eligible; + auto allocate_buffers = [&](auto &shapes, auto &dtypes, auto &alignments) { + if (!group_optimize_for_gemm || with_gemm_swizzled_scales) { + return bulk_allocate(shapes, dtypes, std::nullopt, alignments); + } + + // Compact scales must be releasable after swizzling, independently of data and amax. + const std::vector> scale_shapes(shapes.begin() + num_tensors, + shapes.begin() + 2 * num_tensors); + shapes.erase(shapes.begin() + num_tensors, shapes.begin() + 2 * num_tensors); + dtypes.erase(dtypes.begin() + num_tensors, dtypes.begin() + 2 * num_tensors); + alignments.erase(alignments.begin() + num_tensors, alignments.begin() + 2 * num_tensors); + auto tensors = bulk_allocate(shapes, dtypes, std::nullopt, alignments); + auto scales = + bulk_allocate(scale_shapes, std::vector(num_tensors, torch::kUInt8), + std::nullopt, std::vector(num_tensors, 16)); + tensors.insert(tensors.begin() + num_tensors, scales.begin(), scales.end()); + return tensors; + }; + // Helper function to get size of byte buffer holding FP4 data (last dim divided by 2) auto fp4_byte_shape = [](const std::vector &shape) -> std::vector { NVTE_CHECK(!shape.empty()); @@ -1568,7 +1586,7 @@ std::tuple, std::vector, bool> bulk_alloc dtypes.insert(dtypes.end(), num_tensors, torch::kFloat32); alignments.insert(alignments.end(), num_tensors, 16); } - auto tensors = bulk_allocate(shapes, dtypes, std::nullopt, alignments); + auto tensors = allocate_buffers(shapes, dtypes, alignments); // Split data, scale, and amax tensors for (size_t i = 0; i < num_tensors; ++i) { @@ -1626,7 +1644,7 @@ std::tuple, std::vector, bool> bulk_alloc dtypes.insert(dtypes.end(), num_tensors, torch::kFloat32); alignments.insert(alignments.end(), num_tensors, 16); } - auto tensors = bulk_allocate(shapes, dtypes, std::nullopt, alignments); + auto tensors = allocate_buffers(shapes, dtypes, alignments); // Split data, scale, and amax tensors for (size_t i = 0; i < num_tensors; ++i) { From 9a0068209a939d2ea6b421dfb69de22bc53a557c Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Wed, 4 Nov 2026 23:38:55 +0100 Subject: [PATCH 08/31] Remove the quantized GEMM output reuse test Signed-off-by: Pawel Gadzinski --- tests/pytorch/nvfp4/test_nvfp4_gemm_exact.py | 27 -------------------- 1 file changed, 27 deletions(-) diff --git a/tests/pytorch/nvfp4/test_nvfp4_gemm_exact.py b/tests/pytorch/nvfp4/test_nvfp4_gemm_exact.py index c72c441e43a..5062cf6b42f 100644 --- a/tests/pytorch/nvfp4/test_nvfp4_gemm_exact.py +++ b/tests/pytorch/nvfp4/test_nvfp4_gemm_exact.py @@ -66,33 +66,6 @@ def gemm(w_q, x_q): torch.testing.assert_close(gemm(w_test, x_test), gemm(w_ref, x_ref), atol=0, rtol=0) -@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) -@pytest.mark.parametrize("shape", [(32, 128), (128, 128)]) -@pytest.mark.parametrize("with_rht", [False, True]) -@pytest.mark.parametrize("optimize_for_gemm", [False, True]) -def test_gemm_reuse_quantized_output(shape, with_rht, optimize_for_gemm): - """Requantizing a GEMM output resets its previous scale layout.""" - torch.manual_seed(42) - weight = torch.randn(shape[-1], shape[-1], device="cuda", dtype=torch.bfloat16) - inp = torch.randn(shape, device="cuda", dtype=torch.bfloat16) - next_inp = torch.randn_like(inp) - quantizer = NVFP4Quantizer( - with_rht=with_rht, - with_post_rht_amax=with_rht, - stochastic_rounding=False, - with_random_sign_mask=False, - ) - quantizer.optimize_for_gemm = optimize_for_gemm - out = general_gemm(weight, inp, quantization_params=quantizer)[0] - expected = quantizer(general_gemm(weight, next_inp)[0]) - - actual = general_gemm(weight, next_inp, quantization_params=quantizer, out=out)[0] - - assert actual is out - assert actual._with_gemm_swizzled_scales == optimize_for_gemm - torch.testing.assert_close(actual.dequantize(), expected.dequantize(), atol=0, rtol=0) - - def check_nvfp4_gemm_versus_reference( x_dtype: torch.dtype, w_dtype: torch.dtype, From 431ec18dbaecab3a9675b0acdf28ed45d822f45f Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 5 Nov 2026 00:05:10 +0100 Subject: [PATCH 09/31] Move NVFP4 scale finalization into quantize_impl Signed-off-by: Pawel Gadzinski --- transformer_engine/pytorch/csrc/common.h | 33 +++++++++-------- transformer_engine/pytorch/csrc/extensions.h | 1 + .../pytorch/csrc/extensions/activation.cpp | 13 +++---- .../pytorch/csrc/extensions/bias.cpp | 17 ++++----- .../pytorch/csrc/extensions/cast.cpp | 30 +++++++++------- .../pytorch/csrc/extensions/gemm.cpp | 3 +- .../pytorch/csrc/extensions/normalization.cpp | 21 +++++++---- .../pytorch/csrc/extensions/pybind.cpp | 3 +- .../pytorch/csrc/extensions/swizzle.cpp | 14 ++------ transformer_engine/pytorch/csrc/quantizer.cpp | 35 +++++++++++++------ 10 files changed, 96 insertions(+), 74 deletions(-) diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index c3ecc35eb1e..cdf533afd52 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -131,12 +131,10 @@ class Quantizer { virtual std::pair convert_and_update_tensor( py::object tensor) const = 0; - /*! @brief Convert to a quantized data format */ + /*! @brief Quantize, optionally deferring standalone swizzle to batch finalization. */ virtual void quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag = std::nullopt) = 0; - - /*! @brief Apply the requested GEMM scale layout to a computed tensor. */ - void optimize_tensor_for_gemm(TensorWrapper& tensor, py::handle tensor_py) const; + py::handle out_py, const std::optional& noop_flag = std::nullopt, + bool defer_swizzle = false) = 0; virtual ~Quantizer() = default; @@ -177,7 +175,8 @@ class NoneQuantizer : public Quantizer { std::pair convert_and_update_tensor(py::object tensor) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag = std::nullopt) override; + py::handle out_py, const std::optional& noop_flag = std::nullopt, + bool defer_swizzle = false) override; }; class Float8Quantizer : public Quantizer { @@ -212,7 +211,8 @@ class Float8Quantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag = std::nullopt) override; + py::handle out_py, const std::optional& noop_flag = std::nullopt, + bool defer_swizzle = false) override; }; class Float8CurrentScalingQuantizer : public Quantizer { @@ -252,7 +252,8 @@ class Float8CurrentScalingQuantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag = std::nullopt) override; + py::handle out_py, const std::optional& noop_flag = std::nullopt, + bool defer_swizzle = false) override; /*! @brief Quantize to FP8, skipping local amax computation * @@ -308,7 +309,8 @@ class Float8BlockQuantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag = std::nullopt) override; + py::handle out_py, const std::optional& noop_flag = std::nullopt, + bool defer_swizzle = false) override; std::vector get_scale_shape(const std::vector& shape, bool columnwise) const; }; @@ -337,7 +339,8 @@ class MXFP8Quantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag = std::nullopt) override; + py::handle out_py, const std::optional& noop_flag = std::nullopt, + bool defer_swizzle = false) override; std::vector get_scale_shape(const std::vector& shape, bool columnwise) const; }; @@ -397,9 +400,11 @@ class NVFP4Quantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag = std::nullopt) override; - void quantize_impl(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag, bool compute_amax); + py::handle out_py, const std::optional& noop_flag = std::nullopt, + bool defer_swizzle = false) override; + void quantize_impl(const TensorWrapper& input, TensorWrapper& out, py::handle out_py, + const std::optional& noop_flag, bool compute_amax, + bool defer_swizzle = false); /*! @brief Quantize to NVFP4, skipping local amax computation * @@ -407,7 +412,7 @@ class NVFP4Quantizer : public Quantizer { * local amax. The amax may still be reduced across the amax * reduction group. */ - void quantize_with_amax(TensorWrapper& input, TensorWrapper& out); + void quantize_with_amax(TensorWrapper& input, TensorWrapper& out, py::handle out_py); std::vector get_scale_shape(const std::vector& shape, bool columnwise) const; diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index 7c22e69d92b..a7c6b65c069 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -727,6 +727,7 @@ void fused_multi_row_unpadding(at::Tensor input, at::Tensor output, **************************************************************************************************/ void inplace_swizzle_scale_for_gemm(py::handle &tensor); +void inplace_swizzle_scale_for_gemm(TensorWrapper &tensor, py::handle tensor_py); void inplace_multi_tensor_swizzle_scales_for_gemm(std::vector &tensors, bool rowwise_usage, bool columnwise_usage); diff --git a/transformer_engine/pytorch/csrc/extensions/activation.cpp b/transformer_engine/pytorch/csrc/extensions/activation.cpp index cdd56e2e4a7..178ff9e631e 100644 --- a/transformer_engine/pytorch/csrc/extensions/activation.cpp +++ b/transformer_engine/pytorch/csrc/extensions/activation.cpp @@ -69,7 +69,7 @@ py::object activation_helper(const at::Tensor& input, py::handle quantizer, int act_func(input_nvte.data(), temp_nvte.data(), stream); } }); - quantizer_cpp->quantize(temp_nvte, out_nvte); + quantizer_cpp->quantize(temp_nvte, out_nvte, out_py); } break; case Impl::FULLY_FUSED: @@ -118,14 +118,13 @@ py::object activation_helper(const at::Tensor& input, py::handle quantizer, int act_func(input_nvte.data(), temp_nvte.data(), stream); } }); - nvfp4_quantizer_cpp->quantize_with_amax(temp_nvte, out_nvte); + nvfp4_quantizer_cpp->quantize_with_amax(temp_nvte, out_nvte, out_py); } break; default: NVTE_ERROR("Invalid activation implementation (", static_cast(impl), ")"); } - quantizer_cpp->optimize_tensor_for_gemm(out_nvte, out_py); return out_py; } @@ -186,7 +185,7 @@ py::object dactivation_helper(const at::Tensor& grad_output, const at::Tensor& i dact_func(grad_output_nvte.data(), input_nvte.data(), temp_nvte.data(), stream); } }); - quantizer_cpp->quantize(temp_nvte, grad_input_nvte); + quantizer_cpp->quantize(temp_nvte, grad_input_nvte, grad_input_py); } break; case Impl::FULLY_FUSED: @@ -235,14 +234,13 @@ py::object dactivation_helper(const at::Tensor& grad_output, const at::Tensor& i dact_func(grad_output_nvte.data(), input_nvte.data(), temp_nvte.data(), stream); } }); - nvfp4_quantizer_cpp->quantize_with_amax(temp_nvte, grad_input_nvte); + nvfp4_quantizer_cpp->quantize_with_amax(temp_nvte, grad_input_nvte, grad_input_py); } break; default: NVTE_ERROR("Invalid activation implementation (", static_cast(impl), ")"); } - quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); return grad_input_py; } } // namespace @@ -439,8 +437,7 @@ py::object maybe_quantize(const at::Tensor& tensor, py::handle quantizer) { const std::vector shape(shape_te.data, shape_te.data + shape_te.ndim); auto fake_dtype = GetTransformerEngineDType(tensor.scalar_type()); auto [out_nvte, out_py] = quantizer_cpp->create_tensor(shape, fake_dtype); - quantizer_cpp->quantize(tensor_nvte, out_nvte); - quantizer_cpp->optimize_tensor_for_gemm(out_nvte, out_py); + quantizer_cpp->quantize(tensor_nvte, out_nvte, out_py); return out_py; } diff --git a/transformer_engine/pytorch/csrc/extensions/bias.cpp b/transformer_engine/pytorch/csrc/extensions/bias.cpp index 3631b1641d2..cb914d61aeb 100644 --- a/transformer_engine/pytorch/csrc/extensions/bias.cpp +++ b/transformer_engine/pytorch/csrc/extensions/bias.cpp @@ -67,7 +67,9 @@ std::vector bgrad_quantize(const at::Tensor &grad_output, py::handle // Trivial impl if tensors are empty if (product(shape) == 0) { grad_bias_torch.zero_(); - quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); + if (IsNVFP4Quantizers(quantizer.ptr())) { + quantizer_cpp->quantize(grad_output_nvte, grad_input_nvte, grad_input_py); + } return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } @@ -91,8 +93,7 @@ std::vector bgrad_quantize(const at::Tensor &grad_output, py::handle // Apply unfused impl if fused kernel is not supported if (!with_fused_kernel) { sum_bias_grad(grad_bias_torch, grad_output_torch); - quantizer_cpp->quantize(grad_output_nvte, grad_input_nvte); - quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); + quantizer_cpp->quantize(grad_output_nvte, grad_input_nvte, grad_input_py); return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } @@ -118,7 +119,6 @@ std::vector bgrad_quantize(const at::Tensor &grad_output, py::handle workspace_nvte.data(), stream); }); - quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } @@ -152,7 +152,9 @@ std::vector dact_dbias( // Return immediately if tensors are empty if (product(output_shape) == 0) { grad_bias_torch.zero_(); - quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); + if (detail::IsNVFP4Quantizers(quantizer_py.ptr())) { + quantizer_cpp->quantize(grad_output_nvte, grad_input_nvte, grad_input_py); + } return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } @@ -197,7 +199,7 @@ std::vector dact_dbias( }); const auto temp_torch = temp_py.cast(); sum_bias_grad(grad_bias_torch, temp_torch); - quantizer_cpp->quantize(temp_nvte, grad_input_nvte); + quantizer_cpp->quantize(temp_nvte, grad_input_nvte, grad_input_py); break; } case Impl::FUSED_DACT_DBIAS_QUANTIZE: @@ -256,14 +258,13 @@ std::vector dact_dbias( }); const auto temp_torch = temp_py.cast(); sum_bias_grad(grad_bias_torch, temp_torch); - nvfp4_quantizer_cpp->quantize_with_amax(temp_nvte, grad_input_nvte); + nvfp4_quantizer_cpp->quantize_with_amax(temp_nvte, grad_input_nvte, grad_input_py); break; } default: NVTE_ERROR("Invalid implementation"); } - quantizer_cpp->optimize_tensor_for_gemm(grad_input_nvte, grad_input_py); return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } diff --git a/transformer_engine/pytorch/csrc/extensions/cast.cpp b/transformer_engine/pytorch/csrc/extensions/cast.cpp index f685f6ac727..ea66eeb1caf 100644 --- a/transformer_engine/pytorch/csrc/extensions/cast.cpp +++ b/transformer_engine/pytorch/csrc/extensions/cast.cpp @@ -59,7 +59,9 @@ void optimize_tensors_for_gemm(const std::vector> &qu }); if (!use_batch) { for (size_t i = 0; i < tensors.size(); ++i) { - quantizers[i]->optimize_tensor_for_gemm(tensors[i], tensors_py[i]); + if (quantizers[i]->optimize_for_gemm) { + inplace_swizzle_scale_for_gemm(tensors[i], tensors_py[i]); + } } return; } @@ -138,9 +140,8 @@ py::object quantize(const at::Tensor &tensor, py::handle quantizer, const py::ob } // Perform quantization - quantizer_cpp->quantize(input_cpp, output_cpp, noop_flag_cpp); + quantizer_cpp->quantize(input_cpp, output_cpp, output_py, noop_flag_cpp); - quantizer_cpp->optimize_tensor_for_gemm(output_cpp, output_py); return output_py; } @@ -182,8 +183,7 @@ py::object nvfp4_quantize_with_amax(const at::Tensor &tensor, py::handle quantiz output_py.attr("_amax_columnwise") = py::cast(columnwise_amax); } - nvfp4_quantizer_cpp->quantize_impl(input_cpp, output_cpp, std::nullopt, false); - quantizer_cpp->optimize_tensor_for_gemm(output_cpp, output_py); + nvfp4_quantizer_cpp->quantize_impl(input_cpp, output_cpp, output_py, std::nullopt, false); return output_py; } @@ -1134,7 +1134,8 @@ namespace { void multi_tensor_quantize_impl(const std::vector &input_list, std::vector &quantizer_py_list, std::vector> &quantizer_cpp_list, - std::vector &output_list) { + std::vector &output_list, + const std::vector &output_py_list) { // Check number of tensors const size_t num_tensors = input_list.size(); NVTE_CHECK(quantizer_py_list.size() == num_tensors, "Expected ", num_tensors, @@ -1175,7 +1176,8 @@ void multi_tensor_quantize_impl(const std::vector &input_list, } else { // Quantize kernels individually for (size_t i = 0; i < num_tensors; ++i) { - quantizer_cpp_list[i]->quantize(input_list[i], output_list[i]); + quantizer_cpp_list[i]->quantize(input_list[i], output_list[i], output_py_list[i], + std::nullopt, /*defer_swizzle=*/true); } } } @@ -1216,7 +1218,8 @@ std::vector multi_tensor_quantize(const std::vector &ten } // Perform multi-tensor quantization - multi_tensor_quantize_impl(input_cpp_list, quantizer_list, quantizer_cpp_list, output_cpp_list); + multi_tensor_quantize_impl(input_cpp_list, quantizer_list, quantizer_cpp_list, output_cpp_list, + output_py_list); optimize_tensors_for_gemm(quantizer_cpp_list, output_cpp_list, output_py_list); return output_py_list; @@ -2089,6 +2092,7 @@ void split_quantize_nvfp4_impl_helper(const TensorWrapper &input, void split_quantize_nvfp4_impl(const TensorWrapper &input, const std::vector &input_list, std::vector &output_list, + const std::vector &output_py_list, const std::vector &split_sections, const std::vector &quantizers) { // Check tensor lists @@ -2160,7 +2164,8 @@ void split_quantize_nvfp4_impl(const TensorWrapper &input, if (input_list[i].numel() == 0) { continue; } - quantizers[i]->quantize(input_list[i], output_list[i], std::nullopt); + quantizers[i]->quantize(input_list[i], output_list[i], output_py_list[i], std::nullopt, + /*defer_swizzle=*/true); } } } else { // NVFP4 quantize @@ -2328,13 +2333,14 @@ std::vector split_quantize(const at::Tensor &tensor, for (auto &quantizer : quantizer_cpp_list) { nvfp4_quantizers.push_back(static_cast(quantizer.get())); } - split_quantize_nvfp4_impl(input_nvte, input_list, output_cpp_list, split_sections, - nvfp4_quantizers); + split_quantize_nvfp4_impl(input_nvte, input_list, output_cpp_list, output_py_list, + split_sections, nvfp4_quantizers); break; } default: // General multi-tensor quantization - multi_tensor_quantize_impl(input_list, quantizer_list, quantizer_cpp_list, output_cpp_list); + multi_tensor_quantize_impl(input_list, quantizer_list, quantizer_cpp_list, output_cpp_list, + output_py_list); } optimize_tensors_for_gemm(quantizer_cpp_list, output_cpp_list, output_py_list); diff --git a/transformer_engine/pytorch/csrc/extensions/gemm.cpp b/transformer_engine/pytorch/csrc/extensions/gemm.cpp index 2c27705062f..6a93fc77f75 100644 --- a/transformer_engine/pytorch/csrc/extensions/gemm.cpp +++ b/transformer_engine/pytorch/csrc/extensions/gemm.cpp @@ -427,8 +427,7 @@ std::vector gemm(py::handle A, bool transa, py::handle B, bool trans if (reuse_output) { std::tie(D_tensor, D) = my_quantizer->convert_and_update_tensor(D); } - my_quantizer->quantize(unquantized_D_tensor, D_tensor); - my_quantizer->optimize_tensor_for_gemm(D_tensor, D); + my_quantizer->quantize(unquantized_D_tensor, D_tensor, D); } // Pack outputs std::vector out; diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index 811a32152c8..095e962860a 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -135,6 +135,9 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe } } + const bool cudnn_mxfp8_output = + impl == Impl::FULLY_FUSED && IsMXFP8Quantizers(quantizer.ptr()); + // Output tensor TensorWrapper out_nvte; if (out.is_none()) { @@ -142,7 +145,7 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe } else { std::tie(out_nvte, out) = quantizer_cpp->convert_and_update_tensor(out); } - if (impl == Impl::FULLY_FUSED && IsMXFP8Quantizers(quantizer.ptr())) { + if (cudnn_mxfp8_output) { // cuDNN writes compact scales, including into preallocated outputs. out_nvte.set_with_gemm_swizzled_scales(false); out.attr("_with_gemm_swizzled_scales") = false; @@ -202,7 +205,7 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe // Quantize output if needed switch (impl) { case Impl::UNFUSED: { - quantizer_cpp->quantize(unquantized_out_nvte, out_nvte); + quantizer_cpp->quantize(unquantized_out_nvte, out_nvte, out); } break; case Impl::FUSED_NORM_AMAX_FP8: { auto fp8_quantizer_cpp = static_cast(quantizer_cpp.get()); @@ -210,13 +213,15 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe } break; case Impl::FUSED_NORM_AMAX_NVFP4: { auto nvfp4_quantizer_cpp = static_cast(quantizer_cpp.get()); - nvfp4_quantizer_cpp->quantize_with_amax(unquantized_out_nvte, out_nvte); + nvfp4_quantizer_cpp->quantize_with_amax(unquantized_out_nvte, out_nvte, out); } break; default: { } } - quantizer_cpp->optimize_tensor_for_gemm(out_nvte, out); + if (cudnn_mxfp8_output && quantizer_cpp->optimize_for_gemm) { + inplace_swizzle_scale_for_gemm(out_nvte, out); + } return {out, py::cast(mu_py), py::cast(rsigma_py)}; } @@ -449,7 +454,7 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w // Quantize output if needed switch (impl) { case Impl::UNFUSED: { - quantizer_cpp->quantize(unquantized_out_nvte, out_nvte); + quantizer_cpp->quantize(unquantized_out_nvte, out_nvte, out); } break; case Impl::FUSED_NORM_AMAX_FP8: { auto fp8_quantizer_cpp = static_cast(quantizer_cpp.get()); @@ -457,13 +462,15 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w } break; case Impl::FUSED_NORM_AMAX_NVFP4: { auto nvfp4_quantizer_cpp = static_cast(quantizer_cpp.get()); - nvfp4_quantizer_cpp->quantize_with_amax(unquantized_out_nvte, out_nvte); + nvfp4_quantizer_cpp->quantize_with_amax(unquantized_out_nvte, out_nvte, out); } break; default: { } } - quantizer_cpp->optimize_tensor_for_gemm(out_nvte, out); + if (impl == Impl::FUSED_NORM_QUANT_UNSWIZZLED && quantizer_cpp->optimize_for_gemm) { + inplace_swizzle_scale_for_gemm(out_nvte, out); + } return {out, py::none(), py::cast(rsigma_py)}; } diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index 895b9a96e87..3291af3f342 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -537,7 +537,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { "Fused Multi-tensor padding", py::call_guard()); m.def("fused_multi_row_unpadding", &transformer_engine::pytorch::fused_multi_row_unpadding, "Fused Multi-tensor unpadding", py::call_guard()); - m.def("swizzle_scales_for_gemm_", &transformer_engine::pytorch::inplace_swizzle_scale_for_gemm, + m.def("swizzle_scales_for_gemm_", + py::overload_cast(&transformer_engine::pytorch::inplace_swizzle_scale_for_gemm), "Convert tensor block scales into GEMM swizzled format"); m.def("multi_tensor_swizzle_scales_for_gemm_", &transformer_engine::pytorch::inplace_multi_tensor_swizzle_scales_for_gemm, diff --git a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp index 052c06a8080..ae52f09a616 100644 --- a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp +++ b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp @@ -727,9 +727,7 @@ void inplace_multi_tensor_swizzle_scales_for_gemm_unchecked(std::vector #include "common.h" +#include "extensions.h" #include "common/util/cuda_runtime.h" #include "common/util/system.h" #include "pybind.h" @@ -329,7 +330,8 @@ std::pair NoneQuantizer::convert_and_update_tensor( } void NoneQuantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag) { + py::handle /*out_py*/, const std::optional& noop_flag, + bool /*defer_swizzle*/) { NVTE_ERROR("NoneQuantizer does not support quantization"); } @@ -630,7 +632,8 @@ std::pair Float8Quantizer::convert_and_update_tensor( } void Float8Quantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag) { + py::handle /*out_py*/, const std::optional& noop_flag, + bool /*defer_swizzle*/) { if (input.numel() == 0) { return; } @@ -1033,7 +1036,8 @@ void Float8CurrentScalingQuantizer::quantize_impl(const TensorWrapper& input, Te } void Float8CurrentScalingQuantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag) { + py::handle /*out_py*/, const std::optional& noop_flag, + bool /*defer_swizzle*/) { const auto opts = at::TensorOptions().dtype(torch::kFloat32).device(torch::kCUDA); at::Tensor amax_and_scale = at::empty({2}, opts); this->quantize_impl(input, out, noop_flag, true, amax_and_scale[0], amax_and_scale[1]); @@ -1423,7 +1427,8 @@ std::pair Float8BlockQuantizer::convert_and_update_te } void Float8BlockQuantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag) { + py::handle /*out_py*/, const std::optional& noop_flag, + bool /*defer_swizzle*/) { if (input.numel() == 0) { return; } @@ -1834,7 +1839,8 @@ std::pair MXFP8Quantizer::convert_and_update_tensor( } void MXFP8Quantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag) { + py::handle /*out_py*/, const std::optional& noop_flag, + bool /*defer_swizzle*/) { if (input.numel() == 0) { return; } @@ -2578,8 +2584,14 @@ void NVFP4Quantizer::quantize_with_rht_unfused_helper( } void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& out, + py::handle out_py, const std::optional& noop_flag, - bool compute_amax) { + bool compute_amax, bool defer_swizzle) { + auto swizzle_if_needed = [&]() { + if (!defer_swizzle && optimize_for_gemm && !out.get_with_gemm_swizzled_scales()) { + inplace_swizzle_scale_for_gemm(out, out_py); + } + }; auto reduce_amaxes = [&]() { if (!this->with_amax_reduction || this->disable_second_level_scale) { return; @@ -2614,6 +2626,7 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou if (!compute_amax) { reduce_amaxes(); } + swizzle_if_needed(); return; } @@ -2806,14 +2819,16 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou } else { NVTE_SCOPED_GIL_RELEASE({ nvte_quantize_v2(input.data(), out.data(), quant_config, stream); }); } + swizzle_if_needed(); } void NVFP4Quantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag) { - this->quantize_impl(input, out, noop_flag, true); + py::handle out_py, const std::optional& noop_flag, + bool defer_swizzle) { + this->quantize_impl(input, out, out_py, noop_flag, true, defer_swizzle); } -void NVFP4Quantizer::quantize_with_amax(TensorWrapper& input, TensorWrapper& out) { +void NVFP4Quantizer::quantize_with_amax(TensorWrapper& input, TensorWrapper& out, py::handle out_py) { NVTE_CHECK(!out.get_row_scaled_nvfp4(), "quantize_with_amax is not supported for row-scaled NVFP4 quantization."); // Update output tensor amaxes with input tensor amax @@ -2836,7 +2851,7 @@ void NVFP4Quantizer::quantize_with_amax(TensorWrapper& input, TensorWrapper& out input.set_amax(nullptr, DType::kFloat32, input.defaultShape); // Perform quantization - this->quantize_impl(input, out, std::nullopt, false); + this->quantize_impl(input, out, out_py, std::nullopt, false); } std::vector NVFP4Quantizer::get_scale_shape(const std::vector& shape, From 6d52113321e61e0dde1945e99206d90465710ab5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 23:07:42 +0000 Subject: [PATCH 10/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- transformer_engine/pytorch/csrc/common.h | 28 +++++++++---------- .../pytorch/csrc/extensions/cast.cpp | 3 +- .../pytorch/csrc/extensions/normalization.cpp | 3 +- .../pytorch/csrc/extensions/pybind.cpp | 7 +++-- transformer_engine/pytorch/csrc/quantizer.cpp | 27 +++++++++--------- 5 files changed, 34 insertions(+), 34 deletions(-) diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index cdf533afd52..87f285bb25a 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -132,8 +132,8 @@ class Quantizer { py::object tensor) const = 0; /*! @brief Quantize, optionally deferring standalone swizzle to batch finalization. */ - virtual void quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, const std::optional& noop_flag = std::nullopt, + virtual void quantize(const TensorWrapper& input, TensorWrapper& out, py::handle out_py, + const std::optional& noop_flag = std::nullopt, bool defer_swizzle = false) = 0; virtual ~Quantizer() = default; @@ -174,8 +174,8 @@ class NoneQuantizer : public Quantizer { std::pair convert_and_update_tensor(py::object tensor) const override; - void quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, const std::optional& noop_flag = std::nullopt, + void quantize(const TensorWrapper& input, TensorWrapper& out, py::handle out_py, + const std::optional& noop_flag = std::nullopt, bool defer_swizzle = false) override; }; @@ -210,8 +210,8 @@ class Float8Quantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; - void quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, const std::optional& noop_flag = std::nullopt, + void quantize(const TensorWrapper& input, TensorWrapper& out, py::handle out_py, + const std::optional& noop_flag = std::nullopt, bool defer_swizzle = false) override; }; @@ -251,8 +251,8 @@ class Float8CurrentScalingQuantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; - void quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, const std::optional& noop_flag = std::nullopt, + void quantize(const TensorWrapper& input, TensorWrapper& out, py::handle out_py, + const std::optional& noop_flag = std::nullopt, bool defer_swizzle = false) override; /*! @brief Quantize to FP8, skipping local amax computation @@ -308,8 +308,8 @@ class Float8BlockQuantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; - void quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, const std::optional& noop_flag = std::nullopt, + void quantize(const TensorWrapper& input, TensorWrapper& out, py::handle out_py, + const std::optional& noop_flag = std::nullopt, bool defer_swizzle = false) override; std::vector get_scale_shape(const std::vector& shape, bool columnwise) const; @@ -338,8 +338,8 @@ class MXFP8Quantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; - void quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, const std::optional& noop_flag = std::nullopt, + void quantize(const TensorWrapper& input, TensorWrapper& out, py::handle out_py, + const std::optional& noop_flag = std::nullopt, bool defer_swizzle = false) override; std::vector get_scale_shape(const std::vector& shape, bool columnwise) const; @@ -399,8 +399,8 @@ class NVFP4Quantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; - void quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, const std::optional& noop_flag = std::nullopt, + void quantize(const TensorWrapper& input, TensorWrapper& out, py::handle out_py, + const std::optional& noop_flag = std::nullopt, bool defer_swizzle = false) override; void quantize_impl(const TensorWrapper& input, TensorWrapper& out, py::handle out_py, const std::optional& noop_flag, bool compute_amax, diff --git a/transformer_engine/pytorch/csrc/extensions/cast.cpp b/transformer_engine/pytorch/csrc/extensions/cast.cpp index ea66eeb1caf..b614233ab40 100644 --- a/transformer_engine/pytorch/csrc/extensions/cast.cpp +++ b/transformer_engine/pytorch/csrc/extensions/cast.cpp @@ -142,7 +142,6 @@ py::object quantize(const at::Tensor &tensor, py::handle quantizer, const py::ob // Perform quantization quantizer_cpp->quantize(input_cpp, output_cpp, output_py, noop_flag_cpp); - return output_py; } @@ -2165,7 +2164,7 @@ void split_quantize_nvfp4_impl(const TensorWrapper &input, continue; } quantizers[i]->quantize(input_list[i], output_list[i], output_py_list[i], std::nullopt, - /*defer_swizzle=*/true); + /*defer_swizzle=*/true); } } } else { // NVFP4 quantize diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index 095e962860a..a7a85489138 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -135,8 +135,7 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe } } - const bool cudnn_mxfp8_output = - impl == Impl::FULLY_FUSED && IsMXFP8Quantizers(quantizer.ptr()); + const bool cudnn_mxfp8_output = impl == Impl::FULLY_FUSED && IsMXFP8Quantizers(quantizer.ptr()); // Output tensor TensorWrapper out_nvte; diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index 3291af3f342..c3aa2fe84cf 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -537,9 +537,10 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { "Fused Multi-tensor padding", py::call_guard()); m.def("fused_multi_row_unpadding", &transformer_engine::pytorch::fused_multi_row_unpadding, "Fused Multi-tensor unpadding", py::call_guard()); - m.def("swizzle_scales_for_gemm_", - py::overload_cast(&transformer_engine::pytorch::inplace_swizzle_scale_for_gemm), - "Convert tensor block scales into GEMM swizzled format"); + m.def( + "swizzle_scales_for_gemm_", + py::overload_cast(&transformer_engine::pytorch::inplace_swizzle_scale_for_gemm), + "Convert tensor block scales into GEMM swizzled format"); m.def("multi_tensor_swizzle_scales_for_gemm_", &transformer_engine::pytorch::inplace_multi_tensor_swizzle_scales_for_gemm, "Convert multiple tensors' block scales into GEMM swizzled format", py::arg("tensors"), diff --git a/transformer_engine/pytorch/csrc/quantizer.cpp b/transformer_engine/pytorch/csrc/quantizer.cpp index 61f3bd09ff6..7a91b9ba25c 100644 --- a/transformer_engine/pytorch/csrc/quantizer.cpp +++ b/transformer_engine/pytorch/csrc/quantizer.cpp @@ -8,9 +8,9 @@ #include #include "common.h" -#include "extensions.h" #include "common/util/cuda_runtime.h" #include "common/util/system.h" +#include "extensions.h" #include "pybind.h" #include "torch/torch.h" @@ -329,8 +329,8 @@ std::pair NoneQuantizer::convert_and_update_tensor( return {std::move(out_cpp), std::move(tensor)}; } -void NoneQuantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle /*out_py*/, const std::optional& noop_flag, +void NoneQuantizer::quantize(const TensorWrapper& input, TensorWrapper& out, py::handle /*out_py*/, + const std::optional& noop_flag, bool /*defer_swizzle*/) { NVTE_ERROR("NoneQuantizer does not support quantization"); } @@ -1036,7 +1036,8 @@ void Float8CurrentScalingQuantizer::quantize_impl(const TensorWrapper& input, Te } void Float8CurrentScalingQuantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle /*out_py*/, const std::optional& noop_flag, + py::handle /*out_py*/, + const std::optional& noop_flag, bool /*defer_swizzle*/) { const auto opts = at::TensorOptions().dtype(torch::kFloat32).device(torch::kCUDA); at::Tensor amax_and_scale = at::empty({2}, opts); @@ -1427,7 +1428,8 @@ std::pair Float8BlockQuantizer::convert_and_update_te } void Float8BlockQuantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle /*out_py*/, const std::optional& noop_flag, + py::handle /*out_py*/, + const std::optional& noop_flag, bool /*defer_swizzle*/) { if (input.numel() == 0) { return; @@ -1838,8 +1840,8 @@ std::pair MXFP8Quantizer::convert_and_update_tensor( return {std::move(out_cpp), std::move(tensor)}; } -void MXFP8Quantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle /*out_py*/, const std::optional& noop_flag, +void MXFP8Quantizer::quantize(const TensorWrapper& input, TensorWrapper& out, py::handle /*out_py*/, + const std::optional& noop_flag, bool /*defer_swizzle*/) { if (input.numel() == 0) { return; @@ -2584,8 +2586,7 @@ void NVFP4Quantizer::quantize_with_rht_unfused_helper( } void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, - const std::optional& noop_flag, + py::handle out_py, const std::optional& noop_flag, bool compute_amax, bool defer_swizzle) { auto swizzle_if_needed = [&]() { if (!defer_swizzle && optimize_for_gemm && !out.get_with_gemm_swizzled_scales()) { @@ -2822,13 +2823,13 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou swizzle_if_needed(); } -void NVFP4Quantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, const std::optional& noop_flag, - bool defer_swizzle) { +void NVFP4Quantizer::quantize(const TensorWrapper& input, TensorWrapper& out, py::handle out_py, + const std::optional& noop_flag, bool defer_swizzle) { this->quantize_impl(input, out, out_py, noop_flag, true, defer_swizzle); } -void NVFP4Quantizer::quantize_with_amax(TensorWrapper& input, TensorWrapper& out, py::handle out_py) { +void NVFP4Quantizer::quantize_with_amax(TensorWrapper& input, TensorWrapper& out, + py::handle out_py) { NVTE_CHECK(!out.get_row_scaled_nvfp4(), "quantize_with_amax is not supported for row-scaled NVFP4 quantization."); // Update output tensor amaxes with input tensor amax From 908063a2404d87124b1e7aa7961a26a5fb35b685 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 5 Nov 2026 00:26:54 +0100 Subject: [PATCH 11/31] Keep quantization scale scratch in C++ Signed-off-by: Pawel Gadzinski --- transformer_engine/pytorch/csrc/common.h | 44 ++++-- transformer_engine/pytorch/csrc/extensions.h | 6 +- .../pytorch/csrc/extensions/activation.cpp | 10 +- .../pytorch/csrc/extensions/bias.cpp | 12 +- .../pytorch/csrc/extensions/cast.cpp | 131 ++++------------ .../pytorch/csrc/extensions/gemm.cpp | 2 +- .../pytorch/csrc/extensions/normalization.cpp | 32 ++-- .../pytorch/csrc/extensions/pybind.cpp | 3 +- .../pytorch/csrc/extensions/swizzle.cpp | 146 +++++++++++++++++- transformer_engine/pytorch/csrc/quantizer.cpp | 64 ++++---- 10 files changed, 258 insertions(+), 192 deletions(-) diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index cdf533afd52..50149015b26 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -96,6 +96,16 @@ enum FP8BwdTensors { GRAD_INPUT3 = 5 }; +struct QuantizationWorkspace { + bool defer_swizzle = false; + std::optional final_output; + std::optional rowwise_scales; + std::optional columnwise_scales; + + void prepare(TensorWrapper& output, bool preserve_scales = false); + void finish(TensorWrapper& output); +}; + class Quantizer { public: virtual NVTEScalingMode get_scaling_mode() const = 0; @@ -133,8 +143,8 @@ class Quantizer { /*! @brief Quantize, optionally deferring standalone swizzle to batch finalization. */ virtual void quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, const std::optional& noop_flag = std::nullopt, - bool defer_swizzle = false) = 0; + const std::optional& noop_flag = std::nullopt, + QuantizationWorkspace* workspace = nullptr) = 0; virtual ~Quantizer() = default; @@ -175,8 +185,8 @@ class NoneQuantizer : public Quantizer { std::pair convert_and_update_tensor(py::object tensor) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, const std::optional& noop_flag = std::nullopt, - bool defer_swizzle = false) override; + const std::optional& noop_flag = std::nullopt, + QuantizationWorkspace* workspace = nullptr) override; }; class Float8Quantizer : public Quantizer { @@ -211,8 +221,8 @@ class Float8Quantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, const std::optional& noop_flag = std::nullopt, - bool defer_swizzle = false) override; + const std::optional& noop_flag = std::nullopt, + QuantizationWorkspace* workspace = nullptr) override; }; class Float8CurrentScalingQuantizer : public Quantizer { @@ -252,8 +262,8 @@ class Float8CurrentScalingQuantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, const std::optional& noop_flag = std::nullopt, - bool defer_swizzle = false) override; + const std::optional& noop_flag = std::nullopt, + QuantizationWorkspace* workspace = nullptr) override; /*! @brief Quantize to FP8, skipping local amax computation * @@ -309,8 +319,8 @@ class Float8BlockQuantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, const std::optional& noop_flag = std::nullopt, - bool defer_swizzle = false) override; + const std::optional& noop_flag = std::nullopt, + QuantizationWorkspace* workspace = nullptr) override; std::vector get_scale_shape(const std::vector& shape, bool columnwise) const; }; @@ -339,8 +349,8 @@ class MXFP8Quantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, const std::optional& noop_flag = std::nullopt, - bool defer_swizzle = false) override; + const std::optional& noop_flag = std::nullopt, + QuantizationWorkspace* workspace = nullptr) override; std::vector get_scale_shape(const std::vector& shape, bool columnwise) const; }; @@ -400,11 +410,11 @@ class NVFP4Quantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, const std::optional& noop_flag = std::nullopt, - bool defer_swizzle = false) override; - void quantize_impl(const TensorWrapper& input, TensorWrapper& out, py::handle out_py, + const std::optional& noop_flag = std::nullopt, + QuantizationWorkspace* workspace = nullptr) override; + void quantize_impl(const TensorWrapper& input, TensorWrapper& out, const std::optional& noop_flag, bool compute_amax, - bool defer_swizzle = false); + QuantizationWorkspace* workspace = nullptr); /*! @brief Quantize to NVFP4, skipping local amax computation * @@ -412,7 +422,7 @@ class NVFP4Quantizer : public Quantizer { * local amax. The amax may still be reduced across the amax * reduction group. */ - void quantize_with_amax(TensorWrapper& input, TensorWrapper& out, py::handle out_py); + void quantize_with_amax(TensorWrapper& input, TensorWrapper& out); std::vector get_scale_shape(const std::vector& shape, bool columnwise) const; diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index a7c6b65c069..6b1a1897bd4 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -727,7 +727,11 @@ void fused_multi_row_unpadding(at::Tensor input, at::Tensor output, **************************************************************************************************/ void inplace_swizzle_scale_for_gemm(py::handle &tensor); -void inplace_swizzle_scale_for_gemm(TensorWrapper &tensor, py::handle tensor_py); + +void prepare_quantization_workspaces(std::vector &outputs, + std::vector &workspaces); +void finish_quantization_workspaces(std::vector &outputs, + std::vector &workspaces, bool batched); void inplace_multi_tensor_swizzle_scales_for_gemm(std::vector &tensors, bool rowwise_usage, bool columnwise_usage); diff --git a/transformer_engine/pytorch/csrc/extensions/activation.cpp b/transformer_engine/pytorch/csrc/extensions/activation.cpp index 178ff9e631e..86419f0d3ed 100644 --- a/transformer_engine/pytorch/csrc/extensions/activation.cpp +++ b/transformer_engine/pytorch/csrc/extensions/activation.cpp @@ -69,7 +69,7 @@ py::object activation_helper(const at::Tensor& input, py::handle quantizer, int act_func(input_nvte.data(), temp_nvte.data(), stream); } }); - quantizer_cpp->quantize(temp_nvte, out_nvte, out_py); + quantizer_cpp->quantize(temp_nvte, out_nvte); } break; case Impl::FULLY_FUSED: @@ -118,7 +118,7 @@ py::object activation_helper(const at::Tensor& input, py::handle quantizer, int act_func(input_nvte.data(), temp_nvte.data(), stream); } }); - nvfp4_quantizer_cpp->quantize_with_amax(temp_nvte, out_nvte, out_py); + nvfp4_quantizer_cpp->quantize_with_amax(temp_nvte, out_nvte); } break; default: @@ -185,7 +185,7 @@ py::object dactivation_helper(const at::Tensor& grad_output, const at::Tensor& i dact_func(grad_output_nvte.data(), input_nvte.data(), temp_nvte.data(), stream); } }); - quantizer_cpp->quantize(temp_nvte, grad_input_nvte, grad_input_py); + quantizer_cpp->quantize(temp_nvte, grad_input_nvte); } break; case Impl::FULLY_FUSED: @@ -234,7 +234,7 @@ py::object dactivation_helper(const at::Tensor& grad_output, const at::Tensor& i dact_func(grad_output_nvte.data(), input_nvte.data(), temp_nvte.data(), stream); } }); - nvfp4_quantizer_cpp->quantize_with_amax(temp_nvte, grad_input_nvte, grad_input_py); + nvfp4_quantizer_cpp->quantize_with_amax(temp_nvte, grad_input_nvte); } break; default: @@ -437,7 +437,7 @@ py::object maybe_quantize(const at::Tensor& tensor, py::handle quantizer) { const std::vector shape(shape_te.data, shape_te.data + shape_te.ndim); auto fake_dtype = GetTransformerEngineDType(tensor.scalar_type()); auto [out_nvte, out_py] = quantizer_cpp->create_tensor(shape, fake_dtype); - quantizer_cpp->quantize(tensor_nvte, out_nvte, out_py); + quantizer_cpp->quantize(tensor_nvte, out_nvte); return out_py; } diff --git a/transformer_engine/pytorch/csrc/extensions/bias.cpp b/transformer_engine/pytorch/csrc/extensions/bias.cpp index cb914d61aeb..2dbeb965a1a 100644 --- a/transformer_engine/pytorch/csrc/extensions/bias.cpp +++ b/transformer_engine/pytorch/csrc/extensions/bias.cpp @@ -67,9 +67,6 @@ std::vector bgrad_quantize(const at::Tensor &grad_output, py::handle // Trivial impl if tensors are empty if (product(shape) == 0) { grad_bias_torch.zero_(); - if (IsNVFP4Quantizers(quantizer.ptr())) { - quantizer_cpp->quantize(grad_output_nvte, grad_input_nvte, grad_input_py); - } return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } @@ -93,7 +90,7 @@ std::vector bgrad_quantize(const at::Tensor &grad_output, py::handle // Apply unfused impl if fused kernel is not supported if (!with_fused_kernel) { sum_bias_grad(grad_bias_torch, grad_output_torch); - quantizer_cpp->quantize(grad_output_nvte, grad_input_nvte, grad_input_py); + quantizer_cpp->quantize(grad_output_nvte, grad_input_nvte); return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } @@ -152,9 +149,6 @@ std::vector dact_dbias( // Return immediately if tensors are empty if (product(output_shape) == 0) { grad_bias_torch.zero_(); - if (detail::IsNVFP4Quantizers(quantizer_py.ptr())) { - quantizer_cpp->quantize(grad_output_nvte, grad_input_nvte, grad_input_py); - } return {py::cast(std::move(grad_bias_torch)), std::move(grad_input_py)}; } @@ -199,7 +193,7 @@ std::vector dact_dbias( }); const auto temp_torch = temp_py.cast(); sum_bias_grad(grad_bias_torch, temp_torch); - quantizer_cpp->quantize(temp_nvte, grad_input_nvte, grad_input_py); + quantizer_cpp->quantize(temp_nvte, grad_input_nvte); break; } case Impl::FUSED_DACT_DBIAS_QUANTIZE: @@ -258,7 +252,7 @@ std::vector dact_dbias( }); const auto temp_torch = temp_py.cast(); sum_bias_grad(grad_bias_torch, temp_torch); - nvfp4_quantizer_cpp->quantize_with_amax(temp_nvte, grad_input_nvte, grad_input_py); + nvfp4_quantizer_cpp->quantize_with_amax(temp_nvte, grad_input_nvte); break; } default: diff --git a/transformer_engine/pytorch/csrc/extensions/cast.cpp b/transformer_engine/pytorch/csrc/extensions/cast.cpp index ea66eeb1caf..5d3c70500f8 100644 --- a/transformer_engine/pytorch/csrc/extensions/cast.cpp +++ b/transformer_engine/pytorch/csrc/extensions/cast.cpp @@ -38,65 +38,19 @@ std::vector get_tensor_shape(const TensorWrapper &tensor) { return std::vector(shape.data, shape.data + shape.ndim); } -void optimize_tensors_for_gemm(const std::vector> &quantizers, - std::vector &tensors, - std::vector &tensors_py) { - if (tensors.empty()) { - return; - } - if (std::all_of(tensors.begin(), tensors.end(), - [](const auto &tensor) { return tensor.get_with_gemm_swizzled_scales(); })) { - return; - } +void finish_quantization(const std::vector> &quantizers, + std::vector &tensors, + std::vector &workspaces) { + if (tensors.empty()) return; const auto scaling_mode = tensors.front().scaling_mode(); const bool use_batch = tensors.size() > 1 && - (scaling_mode == NVTE_MXFP8_1D_SCALING || scaling_mode == NVTE_NVFP4_1D_SCALING) && std::all_of(quantizers.begin(), quantizers.end(), [](const auto &quantizer) { return quantizer->optimize_for_gemm; }) && std::all_of(tensors.begin(), tensors.end(), [scaling_mode](const auto &tensor) { return tensor.scaling_mode() == scaling_mode; }); - if (!use_batch) { - for (size_t i = 0; i < tensors.size(); ++i) { - if (quantizers[i]->optimize_for_gemm) { - inplace_swizzle_scale_for_gemm(tensors[i], tensors_py[i]); - } - } - return; - } - - for (bool rowwise : {true, false}) { - const bool has_scales = - std::any_of(tensors.begin(), tensors.end(), [rowwise](const auto &tensor) { - const auto scales = - rowwise ? tensor.get_rowwise_scale_inv() : tensor.get_columnwise_scale_inv(); - return !tensor.get_with_gemm_swizzled_scales() && scales.data_ptr != nullptr; - }); - if (has_scales) { - inplace_multi_tensor_swizzle_scales_for_gemm(tensors_py, rowwise, !rowwise); - } - } - for (size_t i = 0; i < tensors.size(); ++i) { - auto &tensor = tensors[i]; - if (tensor.get_with_gemm_swizzled_scales()) { - continue; - } - for (bool rowwise : {true, false}) { - const auto scales = - rowwise ? tensor.get_rowwise_scale_inv() : tensor.get_columnwise_scale_inv(); - if (scales.data_ptr != nullptr) { - const auto scales_py = tensors_py[i] - .attr(rowwise ? "_rowwise_scale_inv" : "_columnwise_scale_inv") - .cast(); - tensor.set_parameter(rowwise ? kNVTERowwiseScaleInv : kNVTEColumnwiseScaleInv, - scales_py.data_ptr(), static_cast(scales.dtype), scales.shape); - } - } - // Both orientations must be converted before publishing the shared layout flag. - tensors_py[i].attr("_with_gemm_swizzled_scales") = true; - tensor.set_with_gemm_swizzled_scales(true); - } + finish_quantization_workspaces(tensors, workspaces, use_batch); } void allreduce_nvfp4_amax_tensors(NVFP4Quantizer *nvfp4_quantizer_cpp, @@ -140,8 +94,7 @@ py::object quantize(const at::Tensor &tensor, py::handle quantizer, const py::ob } // Perform quantization - quantizer_cpp->quantize(input_cpp, output_cpp, output_py, noop_flag_cpp); - + quantizer_cpp->quantize(input_cpp, output_cpp, noop_flag_cpp); return output_py; } @@ -183,7 +136,7 @@ py::object nvfp4_quantize_with_amax(const at::Tensor &tensor, py::handle quantiz output_py.attr("_amax_columnwise") = py::cast(columnwise_amax); } - nvfp4_quantizer_cpp->quantize_impl(input_cpp, output_cpp, output_py, std::nullopt, false); + nvfp4_quantizer_cpp->quantize_impl(input_cpp, output_cpp, std::nullopt, false); return output_py; } @@ -1135,7 +1088,7 @@ void multi_tensor_quantize_impl(const std::vector &input_list, std::vector &quantizer_py_list, std::vector> &quantizer_cpp_list, std::vector &output_list, - const std::vector &output_py_list) { + std::vector &workspaces) { // Check number of tensors const size_t num_tensors = input_list.size(); NVTE_CHECK(quantizer_py_list.size() == num_tensors, "Expected ", num_tensors, @@ -1176,8 +1129,8 @@ void multi_tensor_quantize_impl(const std::vector &input_list, } else { // Quantize kernels individually for (size_t i = 0; i < num_tensors; ++i) { - quantizer_cpp_list[i]->quantize(input_list[i], output_list[i], output_py_list[i], - std::nullopt, /*defer_swizzle=*/true); + workspaces[i].defer_swizzle = true; + quantizer_cpp_list[i]->quantize(input_list[i], output_list[i], std::nullopt, &workspaces[i]); } } } @@ -1218,10 +1171,11 @@ std::vector multi_tensor_quantize(const std::vector &ten } // Perform multi-tensor quantization + std::vector workspaces(num_tensors); multi_tensor_quantize_impl(input_cpp_list, quantizer_list, quantizer_cpp_list, output_cpp_list, - output_py_list); + workspaces); - optimize_tensors_for_gemm(quantizer_cpp_list, output_cpp_list, output_py_list); + finish_quantization(quantizer_cpp_list, output_cpp_list, workspaces); return output_py_list; } @@ -1448,7 +1402,8 @@ std::tuple, std::vector> bulk_allocate_mx // amax buffer will be zeroed out by later amax kernels, so we can use empty to allocate std::tuple, std::vector, bool> bulk_allocate_nvfp4_tensors( std::vector> &shape_list, std::vector &quantizer_py_list, - std::vector &quantizer_cpp_list) { + std::vector &quantizer_cpp_list, + std::vector &workspaces) { init_extension(); std::tuple, std::vector, bool> retval; auto &tensor_py_list = std::get<0>(retval); @@ -1476,15 +1431,6 @@ std::tuple, std::vector, bool> bulk_alloc const auto fp4_dtype = quantizer_cpp_list[0]->dtype; const auto scale_dtype = quantizer_cpp_list[0]->scale_dtype; - // with_gemm_swizzled_scales is a single group-wide boolean baked - // into every output tensor. We can safely request it only when - // (a) every quantizer in the group has optimize_for_gemm and - // with_rht set, and (b) every tensor's shape qualifies for RHT - // cast-fusion. Disagreement among quantizers would silently give - // some outputs a layout that their own quantizer did not request; - // the NVTE_CHECK loop below turns that into a loud error. The - // final flag ANDs all three predicates (the shape-eligibility one - // is computed by the loop further below). const bool group_optimize_for_gemm = quantizer_cpp_list[0]->optimize_for_gemm; const bool group_with_rht = quantizer_cpp_list[0]->with_rht; for (size_t i = 1; i < num_tensors; ++i) { @@ -1509,27 +1455,9 @@ std::tuple, std::vector, bool> bulk_alloc break; } } - const bool with_gemm_swizzled_scales = + const bool kernel_swizzles = group_optimize_for_gemm && group_with_rht && all_tensors_rht_cast_fusion_eligible; - - auto allocate_buffers = [&](auto &shapes, auto &dtypes, auto &alignments) { - if (!group_optimize_for_gemm || with_gemm_swizzled_scales) { - return bulk_allocate(shapes, dtypes, std::nullopt, alignments); - } - - // Compact scales must be releasable after swizzling, independently of data and amax. - const std::vector> scale_shapes(shapes.begin() + num_tensors, - shapes.begin() + 2 * num_tensors); - shapes.erase(shapes.begin() + num_tensors, shapes.begin() + 2 * num_tensors); - dtypes.erase(dtypes.begin() + num_tensors, dtypes.begin() + 2 * num_tensors); - alignments.erase(alignments.begin() + num_tensors, alignments.begin() + 2 * num_tensors); - auto tensors = bulk_allocate(shapes, dtypes, std::nullopt, alignments); - auto scales = - bulk_allocate(scale_shapes, std::vector(num_tensors, torch::kUInt8), - std::nullopt, std::vector(num_tensors, 16)); - tensors.insert(tensors.begin() + num_tensors, scales.begin(), scales.end()); - return tensors; - }; + const bool with_gemm_swizzled_scales = group_optimize_for_gemm; // Helper function to get size of byte buffer holding FP4 data (last dim divided by 2) auto fp4_byte_shape = [](const std::vector &shape) -> std::vector { @@ -1589,7 +1517,7 @@ std::tuple, std::vector, bool> bulk_alloc dtypes.insert(dtypes.end(), num_tensors, torch::kFloat32); alignments.insert(alignments.end(), num_tensors, 16); } - auto tensors = allocate_buffers(shapes, dtypes, alignments); + auto tensors = bulk_allocate(shapes, dtypes, std::nullopt, alignments); // Split data, scale, and amax tensors for (size_t i = 0; i < num_tensors; ++i) { @@ -1647,7 +1575,7 @@ std::tuple, std::vector, bool> bulk_alloc dtypes.insert(dtypes.end(), num_tensors, torch::kFloat32); alignments.insert(alignments.end(), num_tensors, 16); } - auto tensors = allocate_buffers(shapes, dtypes, alignments); + auto tensors = bulk_allocate(shapes, dtypes, std::nullopt, alignments); // Split data, scale, and amax tensors for (size_t i = 0; i < num_tensors; ++i) { @@ -1720,6 +1648,10 @@ std::tuple, std::vector, bool> bulk_alloc } } + if (group_optimize_for_gemm && !kernel_swizzles) { + prepare_quantization_workspaces(tensor_cpp_list, workspaces); + } + return retval; } @@ -2092,7 +2024,7 @@ void split_quantize_nvfp4_impl_helper(const TensorWrapper &input, void split_quantize_nvfp4_impl(const TensorWrapper &input, const std::vector &input_list, std::vector &output_list, - const std::vector &output_py_list, + std::vector &workspaces, const std::vector &split_sections, const std::vector &quantizers) { // Check tensor lists @@ -2164,8 +2096,8 @@ void split_quantize_nvfp4_impl(const TensorWrapper &input, if (input_list[i].numel() == 0) { continue; } - quantizers[i]->quantize(input_list[i], output_list[i], output_py_list[i], std::nullopt, - /*defer_swizzle=*/true); + workspaces[i].defer_swizzle = true; + quantizers[i]->quantize(input_list[i], output_list[i], std::nullopt, &workspaces[i]); } } } else { // NVFP4 quantize @@ -2264,6 +2196,7 @@ std::vector split_quantize(const at::Tensor &tensor, } // Allocate output tensors + std::vector workspaces(num_splits); std::vector output_cpp_list; std::vector output_py_list; switch (allocation_method) { @@ -2295,7 +2228,7 @@ std::vector split_quantize(const at::Tensor &tensor, } bool contiguous_data_and_scale = false; std::tie(output_py_list, output_cpp_list, contiguous_data_and_scale) = - bulk_allocate_nvfp4_tensors(split_shapes, quantizer_list, nvfp4_quantizers); + bulk_allocate_nvfp4_tensors(split_shapes, quantizer_list, nvfp4_quantizers, workspaces); if (quantization_method == QuantizationMethod::FUSED_NVFP4 && !input_shape.empty() && input_shape.back() % 128 != 0) { static std::once_flag once_unfused_nvfp4_fallback_warning; @@ -2333,17 +2266,17 @@ std::vector split_quantize(const at::Tensor &tensor, for (auto &quantizer : quantizer_cpp_list) { nvfp4_quantizers.push_back(static_cast(quantizer.get())); } - split_quantize_nvfp4_impl(input_nvte, input_list, output_cpp_list, output_py_list, - split_sections, nvfp4_quantizers); + split_quantize_nvfp4_impl(input_nvte, input_list, output_cpp_list, workspaces, split_sections, + nvfp4_quantizers); break; } default: // General multi-tensor quantization multi_tensor_quantize_impl(input_list, quantizer_list, quantizer_cpp_list, output_cpp_list, - output_py_list); + workspaces); } - optimize_tensors_for_gemm(quantizer_cpp_list, output_cpp_list, output_py_list); + finish_quantization(quantizer_cpp_list, output_cpp_list, workspaces); return output_py_list; } diff --git a/transformer_engine/pytorch/csrc/extensions/gemm.cpp b/transformer_engine/pytorch/csrc/extensions/gemm.cpp index 6a93fc77f75..28fe93376f4 100644 --- a/transformer_engine/pytorch/csrc/extensions/gemm.cpp +++ b/transformer_engine/pytorch/csrc/extensions/gemm.cpp @@ -427,7 +427,7 @@ std::vector gemm(py::handle A, bool transa, py::handle B, bool trans if (reuse_output) { std::tie(D_tensor, D) = my_quantizer->convert_and_update_tensor(D); } - my_quantizer->quantize(unquantized_D_tensor, D_tensor, D); + my_quantizer->quantize(unquantized_D_tensor, D_tensor); } // Pack outputs std::vector out; diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index 095e962860a..b53f70f6a9c 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -135,8 +135,7 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe } } - const bool cudnn_mxfp8_output = - impl == Impl::FULLY_FUSED && IsMXFP8Quantizers(quantizer.ptr()); + const bool cudnn_mxfp8_output = impl == Impl::FULLY_FUSED && IsMXFP8Quantizers(quantizer.ptr()); // Output tensor TensorWrapper out_nvte; @@ -145,10 +144,9 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe } else { std::tie(out_nvte, out) = quantizer_cpp->convert_and_update_tensor(out); } - if (cudnn_mxfp8_output) { - // cuDNN writes compact scales, including into preallocated outputs. - out_nvte.set_with_gemm_swizzled_scales(false); - out.attr("_with_gemm_swizzled_scales") = false; + QuantizationWorkspace scale_workspace; + if (cudnn_mxfp8_output && quantizer_cpp->optimize_for_gemm) { + scale_workspace.prepare(out_nvte); } // Construct unquantized output tensor if needed @@ -205,7 +203,7 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe // Quantize output if needed switch (impl) { case Impl::UNFUSED: { - quantizer_cpp->quantize(unquantized_out_nvte, out_nvte, out); + quantizer_cpp->quantize(unquantized_out_nvte, out_nvte); } break; case Impl::FUSED_NORM_AMAX_FP8: { auto fp8_quantizer_cpp = static_cast(quantizer_cpp.get()); @@ -213,15 +211,13 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe } break; case Impl::FUSED_NORM_AMAX_NVFP4: { auto nvfp4_quantizer_cpp = static_cast(quantizer_cpp.get()); - nvfp4_quantizer_cpp->quantize_with_amax(unquantized_out_nvte, out_nvte, out); + nvfp4_quantizer_cpp->quantize_with_amax(unquantized_out_nvte, out_nvte); } break; default: { } } - if (cudnn_mxfp8_output && quantizer_cpp->optimize_for_gemm) { - inplace_swizzle_scale_for_gemm(out_nvte, out); - } + scale_workspace.finish(out_nvte); return {out, py::cast(mu_py), py::cast(rsigma_py)}; } @@ -397,9 +393,9 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w } else { std::tie(out_nvte, out) = quantizer_cpp->convert_and_update_tensor(out); } - if (impl == Impl::FUSED_NORM_QUANT_UNSWIZZLED) { - out_nvte.set_with_gemm_swizzled_scales(false); - out.attr("_with_gemm_swizzled_scales") = false; + QuantizationWorkspace scale_workspace; + if (impl == Impl::FUSED_NORM_QUANT_UNSWIZZLED && quantizer_cpp->optimize_for_gemm) { + scale_workspace.prepare(out_nvte); } // Construct unquantized output tensor if needed @@ -454,7 +450,7 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w // Quantize output if needed switch (impl) { case Impl::UNFUSED: { - quantizer_cpp->quantize(unquantized_out_nvte, out_nvte, out); + quantizer_cpp->quantize(unquantized_out_nvte, out_nvte); } break; case Impl::FUSED_NORM_AMAX_FP8: { auto fp8_quantizer_cpp = static_cast(quantizer_cpp.get()); @@ -462,15 +458,13 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w } break; case Impl::FUSED_NORM_AMAX_NVFP4: { auto nvfp4_quantizer_cpp = static_cast(quantizer_cpp.get()); - nvfp4_quantizer_cpp->quantize_with_amax(unquantized_out_nvte, out_nvte, out); + nvfp4_quantizer_cpp->quantize_with_amax(unquantized_out_nvte, out_nvte); } break; default: { } } - if (impl == Impl::FUSED_NORM_QUANT_UNSWIZZLED && quantizer_cpp->optimize_for_gemm) { - inplace_swizzle_scale_for_gemm(out_nvte, out); - } + scale_workspace.finish(out_nvte); return {out, py::none(), py::cast(rsigma_py)}; } diff --git a/transformer_engine/pytorch/csrc/extensions/pybind.cpp b/transformer_engine/pytorch/csrc/extensions/pybind.cpp index 3291af3f342..895b9a96e87 100644 --- a/transformer_engine/pytorch/csrc/extensions/pybind.cpp +++ b/transformer_engine/pytorch/csrc/extensions/pybind.cpp @@ -537,8 +537,7 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { "Fused Multi-tensor padding", py::call_guard()); m.def("fused_multi_row_unpadding", &transformer_engine::pytorch::fused_multi_row_unpadding, "Fused Multi-tensor unpadding", py::call_guard()); - m.def("swizzle_scales_for_gemm_", - py::overload_cast(&transformer_engine::pytorch::inplace_swizzle_scale_for_gemm), + m.def("swizzle_scales_for_gemm_", &transformer_engine::pytorch::inplace_swizzle_scale_for_gemm, "Convert tensor block scales into GEMM swizzled format"); m.def("multi_tensor_swizzle_scales_for_gemm_", &transformer_engine::pytorch::inplace_multi_tensor_swizzle_scales_for_gemm, diff --git a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp index ae52f09a616..57832638838 100644 --- a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp +++ b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp @@ -42,8 +42,146 @@ bool is_empty_grouped_tensor_param(const NVTEBasicTensor &t) { return t.shape.ndim == 1 && t.shape.data[0] == 0; } +TensorWrapper copy_tensor_view(const TensorWrapper &tensor) { + TensorWrapper view(tensor.scaling_mode()); + for (int i = kNVTERowwiseData; i <= kNVTEColumnwiseAmax; ++i) { + const auto param = static_cast(i); + const auto value = tensor.get_parameter(param); + view.set_parameter(param, value.data_ptr, static_cast(value.dtype), value.shape); + } + view.set_with_gemm_swizzled_scales(tensor.get_with_gemm_swizzled_scales()); + view.set_row_scaled_nvfp4(tensor.get_row_scaled_nvfp4()); + view.set_nvfp4_e4m3_max(tensor.get_nvfp4_e4m3_max()); + return view; +} + +TensorWrapper scale_view(const TensorWrapper &tensor, bool rowwise) { + const auto data = rowwise ? tensor.get_rowwise_data() : tensor.get_columnwise_data(); + const auto scales = rowwise ? tensor.get_rowwise_scale_inv() : tensor.get_columnwise_scale_inv(); + TensorWrapper view(tensor.scaling_mode()); + view.set_parameter(rowwise ? kNVTERowwiseData : kNVTEColumnwiseData, nullptr, + static_cast(data.dtype), data.shape); + view.set_parameter(rowwise ? kNVTERowwiseScaleInv : kNVTEColumnwiseScaleInv, scales.data_ptr, + static_cast(scales.dtype), scales.shape); + view.set_with_gemm_swizzled_scales(tensor.get_with_gemm_swizzled_scales()); + return view; +} + } // namespace +void QuantizationWorkspace::prepare(TensorWrapper &output, bool preserve_scales) { + NVTE_CHECK(!final_output.has_value(), "Quantization workspace is already in use"); + NVTE_CHECK(output.get_with_gemm_swizzled_scales(), "Expected a GEMM-ready output"); + auto compact_output = copy_tensor_view(output); + compact_output.set_with_gemm_swizzled_scales(false); + for (bool rowwise : {true, false}) { + const auto scales = + rowwise ? output.get_rowwise_scale_inv() : output.get_columnwise_scale_inv(); + if (scales.data_ptr == nullptr) continue; + auto &scratch = rowwise ? rowwise_scales : columnwise_scales; + if (!scratch.has_value()) { + scratch = allocateSpace(scales.shape, DType::kByte, false); + } + compact_output.set_parameter(rowwise ? kNVTERowwiseScaleInv : kNVTEColumnwiseScaleInv, + scratch->data_ptr(), static_cast(scales.dtype), + scales.shape); + if (preserve_scales) { + // A GPU no-op must preserve the previous scales through finalization. + auto input_scales = scale_view(output, rowwise); + auto output_scales = scale_view(compact_output, rowwise); + NVTE_SCOPED_GIL_RELEASE({ + nvte_unswizzle_scaling_factors(input_scales.data(), output_scales.data(), + at::cuda::getCurrentCUDAStream()); + }); + } + } + final_output.emplace(std::move(output)); + output = std::move(compact_output); +} + +void QuantizationWorkspace::finish(TensorWrapper &output) { + if (!final_output.has_value()) return; + for (bool rowwise : {true, false}) { + const auto scales = + rowwise ? output.get_rowwise_scale_inv() : output.get_columnwise_scale_inv(); + if (scales.data_ptr == nullptr) continue; + auto input_scales = scale_view(output, rowwise); + auto output_scales = scale_view(*final_output, rowwise); + NVTE_SCOPED_GIL_RELEASE({ + nvte_swizzle_scaling_factors(input_scales.data(), output_scales.data(), + at::cuda::getCurrentCUDAStream()); + }); + } + output = std::move(*final_output); + final_output.reset(); + rowwise_scales.reset(); + columnwise_scales.reset(); +} + +void prepare_quantization_workspaces(std::vector &outputs, + std::vector &workspaces) { + NVTE_CHECK(outputs.size() == workspaces.size(), "Mismatched quantization workspaces"); + for (bool rowwise : {true, false}) { + std::vector> shapes; + std::vector indices; + for (size_t i = 0; i < outputs.size(); ++i) { + const auto scales = + rowwise ? outputs[i].get_rowwise_scale_inv() : outputs[i].get_columnwise_scale_inv(); + if (outputs[i].numel() != 0 && scales.data_ptr != nullptr) { + shapes.emplace_back(scales.shape.data, scales.shape.data + scales.shape.ndim); + indices.push_back(i); + } + } + if (indices.empty()) continue; + auto scratch = bulk_allocate(shapes, std::vector(shapes.size(), at::kByte), + std::nullopt, std::vector(shapes.size(), 16)); + for (size_t i = 0; i < indices.size(); ++i) { + auto &workspace = workspaces[indices[i]]; + (rowwise ? workspace.rowwise_scales : workspace.columnwise_scales) = std::move(scratch[i]); + } + } + for (size_t i = 0; i < outputs.size(); ++i) { + workspaces[i].defer_swizzle = true; + if (outputs[i].numel() != 0) workspaces[i].prepare(outputs[i]); + } +} + +void finish_quantization_workspaces(std::vector &outputs, + std::vector &workspaces, bool batched) { + NVTE_CHECK(outputs.size() == workspaces.size(), "Mismatched quantization workspaces"); + if (!batched) { + for (size_t i = 0; i < outputs.size(); ++i) workspaces[i].finish(outputs[i]); + return; + } + for (bool rowwise : {true, false}) { + std::vector input_views, output_views; + std::vector inputs, destinations; + for (size_t i = 0; i < outputs.size(); ++i) { + const auto scales = + rowwise ? outputs[i].get_rowwise_scale_inv() : outputs[i].get_columnwise_scale_inv(); + if (!workspaces[i].final_output.has_value() || scales.data_ptr == nullptr) continue; + input_views.emplace_back(scale_view(outputs[i], rowwise)); + output_views.emplace_back(scale_view(*workspaces[i].final_output, rowwise)); + inputs.push_back(input_views.back().data()); + destinations.push_back(output_views.back().data()); + } + if (!inputs.empty()) { + NVTE_SCOPED_GIL_RELEASE({ + nvte_multi_tensor_swizzle_scaling_factors(inputs.data(), destinations.data(), inputs.size(), + at::cuda::getCurrentCUDAStream()); + }); + } + } + for (size_t i = 0; i < outputs.size(); ++i) { + auto &workspace = workspaces[i]; + if (!workspace.final_output.has_value()) continue; + outputs[i] = std::move(*workspace.final_output); + workspace.final_output.reset(); + workspace.rowwise_scales.reset(); + workspace.columnwise_scales.reset(); + } +} + std::tuple, std::optional> swizzle_scales_for_gemm( transformer_engine::TensorWrapper &tensor, bool rowwise_usage, bool columnwise_usage) { // Return early if scale swizzling is not required @@ -727,7 +865,8 @@ void inplace_multi_tensor_swizzle_scales_for_gemm_unchecked(std::vector #include "common.h" -#include "extensions.h" #include "common/util/cuda_runtime.h" #include "common/util/system.h" #include "pybind.h" @@ -330,8 +329,8 @@ std::pair NoneQuantizer::convert_and_update_tensor( } void NoneQuantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle /*out_py*/, const std::optional& noop_flag, - bool /*defer_swizzle*/) { + const std::optional& noop_flag, + QuantizationWorkspace* /*workspace*/) { NVTE_ERROR("NoneQuantizer does not support quantization"); } @@ -632,8 +631,8 @@ std::pair Float8Quantizer::convert_and_update_tensor( } void Float8Quantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle /*out_py*/, const std::optional& noop_flag, - bool /*defer_swizzle*/) { + const std::optional& noop_flag, + QuantizationWorkspace* /*workspace*/) { if (input.numel() == 0) { return; } @@ -1036,8 +1035,8 @@ void Float8CurrentScalingQuantizer::quantize_impl(const TensorWrapper& input, Te } void Float8CurrentScalingQuantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle /*out_py*/, const std::optional& noop_flag, - bool /*defer_swizzle*/) { + const std::optional& noop_flag, + QuantizationWorkspace* /*workspace*/) { const auto opts = at::TensorOptions().dtype(torch::kFloat32).device(torch::kCUDA); at::Tensor amax_and_scale = at::empty({2}, opts); this->quantize_impl(input, out, noop_flag, true, amax_and_scale[0], amax_and_scale[1]); @@ -1427,8 +1426,8 @@ std::pair Float8BlockQuantizer::convert_and_update_te } void Float8BlockQuantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle /*out_py*/, const std::optional& noop_flag, - bool /*defer_swizzle*/) { + const std::optional& noop_flag, + QuantizationWorkspace* /*workspace*/) { if (input.numel() == 0) { return; } @@ -1839,8 +1838,8 @@ std::pair MXFP8Quantizer::convert_and_update_tensor( } void MXFP8Quantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle /*out_py*/, const std::optional& noop_flag, - bool /*defer_swizzle*/) { + const std::optional& noop_flag, + QuantizationWorkspace* /*workspace*/) { if (input.numel() == 0) { return; } @@ -2017,10 +2016,7 @@ std::pair NVFP4Quantizer::create_tensor( const std::vector shape_int64(shape.begin(), shape.end()); const auto [flat_first_dim, flat_last_dim] = get_2d_dims(shape); - // Swizzled SF is emitted directly by the RHT cast-fusion kernel or by the plain - // 2D quantize kernel (for aligned shapes); other quantize paths reject it and go - // through the standalone post-quantize swizzle pass instead. - const bool with_gemm_swizzled_scales = emits_gemm_swizzled_scales(shape); + const bool with_gemm_swizzled_scales = this->optimize_for_gemm; NVTE_CHECK(flat_first_dim % NVFP4_BLOCK_SIZE == 0, "First dim for NVFP4 must be divisible by ", NVFP4_BLOCK_SIZE, " (got shape=", shape, ")"); NVTE_CHECK(flat_last_dim % NVFP4_BLOCK_SIZE == 0, @@ -2370,10 +2366,7 @@ std::pair NVFP4Quantizer::convert_and_update_tensor( const auto [flat_first_dim, flat_last_dim] = get_2d_dims(shape); - // Swizzled SF is emitted directly by the RHT cast-fusion kernel or by the plain - // 2D quantize kernel (for aligned shapes); other quantize paths reject it and go - // through the standalone post-quantize swizzle pass instead. - const bool with_gemm_swizzled_scales = emits_gemm_swizzled_scales(shape); + const bool with_gemm_swizzled_scales = this->optimize_for_gemm; const bool row_scaled_nvfp4 = this->row_scaled_nvfp4; const bool disable_second_level_scale = this->disable_second_level_scale; @@ -2584,13 +2577,12 @@ void NVFP4Quantizer::quantize_with_rht_unfused_helper( } void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, - const std::optional& noop_flag, - bool compute_amax, bool defer_swizzle) { - auto swizzle_if_needed = [&]() { - if (!defer_swizzle && optimize_for_gemm && !out.get_with_gemm_swizzled_scales()) { - inplace_swizzle_scale_for_gemm(out, out_py); - } + const std::optional& noop_flag, bool compute_amax, + QuantizationWorkspace* workspace) { + QuantizationWorkspace local_workspace; + auto& scale_workspace = workspace ? *workspace : local_workspace; + auto finish = [&]() { + if (!scale_workspace.defer_swizzle) scale_workspace.finish(out); }; auto reduce_amaxes = [&]() { if (!this->with_amax_reduction || this->disable_second_level_scale) { @@ -2626,10 +2618,16 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou if (!compute_amax) { reduce_amaxes(); } - swizzle_if_needed(); + finish(); return; } + const bool kernel_swizzles = emits_gemm_swizzled_scales(convertShape(input.shape())) && + (!with_rht || input.dtype() == DType::kBFloat16); + if (out.get_with_gemm_swizzled_scales() && !kernel_swizzles) { + scale_workspace.prepare(out, noop_flag.has_value()); + } + auto stream = at::cuda::getCurrentCUDAStream(); QuantizationConfigWrapper quant_config; @@ -2819,16 +2817,16 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou } else { NVTE_SCOPED_GIL_RELEASE({ nvte_quantize_v2(input.data(), out.data(), quant_config, stream); }); } - swizzle_if_needed(); + finish(); } void NVFP4Quantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - py::handle out_py, const std::optional& noop_flag, - bool defer_swizzle) { - this->quantize_impl(input, out, out_py, noop_flag, true, defer_swizzle); + const std::optional& noop_flag, + QuantizationWorkspace* workspace) { + this->quantize_impl(input, out, noop_flag, true, workspace); } -void NVFP4Quantizer::quantize_with_amax(TensorWrapper& input, TensorWrapper& out, py::handle out_py) { +void NVFP4Quantizer::quantize_with_amax(TensorWrapper& input, TensorWrapper& out) { NVTE_CHECK(!out.get_row_scaled_nvfp4(), "quantize_with_amax is not supported for row-scaled NVFP4 quantization."); // Update output tensor amaxes with input tensor amax @@ -2851,7 +2849,7 @@ void NVFP4Quantizer::quantize_with_amax(TensorWrapper& input, TensorWrapper& out input.set_amax(nullptr, DType::kFloat32, input.defaultShape); // Perform quantization - this->quantize_impl(input, out, out_py, std::nullopt, false); + this->quantize_impl(input, out, std::nullopt, false); } std::vector NVFP4Quantizer::get_scale_shape(const std::vector& shape, From 18d711094df4d08fc30c852319cc073e4b49552a Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 5 Nov 2026 12:11:02 +0100 Subject: [PATCH 12/31] Replace quantization workspace with local output views Signed-off-by: Pawel Gadzinski --- transformer_engine/pytorch/csrc/common.h | 38 +++----- transformer_engine/pytorch/csrc/extensions.h | 12 ++- .../pytorch/csrc/extensions/cast.cpp | 94 ++++++++++--------- .../pytorch/csrc/extensions/normalization.cpp | 18 ++-- .../pytorch/csrc/extensions/swizzle.cpp | 91 ++++++++---------- transformer_engine/pytorch/csrc/quantizer.cpp | 54 +++++------ 6 files changed, 144 insertions(+), 163 deletions(-) diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index 50149015b26..053d5b45b22 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -96,16 +96,6 @@ enum FP8BwdTensors { GRAD_INPUT3 = 5 }; -struct QuantizationWorkspace { - bool defer_swizzle = false; - std::optional final_output; - std::optional rowwise_scales; - std::optional columnwise_scales; - - void prepare(TensorWrapper& output, bool preserve_scales = false); - void finish(TensorWrapper& output); -}; - class Quantizer { public: virtual NVTEScalingMode get_scaling_mode() const = 0; @@ -141,10 +131,9 @@ class Quantizer { virtual std::pair convert_and_update_tensor( py::object tensor) const = 0; - /*! @brief Quantize, optionally deferring standalone swizzle to batch finalization. */ + /*! @brief Quantize into the layout requested by the output tensor. */ virtual void quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag = std::nullopt, - QuantizationWorkspace* workspace = nullptr) = 0; + const std::optional& noop_flag = std::nullopt) = 0; virtual ~Quantizer() = default; @@ -185,8 +174,7 @@ class NoneQuantizer : public Quantizer { std::pair convert_and_update_tensor(py::object tensor) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag = std::nullopt, - QuantizationWorkspace* workspace = nullptr) override; + const std::optional& noop_flag = std::nullopt) override; }; class Float8Quantizer : public Quantizer { @@ -221,8 +209,7 @@ class Float8Quantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag = std::nullopt, - QuantizationWorkspace* workspace = nullptr) override; + const std::optional& noop_flag = std::nullopt) override; }; class Float8CurrentScalingQuantizer : public Quantizer { @@ -262,8 +249,7 @@ class Float8CurrentScalingQuantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag = std::nullopt, - QuantizationWorkspace* workspace = nullptr) override; + const std::optional& noop_flag = std::nullopt) override; /*! @brief Quantize to FP8, skipping local amax computation * @@ -319,8 +305,7 @@ class Float8BlockQuantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag = std::nullopt, - QuantizationWorkspace* workspace = nullptr) override; + const std::optional& noop_flag = std::nullopt) override; std::vector get_scale_shape(const std::vector& shape, bool columnwise) const; }; @@ -349,8 +334,7 @@ class MXFP8Quantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag = std::nullopt, - QuantizationWorkspace* workspace = nullptr) override; + const std::optional& noop_flag = std::nullopt) override; std::vector get_scale_shape(const std::vector& shape, bool columnwise) const; }; @@ -410,11 +394,11 @@ class NVFP4Quantizer : public Quantizer { std::pair convert_and_update_tensor(py::object shape) const override; void quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag = std::nullopt, - QuantizationWorkspace* workspace = nullptr) override; + const std::optional& noop_flag = std::nullopt) override; void quantize_impl(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag, bool compute_amax, - QuantizationWorkspace* workspace = nullptr); + const std::optional& noop_flag, bool compute_amax); + void quantize_kernel(const TensorWrapper& input, TensorWrapper& out, + const std::optional& noop_flag, bool compute_amax); /*! @brief Quantize to NVFP4, skipping local amax computation * diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index 6b1a1897bd4..abd9c581c1d 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -728,10 +728,14 @@ void fused_multi_row_unpadding(at::Tensor input, at::Tensor output, void inplace_swizzle_scale_for_gemm(py::handle &tensor); -void prepare_quantization_workspaces(std::vector &outputs, - std::vector &workspaces); -void finish_quantization_workspaces(std::vector &outputs, - std::vector &workspaces, bool batched); +TensorWrapper make_compact_output_view(const TensorWrapper &output, at::Tensor &rowwise_scales, + at::Tensor &columnwise_scales, bool preserve_scales = false); +std::vector make_compact_output_views(const std::vector &outputs, + const std::vector &needs_swizzle, + std::vector &scratch); +void swizzle_scales_into(const TensorWrapper &input, const TensorWrapper &output); +void multi_tensor_swizzle_scales_into(const std::vector &inputs, + const std::vector &outputs); void inplace_multi_tensor_swizzle_scales_for_gemm(std::vector &tensors, bool rowwise_usage, bool columnwise_usage); diff --git a/transformer_engine/pytorch/csrc/extensions/cast.cpp b/transformer_engine/pytorch/csrc/extensions/cast.cpp index 5d3c70500f8..d658b3fde85 100644 --- a/transformer_engine/pytorch/csrc/extensions/cast.cpp +++ b/transformer_engine/pytorch/csrc/extensions/cast.cpp @@ -38,19 +38,36 @@ std::vector get_tensor_shape(const TensorWrapper &tensor) { return std::vector(shape.data, shape.data + shape.ndim); } -void finish_quantization(const std::vector> &quantizers, - std::vector &tensors, - std::vector &workspaces) { - if (tensors.empty()) return; - const auto scaling_mode = tensors.front().scaling_mode(); - const bool use_batch = - tensors.size() > 1 && - std::all_of(quantizers.begin(), quantizers.end(), - [](const auto &quantizer) { return quantizer->optimize_for_gemm; }) && - std::all_of(tensors.begin(), tensors.end(), [scaling_mode](const auto &tensor) { - return tensor.scaling_mode() == scaling_mode; +std::vector make_quantization_output_views( + const std::vector &inputs, + const std::vector> &quantizers, + const std::vector &outputs, std::vector &scratch, + bool grouped_kernel = false) { + if (outputs.empty() || (outputs.size() == 1 && !grouped_kernel) || + !std::all_of(quantizers.begin(), quantizers.end(), [](const auto &quantizer) { + return quantizer->optimize_for_gemm && + quantizer->get_scaling_mode() == NVTE_NVFP4_1D_SCALING; + })) { + return {}; + } + const bool grouped_swizzles = + grouped_kernel && static_cast(quantizers.front().get())->with_rht && + std::all_of(inputs.begin(), inputs.end(), [](const auto &input) { + return NVFP4Quantizer::is_eligible_for_rht_cast_fusion(get_tensor_shape(input), + /*for_grouped_kernel=*/true); }); - finish_quantization_workspaces(tensors, workspaces, use_batch); + std::vector needs_swizzle(outputs.size()); + for (size_t i = 0; i < outputs.size(); ++i) { + const auto &quantizer = *static_cast(quantizers[i].get()); + needs_swizzle[i] = + inputs[i].numel() != 0 && + !(grouped_kernel ? grouped_swizzles + : quantizer.emits_gemm_swizzled_scales(get_tensor_shape(inputs[i]))); + } + if (std::none_of(needs_swizzle.begin(), needs_swizzle.end(), [](bool value) { return value; })) { + return {}; + } + return make_compact_output_views(outputs, needs_swizzle, scratch); } void allreduce_nvfp4_amax_tensors(NVFP4Quantizer *nvfp4_quantizer_cpp, @@ -1087,8 +1104,7 @@ namespace { void multi_tensor_quantize_impl(const std::vector &input_list, std::vector &quantizer_py_list, std::vector> &quantizer_cpp_list, - std::vector &output_list, - std::vector &workspaces) { + std::vector &output_list) { // Check number of tensors const size_t num_tensors = input_list.size(); NVTE_CHECK(quantizer_py_list.size() == num_tensors, "Expected ", num_tensors, @@ -1129,8 +1145,7 @@ void multi_tensor_quantize_impl(const std::vector &input_list, } else { // Quantize kernels individually for (size_t i = 0; i < num_tensors; ++i) { - workspaces[i].defer_swizzle = true; - quantizer_cpp_list[i]->quantize(input_list[i], output_list[i], std::nullopt, &workspaces[i]); + quantizer_cpp_list[i]->quantize(input_list[i], output_list[i]); } } } @@ -1171,11 +1186,12 @@ std::vector multi_tensor_quantize(const std::vector &ten } // Perform multi-tensor quantization - std::vector workspaces(num_tensors); - multi_tensor_quantize_impl(input_cpp_list, quantizer_list, quantizer_cpp_list, output_cpp_list, - workspaces); - - finish_quantization(quantizer_cpp_list, output_cpp_list, workspaces); + std::vector scratch; + auto kernel_outputs = + make_quantization_output_views(input_cpp_list, quantizer_cpp_list, output_cpp_list, scratch); + multi_tensor_quantize_impl(input_cpp_list, quantizer_list, quantizer_cpp_list, + kernel_outputs.empty() ? output_cpp_list : kernel_outputs); + if (!kernel_outputs.empty()) multi_tensor_swizzle_scales_into(kernel_outputs, output_cpp_list); return output_py_list; } @@ -1402,8 +1418,7 @@ std::tuple, std::vector> bulk_allocate_mx // amax buffer will be zeroed out by later amax kernels, so we can use empty to allocate std::tuple, std::vector, bool> bulk_allocate_nvfp4_tensors( std::vector> &shape_list, std::vector &quantizer_py_list, - std::vector &quantizer_cpp_list, - std::vector &workspaces) { + std::vector &quantizer_cpp_list) { init_extension(); std::tuple, std::vector, bool> retval; auto &tensor_py_list = std::get<0>(retval); @@ -1447,16 +1462,6 @@ std::tuple, std::vector, bool> bulk_alloc "NVFP4 bulk allocation requires all quantizers in the group to share " "the same disable_second_level_scale value."); } - bool all_tensors_rht_cast_fusion_eligible = true; - for (size_t i = 0; i < num_tensors; ++i) { - if (!NVFP4Quantizer::is_eligible_for_rht_cast_fusion(shape_list[i], - /*for_grouped_kernel=*/true)) { - all_tensors_rht_cast_fusion_eligible = false; - break; - } - } - const bool kernel_swizzles = - group_optimize_for_gemm && group_with_rht && all_tensors_rht_cast_fusion_eligible; const bool with_gemm_swizzled_scales = group_optimize_for_gemm; // Helper function to get size of byte buffer holding FP4 data (last dim divided by 2) @@ -1648,10 +1653,6 @@ std::tuple, std::vector, bool> bulk_alloc } } - if (group_optimize_for_gemm && !kernel_swizzles) { - prepare_quantization_workspaces(tensor_cpp_list, workspaces); - } - return retval; } @@ -2024,7 +2025,6 @@ void split_quantize_nvfp4_impl_helper(const TensorWrapper &input, void split_quantize_nvfp4_impl(const TensorWrapper &input, const std::vector &input_list, std::vector &output_list, - std::vector &workspaces, const std::vector &split_sections, const std::vector &quantizers) { // Check tensor lists @@ -2096,8 +2096,7 @@ void split_quantize_nvfp4_impl(const TensorWrapper &input, if (input_list[i].numel() == 0) { continue; } - workspaces[i].defer_swizzle = true; - quantizers[i]->quantize(input_list[i], output_list[i], std::nullopt, &workspaces[i]); + quantizers[i]->quantize(input_list[i], output_list[i]); } } } else { // NVFP4 quantize @@ -2196,7 +2195,6 @@ std::vector split_quantize(const at::Tensor &tensor, } // Allocate output tensors - std::vector workspaces(num_splits); std::vector output_cpp_list; std::vector output_py_list; switch (allocation_method) { @@ -2228,7 +2226,7 @@ std::vector split_quantize(const at::Tensor &tensor, } bool contiguous_data_and_scale = false; std::tie(output_py_list, output_cpp_list, contiguous_data_and_scale) = - bulk_allocate_nvfp4_tensors(split_shapes, quantizer_list, nvfp4_quantizers, workspaces); + bulk_allocate_nvfp4_tensors(split_shapes, quantizer_list, nvfp4_quantizers); if (quantization_method == QuantizationMethod::FUSED_NVFP4 && !input_shape.empty() && input_shape.back() % 128 != 0) { static std::once_flag once_unfused_nvfp4_fallback_warning; @@ -2258,6 +2256,11 @@ std::vector split_quantize(const at::Tensor &tensor, } // Quantize into output tensors + std::vector scratch; + auto kernel_outputs = + make_quantization_output_views(input_list, quantizer_cpp_list, output_cpp_list, scratch, + quantization_method == QuantizationMethod::FUSED_NVFP4); + auto &quantized_outputs = kernel_outputs.empty() ? output_cpp_list : kernel_outputs; switch (quantization_method) { case QuantizationMethod::FUSED_NVFP4: { // Fused NVFP4 quantize kernel @@ -2266,17 +2269,16 @@ std::vector split_quantize(const at::Tensor &tensor, for (auto &quantizer : quantizer_cpp_list) { nvfp4_quantizers.push_back(static_cast(quantizer.get())); } - split_quantize_nvfp4_impl(input_nvte, input_list, output_cpp_list, workspaces, split_sections, + split_quantize_nvfp4_impl(input_nvte, input_list, quantized_outputs, split_sections, nvfp4_quantizers); break; } default: // General multi-tensor quantization - multi_tensor_quantize_impl(input_list, quantizer_list, quantizer_cpp_list, output_cpp_list, - workspaces); + multi_tensor_quantize_impl(input_list, quantizer_list, quantizer_cpp_list, quantized_outputs); } - finish_quantization(quantizer_cpp_list, output_cpp_list, workspaces); + if (!kernel_outputs.empty()) multi_tensor_swizzle_scales_into(kernel_outputs, output_cpp_list); return output_py_list; } diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index b53f70f6a9c..d8ce687b97b 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -144,16 +144,17 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe } else { std::tie(out_nvte, out) = quantizer_cpp->convert_and_update_tensor(out); } - QuantizationWorkspace scale_workspace; + at::Tensor rowwise_scales, columnwise_scales; + std::optional compact_out_nvte; if (cudnn_mxfp8_output && quantizer_cpp->optimize_for_gemm) { - scale_workspace.prepare(out_nvte); + compact_out_nvte.emplace(make_compact_output_view(out_nvte, rowwise_scales, columnwise_scales)); } // Construct unquantized output tensor if needed TensorWrapper unquantized_out_nvte; py::object unquantized_out; at::Tensor amax_buf; - TensorWrapper *kernel_out_nvte = &out_nvte; + TensorWrapper *kernel_out_nvte = compact_out_nvte ? &*compact_out_nvte : &out_nvte; switch (impl) { case Impl::UNFUSED: { NoneQuantizer q{none}; @@ -217,7 +218,7 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe } } - scale_workspace.finish(out_nvte); + if (compact_out_nvte) swizzle_scales_into(*compact_out_nvte, out_nvte); return {out, py::cast(mu_py), py::cast(rsigma_py)}; } @@ -393,16 +394,17 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w } else { std::tie(out_nvte, out) = quantizer_cpp->convert_and_update_tensor(out); } - QuantizationWorkspace scale_workspace; + at::Tensor rowwise_scales, columnwise_scales; + std::optional compact_out_nvte; if (impl == Impl::FUSED_NORM_QUANT_UNSWIZZLED && quantizer_cpp->optimize_for_gemm) { - scale_workspace.prepare(out_nvte); + compact_out_nvte.emplace(make_compact_output_view(out_nvte, rowwise_scales, columnwise_scales)); } // Construct unquantized output tensor if needed TensorWrapper unquantized_out_nvte; py::object unquantized_out; at::Tensor amax_buf; - TensorWrapper *kernel_out_nvte = &out_nvte; + TensorWrapper *kernel_out_nvte = compact_out_nvte ? &*compact_out_nvte : &out_nvte; switch (impl) { case Impl::UNFUSED: { NoneQuantizer q{none}; @@ -464,7 +466,7 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w } } - scale_workspace.finish(out_nvte); + if (compact_out_nvte) swizzle_scales_into(*compact_out_nvte, out_nvte); return {out, py::none(), py::cast(rsigma_py)}; } diff --git a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp index 57832638838..1f0bd15f4b1 100644 --- a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp +++ b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp @@ -69,8 +69,8 @@ TensorWrapper scale_view(const TensorWrapper &tensor, bool rowwise) { } // namespace -void QuantizationWorkspace::prepare(TensorWrapper &output, bool preserve_scales) { - NVTE_CHECK(!final_output.has_value(), "Quantization workspace is already in use"); +TensorWrapper make_compact_output_view(const TensorWrapper &output, at::Tensor &rowwise_scales, + at::Tensor &columnwise_scales, bool preserve_scales) { NVTE_CHECK(output.get_with_gemm_swizzled_scales(), "Expected a GEMM-ready output"); auto compact_output = copy_tensor_view(output); compact_output.set_with_gemm_swizzled_scales(false); @@ -79,11 +79,9 @@ void QuantizationWorkspace::prepare(TensorWrapper &output, bool preserve_scales) rowwise ? output.get_rowwise_scale_inv() : output.get_columnwise_scale_inv(); if (scales.data_ptr == nullptr) continue; auto &scratch = rowwise ? rowwise_scales : columnwise_scales; - if (!scratch.has_value()) { - scratch = allocateSpace(scales.shape, DType::kByte, false); - } + if (!scratch.defined()) scratch = allocateSpace(scales.shape, DType::kByte, false); compact_output.set_parameter(rowwise ? kNVTERowwiseScaleInv : kNVTEColumnwiseScaleInv, - scratch->data_ptr(), static_cast(scales.dtype), + scratch.data_ptr(), static_cast(scales.dtype), scales.shape); if (preserve_scales) { // A GPU no-op must preserve the previous scales through finalization. @@ -95,91 +93,84 @@ void QuantizationWorkspace::prepare(TensorWrapper &output, bool preserve_scales) }); } } - final_output.emplace(std::move(output)); - output = std::move(compact_output); + return compact_output; } -void QuantizationWorkspace::finish(TensorWrapper &output) { - if (!final_output.has_value()) return; +void swizzle_scales_into(const TensorWrapper &input, const TensorWrapper &output) { + if (input.get_with_gemm_swizzled_scales() || input.numel() == 0) return; for (bool rowwise : {true, false}) { - const auto scales = - rowwise ? output.get_rowwise_scale_inv() : output.get_columnwise_scale_inv(); + const auto scales = rowwise ? input.get_rowwise_scale_inv() : input.get_columnwise_scale_inv(); if (scales.data_ptr == nullptr) continue; - auto input_scales = scale_view(output, rowwise); - auto output_scales = scale_view(*final_output, rowwise); + auto input_scales = scale_view(input, rowwise); + auto output_scales = scale_view(output, rowwise); NVTE_SCOPED_GIL_RELEASE({ nvte_swizzle_scaling_factors(input_scales.data(), output_scales.data(), at::cuda::getCurrentCUDAStream()); }); } - output = std::move(*final_output); - final_output.reset(); - rowwise_scales.reset(); - columnwise_scales.reset(); } -void prepare_quantization_workspaces(std::vector &outputs, - std::vector &workspaces) { - NVTE_CHECK(outputs.size() == workspaces.size(), "Mismatched quantization workspaces"); +std::vector make_compact_output_views(const std::vector &outputs, + const std::vector &needs_swizzle, + std::vector &scratch) { + NVTE_CHECK(outputs.size() == needs_swizzle.size(), "Mismatched output layouts"); + scratch.resize(2 * outputs.size()); for (bool rowwise : {true, false}) { std::vector> shapes; std::vector indices; for (size_t i = 0; i < outputs.size(); ++i) { const auto scales = rowwise ? outputs[i].get_rowwise_scale_inv() : outputs[i].get_columnwise_scale_inv(); - if (outputs[i].numel() != 0 && scales.data_ptr != nullptr) { + if (needs_swizzle[i] && scales.data_ptr != nullptr) { shapes.emplace_back(scales.shape.data, scales.shape.data + scales.shape.ndim); indices.push_back(i); } } if (indices.empty()) continue; - auto scratch = bulk_allocate(shapes, std::vector(shapes.size(), at::kByte), + auto buffers = bulk_allocate(shapes, std::vector(shapes.size(), at::kByte), std::nullopt, std::vector(shapes.size(), 16)); for (size_t i = 0; i < indices.size(); ++i) { - auto &workspace = workspaces[indices[i]]; - (rowwise ? workspace.rowwise_scales : workspace.columnwise_scales) = std::move(scratch[i]); + scratch[2 * indices[i] + !rowwise] = std::move(buffers[i]); } } + std::vector views; + views.reserve(outputs.size()); for (size_t i = 0; i < outputs.size(); ++i) { - workspaces[i].defer_swizzle = true; - if (outputs[i].numel() != 0) workspaces[i].prepare(outputs[i]); + views.emplace_back( + needs_swizzle[i] ? make_compact_output_view(outputs[i], scratch[2 * i], scratch[2 * i + 1]) + : copy_tensor_view(outputs[i])); } + return views; } -void finish_quantization_workspaces(std::vector &outputs, - std::vector &workspaces, bool batched) { - NVTE_CHECK(outputs.size() == workspaces.size(), "Mismatched quantization workspaces"); - if (!batched) { - for (size_t i = 0; i < outputs.size(); ++i) workspaces[i].finish(outputs[i]); +void multi_tensor_swizzle_scales_into(const std::vector &inputs, + const std::vector &outputs) { + NVTE_CHECK(inputs.size() == outputs.size(), "Mismatched quantization outputs"); + if (inputs.size() == 1) { + swizzle_scales_into(inputs.front(), outputs.front()); return; } for (bool rowwise : {true, false}) { std::vector input_views, output_views; - std::vector inputs, destinations; - for (size_t i = 0; i < outputs.size(); ++i) { + std::vector sources, destinations; + for (size_t i = 0; i < inputs.size(); ++i) { const auto scales = - rowwise ? outputs[i].get_rowwise_scale_inv() : outputs[i].get_columnwise_scale_inv(); - if (!workspaces[i].final_output.has_value() || scales.data_ptr == nullptr) continue; - input_views.emplace_back(scale_view(outputs[i], rowwise)); - output_views.emplace_back(scale_view(*workspaces[i].final_output, rowwise)); - inputs.push_back(input_views.back().data()); + rowwise ? inputs[i].get_rowwise_scale_inv() : inputs[i].get_columnwise_scale_inv(); + if (inputs[i].get_with_gemm_swizzled_scales() || inputs[i].numel() == 0 || + scales.data_ptr == nullptr) + continue; + input_views.emplace_back(scale_view(inputs[i], rowwise)); + output_views.emplace_back(scale_view(outputs[i], rowwise)); + sources.push_back(input_views.back().data()); destinations.push_back(output_views.back().data()); } - if (!inputs.empty()) { + if (!sources.empty()) { NVTE_SCOPED_GIL_RELEASE({ - nvte_multi_tensor_swizzle_scaling_factors(inputs.data(), destinations.data(), inputs.size(), - at::cuda::getCurrentCUDAStream()); + nvte_multi_tensor_swizzle_scaling_factors(sources.data(), destinations.data(), + sources.size(), at::cuda::getCurrentCUDAStream()); }); } } - for (size_t i = 0; i < outputs.size(); ++i) { - auto &workspace = workspaces[i]; - if (!workspace.final_output.has_value()) continue; - outputs[i] = std::move(*workspace.final_output); - workspace.final_output.reset(); - workspace.rowwise_scales.reset(); - workspace.columnwise_scales.reset(); - } } std::tuple, std::optional> swizzle_scales_for_gemm( diff --git a/transformer_engine/pytorch/csrc/quantizer.cpp b/transformer_engine/pytorch/csrc/quantizer.cpp index e952082a5f3..ea623aef426 100644 --- a/transformer_engine/pytorch/csrc/quantizer.cpp +++ b/transformer_engine/pytorch/csrc/quantizer.cpp @@ -10,6 +10,7 @@ #include "common.h" #include "common/util/cuda_runtime.h" #include "common/util/system.h" +#include "extensions.h" #include "pybind.h" #include "torch/torch.h" @@ -329,8 +330,7 @@ std::pair NoneQuantizer::convert_and_update_tensor( } void NoneQuantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag, - QuantizationWorkspace* /*workspace*/) { + const std::optional& noop_flag) { NVTE_ERROR("NoneQuantizer does not support quantization"); } @@ -631,8 +631,7 @@ std::pair Float8Quantizer::convert_and_update_tensor( } void Float8Quantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag, - QuantizationWorkspace* /*workspace*/) { + const std::optional& noop_flag) { if (input.numel() == 0) { return; } @@ -1035,8 +1034,7 @@ void Float8CurrentScalingQuantizer::quantize_impl(const TensorWrapper& input, Te } void Float8CurrentScalingQuantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag, - QuantizationWorkspace* /*workspace*/) { + const std::optional& noop_flag) { const auto opts = at::TensorOptions().dtype(torch::kFloat32).device(torch::kCUDA); at::Tensor amax_and_scale = at::empty({2}, opts); this->quantize_impl(input, out, noop_flag, true, amax_and_scale[0], amax_and_scale[1]); @@ -1426,8 +1424,7 @@ std::pair Float8BlockQuantizer::convert_and_update_te } void Float8BlockQuantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag, - QuantizationWorkspace* /*workspace*/) { + const std::optional& noop_flag) { if (input.numel() == 0) { return; } @@ -1838,8 +1835,7 @@ std::pair MXFP8Quantizer::convert_and_update_tensor( } void MXFP8Quantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag, - QuantizationWorkspace* /*workspace*/) { + const std::optional& noop_flag) { if (input.numel() == 0) { return; } @@ -2577,13 +2573,24 @@ void NVFP4Quantizer::quantize_with_rht_unfused_helper( } void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag, bool compute_amax, - QuantizationWorkspace* workspace) { - QuantizationWorkspace local_workspace; - auto& scale_workspace = workspace ? *workspace : local_workspace; - auto finish = [&]() { - if (!scale_workspace.defer_swizzle) scale_workspace.finish(out); - }; + const std::optional& noop_flag, + bool compute_amax) { + const bool needs_swizzle = input.numel() != 0 && out.get_with_gemm_swizzled_scales() && + !emits_gemm_swizzled_scales(convertShape(input.shape())); + if (!needs_swizzle) { + quantize_kernel(input, out, noop_flag, compute_amax); + return; + } + at::Tensor rowwise_scales, columnwise_scales; + auto kernel_out = + make_compact_output_view(out, rowwise_scales, columnwise_scales, noop_flag.has_value()); + quantize_kernel(input, kernel_out, noop_flag, compute_amax); + swizzle_scales_into(kernel_out, out); +} + +void NVFP4Quantizer::quantize_kernel(const TensorWrapper& input, TensorWrapper& out, + const std::optional& noop_flag, + bool compute_amax) { auto reduce_amaxes = [&]() { if (!this->with_amax_reduction || this->disable_second_level_scale) { return; @@ -2618,16 +2625,9 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou if (!compute_amax) { reduce_amaxes(); } - finish(); return; } - const bool kernel_swizzles = emits_gemm_swizzled_scales(convertShape(input.shape())) && - (!with_rht || input.dtype() == DType::kBFloat16); - if (out.get_with_gemm_swizzled_scales() && !kernel_swizzles) { - scale_workspace.prepare(out, noop_flag.has_value()); - } - auto stream = at::cuda::getCurrentCUDAStream(); QuantizationConfigWrapper quant_config; @@ -2817,13 +2817,11 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou } else { NVTE_SCOPED_GIL_RELEASE({ nvte_quantize_v2(input.data(), out.data(), quant_config, stream); }); } - finish(); } void NVFP4Quantizer::quantize(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag, - QuantizationWorkspace* workspace) { - this->quantize_impl(input, out, noop_flag, true, workspace); + const std::optional& noop_flag) { + this->quantize_impl(input, out, noop_flag, true); } void NVFP4Quantizer::quantize_with_amax(TensorWrapper& input, TensorWrapper& out) { From ce64f5ea2cda38f1b20fc2944e54e5f9d4401574 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 5 Nov 2026 12:14:22 +0100 Subject: [PATCH 13/31] Allow batched quantization to query native scale layout Signed-off-by: Pawel Gadzinski --- transformer_engine/pytorch/csrc/common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index 053d5b45b22..d87b5fff509 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -425,6 +425,7 @@ class NVFP4Quantizer : public Quantizer { * skipped entirely. */ static bool is_eligible_for_2d_swizzle_fusion(const std::vector& shape); + bool emits_gemm_swizzled_scales(const std::vector& shape) const; // Flattened row/column alignment for directly emitted GEMM-swizzled scales, or no such path. static ShapeAlignment gemm_swizzled_scales_alignment(bool optimize_for_gemm, bool with_rht, @@ -439,7 +440,6 @@ class NVFP4Quantizer : public Quantizer { bool for_grouped_kernel = false); static ShapeAlignment get_2d_swizzle_fusion_alignment(int sm_arch); static bool meets_alignment(const std::vector& shape, const ShapeAlignment& alignment); - bool emits_gemm_swizzled_scales(const std::vector& shape) const; void quantize_with_rht_unfused_helper(const TensorWrapper& input, TensorWrapper& out, TensorWrapper& rht_output_t_cpp, From 663a4855d56b37ea21ab7b280b3f6f1f436066e2 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 5 Nov 2026 14:26:07 +0100 Subject: [PATCH 14/31] Keep NVFP4 quantization in quantize_impl Signed-off-by: Pawel Gadzinski --- transformer_engine/pytorch/csrc/common.h | 4 +--- transformer_engine/pytorch/csrc/quantizer.cpp | 24 +++++++------------ 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index d87b5fff509..e28e12dec77 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -395,10 +395,8 @@ class NVFP4Quantizer : public Quantizer { void quantize(const TensorWrapper& input, TensorWrapper& out, const std::optional& noop_flag = std::nullopt) override; - void quantize_impl(const TensorWrapper& input, TensorWrapper& out, + void quantize_impl(const TensorWrapper& input, TensorWrapper& output, const std::optional& noop_flag, bool compute_amax); - void quantize_kernel(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag, bool compute_amax); /*! @brief Quantize to NVFP4, skipping local amax computation * diff --git a/transformer_engine/pytorch/csrc/quantizer.cpp b/transformer_engine/pytorch/csrc/quantizer.cpp index ea623aef426..c3f850c4836 100644 --- a/transformer_engine/pytorch/csrc/quantizer.cpp +++ b/transformer_engine/pytorch/csrc/quantizer.cpp @@ -2572,25 +2572,18 @@ void NVFP4Quantizer::quantize_with_rht_unfused_helper( } } -void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& out, +void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& output, const std::optional& noop_flag, bool compute_amax) { - const bool needs_swizzle = input.numel() != 0 && out.get_with_gemm_swizzled_scales() && - !emits_gemm_swizzled_scales(convertShape(input.shape())); - if (!needs_swizzle) { - quantize_kernel(input, out, noop_flag, compute_amax); - return; - } at::Tensor rowwise_scales, columnwise_scales; - auto kernel_out = - make_compact_output_view(out, rowwise_scales, columnwise_scales, noop_flag.has_value()); - quantize_kernel(input, kernel_out, noop_flag, compute_amax); - swizzle_scales_into(kernel_out, out); -} + std::optional compact_out; + if (input.numel() != 0 && output.get_with_gemm_swizzled_scales() && + !emits_gemm_swizzled_scales(convertShape(input.shape()))) { + compact_out.emplace( + make_compact_output_view(output, rowwise_scales, columnwise_scales, noop_flag.has_value())); + } + TensorWrapper& out = compact_out ? *compact_out : output; -void NVFP4Quantizer::quantize_kernel(const TensorWrapper& input, TensorWrapper& out, - const std::optional& noop_flag, - bool compute_amax) { auto reduce_amaxes = [&]() { if (!this->with_amax_reduction || this->disable_second_level_scale) { return; @@ -2817,6 +2810,7 @@ void NVFP4Quantizer::quantize_kernel(const TensorWrapper& input, TensorWrapper& } else { NVTE_SCOPED_GIL_RELEASE({ nvte_quantize_v2(input.data(), out.data(), quant_config, stream); }); } + if (compact_out) swizzle_scales_into(*compact_out, output); } void NVFP4Quantizer::quantize(const TensorWrapper& input, TensorWrapper& out, From 5ba83966f3d6b6d8589d2c9ecf1d54a81783e1a1 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 5 Nov 2026 14:47:05 +0100 Subject: [PATCH 15/31] Remove redundant layout aliases and incidental changes Signed-off-by: Pawel Gadzinski --- transformer_engine/pytorch/csrc/common.h | 2 +- .../pytorch/csrc/extensions/cast.cpp | 14 +++++++------- .../pytorch/csrc/extensions/normalization.cpp | 5 ++--- .../pytorch/csrc/extensions/swizzle.cpp | 2 ++ transformer_engine/pytorch/csrc/quantizer.cpp | 13 +++++-------- transformer_engine/pytorch/quantized_tensor.py | 3 --- 6 files changed, 17 insertions(+), 22 deletions(-) diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index e28e12dec77..17a02cffc9e 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -131,7 +131,7 @@ class Quantizer { virtual std::pair convert_and_update_tensor( py::object tensor) const = 0; - /*! @brief Quantize into the layout requested by the output tensor. */ + /*! @brief Convert to a quantized data format */ virtual void quantize(const TensorWrapper& input, TensorWrapper& out, const std::optional& noop_flag = std::nullopt) = 0; diff --git a/transformer_engine/pytorch/csrc/extensions/cast.cpp b/transformer_engine/pytorch/csrc/extensions/cast.cpp index d658b3fde85..37f84c1b5eb 100644 --- a/transformer_engine/pytorch/csrc/extensions/cast.cpp +++ b/transformer_engine/pytorch/csrc/extensions/cast.cpp @@ -58,11 +58,11 @@ std::vector make_quantization_output_views( }); std::vector needs_swizzle(outputs.size()); for (size_t i = 0; i < outputs.size(); ++i) { - const auto &quantizer = *static_cast(quantizers[i].get()); needs_swizzle[i] = inputs[i].numel() != 0 && !(grouped_kernel ? grouped_swizzles - : quantizer.emits_gemm_swizzled_scales(get_tensor_shape(inputs[i]))); + : static_cast(quantizers[i].get()) + ->emits_gemm_swizzled_scales(get_tensor_shape(inputs[i]))); } if (std::none_of(needs_swizzle.begin(), needs_swizzle.end(), [](bool value) { return value; })) { return {}; @@ -1414,7 +1414,8 @@ std::tuple, std::vector> bulk_allocate_mx return retval; } -// Bulk-allocate FP4 data, scales, and amax values. +// allocate fp4 data, fp8 scalings, and amax values +// layout: [fp4_data0, ..., fp4_dataN, fp8_scaling0, ..., fp8_scalingN, amax0, ..., amaxN] // amax buffer will be zeroed out by later amax kernels, so we can use empty to allocate std::tuple, std::vector, bool> bulk_allocate_nvfp4_tensors( std::vector> &shape_list, std::vector &quantizer_py_list, @@ -1462,7 +1463,6 @@ std::tuple, std::vector, bool> bulk_alloc "NVFP4 bulk allocation requires all quantizers in the group to share " "the same disable_second_level_scale value."); } - const bool with_gemm_swizzled_scales = group_optimize_for_gemm; // Helper function to get size of byte buffer holding FP4 data (last dim divided by 2) auto fp4_byte_shape = [](const std::vector &shape) -> std::vector { @@ -1614,7 +1614,7 @@ std::tuple, std::vector, bool> bulk_alloc NVFP4TensorClass(rowwise_data, rowwise_scale, columnwise_data, columnwise_scale, amax_rowwise, amax_columnwise, MakePythonDType(fp4_dtype), MakePythonDType(scale_dtype), quantizer_py_list[i], - with_gemm_swizzled_scales, py::arg("row_scaled_nvfp4") = row_scaled_nvfp4, + group_optimize_for_gemm, py::arg("row_scaled_nvfp4") = row_scaled_nvfp4, py::arg("nvfp4_use_4over6") = nvfp4_use_4over6, py::arg("nvfp4_e4m3_max") = nvfp4_e4m3_max)); @@ -1643,7 +1643,7 @@ std::tuple, std::vector, bool> bulk_alloc getTensorShape(amax_columnwise_list[i])); } } - tensor_wrapper.set_with_gemm_swizzled_scales(with_gemm_swizzled_scales); + tensor_wrapper.set_with_gemm_swizzled_scales(group_optimize_for_gemm); tensor_wrapper.set_row_scaled_nvfp4(row_scaled_nvfp4); if (nvfp4_e4m3_max != 0) { tensor_wrapper.set_nvfp4_e4m3_max(nvfp4_e4m3_max); @@ -2096,7 +2096,7 @@ void split_quantize_nvfp4_impl(const TensorWrapper &input, if (input_list[i].numel() == 0) { continue; } - quantizers[i]->quantize(input_list[i], output_list[i]); + quantizers[i]->quantize(input_list[i], output_list[i], std::nullopt); } } } else { // NVFP4 quantize diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index d8ce687b97b..c92afabcb84 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -135,8 +135,6 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe } } - const bool cudnn_mxfp8_output = impl == Impl::FULLY_FUSED && IsMXFP8Quantizers(quantizer.ptr()); - // Output tensor TensorWrapper out_nvte; if (out.is_none()) { @@ -146,7 +144,8 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe } at::Tensor rowwise_scales, columnwise_scales; std::optional compact_out_nvte; - if (cudnn_mxfp8_output && quantizer_cpp->optimize_for_gemm) { + if (impl == Impl::FULLY_FUSED && IsMXFP8Quantizers(quantizer.ptr()) && + quantizer_cpp->optimize_for_gemm) { compact_out_nvte.emplace(make_compact_output_view(out_nvte, rowwise_scales, columnwise_scales)); } diff --git a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp index 1f0bd15f4b1..8c2feffa809 100644 --- a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp +++ b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp @@ -857,7 +857,9 @@ void inplace_multi_tensor_swizzle_scales_for_gemm_unchecked(std::vector NVFP4Quantizer::create_tensor( const std::vector shape_int64(shape.begin(), shape.end()); const auto [flat_first_dim, flat_last_dim] = get_2d_dims(shape); - const bool with_gemm_swizzled_scales = this->optimize_for_gemm; NVTE_CHECK(flat_first_dim % NVFP4_BLOCK_SIZE == 0, "First dim for NVFP4 must be divisible by ", NVFP4_BLOCK_SIZE, " (got shape=", shape, ")"); NVTE_CHECK(flat_last_dim % NVFP4_BLOCK_SIZE == 0, @@ -2091,7 +2090,7 @@ std::pair NVFP4Quantizer::create_tensor( kwargs["fp4_dtype"] = MakePythonDType(this->dtype); kwargs["scale_dtype"] = MakePythonDType(this->scale_dtype); kwargs["quantizer"] = this->quantizer; - kwargs["with_gemm_swizzled_scales"] = py::cast(with_gemm_swizzled_scales); + kwargs["with_gemm_swizzled_scales"] = py::cast(this->optimize_for_gemm); kwargs["row_scaled_nvfp4"] = py::cast(row_scaled_nvfp4); kwargs["nvfp4_use_4over6"] = py::cast(nvfp4_use_4over6); kwargs["nvfp4_e4m3_max"] = py::cast(this->nvfp4_e4m3_max); @@ -2123,7 +2122,7 @@ std::pair NVFP4Quantizer::create_tensor( kwargs["fp4_dtype"] = MakePythonDType(this->dtype); kwargs["scale_dtype"] = MakePythonDType(this->scale_dtype); kwargs["quantizer"] = this->quantizer; - kwargs["with_gemm_swizzled_scales"] = py::cast(with_gemm_swizzled_scales); + kwargs["with_gemm_swizzled_scales"] = py::cast(this->optimize_for_gemm); kwargs["device"] = py::cast(device); kwargs["row_scaled_nvfp4"] = py::cast(row_scaled_nvfp4); kwargs["nvfp4_use_4over6"] = py::cast(nvfp4_use_4over6); @@ -2163,7 +2162,7 @@ std::pair NVFP4Quantizer::create_tensor( getTensorShape(amax_columnwise)); } } - out_cpp.set_with_gemm_swizzled_scales(with_gemm_swizzled_scales); + out_cpp.set_with_gemm_swizzled_scales(this->optimize_for_gemm); out_cpp.set_row_scaled_nvfp4(row_scaled_nvfp4); if (this->nvfp4_e4m3_max != 0) { out_cpp.set_nvfp4_e4m3_max(this->nvfp4_e4m3_max); @@ -2362,8 +2361,6 @@ std::pair NVFP4Quantizer::convert_and_update_tensor( const auto [flat_first_dim, flat_last_dim] = get_2d_dims(shape); - const bool with_gemm_swizzled_scales = this->optimize_for_gemm; - const bool row_scaled_nvfp4 = this->row_scaled_nvfp4; const bool disable_second_level_scale = this->disable_second_level_scale; const bool nvfp4_use_4over6 = this->nvfp4_4over6_mode != kNVTENVFP44Over6Disabled; @@ -2371,7 +2368,7 @@ std::pair NVFP4Quantizer::convert_and_update_tensor( NVTE_CHECK(rowwise_usage, "Row-scaled NVFP4 quantization requires rowwise usage."); } tensor.attr("_row_scaled_nvfp4") = row_scaled_nvfp4; - tensor.attr("_with_gemm_swizzled_scales") = with_gemm_swizzled_scales; + tensor.attr("_with_gemm_swizzled_scales") = this->optimize_for_gemm; tensor.attr("_nvfp4_use_4over6") = py::cast(nvfp4_use_4over6); tensor.attr("_nvfp4_e4m3_max") = py::cast(this->nvfp4_e4m3_max); tensor.attr("_scale_dtype") = MakePythonDType(this->scale_dtype); @@ -2488,7 +2485,7 @@ std::pair NVFP4Quantizer::convert_and_update_tensor( getTensorShape(*amax_columnwise)); } } - out_cpp.set_with_gemm_swizzled_scales(with_gemm_swizzled_scales); + out_cpp.set_with_gemm_swizzled_scales(this->optimize_for_gemm); out_cpp.set_row_scaled_nvfp4(row_scaled_nvfp4); if (this->nvfp4_e4m3_max != 0) { out_cpp.set_nvfp4_e4m3_max(this->nvfp4_e4m3_max); diff --git a/transformer_engine/pytorch/quantized_tensor.py b/transformer_engine/pytorch/quantized_tensor.py index c486bfc7586..204e703e1d9 100644 --- a/transformer_engine/pytorch/quantized_tensor.py +++ b/transformer_engine/pytorch/quantized_tensor.py @@ -450,9 +450,6 @@ class Quantizer(abc.ABC): """Whether to solely optimize for matrix multiplication - MXFP8 and NVFP4 outputs use GEMM-swizzled scales even when a separate - swizzle is needed after quantization. - The resulting quantized tensors are not guaranteed to support any operation other than matrix multiplication. Use with care since this is likely to break communication, checkpointing, and many From 843b7ea281f16861bcf960eaa50a7c8832e6fa25 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 5 Nov 2026 15:00:14 +0100 Subject: [PATCH 16/31] Select compact-scale normalization via Impl enum Add FUSED_NORM_QUANT_UNSWIZZLED to layernorm_fwd and handle compact scale views and swizzling in the existing Impl switches of both normalization functions instead of a separate optional path. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Pawel Gadzinski --- .../pytorch/csrc/extensions/normalization.cpp | 54 ++++++++++--------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index c92afabcb84..b75e9966208 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -97,6 +97,8 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe UNFUSED, // Compute norm directly FULLY_FUSED, + // Compute norm directly with compact scales, then swizzle them for GEMM + FUSED_NORM_QUANT_UNSWIZZLED, // Compute norm and amax in high precision, then quantize to FP8 FUSED_NORM_AMAX_FP8, // Compute norm and amax in high precision, then quantize to NVFP4 @@ -112,7 +114,8 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe if (!mxfp8_quantizer_cpp->with_2d_quantization && transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN") && outer_size % 128 == 0 && inner_size % 128 == 0) { - impl = Impl::FULLY_FUSED; + impl = quantizer_cpp->optimize_for_gemm ? Impl::FUSED_NORM_QUANT_UNSWIZZLED + : Impl::FULLY_FUSED; } } else if (detail::IsFloat8CurrentScalingQuantizers(quantizer.ptr()) && !transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")) { @@ -142,24 +145,24 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe } else { std::tie(out_nvte, out) = quantizer_cpp->convert_and_update_tensor(out); } - at::Tensor rowwise_scales, columnwise_scales; - std::optional compact_out_nvte; - if (impl == Impl::FULLY_FUSED && IsMXFP8Quantizers(quantizer.ptr()) && - quantizer_cpp->optimize_for_gemm) { - compact_out_nvte.emplace(make_compact_output_view(out_nvte, rowwise_scales, columnwise_scales)); - } // Construct unquantized output tensor if needed TensorWrapper unquantized_out_nvte; py::object unquantized_out; at::Tensor amax_buf; - TensorWrapper *kernel_out_nvte = compact_out_nvte ? &*compact_out_nvte : &out_nvte; + TensorWrapper compact_out_nvte; + at::Tensor rowwise_scales, columnwise_scales; + TensorWrapper *kernel_out_nvte = &out_nvte; switch (impl) { case Impl::UNFUSED: { NoneQuantizer q{none}; std::tie(unquantized_out_nvte, unquantized_out) = q.create_tensor(shape, out_dtype); kernel_out_nvte = &unquantized_out_nvte; } break; + case Impl::FUSED_NORM_QUANT_UNSWIZZLED: { + compact_out_nvte = make_compact_output_view(out_nvte, rowwise_scales, columnwise_scales); + kernel_out_nvte = &compact_out_nvte; + } break; case Impl::FUSED_NORM_AMAX_FP8: { auto fp8_quantizer_cpp = static_cast(quantizer_cpp.get()); std::tie(unquantized_out_nvte, unquantized_out, amax_buf) = @@ -205,6 +208,9 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe case Impl::UNFUSED: { quantizer_cpp->quantize(unquantized_out_nvte, out_nvte); } break; + case Impl::FUSED_NORM_QUANT_UNSWIZZLED: { + swizzle_scales_into(compact_out_nvte, out_nvte); + } break; case Impl::FUSED_NORM_AMAX_FP8: { auto fp8_quantizer_cpp = static_cast(quantizer_cpp.get()); fp8_quantizer_cpp->quantize_with_amax(unquantized_out_nvte, out_nvte, amax_buf); @@ -216,8 +222,6 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe default: { } } - - if (compact_out_nvte) swizzle_scales_into(*compact_out_nvte, out_nvte); return {out, py::cast(mu_py), py::cast(rsigma_py)}; } @@ -342,7 +346,7 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w UNFUSED, // Compute norm directly FULLY_FUSED, - // Compute norm directly, with quantization scales in compact format + // Compute norm directly with compact scales, then swizzle them for GEMM FUSED_NORM_QUANT_UNSWIZZLED, // Compute norm and amax in high precision, then quantize to FP8 FUSED_NORM_AMAX_FP8, @@ -358,12 +362,10 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w // The fused MXFP8 kernels quantize 1D blocks and require full 128x128 tiles if (!mxfp8_quantizer_cpp->with_2d_quantization && outer_size % 128 == 0 && inner_size % 128 == 0) { - if (transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")) { - impl = Impl::FUSED_NORM_QUANT_UNSWIZZLED; - } else { - // Transformer Engine's fused RMSNorm + MXFP8 kernel - impl = Impl::FULLY_FUSED; - } + // cuDNN emits compact scales + const bool use_cudnn = transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN"); + impl = use_cudnn && quantizer_cpp->optimize_for_gemm ? Impl::FUSED_NORM_QUANT_UNSWIZZLED + : Impl::FULLY_FUSED; } } else if (detail::IsFloat8CurrentScalingQuantizers(quantizer.ptr()) && !transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")) { @@ -393,23 +395,24 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w } else { std::tie(out_nvte, out) = quantizer_cpp->convert_and_update_tensor(out); } - at::Tensor rowwise_scales, columnwise_scales; - std::optional compact_out_nvte; - if (impl == Impl::FUSED_NORM_QUANT_UNSWIZZLED && quantizer_cpp->optimize_for_gemm) { - compact_out_nvte.emplace(make_compact_output_view(out_nvte, rowwise_scales, columnwise_scales)); - } // Construct unquantized output tensor if needed TensorWrapper unquantized_out_nvte; py::object unquantized_out; at::Tensor amax_buf; - TensorWrapper *kernel_out_nvte = compact_out_nvte ? &*compact_out_nvte : &out_nvte; + TensorWrapper compact_out_nvte; + at::Tensor rowwise_scales, columnwise_scales; + TensorWrapper *kernel_out_nvte = &out_nvte; switch (impl) { case Impl::UNFUSED: { NoneQuantizer q{none}; std::tie(unquantized_out_nvte, unquantized_out) = q.create_tensor(shape, out_dtype); kernel_out_nvte = &unquantized_out_nvte; } break; + case Impl::FUSED_NORM_QUANT_UNSWIZZLED: { + compact_out_nvte = make_compact_output_view(out_nvte, rowwise_scales, columnwise_scales); + kernel_out_nvte = &compact_out_nvte; + } break; case Impl::FUSED_NORM_AMAX_FP8: { auto fp8_quantizer_cpp = static_cast(quantizer_cpp.get()); std::tie(unquantized_out_nvte, unquantized_out, amax_buf) = @@ -453,6 +456,9 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w case Impl::UNFUSED: { quantizer_cpp->quantize(unquantized_out_nvte, out_nvte); } break; + case Impl::FUSED_NORM_QUANT_UNSWIZZLED: { + swizzle_scales_into(compact_out_nvte, out_nvte); + } break; case Impl::FUSED_NORM_AMAX_FP8: { auto fp8_quantizer_cpp = static_cast(quantizer_cpp.get()); fp8_quantizer_cpp->quantize_with_amax(unquantized_out_nvte, out_nvte, amax_buf); @@ -464,8 +470,6 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w default: { } } - - if (compact_out_nvte) swizzle_scales_into(*compact_out_nvte, out_nvte); return {out, py::none(), py::cast(rsigma_py)}; } From 5aa752b8fb6523aefeba6311ee59ef0b448831c0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 14:01:57 +0000 Subject: [PATCH 17/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- transformer_engine/pytorch/csrc/extensions/normalization.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index b75e9966208..ed6d19e19ac 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -114,8 +114,8 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe if (!mxfp8_quantizer_cpp->with_2d_quantization && transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN") && outer_size % 128 == 0 && inner_size % 128 == 0) { - impl = quantizer_cpp->optimize_for_gemm ? Impl::FUSED_NORM_QUANT_UNSWIZZLED - : Impl::FULLY_FUSED; + impl = + quantizer_cpp->optimize_for_gemm ? Impl::FUSED_NORM_QUANT_UNSWIZZLED : Impl::FULLY_FUSED; } } else if (detail::IsFloat8CurrentScalingQuantizers(quantizer.ptr()) && !transformer_engine::getenv("NVTE_NORM_FWD_USE_CUDNN")) { From 832ec271a8b989eb4c307e05c824ce27d9eb10a3 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 5 Nov 2026 15:09:59 +0100 Subject: [PATCH 18/31] Simplify GEMM output conversion and batched scale swizzling Convert a reused GEMM output with the quantizer where the output tensor is constructed instead of after the GEMM. Limit compact scale views and batched swizzling to the grouped NVFP4 split-quantize kernel; other multi-tensor paths swizzle per tensor in quantize(). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Pawel Gadzinski --- transformer_engine/pytorch/csrc/common.h | 2 +- transformer_engine/pytorch/csrc/extensions.h | 1 - .../pytorch/csrc/extensions/cast.cpp | 68 ++++++------------- .../pytorch/csrc/extensions/gemm.cpp | 36 +++++----- .../pytorch/csrc/extensions/swizzle.cpp | 8 +-- 5 files changed, 42 insertions(+), 73 deletions(-) diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index 17a02cffc9e..4f7480a085a 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -423,7 +423,6 @@ class NVFP4Quantizer : public Quantizer { * skipped entirely. */ static bool is_eligible_for_2d_swizzle_fusion(const std::vector& shape); - bool emits_gemm_swizzled_scales(const std::vector& shape) const; // Flattened row/column alignment for directly emitted GEMM-swizzled scales, or no such path. static ShapeAlignment gemm_swizzled_scales_alignment(bool optimize_for_gemm, bool with_rht, @@ -438,6 +437,7 @@ class NVFP4Quantizer : public Quantizer { bool for_grouped_kernel = false); static ShapeAlignment get_2d_swizzle_fusion_alignment(int sm_arch); static bool meets_alignment(const std::vector& shape, const ShapeAlignment& alignment); + bool emits_gemm_swizzled_scales(const std::vector& shape) const; void quantize_with_rht_unfused_helper(const TensorWrapper& input, TensorWrapper& out, TensorWrapper& rht_output_t_cpp, diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index abd9c581c1d..9d5c85e0ce0 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -731,7 +731,6 @@ void inplace_swizzle_scale_for_gemm(py::handle &tensor); TensorWrapper make_compact_output_view(const TensorWrapper &output, at::Tensor &rowwise_scales, at::Tensor &columnwise_scales, bool preserve_scales = false); std::vector make_compact_output_views(const std::vector &outputs, - const std::vector &needs_swizzle, std::vector &scratch); void swizzle_scales_into(const TensorWrapper &input, const TensorWrapper &output); void multi_tensor_swizzle_scales_into(const std::vector &inputs, diff --git a/transformer_engine/pytorch/csrc/extensions/cast.cpp b/transformer_engine/pytorch/csrc/extensions/cast.cpp index 37f84c1b5eb..0b033b249d5 100644 --- a/transformer_engine/pytorch/csrc/extensions/cast.cpp +++ b/transformer_engine/pytorch/csrc/extensions/cast.cpp @@ -38,38 +38,6 @@ std::vector get_tensor_shape(const TensorWrapper &tensor) { return std::vector(shape.data, shape.data + shape.ndim); } -std::vector make_quantization_output_views( - const std::vector &inputs, - const std::vector> &quantizers, - const std::vector &outputs, std::vector &scratch, - bool grouped_kernel = false) { - if (outputs.empty() || (outputs.size() == 1 && !grouped_kernel) || - !std::all_of(quantizers.begin(), quantizers.end(), [](const auto &quantizer) { - return quantizer->optimize_for_gemm && - quantizer->get_scaling_mode() == NVTE_NVFP4_1D_SCALING; - })) { - return {}; - } - const bool grouped_swizzles = - grouped_kernel && static_cast(quantizers.front().get())->with_rht && - std::all_of(inputs.begin(), inputs.end(), [](const auto &input) { - return NVFP4Quantizer::is_eligible_for_rht_cast_fusion(get_tensor_shape(input), - /*for_grouped_kernel=*/true); - }); - std::vector needs_swizzle(outputs.size()); - for (size_t i = 0; i < outputs.size(); ++i) { - needs_swizzle[i] = - inputs[i].numel() != 0 && - !(grouped_kernel ? grouped_swizzles - : static_cast(quantizers[i].get()) - ->emits_gemm_swizzled_scales(get_tensor_shape(inputs[i]))); - } - if (std::none_of(needs_swizzle.begin(), needs_swizzle.end(), [](bool value) { return value; })) { - return {}; - } - return make_compact_output_views(outputs, needs_swizzle, scratch); -} - void allreduce_nvfp4_amax_tensors(NVFP4Quantizer *nvfp4_quantizer_cpp, std::vector &&amax_tensors) { if (!nvfp4_quantizer_cpp->with_amax_reduction || amax_tensors.empty()) { @@ -1186,12 +1154,8 @@ std::vector multi_tensor_quantize(const std::vector &ten } // Perform multi-tensor quantization - std::vector scratch; - auto kernel_outputs = - make_quantization_output_views(input_cpp_list, quantizer_cpp_list, output_cpp_list, scratch); - multi_tensor_quantize_impl(input_cpp_list, quantizer_list, quantizer_cpp_list, - kernel_outputs.empty() ? output_cpp_list : kernel_outputs); - if (!kernel_outputs.empty()) multi_tensor_swizzle_scales_into(kernel_outputs, output_cpp_list); + multi_tensor_quantize_impl(input_cpp_list, quantizer_list, quantizer_cpp_list, output_cpp_list); + return output_py_list; } @@ -2256,11 +2220,6 @@ std::vector split_quantize(const at::Tensor &tensor, } // Quantize into output tensors - std::vector scratch; - auto kernel_outputs = - make_quantization_output_views(input_list, quantizer_cpp_list, output_cpp_list, scratch, - quantization_method == QuantizationMethod::FUSED_NVFP4); - auto &quantized_outputs = kernel_outputs.empty() ? output_cpp_list : kernel_outputs; switch (quantization_method) { case QuantizationMethod::FUSED_NVFP4: { // Fused NVFP4 quantize kernel @@ -2269,16 +2228,31 @@ std::vector split_quantize(const at::Tensor &tensor, for (auto &quantizer : quantizer_cpp_list) { nvfp4_quantizers.push_back(static_cast(quantizer.get())); } - split_quantize_nvfp4_impl(input_nvte, input_list, quantized_outputs, split_sections, - nvfp4_quantizers); + // Grouped kernels emit GEMM-swizzled scales only with RHT cast fusion + const auto &quantizer = *nvfp4_quantizers.front(); + const bool emits_gemm_swizzled_scales = + quantizer.with_rht && + std::all_of(input_list.begin(), input_list.end(), [](const auto &input) { + return NVFP4Quantizer::is_eligible_for_rht_cast_fusion(get_tensor_shape(input), + /*for_grouped_kernel=*/true); + }); + if (quantizer.optimize_for_gemm && !emits_gemm_swizzled_scales) { + std::vector scratch; + auto compact_outputs = make_compact_output_views(output_cpp_list, scratch); + split_quantize_nvfp4_impl(input_nvte, input_list, compact_outputs, split_sections, + nvfp4_quantizers); + multi_tensor_swizzle_scales_into(compact_outputs, output_cpp_list); + } else { + split_quantize_nvfp4_impl(input_nvte, input_list, output_cpp_list, split_sections, + nvfp4_quantizers); + } break; } default: // General multi-tensor quantization - multi_tensor_quantize_impl(input_list, quantizer_list, quantizer_cpp_list, quantized_outputs); + multi_tensor_quantize_impl(input_list, quantizer_list, quantizer_cpp_list, output_cpp_list); } - if (!kernel_outputs.empty()) multi_tensor_swizzle_scales_into(kernel_outputs, output_cpp_list); return output_py_list; } diff --git a/transformer_engine/pytorch/csrc/extensions/gemm.cpp b/transformer_engine/pytorch/csrc/extensions/gemm.cpp index 28fe93376f4..6911ba84805 100644 --- a/transformer_engine/pytorch/csrc/extensions/gemm.cpp +++ b/transformer_engine/pytorch/csrc/extensions/gemm.cpp @@ -226,13 +226,29 @@ std::vector gemm(py::handle A, bool transa, py::handle B, bool trans } DType output_dtype = out_dtype ? *out_dtype : A_tensor.dtype(); + // Unfused quantization is needed in the following cases + // 1. Inputs: BF16, Output: FP8 (GEMM output has to be BF16, so FP8 quantization needed after that) + // 2. Inputs: FP8, Output: FP8 (For any quantization apart from delayed scaling, + // GEMM Output needs to be in BF16, to allow for unfused quantization) + bool unfused_quantization_needed = !quantizer.is_none(); + if (low_precision) { + // At the moment, only use-case for fused GEMM: + // Delayed scaling quantizer with per-tensor scaling inputs + bool is_per_tensor_scaling_input = IsFloat8Tensor(A.ptr()) || IsFloat8Tensor(B.ptr()); + if (IsFloat8Quantizers(quantizer.ptr()) && is_per_tensor_scaling_input) + unfused_quantization_needed = false; + } + // Output tensor - const bool reuse_output = !D.is_none(); TensorWrapper D_tensor; if (D.is_none()) { std::tie(D_tensor, D) = createOutputTensor(D_shape, output_dtype, quantizer); } else { - D_tensor = makeTransformerEngineTensor(D, quantizer); + if (unfused_quantization_needed) { + std::tie(D_tensor, D) = convert_quantizer(quantizer)->convert_and_update_tensor(D); + } else { + D_tensor = makeTransformerEngineTensor(D, quantizer); + } NVTE_CHECK(detail::checkGemmShape(D_shape, D_tensor.shape()), "GEMM output has invalid dims (expected ", std::to_string(D_shape), ", got ", std::to_string(D_tensor.shape()), ")"); @@ -245,19 +261,6 @@ std::vector gemm(py::handle A, bool transa, py::handle B, bool trans // maintain unquantized tensor in case we need unfused quantization support. TensorWrapper unquantized_D_tensor; py::object unquantized_out; - // Unfused quantization is needed in the following cases - // 1. Inputs: BF16, Output: FP8 (GEMM output has to be BF16, so FP8 quantization needed after that) - // 2. Inputs: FP8, Output: FP8 (For any quantization apart from delayed scaling, - // GEMM Output needs to be in BF16, to allow for unfused quantization) - bool unfused_quantization_needed = !quantizer.is_none(); - if (low_precision) { - // At the moment, only use-case for fused GEMM: - // Delayed scaling quantizer with per-tensor scaling inputs - bool is_per_tensor_scaling_input = IsFloat8Tensor(A.ptr()) || IsFloat8Tensor(B.ptr()); - if (IsFloat8Quantizers(quantizer.ptr()) && is_per_tensor_scaling_input) - unfused_quantization_needed = false; - } - if (unfused_quantization_needed) { NoneQuantizer q{none}; std::tie(unquantized_D_tensor, unquantized_out) = q.create_tensor(D_shape, output_dtype); @@ -424,9 +427,6 @@ std::vector gemm(py::handle A, bool transa, py::handle B, bool trans if (unfused_quantization_needed) { // Quantize the output std::unique_ptr my_quantizer = convert_quantizer(quantizer); - if (reuse_output) { - std::tie(D_tensor, D) = my_quantizer->convert_and_update_tensor(D); - } my_quantizer->quantize(unquantized_D_tensor, D_tensor); } // Pack outputs diff --git a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp index 8c2feffa809..dcfc9bcf43e 100644 --- a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp +++ b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp @@ -111,9 +111,7 @@ void swizzle_scales_into(const TensorWrapper &input, const TensorWrapper &output } std::vector make_compact_output_views(const std::vector &outputs, - const std::vector &needs_swizzle, std::vector &scratch) { - NVTE_CHECK(outputs.size() == needs_swizzle.size(), "Mismatched output layouts"); scratch.resize(2 * outputs.size()); for (bool rowwise : {true, false}) { std::vector> shapes; @@ -121,7 +119,7 @@ std::vector make_compact_output_views(const std::vector make_compact_output_views(const std::vector views; views.reserve(outputs.size()); for (size_t i = 0; i < outputs.size(); ++i) { - views.emplace_back( - needs_swizzle[i] ? make_compact_output_view(outputs[i], scratch[2 * i], scratch[2 * i + 1]) - : copy_tensor_view(outputs[i])); + views.emplace_back(make_compact_output_view(outputs[i], scratch[2 * i], scratch[2 * i + 1])); } return views; } From d106f02d3195390c4e8f7ae427cb220b6a63f763 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 5 Nov 2026 16:27:11 +0100 Subject: [PATCH 19/31] Batch scale swizzling for compact multi-tensor outputs Quantize NVFP4 outputs whose kernel cannot write GEMM-swizzled scales into compact scratch views and swizzle them with one kernel in multi_tensor_quantize and split_quantize, while outputs with in-kernel swizzling are written directly. Declare the internal swizzle helpers in util.h next to the existing ones. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Pawel Gadzinski --- transformer_engine/pytorch/csrc/common.h | 2 +- transformer_engine/pytorch/csrc/extensions.h | 8 --- .../pytorch/csrc/extensions/cast.cpp | 60 +++++++++++++++---- .../pytorch/csrc/extensions/normalization.cpp | 1 + .../pytorch/csrc/extensions/swizzle.cpp | 8 ++- transformer_engine/pytorch/csrc/quantizer.cpp | 2 +- transformer_engine/pytorch/csrc/util.h | 15 +++++ 7 files changed, 72 insertions(+), 24 deletions(-) diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index 4f7480a085a..17a02cffc9e 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -423,6 +423,7 @@ class NVFP4Quantizer : public Quantizer { * skipped entirely. */ static bool is_eligible_for_2d_swizzle_fusion(const std::vector& shape); + bool emits_gemm_swizzled_scales(const std::vector& shape) const; // Flattened row/column alignment for directly emitted GEMM-swizzled scales, or no such path. static ShapeAlignment gemm_swizzled_scales_alignment(bool optimize_for_gemm, bool with_rht, @@ -437,7 +438,6 @@ class NVFP4Quantizer : public Quantizer { bool for_grouped_kernel = false); static ShapeAlignment get_2d_swizzle_fusion_alignment(int sm_arch); static bool meets_alignment(const std::vector& shape, const ShapeAlignment& alignment); - bool emits_gemm_swizzled_scales(const std::vector& shape) const; void quantize_with_rht_unfused_helper(const TensorWrapper& input, TensorWrapper& out, TensorWrapper& rht_output_t_cpp, diff --git a/transformer_engine/pytorch/csrc/extensions.h b/transformer_engine/pytorch/csrc/extensions.h index 9d5c85e0ce0..7c22e69d92b 100644 --- a/transformer_engine/pytorch/csrc/extensions.h +++ b/transformer_engine/pytorch/csrc/extensions.h @@ -728,14 +728,6 @@ void fused_multi_row_unpadding(at::Tensor input, at::Tensor output, void inplace_swizzle_scale_for_gemm(py::handle &tensor); -TensorWrapper make_compact_output_view(const TensorWrapper &output, at::Tensor &rowwise_scales, - at::Tensor &columnwise_scales, bool preserve_scales = false); -std::vector make_compact_output_views(const std::vector &outputs, - std::vector &scratch); -void swizzle_scales_into(const TensorWrapper &input, const TensorWrapper &output); -void multi_tensor_swizzle_scales_into(const std::vector &inputs, - const std::vector &outputs); - void inplace_multi_tensor_swizzle_scales_for_gemm(std::vector &tensors, bool rowwise_usage, bool columnwise_usage); diff --git a/transformer_engine/pytorch/csrc/extensions/cast.cpp b/transformer_engine/pytorch/csrc/extensions/cast.cpp index 0b033b249d5..93fcec7aeb7 100644 --- a/transformer_engine/pytorch/csrc/extensions/cast.cpp +++ b/transformer_engine/pytorch/csrc/extensions/cast.cpp @@ -27,6 +27,7 @@ #include "transformer_engine/multi_tensor.h" #include "transformer_engine/recipe.h" #include "transformer_engine/transformer_engine.h" +#include "util.h" namespace transformer_engine { namespace pytorch { @@ -38,6 +39,29 @@ std::vector get_tensor_shape(const TensorWrapper &tensor) { return std::vector(shape.data, shape.data + shape.ndim); } +bool needs_compact_scales(const Quantizer &quantizer, const TensorWrapper &input) { + if (!quantizer.optimize_for_gemm || input.numel() == 0 || + quantizer.get_scaling_mode() != NVTE_NVFP4_1D_SCALING) { + return false; + } + return !static_cast(quantizer).emits_gemm_swizzled_scales( + get_tensor_shape(input)); +} + +// Compact-scale outputs are quantized into scratch and swizzled with one kernel +template +void quantize_with_batched_swizzle(std::vector &outputs, + const std::vector &compact_scales, QuantizeFn &&quantize) { + if (std::none_of(compact_scales.begin(), compact_scales.end(), [](bool b) { return b; })) { + quantize(outputs); + return; + } + std::vector scratch; + auto kernel_outputs = make_compact_output_views(outputs, compact_scales, scratch); + quantize(kernel_outputs); + multi_tensor_swizzle_scales_into(kernel_outputs, outputs); +} + void allreduce_nvfp4_amax_tensors(NVFP4Quantizer *nvfp4_quantizer_cpp, std::vector &&amax_tensors) { if (!nvfp4_quantizer_cpp->with_amax_reduction || amax_tensors.empty()) { @@ -1154,7 +1178,13 @@ std::vector multi_tensor_quantize(const std::vector &ten } // Perform multi-tensor quantization - multi_tensor_quantize_impl(input_cpp_list, quantizer_list, quantizer_cpp_list, output_cpp_list); + std::vector compact_scales(num_tensors); + for (size_t i = 0; i < num_tensors; ++i) { + compact_scales[i] = needs_compact_scales(*quantizer_cpp_list[i], input_cpp_list[i]); + } + quantize_with_batched_swizzle(output_cpp_list, compact_scales, [&](auto &outputs) { + multi_tensor_quantize_impl(input_cpp_list, quantizer_list, quantizer_cpp_list, outputs); + }); return output_py_list; } @@ -2236,21 +2266,27 @@ std::vector split_quantize(const at::Tensor &tensor, return NVFP4Quantizer::is_eligible_for_rht_cast_fusion(get_tensor_shape(input), /*for_grouped_kernel=*/true); }); - if (quantizer.optimize_for_gemm && !emits_gemm_swizzled_scales) { - std::vector scratch; - auto compact_outputs = make_compact_output_views(output_cpp_list, scratch); - split_quantize_nvfp4_impl(input_nvte, input_list, compact_outputs, split_sections, - nvfp4_quantizers); - multi_tensor_swizzle_scales_into(compact_outputs, output_cpp_list); - } else { - split_quantize_nvfp4_impl(input_nvte, input_list, output_cpp_list, split_sections, - nvfp4_quantizers); + std::vector compact_scales(num_splits); + for (size_t i = 0; i < num_splits; ++i) { + compact_scales[i] = quantizer.optimize_for_gemm && !emits_gemm_swizzled_scales && + input_list[i].numel() != 0; } + quantize_with_batched_swizzle(output_cpp_list, compact_scales, [&](auto &outputs) { + split_quantize_nvfp4_impl(input_nvte, input_list, outputs, split_sections, + nvfp4_quantizers); + }); break; } - default: + default: { // General multi-tensor quantization - multi_tensor_quantize_impl(input_list, quantizer_list, quantizer_cpp_list, output_cpp_list); + std::vector compact_scales(num_splits); + for (size_t i = 0; i < num_splits; ++i) { + compact_scales[i] = needs_compact_scales(*quantizer_cpp_list[i], input_list[i]); + } + quantize_with_batched_swizzle(output_cpp_list, compact_scales, [&](auto &outputs) { + multi_tensor_quantize_impl(input_list, quantizer_list, quantizer_cpp_list, outputs); + }); + } } return output_py_list; diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index ed6d19e19ac..3271941c148 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -7,6 +7,7 @@ #include "../extensions.h" #include "common/util/system.h" #include "pybind.h" +#include "util.h" namespace transformer_engine::pytorch { diff --git a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp index dcfc9bcf43e..5ec9888df49 100644 --- a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp +++ b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp @@ -111,7 +111,9 @@ void swizzle_scales_into(const TensorWrapper &input, const TensorWrapper &output } std::vector make_compact_output_views(const std::vector &outputs, + const std::vector &compact_scales, std::vector &scratch) { + NVTE_CHECK(outputs.size() == compact_scales.size(), "Mismatched output layouts"); scratch.resize(2 * outputs.size()); for (bool rowwise : {true, false}) { std::vector> shapes; @@ -119,7 +121,7 @@ std::vector make_compact_output_views(const std::vector make_compact_output_views(const std::vector views; views.reserve(outputs.size()); for (size_t i = 0; i < outputs.size(); ++i) { - views.emplace_back(make_compact_output_view(outputs[i], scratch[2 * i], scratch[2 * i + 1])); + views.emplace_back(compact_scales[i] ? make_compact_output_view(outputs[i], scratch[2 * i], + scratch[2 * i + 1]) + : copy_tensor_view(outputs[i])); } return views; } diff --git a/transformer_engine/pytorch/csrc/quantizer.cpp b/transformer_engine/pytorch/csrc/quantizer.cpp index 28c7ef72bed..0afe7b81528 100644 --- a/transformer_engine/pytorch/csrc/quantizer.cpp +++ b/transformer_engine/pytorch/csrc/quantizer.cpp @@ -10,9 +10,9 @@ #include "common.h" #include "common/util/cuda_runtime.h" #include "common/util/system.h" -#include "extensions.h" #include "pybind.h" #include "torch/torch.h" +#include "util.h" namespace transformer_engine::pytorch { diff --git a/transformer_engine/pytorch/csrc/util.h b/transformer_engine/pytorch/csrc/util.h index 1a4e4eb57ab..5c067dce3d8 100644 --- a/transformer_engine/pytorch/csrc/util.h +++ b/transformer_engine/pytorch/csrc/util.h @@ -36,6 +36,21 @@ std::optional multi_tensor_swizzle_scales_for_gemm(std::vector multi_tensor_swizzle_scales_for_gemm_unchecked( std::vector& tensors, bool rowwise_usage, bool columnwise_usage); +/*! \brief View of a GEMM-swizzled output whose scales are written to compact scratch buffers. */ +TensorWrapper make_compact_output_view(const TensorWrapper& output, at::Tensor& rowwise_scales, + at::Tensor& columnwise_scales, bool preserve_scales = false); + +/*! \brief Output views with compact scratch scales where compact_scales is set. */ +std::vector make_compact_output_views(const std::vector& outputs, + const std::vector& compact_scales, + std::vector& scratch); + +/*! \brief Swizzle compact scales into the scale buffers of a GEMM-swizzled output. */ +void swizzle_scales_into(const TensorWrapper& input, const TensorWrapper& output); + +void multi_tensor_swizzle_scales_into(const std::vector& inputs, + const std::vector& outputs); + using SwizzledGroupedScales = std::pair, std::optional>; /*! \brief Swizzle grouped tensor scales for GEMM if needed. From 24ab3e52b85965ea919b2dbc6cb55bbeca3b4a7c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:30:07 +0000 Subject: [PATCH 20/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- transformer_engine/pytorch/csrc/extensions/swizzle.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp index 5ec9888df49..5fb932ef24c 100644 --- a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp +++ b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp @@ -136,9 +136,9 @@ std::vector make_compact_output_views(const std::vector views; views.reserve(outputs.size()); for (size_t i = 0; i < outputs.size(); ++i) { - views.emplace_back(compact_scales[i] ? make_compact_output_view(outputs[i], scratch[2 * i], - scratch[2 * i + 1]) - : copy_tensor_view(outputs[i])); + views.emplace_back( + compact_scales[i] ? make_compact_output_view(outputs[i], scratch[2 * i], scratch[2 * i + 1]) + : copy_tensor_view(outputs[i])); } return views; } From a108558e8b8d375708ae7edf62c95bc1bb677d4f Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 5 Nov 2026 17:54:14 +0100 Subject: [PATCH 21/31] Keep quantize_impl output parameter named out Co-Authored-By: Claude Opus 5.5 Signed-off-by: Pawel Gadzinski --- transformer_engine/pytorch/csrc/common.h | 2 +- transformer_engine/pytorch/csrc/quantizer.cpp | 40 ++++++++++--------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index 17a02cffc9e..e85ebb491b4 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -395,7 +395,7 @@ class NVFP4Quantizer : public Quantizer { void quantize(const TensorWrapper& input, TensorWrapper& out, const std::optional& noop_flag = std::nullopt) override; - void quantize_impl(const TensorWrapper& input, TensorWrapper& output, + void quantize_impl(const TensorWrapper& input, TensorWrapper& out, const std::optional& noop_flag, bool compute_amax); /*! @brief Quantize to NVFP4, skipping local amax computation diff --git a/transformer_engine/pytorch/csrc/quantizer.cpp b/transformer_engine/pytorch/csrc/quantizer.cpp index 0afe7b81528..5e2ea9d341b 100644 --- a/transformer_engine/pytorch/csrc/quantizer.cpp +++ b/transformer_engine/pytorch/csrc/quantizer.cpp @@ -2569,17 +2569,17 @@ void NVFP4Quantizer::quantize_with_rht_unfused_helper( } } -void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& output, +void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& out, const std::optional& noop_flag, bool compute_amax) { at::Tensor rowwise_scales, columnwise_scales; std::optional compact_out; - if (input.numel() != 0 && output.get_with_gemm_swizzled_scales() && + if (input.numel() != 0 && out.get_with_gemm_swizzled_scales() && !emits_gemm_swizzled_scales(convertShape(input.shape()))) { compact_out.emplace( - make_compact_output_view(output, rowwise_scales, columnwise_scales, noop_flag.has_value())); + make_compact_output_view(out, rowwise_scales, columnwise_scales, noop_flag.has_value())); } - TensorWrapper& out = compact_out ? *compact_out : output; + TensorWrapper& kernel_out = compact_out ? *compact_out : out; auto reduce_amaxes = [&]() { if (!this->with_amax_reduction || this->disable_second_level_scale) { @@ -2595,10 +2595,10 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou at::device(at::kCUDA).dtype(torch::kFloat32)); }; if (rowwise_usage) { - amax_tensors.push_back(make_amax_tensor(out.get_amax().data_ptr)); + amax_tensors.push_back(make_amax_tensor(kernel_out.get_amax().data_ptr)); } if (columnwise_usage) { - amax_tensors.push_back(make_amax_tensor(out.get_columnwise_amax().data_ptr)); + amax_tensors.push_back(make_amax_tensor(kernel_out.get_columnwise_amax().data_ptr)); } if (amax_tensors.empty()) { return; @@ -2641,7 +2641,7 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou // flat first dim and last dim for multi dimensional input const auto [rows, cols] = get_2d_dims(input.shape()); - const bool row_scaled_nvfp4 = out.get_row_scaled_nvfp4(); + const bool row_scaled_nvfp4 = kernel_out.get_row_scaled_nvfp4(); if (row_scaled_nvfp4) { NVTE_CHECK(!this->with_rht, "Row-scaled NVFP4 quantization does not support RHT."); NVTE_CHECK(!this->with_2d_quantization, @@ -2714,7 +2714,7 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou // 2. Columnwise amax = amax for RHT(input.t) if (compute_amax && !this->disable_second_level_scale) { NVTE_SCOPED_GIL_RELEASE({ - nvte_hadamard_transform_amax(input.data(), out.data(), 0, + nvte_hadamard_transform_amax(input.data(), kernel_out.data(), 0, this->rht_matrix_random_sign_mask_t, stream); }); } @@ -2727,16 +2727,17 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou } else { // Without RHT if (compute_amax && !row_scaled_nvfp4 && !this->disable_second_level_scale) { // Amax pointers - auto rowwise_amax_ptr = out.get_amax().data_ptr; - auto columnwise_amax_ptr = out.get_columnwise_amax().data_ptr; + auto rowwise_amax_ptr = kernel_out.get_amax().data_ptr; + auto columnwise_amax_ptr = kernel_out.get_columnwise_amax().data_ptr; void* amax_ptr = rowwise_amax_ptr != nullptr ? rowwise_amax_ptr : columnwise_amax_ptr; NVTE_CHECK(amax_ptr != nullptr, "Could not find amax pointer"); // Compute amax of input tensor - out.set_amax(amax_ptr, DType::kFloat32, std::vector{1}); - NVTE_SCOPED_GIL_RELEASE( - { nvte_compute_amax_with_config(input.data(), out.data(), quant_config, stream); }); - out.set_amax(rowwise_amax_ptr, DType::kFloat32, std::vector{1}); + kernel_out.set_amax(amax_ptr, DType::kFloat32, std::vector{1}); + NVTE_SCOPED_GIL_RELEASE({ + nvte_compute_amax_with_config(input.data(), kernel_out.data(), quant_config, stream); + }); + kernel_out.set_amax(rowwise_amax_ptr, DType::kFloat32, std::vector{1}); // Make sure row-wise and column-wise amaxes match if (rowwise_amax_ptr != amax_ptr && rowwise_amax_ptr != nullptr) { @@ -2782,8 +2783,8 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou // 1. Rowwise quantization // 2. RHT followed by columnwise quantization & transpose NVTE_SCOPED_GIL_RELEASE({ - nvte_quantize_with_hadamard_transform(input.data(), out.data(), rht_matrix_nvte.data(), - quant_config, stream); + nvte_quantize_with_hadamard_transform(input.data(), kernel_out.data(), + rht_matrix_nvte.data(), quant_config, stream); }); } else { // Use separate RNG state for columnwise to ensure different random numbers than rowwise @@ -2801,13 +2802,14 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou // result of transposed RHT to the output of rowwise. rht_output_t_cpp.set_rowwise_data(rht_output_t.data_ptr(), input.dtype(), std::vector{cols, rows}); - this->quantize_with_rht_unfused_helper(input, out, rht_output_t_cpp, quant_config, + this->quantize_with_rht_unfused_helper(input, kernel_out, rht_output_t_cpp, quant_config, columnwise_quant_config_to_use, stream); } } else { - NVTE_SCOPED_GIL_RELEASE({ nvte_quantize_v2(input.data(), out.data(), quant_config, stream); }); + NVTE_SCOPED_GIL_RELEASE( + { nvte_quantize_v2(input.data(), kernel_out.data(), quant_config, stream); }); } - if (compact_out) swizzle_scales_into(*compact_out, output); + if (compact_out) swizzle_scales_into(*compact_out, out); } void NVFP4Quantizer::quantize(const TensorWrapper& input, TensorWrapper& out, From 497f26d0c99f26c6f3538219cd794205b85c0d53 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 5 Nov 2026 18:05:03 +0100 Subject: [PATCH 22/31] Swizzle only outputs quantized with compact scales Co-Authored-By: Claude Opus 5.5 Signed-off-by: Pawel Gadzinski --- transformer_engine/pytorch/csrc/extensions/cast.cpp | 2 +- .../pytorch/csrc/extensions/swizzle.cpp | 12 ++++++------ transformer_engine/pytorch/csrc/util.h | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/transformer_engine/pytorch/csrc/extensions/cast.cpp b/transformer_engine/pytorch/csrc/extensions/cast.cpp index 93fcec7aeb7..7174168901e 100644 --- a/transformer_engine/pytorch/csrc/extensions/cast.cpp +++ b/transformer_engine/pytorch/csrc/extensions/cast.cpp @@ -59,7 +59,7 @@ void quantize_with_batched_swizzle(std::vector &outputs, std::vector scratch; auto kernel_outputs = make_compact_output_views(outputs, compact_scales, scratch); quantize(kernel_outputs); - multi_tensor_swizzle_scales_into(kernel_outputs, outputs); + multi_tensor_swizzle_scales_into(kernel_outputs, outputs, compact_scales); } void allreduce_nvfp4_amax_tensors(NVFP4Quantizer *nvfp4_quantizer_cpp, diff --git a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp index 5fb932ef24c..4d23bd9ddb7 100644 --- a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp +++ b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp @@ -144,10 +144,12 @@ std::vector make_compact_output_views(const std::vector &inputs, - const std::vector &outputs) { - NVTE_CHECK(inputs.size() == outputs.size(), "Mismatched quantization outputs"); + const std::vector &outputs, + const std::vector &compact_scales) { + NVTE_CHECK(inputs.size() == outputs.size() && inputs.size() == compact_scales.size(), + "Mismatched quantization outputs"); if (inputs.size() == 1) { - swizzle_scales_into(inputs.front(), outputs.front()); + if (compact_scales.front()) swizzle_scales_into(inputs.front(), outputs.front()); return; } for (bool rowwise : {true, false}) { @@ -156,9 +158,7 @@ void multi_tensor_swizzle_scales_into(const std::vector &inputs, for (size_t i = 0; i < inputs.size(); ++i) { const auto scales = rowwise ? inputs[i].get_rowwise_scale_inv() : inputs[i].get_columnwise_scale_inv(); - if (inputs[i].get_with_gemm_swizzled_scales() || inputs[i].numel() == 0 || - scales.data_ptr == nullptr) - continue; + if (!compact_scales[i] || inputs[i].numel() == 0 || scales.data_ptr == nullptr) continue; input_views.emplace_back(scale_view(inputs[i], rowwise)); output_views.emplace_back(scale_view(outputs[i], rowwise)); sources.push_back(input_views.back().data()); diff --git a/transformer_engine/pytorch/csrc/util.h b/transformer_engine/pytorch/csrc/util.h index 5c067dce3d8..4c0329bfd7f 100644 --- a/transformer_engine/pytorch/csrc/util.h +++ b/transformer_engine/pytorch/csrc/util.h @@ -49,7 +49,8 @@ std::vector make_compact_output_views(const std::vector& inputs, - const std::vector& outputs); + const std::vector& outputs, + const std::vector& compact_scales); using SwizzledGroupedScales = std::pair, std::optional>; From f16c6adc10f8b359568a2995d736561f34e31db5 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 5 Nov 2026 19:41:05 +0100 Subject: [PATCH 23/31] Quantize into outputs with temporarily redirected compact scales Co-Authored-By: Claude Opus 5.5 Signed-off-by: Pawel Gadzinski --- .../pytorch/csrc/extensions/cast.cpp | 26 +-- .../pytorch/csrc/extensions/normalization.cpp | 118 +++++++------ .../pytorch/csrc/extensions/swizzle.cpp | 163 ++++++++++-------- transformer_engine/pytorch/csrc/quantizer.cpp | 39 ++--- transformer_engine/pytorch/csrc/util.h | 26 ++- 5 files changed, 183 insertions(+), 189 deletions(-) diff --git a/transformer_engine/pytorch/csrc/extensions/cast.cpp b/transformer_engine/pytorch/csrc/extensions/cast.cpp index 7174168901e..0c36a370dc4 100644 --- a/transformer_engine/pytorch/csrc/extensions/cast.cpp +++ b/transformer_engine/pytorch/csrc/extensions/cast.cpp @@ -48,20 +48,6 @@ bool needs_compact_scales(const Quantizer &quantizer, const TensorWrapper &input get_tensor_shape(input)); } -// Compact-scale outputs are quantized into scratch and swizzled with one kernel -template -void quantize_with_batched_swizzle(std::vector &outputs, - const std::vector &compact_scales, QuantizeFn &&quantize) { - if (std::none_of(compact_scales.begin(), compact_scales.end(), [](bool b) { return b; })) { - quantize(outputs); - return; - } - std::vector scratch; - auto kernel_outputs = make_compact_output_views(outputs, compact_scales, scratch); - quantize(kernel_outputs); - multi_tensor_swizzle_scales_into(kernel_outputs, outputs, compact_scales); -} - void allreduce_nvfp4_amax_tensors(NVFP4Quantizer *nvfp4_quantizer_cpp, std::vector &&amax_tensors) { if (!nvfp4_quantizer_cpp->with_amax_reduction || amax_tensors.empty()) { @@ -1182,8 +1168,8 @@ std::vector multi_tensor_quantize(const std::vector &ten for (size_t i = 0; i < num_tensors; ++i) { compact_scales[i] = needs_compact_scales(*quantizer_cpp_list[i], input_cpp_list[i]); } - quantize_with_batched_swizzle(output_cpp_list, compact_scales, [&](auto &outputs) { - multi_tensor_quantize_impl(input_cpp_list, quantizer_list, quantizer_cpp_list, outputs); + quantize_with_compact_scales(output_cpp_list, compact_scales, [&] { + multi_tensor_quantize_impl(input_cpp_list, quantizer_list, quantizer_cpp_list, output_cpp_list); }); return output_py_list; @@ -2271,8 +2257,8 @@ std::vector split_quantize(const at::Tensor &tensor, compact_scales[i] = quantizer.optimize_for_gemm && !emits_gemm_swizzled_scales && input_list[i].numel() != 0; } - quantize_with_batched_swizzle(output_cpp_list, compact_scales, [&](auto &outputs) { - split_quantize_nvfp4_impl(input_nvte, input_list, outputs, split_sections, + quantize_with_compact_scales(output_cpp_list, compact_scales, [&] { + split_quantize_nvfp4_impl(input_nvte, input_list, output_cpp_list, split_sections, nvfp4_quantizers); }); break; @@ -2283,8 +2269,8 @@ std::vector split_quantize(const at::Tensor &tensor, for (size_t i = 0; i < num_splits; ++i) { compact_scales[i] = needs_compact_scales(*quantizer_cpp_list[i], input_list[i]); } - quantize_with_batched_swizzle(output_cpp_list, compact_scales, [&](auto &outputs) { - multi_tensor_quantize_impl(input_list, quantizer_list, quantizer_cpp_list, outputs); + quantize_with_compact_scales(output_cpp_list, compact_scales, [&] { + multi_tensor_quantize_impl(input_list, quantizer_list, quantizer_cpp_list, output_cpp_list); }); } } diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index 3271941c148..79311e255ed 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -151,8 +151,6 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe TensorWrapper unquantized_out_nvte; py::object unquantized_out; at::Tensor amax_buf; - TensorWrapper compact_out_nvte; - at::Tensor rowwise_scales, columnwise_scales; TensorWrapper *kernel_out_nvte = &out_nvte; switch (impl) { case Impl::UNFUSED: { @@ -160,10 +158,6 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe std::tie(unquantized_out_nvte, unquantized_out) = q.create_tensor(shape, out_dtype); kernel_out_nvte = &unquantized_out_nvte; } break; - case Impl::FUSED_NORM_QUANT_UNSWIZZLED: { - compact_out_nvte = make_compact_output_view(out_nvte, rowwise_scales, columnwise_scales); - kernel_out_nvte = &compact_out_nvte; - } break; case Impl::FUSED_NORM_AMAX_FP8: { auto fp8_quantizer_cpp = static_cast(quantizer_cpp.get()); std::tie(unquantized_out_nvte, unquantized_out, amax_buf) = @@ -180,38 +174,43 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe } } - // Query workspace size - TensorWrapper workspace; - NVTE_SCOPED_GIL_RELEASE({ - nvte_layernorm_fwd(input_nvte.data(), weight_nvte.data(), bias_nvte.data(), eps, - kernel_out_nvte->data(), mu_nvte.data(), rsigma_nvte.data(), - workspace.data(), - at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin, - zero_centered_gamma, at::cuda::getCurrentCUDAStream()); - }); - - // Allocate workspace - auto workspace_data = allocateSpace(workspace.shape(), workspace.dtype()); - workspace = - makeTransformerEngineTensor(workspace_data.data_ptr(), workspace.shape(), workspace.dtype()); - - // Launch kernel - NVTE_SCOPED_GIL_RELEASE({ - nvte_layernorm_fwd(input_nvte.data(), weight_nvte.data(), bias_nvte.data(), eps, - kernel_out_nvte->data(), mu_nvte.data(), rsigma_nvte.data(), - workspace.data(), - at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin, - zero_centered_gamma, at::cuda::getCurrentCUDAStream()); - }); + // Launch normalization kernel + auto launch_norm = [&]() { + // Query workspace size + TensorWrapper workspace; + NVTE_SCOPED_GIL_RELEASE({ + nvte_layernorm_fwd(input_nvte.data(), weight_nvte.data(), bias_nvte.data(), eps, + kernel_out_nvte->data(), mu_nvte.data(), rsigma_nvte.data(), + workspace.data(), + at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin, + zero_centered_gamma, at::cuda::getCurrentCUDAStream()); + }); + + // Allocate workspace + auto workspace_data = allocateSpace(workspace.shape(), workspace.dtype()); + workspace = makeTransformerEngineTensor(workspace_data.data_ptr(), workspace.shape(), + workspace.dtype()); + + // Launch kernel + NVTE_SCOPED_GIL_RELEASE({ + nvte_layernorm_fwd(input_nvte.data(), weight_nvte.data(), bias_nvte.data(), eps, + kernel_out_nvte->data(), mu_nvte.data(), rsigma_nvte.data(), + workspace.data(), + at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin, + zero_centered_gamma, at::cuda::getCurrentCUDAStream()); + }); + }; + if (impl == Impl::FUSED_NORM_QUANT_UNSWIZZLED) { + quantize_with_compact_scales(out_nvte, false, launch_norm); + } else { + launch_norm(); + } // Quantize output if needed switch (impl) { case Impl::UNFUSED: { quantizer_cpp->quantize(unquantized_out_nvte, out_nvte); } break; - case Impl::FUSED_NORM_QUANT_UNSWIZZLED: { - swizzle_scales_into(compact_out_nvte, out_nvte); - } break; case Impl::FUSED_NORM_AMAX_FP8: { auto fp8_quantizer_cpp = static_cast(quantizer_cpp.get()); fp8_quantizer_cpp->quantize_with_amax(unquantized_out_nvte, out_nvte, amax_buf); @@ -401,8 +400,6 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w TensorWrapper unquantized_out_nvte; py::object unquantized_out; at::Tensor amax_buf; - TensorWrapper compact_out_nvte; - at::Tensor rowwise_scales, columnwise_scales; TensorWrapper *kernel_out_nvte = &out_nvte; switch (impl) { case Impl::UNFUSED: { @@ -410,10 +407,6 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w std::tie(unquantized_out_nvte, unquantized_out) = q.create_tensor(shape, out_dtype); kernel_out_nvte = &unquantized_out_nvte; } break; - case Impl::FUSED_NORM_QUANT_UNSWIZZLED: { - compact_out_nvte = make_compact_output_view(out_nvte, rowwise_scales, columnwise_scales); - kernel_out_nvte = &compact_out_nvte; - } break; case Impl::FUSED_NORM_AMAX_FP8: { auto fp8_quantizer_cpp = static_cast(quantizer_cpp.get()); std::tie(unquantized_out_nvte, unquantized_out, amax_buf) = @@ -430,36 +423,41 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w } } - // Query workspace size - TensorWrapper workspace; - NVTE_SCOPED_GIL_RELEASE({ - nvte_rmsnorm_fwd(input_nvte.data(), weight_nvte.data(), eps, kernel_out_nvte->data(), - rsigma_nvte.data(), workspace.data(), - at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin, - zero_centered_gamma, at::cuda::getCurrentCUDAStream()); - }); + // Launch normalization kernel + auto launch_norm = [&]() { + // Query workspace size + TensorWrapper workspace; + NVTE_SCOPED_GIL_RELEASE({ + nvte_rmsnorm_fwd(input_nvte.data(), weight_nvte.data(), eps, kernel_out_nvte->data(), + rsigma_nvte.data(), workspace.data(), + at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin, + zero_centered_gamma, at::cuda::getCurrentCUDAStream()); + }); - // Allocate workspace - auto workspace_data = allocateSpace(workspace.shape(), workspace.dtype()); - workspace = - makeTransformerEngineTensor(workspace_data.data_ptr(), workspace.shape(), workspace.dtype()); + // Allocate workspace + auto workspace_data = allocateSpace(workspace.shape(), workspace.dtype()); + workspace = makeTransformerEngineTensor(workspace_data.data_ptr(), workspace.shape(), + workspace.dtype()); - // Launch kernel - NVTE_SCOPED_GIL_RELEASE({ - nvte_rmsnorm_fwd(input_nvte.data(), weight_nvte.data(), eps, kernel_out_nvte->data(), - rsigma_nvte.data(), workspace.data(), - at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin, - zero_centered_gamma, at::cuda::getCurrentCUDAStream()); - }); + // Launch kernel + NVTE_SCOPED_GIL_RELEASE({ + nvte_rmsnorm_fwd(input_nvte.data(), weight_nvte.data(), eps, kernel_out_nvte->data(), + rsigma_nvte.data(), workspace.data(), + at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin, + zero_centered_gamma, at::cuda::getCurrentCUDAStream()); + }); + }; + if (impl == Impl::FUSED_NORM_QUANT_UNSWIZZLED) { + quantize_with_compact_scales(out_nvte, false, launch_norm); + } else { + launch_norm(); + } // Quantize output if needed switch (impl) { case Impl::UNFUSED: { quantizer_cpp->quantize(unquantized_out_nvte, out_nvte); } break; - case Impl::FUSED_NORM_QUANT_UNSWIZZLED: { - swizzle_scales_into(compact_out_nvte, out_nvte); - } break; case Impl::FUSED_NORM_AMAX_FP8: { auto fp8_quantizer_cpp = static_cast(quantizer_cpp.get()); fp8_quantizer_cpp->quantize_with_amax(unquantized_out_nvte, out_nvte, amax_buf); diff --git a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp index 4d23bd9ddb7..99bb809847f 100644 --- a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp +++ b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp @@ -4,6 +4,8 @@ * See LICENSE for license information. ************************************************************************/ +#include +#include #include #include #include @@ -42,67 +44,81 @@ bool is_empty_grouped_tensor_param(const NVTEBasicTensor &t) { return t.shape.ndim == 1 && t.shape.data[0] == 0; } -TensorWrapper copy_tensor_view(const TensorWrapper &tensor) { - TensorWrapper view(tensor.scaling_mode()); - for (int i = kNVTERowwiseData; i <= kNVTEColumnwiseAmax; ++i) { - const auto param = static_cast(i); - const auto value = tensor.get_parameter(param); - view.set_parameter(param, value.data_ptr, static_cast(value.dtype), value.shape); - } - view.set_with_gemm_swizzled_scales(tensor.get_with_gemm_swizzled_scales()); - view.set_row_scaled_nvfp4(tensor.get_row_scaled_nvfp4()); - view.set_nvfp4_e4m3_max(tensor.get_nvfp4_e4m3_max()); - return view; +NVTEBasicTensor get_scales(const TensorWrapper &tensor, bool rowwise) { + return rowwise ? tensor.get_rowwise_scale_inv() : tensor.get_columnwise_scale_inv(); +} + +void set_scales_ptr(TensorWrapper &tensor, bool rowwise, void *ptr) { + const auto scales = get_scales(tensor, rowwise); + tensor.set_parameter(rowwise ? kNVTERowwiseScaleInv : kNVTEColumnwiseScaleInv, ptr, + static_cast(scales.dtype), scales.shape); } -TensorWrapper scale_view(const TensorWrapper &tensor, bool rowwise) { +// Tensor holding only one direction's scales, as consumed by the swizzle kernels +TensorWrapper scale_view(const TensorWrapper &tensor, bool rowwise, void *scales_ptr, + bool swizzled) { const auto data = rowwise ? tensor.get_rowwise_data() : tensor.get_columnwise_data(); - const auto scales = rowwise ? tensor.get_rowwise_scale_inv() : tensor.get_columnwise_scale_inv(); TensorWrapper view(tensor.scaling_mode()); view.set_parameter(rowwise ? kNVTERowwiseData : kNVTEColumnwiseData, nullptr, static_cast(data.dtype), data.shape); - view.set_parameter(rowwise ? kNVTERowwiseScaleInv : kNVTEColumnwiseScaleInv, scales.data_ptr, + const auto scales = get_scales(tensor, rowwise); + view.set_parameter(rowwise ? kNVTERowwiseScaleInv : kNVTEColumnwiseScaleInv, scales_ptr, static_cast(scales.dtype), scales.shape); - view.set_with_gemm_swizzled_scales(tensor.get_with_gemm_swizzled_scales()); + view.set_with_gemm_swizzled_scales(swizzled); return view; } +// Compact scratch scales of an output and its own GEMM-swizzled scale buffers +struct RedirectedScales { + at::Tensor scratch[2]; // rowwise, columnwise + void *swizzled[2] = {nullptr, nullptr}; +}; + +void redirect_scales(TensorWrapper &output, bool rowwise, at::Tensor scratch, + RedirectedScales &redirected, bool preserve_scales) { + void *swizzled = get_scales(output, rowwise).data_ptr; + if (preserve_scales) { + // A GPU no-op must preserve the previous scales through finalization. + auto input_scales = scale_view(output, rowwise, swizzled, true); + auto output_scales = scale_view(output, rowwise, scratch.data_ptr(), false); + NVTE_SCOPED_GIL_RELEASE({ + nvte_unswizzle_scaling_factors(input_scales.data(), output_scales.data(), + at::cuda::getCurrentCUDAStream()); + }); + } + set_scales_ptr(output, rowwise, scratch.data_ptr()); + redirected.swizzled[!rowwise] = swizzled; + redirected.scratch[!rowwise] = std::move(scratch); +} + +// Restores the output's scale buffers and returns the compact scratch scales +void *restore_scales(TensorWrapper &output, bool rowwise, const RedirectedScales &redirected) { + const auto &scratch = redirected.scratch[!rowwise]; + if (!scratch.defined()) return nullptr; + set_scales_ptr(output, rowwise, redirected.swizzled[!rowwise]); + return scratch.data_ptr(); +} + } // namespace -TensorWrapper make_compact_output_view(const TensorWrapper &output, at::Tensor &rowwise_scales, - at::Tensor &columnwise_scales, bool preserve_scales) { +void quantize_with_compact_scales(TensorWrapper &output, bool preserve_scales, + const std::function &quantize) { NVTE_CHECK(output.get_with_gemm_swizzled_scales(), "Expected a GEMM-ready output"); - auto compact_output = copy_tensor_view(output); - compact_output.set_with_gemm_swizzled_scales(false); + RedirectedScales redirected; for (bool rowwise : {true, false}) { - const auto scales = - rowwise ? output.get_rowwise_scale_inv() : output.get_columnwise_scale_inv(); + const auto scales = get_scales(output, rowwise); if (scales.data_ptr == nullptr) continue; - auto &scratch = rowwise ? rowwise_scales : columnwise_scales; - if (!scratch.defined()) scratch = allocateSpace(scales.shape, DType::kByte, false); - compact_output.set_parameter(rowwise ? kNVTERowwiseScaleInv : kNVTEColumnwiseScaleInv, - scratch.data_ptr(), static_cast(scales.dtype), - scales.shape); - if (preserve_scales) { - // A GPU no-op must preserve the previous scales through finalization. - auto input_scales = scale_view(output, rowwise); - auto output_scales = scale_view(compact_output, rowwise); - NVTE_SCOPED_GIL_RELEASE({ - nvte_unswizzle_scaling_factors(input_scales.data(), output_scales.data(), - at::cuda::getCurrentCUDAStream()); - }); - } + redirect_scales(output, rowwise, allocateSpace(scales.shape, DType::kByte, false), redirected, + preserve_scales); } - return compact_output; -} - -void swizzle_scales_into(const TensorWrapper &input, const TensorWrapper &output) { - if (input.get_with_gemm_swizzled_scales() || input.numel() == 0) return; + output.set_with_gemm_swizzled_scales(false); + quantize(); + output.set_with_gemm_swizzled_scales(true); for (bool rowwise : {true, false}) { - const auto scales = rowwise ? input.get_rowwise_scale_inv() : input.get_columnwise_scale_inv(); - if (scales.data_ptr == nullptr) continue; - auto input_scales = scale_view(input, rowwise); - auto output_scales = scale_view(output, rowwise); + void *compact_scales = restore_scales(output, rowwise, redirected); + if (compact_scales == nullptr || output.numel() == 0) continue; + auto input_scales = scale_view(output, rowwise, compact_scales, false); + auto output_scales = scale_view(output, rowwise, get_scales(output, rowwise).data_ptr, true); NVTE_SCOPED_GIL_RELEASE({ nvte_swizzle_scaling_factors(input_scales.data(), output_scales.data(), at::cuda::getCurrentCUDAStream()); @@ -110,17 +126,26 @@ void swizzle_scales_into(const TensorWrapper &input, const TensorWrapper &output } } -std::vector make_compact_output_views(const std::vector &outputs, - const std::vector &compact_scales, - std::vector &scratch) { +void quantize_with_compact_scales(std::vector &outputs, + const std::vector &compact_scales, + const std::function &quantize) { NVTE_CHECK(outputs.size() == compact_scales.size(), "Mismatched output layouts"); - scratch.resize(2 * outputs.size()); + if (std::none_of(compact_scales.begin(), compact_scales.end(), [](bool b) { return b; })) { + quantize(); + return; + } + for (size_t i = 0; i < outputs.size(); ++i) { + NVTE_CHECK(!compact_scales[i] || outputs[i].get_with_gemm_swizzled_scales(), + "Expected a GEMM-ready output"); + } + + // Redirect scales to scratch buffers, allocated together per usage + std::vector redirected(outputs.size()); for (bool rowwise : {true, false}) { std::vector> shapes; std::vector indices; for (size_t i = 0; i < outputs.size(); ++i) { - const auto scales = - rowwise ? outputs[i].get_rowwise_scale_inv() : outputs[i].get_columnwise_scale_inv(); + const auto scales = get_scales(outputs[i], rowwise); if (compact_scales[i] && scales.data_ptr != nullptr) { shapes.emplace_back(scales.shape.data, scales.shape.data + scales.shape.ndim); indices.push_back(i); @@ -129,38 +154,30 @@ std::vector make_compact_output_views(const std::vector(shapes.size(), at::kByte), std::nullopt, std::vector(shapes.size(), 16)); - for (size_t i = 0; i < indices.size(); ++i) { - scratch[2 * indices[i] + !rowwise] = std::move(buffers[i]); + for (size_t j = 0; j < indices.size(); ++j) { + redirect_scales(outputs[indices[j]], rowwise, std::move(buffers[j]), redirected[indices[j]], + false); } } - std::vector views; - views.reserve(outputs.size()); for (size_t i = 0; i < outputs.size(); ++i) { - views.emplace_back( - compact_scales[i] ? make_compact_output_view(outputs[i], scratch[2 * i], scratch[2 * i + 1]) - : copy_tensor_view(outputs[i])); + if (compact_scales[i]) outputs[i].set_with_gemm_swizzled_scales(false); } - return views; -} -void multi_tensor_swizzle_scales_into(const std::vector &inputs, - const std::vector &outputs, - const std::vector &compact_scales) { - NVTE_CHECK(inputs.size() == outputs.size() && inputs.size() == compact_scales.size(), - "Mismatched quantization outputs"); - if (inputs.size() == 1) { - if (compact_scales.front()) swizzle_scales_into(inputs.front(), outputs.front()); - return; + quantize(); + + // Restore the outputs and swizzle all compact scales with one kernel per usage + for (size_t i = 0; i < outputs.size(); ++i) { + if (compact_scales[i]) outputs[i].set_with_gemm_swizzled_scales(true); } for (bool rowwise : {true, false}) { std::vector input_views, output_views; std::vector sources, destinations; - for (size_t i = 0; i < inputs.size(); ++i) { - const auto scales = - rowwise ? inputs[i].get_rowwise_scale_inv() : inputs[i].get_columnwise_scale_inv(); - if (!compact_scales[i] || inputs[i].numel() == 0 || scales.data_ptr == nullptr) continue; - input_views.emplace_back(scale_view(inputs[i], rowwise)); - output_views.emplace_back(scale_view(outputs[i], rowwise)); + for (size_t i = 0; i < outputs.size(); ++i) { + void *compact = restore_scales(outputs[i], rowwise, redirected[i]); + if (compact == nullptr || outputs[i].numel() == 0) continue; + input_views.emplace_back(scale_view(outputs[i], rowwise, compact, false)); + output_views.emplace_back( + scale_view(outputs[i], rowwise, get_scales(outputs[i], rowwise).data_ptr, true)); sources.push_back(input_views.back().data()); destinations.push_back(output_views.back().data()); } diff --git a/transformer_engine/pytorch/csrc/quantizer.cpp b/transformer_engine/pytorch/csrc/quantizer.cpp index 5e2ea9d341b..3dc1bee8e65 100644 --- a/transformer_engine/pytorch/csrc/quantizer.cpp +++ b/transformer_engine/pytorch/csrc/quantizer.cpp @@ -2572,14 +2572,12 @@ void NVFP4Quantizer::quantize_with_rht_unfused_helper( void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& out, const std::optional& noop_flag, bool compute_amax) { - at::Tensor rowwise_scales, columnwise_scales; - std::optional compact_out; if (input.numel() != 0 && out.get_with_gemm_swizzled_scales() && !emits_gemm_swizzled_scales(convertShape(input.shape()))) { - compact_out.emplace( - make_compact_output_view(out, rowwise_scales, columnwise_scales, noop_flag.has_value())); + quantize_with_compact_scales(out, noop_flag.has_value(), + [&] { quantize_impl(input, out, noop_flag, compute_amax); }); + return; } - TensorWrapper& kernel_out = compact_out ? *compact_out : out; auto reduce_amaxes = [&]() { if (!this->with_amax_reduction || this->disable_second_level_scale) { @@ -2595,10 +2593,10 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou at::device(at::kCUDA).dtype(torch::kFloat32)); }; if (rowwise_usage) { - amax_tensors.push_back(make_amax_tensor(kernel_out.get_amax().data_ptr)); + amax_tensors.push_back(make_amax_tensor(out.get_amax().data_ptr)); } if (columnwise_usage) { - amax_tensors.push_back(make_amax_tensor(kernel_out.get_columnwise_amax().data_ptr)); + amax_tensors.push_back(make_amax_tensor(out.get_columnwise_amax().data_ptr)); } if (amax_tensors.empty()) { return; @@ -2641,7 +2639,7 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou // flat first dim and last dim for multi dimensional input const auto [rows, cols] = get_2d_dims(input.shape()); - const bool row_scaled_nvfp4 = kernel_out.get_row_scaled_nvfp4(); + const bool row_scaled_nvfp4 = out.get_row_scaled_nvfp4(); if (row_scaled_nvfp4) { NVTE_CHECK(!this->with_rht, "Row-scaled NVFP4 quantization does not support RHT."); NVTE_CHECK(!this->with_2d_quantization, @@ -2714,7 +2712,7 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou // 2. Columnwise amax = amax for RHT(input.t) if (compute_amax && !this->disable_second_level_scale) { NVTE_SCOPED_GIL_RELEASE({ - nvte_hadamard_transform_amax(input.data(), kernel_out.data(), 0, + nvte_hadamard_transform_amax(input.data(), out.data(), 0, this->rht_matrix_random_sign_mask_t, stream); }); } @@ -2727,17 +2725,16 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou } else { // Without RHT if (compute_amax && !row_scaled_nvfp4 && !this->disable_second_level_scale) { // Amax pointers - auto rowwise_amax_ptr = kernel_out.get_amax().data_ptr; - auto columnwise_amax_ptr = kernel_out.get_columnwise_amax().data_ptr; + auto rowwise_amax_ptr = out.get_amax().data_ptr; + auto columnwise_amax_ptr = out.get_columnwise_amax().data_ptr; void* amax_ptr = rowwise_amax_ptr != nullptr ? rowwise_amax_ptr : columnwise_amax_ptr; NVTE_CHECK(amax_ptr != nullptr, "Could not find amax pointer"); // Compute amax of input tensor - kernel_out.set_amax(amax_ptr, DType::kFloat32, std::vector{1}); - NVTE_SCOPED_GIL_RELEASE({ - nvte_compute_amax_with_config(input.data(), kernel_out.data(), quant_config, stream); - }); - kernel_out.set_amax(rowwise_amax_ptr, DType::kFloat32, std::vector{1}); + out.set_amax(amax_ptr, DType::kFloat32, std::vector{1}); + NVTE_SCOPED_GIL_RELEASE( + { nvte_compute_amax_with_config(input.data(), out.data(), quant_config, stream); }); + out.set_amax(rowwise_amax_ptr, DType::kFloat32, std::vector{1}); // Make sure row-wise and column-wise amaxes match if (rowwise_amax_ptr != amax_ptr && rowwise_amax_ptr != nullptr) { @@ -2783,8 +2780,8 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou // 1. Rowwise quantization // 2. RHT followed by columnwise quantization & transpose NVTE_SCOPED_GIL_RELEASE({ - nvte_quantize_with_hadamard_transform(input.data(), kernel_out.data(), - rht_matrix_nvte.data(), quant_config, stream); + nvte_quantize_with_hadamard_transform(input.data(), out.data(), rht_matrix_nvte.data(), + quant_config, stream); }); } else { // Use separate RNG state for columnwise to ensure different random numbers than rowwise @@ -2802,14 +2799,12 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou // result of transposed RHT to the output of rowwise. rht_output_t_cpp.set_rowwise_data(rht_output_t.data_ptr(), input.dtype(), std::vector{cols, rows}); - this->quantize_with_rht_unfused_helper(input, kernel_out, rht_output_t_cpp, quant_config, + this->quantize_with_rht_unfused_helper(input, out, rht_output_t_cpp, quant_config, columnwise_quant_config_to_use, stream); } } else { - NVTE_SCOPED_GIL_RELEASE( - { nvte_quantize_v2(input.data(), kernel_out.data(), quant_config, stream); }); + NVTE_SCOPED_GIL_RELEASE({ nvte_quantize_v2(input.data(), out.data(), quant_config, stream); }); } - if (compact_out) swizzle_scales_into(*compact_out, out); } void NVFP4Quantizer::quantize(const TensorWrapper& input, TensorWrapper& out, diff --git a/transformer_engine/pytorch/csrc/util.h b/transformer_engine/pytorch/csrc/util.h index 4c0329bfd7f..f3dcf841295 100644 --- a/transformer_engine/pytorch/csrc/util.h +++ b/transformer_engine/pytorch/csrc/util.h @@ -9,6 +9,7 @@ #include +#include #include #include #include @@ -36,21 +37,18 @@ std::optional multi_tensor_swizzle_scales_for_gemm(std::vector multi_tensor_swizzle_scales_for_gemm_unchecked( std::vector& tensors, bool rowwise_usage, bool columnwise_usage); -/*! \brief View of a GEMM-swizzled output whose scales are written to compact scratch buffers. */ -TensorWrapper make_compact_output_view(const TensorWrapper& output, at::Tensor& rowwise_scales, - at::Tensor& columnwise_scales, bool preserve_scales = false); - -/*! \brief Output views with compact scratch scales where compact_scales is set. */ -std::vector make_compact_output_views(const std::vector& outputs, - const std::vector& compact_scales, - std::vector& scratch); - -/*! \brief Swizzle compact scales into the scale buffers of a GEMM-swizzled output. */ -void swizzle_scales_into(const TensorWrapper& input, const TensorWrapper& output); +/*! \brief Run quantize on a GEMM-swizzled output with compact scratch scales, then swizzle them. + * + * For kernels that only emit compact scales. preserve_scales keeps the previous scales + * if quantize is a GPU no-op. + */ +void quantize_with_compact_scales(TensorWrapper& output, bool preserve_scales, + const std::function& quantize); -void multi_tensor_swizzle_scales_into(const std::vector& inputs, - const std::vector& outputs, - const std::vector& compact_scales); +/*! \brief Same for the outputs where compact_scales is set, with one swizzle kernel per usage. */ +void quantize_with_compact_scales(std::vector& outputs, + const std::vector& compact_scales, + const std::function& quantize); using SwizzledGroupedScales = std::pair, std::optional>; From 372b528f8dfbc0a83b0b860654d311295b54e81d Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 5 Nov 2026 20:02:46 +0100 Subject: [PATCH 24/31] Redirect output scales to scratch in explicit steps Co-Authored-By: Claude Opus 5.5 Signed-off-by: Pawel Gadzinski --- .../pytorch/csrc/extensions/cast.cpp | 20 +-- .../pytorch/csrc/extensions/normalization.cpp | 114 +++++++++--------- .../pytorch/csrc/extensions/swizzle.cpp | 51 ++++---- transformer_engine/pytorch/csrc/quantizer.cpp | 6 +- transformer_engine/pytorch/csrc/util.h | 32 +++-- 5 files changed, 113 insertions(+), 110 deletions(-) diff --git a/transformer_engine/pytorch/csrc/extensions/cast.cpp b/transformer_engine/pytorch/csrc/extensions/cast.cpp index 0c36a370dc4..0f539d03527 100644 --- a/transformer_engine/pytorch/csrc/extensions/cast.cpp +++ b/transformer_engine/pytorch/csrc/extensions/cast.cpp @@ -1168,9 +1168,9 @@ std::vector multi_tensor_quantize(const std::vector &ten for (size_t i = 0; i < num_tensors; ++i) { compact_scales[i] = needs_compact_scales(*quantizer_cpp_list[i], input_cpp_list[i]); } - quantize_with_compact_scales(output_cpp_list, compact_scales, [&] { - multi_tensor_quantize_impl(input_cpp_list, quantizer_list, quantizer_cpp_list, output_cpp_list); - }); + auto redirected_scales = redirect_scales_to_scratch(output_cpp_list, compact_scales); + multi_tensor_quantize_impl(input_cpp_list, quantizer_list, quantizer_cpp_list, output_cpp_list); + restore_and_swizzle_scales(output_cpp_list, redirected_scales); return output_py_list; } @@ -2257,10 +2257,10 @@ std::vector split_quantize(const at::Tensor &tensor, compact_scales[i] = quantizer.optimize_for_gemm && !emits_gemm_swizzled_scales && input_list[i].numel() != 0; } - quantize_with_compact_scales(output_cpp_list, compact_scales, [&] { - split_quantize_nvfp4_impl(input_nvte, input_list, output_cpp_list, split_sections, - nvfp4_quantizers); - }); + auto redirected_scales = redirect_scales_to_scratch(output_cpp_list, compact_scales); + split_quantize_nvfp4_impl(input_nvte, input_list, output_cpp_list, split_sections, + nvfp4_quantizers); + restore_and_swizzle_scales(output_cpp_list, redirected_scales); break; } default: { @@ -2269,9 +2269,9 @@ std::vector split_quantize(const at::Tensor &tensor, for (size_t i = 0; i < num_splits; ++i) { compact_scales[i] = needs_compact_scales(*quantizer_cpp_list[i], input_list[i]); } - quantize_with_compact_scales(output_cpp_list, compact_scales, [&] { - multi_tensor_quantize_impl(input_list, quantizer_list, quantizer_cpp_list, output_cpp_list); - }); + auto redirected_scales = redirect_scales_to_scratch(output_cpp_list, compact_scales); + multi_tensor_quantize_impl(input_list, quantizer_list, quantizer_cpp_list, output_cpp_list); + restore_and_swizzle_scales(output_cpp_list, redirected_scales); } } diff --git a/transformer_engine/pytorch/csrc/extensions/normalization.cpp b/transformer_engine/pytorch/csrc/extensions/normalization.cpp index 79311e255ed..d929a096fe0 100644 --- a/transformer_engine/pytorch/csrc/extensions/normalization.cpp +++ b/transformer_engine/pytorch/csrc/extensions/normalization.cpp @@ -151,6 +151,7 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe TensorWrapper unquantized_out_nvte; py::object unquantized_out; at::Tensor amax_buf; + RedirectedScales redirected_scales; TensorWrapper *kernel_out_nvte = &out_nvte; switch (impl) { case Impl::UNFUSED: { @@ -158,6 +159,9 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe std::tie(unquantized_out_nvte, unquantized_out) = q.create_tensor(shape, out_dtype); kernel_out_nvte = &unquantized_out_nvte; } break; + case Impl::FUSED_NORM_QUANT_UNSWIZZLED: { + redirected_scales = redirect_scales_to_scratch(out_nvte); + } break; case Impl::FUSED_NORM_AMAX_FP8: { auto fp8_quantizer_cpp = static_cast(quantizer_cpp.get()); std::tie(unquantized_out_nvte, unquantized_out, amax_buf) = @@ -174,43 +178,38 @@ std::vector layernorm_fwd(py::handle input, py::handle weight, Maybe } } - // Launch normalization kernel - auto launch_norm = [&]() { - // Query workspace size - TensorWrapper workspace; - NVTE_SCOPED_GIL_RELEASE({ - nvte_layernorm_fwd(input_nvte.data(), weight_nvte.data(), bias_nvte.data(), eps, - kernel_out_nvte->data(), mu_nvte.data(), rsigma_nvte.data(), - workspace.data(), - at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin, - zero_centered_gamma, at::cuda::getCurrentCUDAStream()); - }); - - // Allocate workspace - auto workspace_data = allocateSpace(workspace.shape(), workspace.dtype()); - workspace = makeTransformerEngineTensor(workspace_data.data_ptr(), workspace.shape(), - workspace.dtype()); - - // Launch kernel - NVTE_SCOPED_GIL_RELEASE({ - nvte_layernorm_fwd(input_nvte.data(), weight_nvte.data(), bias_nvte.data(), eps, - kernel_out_nvte->data(), mu_nvte.data(), rsigma_nvte.data(), - workspace.data(), - at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin, - zero_centered_gamma, at::cuda::getCurrentCUDAStream()); - }); - }; - if (impl == Impl::FUSED_NORM_QUANT_UNSWIZZLED) { - quantize_with_compact_scales(out_nvte, false, launch_norm); - } else { - launch_norm(); - } + // Query workspace size + TensorWrapper workspace; + NVTE_SCOPED_GIL_RELEASE({ + nvte_layernorm_fwd(input_nvte.data(), weight_nvte.data(), bias_nvte.data(), eps, + kernel_out_nvte->data(), mu_nvte.data(), rsigma_nvte.data(), + workspace.data(), + at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin, + zero_centered_gamma, at::cuda::getCurrentCUDAStream()); + }); + + // Allocate workspace + auto workspace_data = allocateSpace(workspace.shape(), workspace.dtype()); + workspace = + makeTransformerEngineTensor(workspace_data.data_ptr(), workspace.shape(), workspace.dtype()); + + // Launch kernel + NVTE_SCOPED_GIL_RELEASE({ + nvte_layernorm_fwd(input_nvte.data(), weight_nvte.data(), bias_nvte.data(), eps, + kernel_out_nvte->data(), mu_nvte.data(), rsigma_nvte.data(), + workspace.data(), + at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin, + zero_centered_gamma, at::cuda::getCurrentCUDAStream()); + }); // Quantize output if needed switch (impl) { case Impl::UNFUSED: { quantizer_cpp->quantize(unquantized_out_nvte, out_nvte); } break; + case Impl::FUSED_NORM_QUANT_UNSWIZZLED: { + restore_and_swizzle_scales(out_nvte, redirected_scales); + } break; case Impl::FUSED_NORM_AMAX_FP8: { auto fp8_quantizer_cpp = static_cast(quantizer_cpp.get()); fp8_quantizer_cpp->quantize_with_amax(unquantized_out_nvte, out_nvte, amax_buf); @@ -400,6 +399,7 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w TensorWrapper unquantized_out_nvte; py::object unquantized_out; at::Tensor amax_buf; + RedirectedScales redirected_scales; TensorWrapper *kernel_out_nvte = &out_nvte; switch (impl) { case Impl::UNFUSED: { @@ -407,6 +407,9 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w std::tie(unquantized_out_nvte, unquantized_out) = q.create_tensor(shape, out_dtype); kernel_out_nvte = &unquantized_out_nvte; } break; + case Impl::FUSED_NORM_QUANT_UNSWIZZLED: { + redirected_scales = redirect_scales_to_scratch(out_nvte); + } break; case Impl::FUSED_NORM_AMAX_FP8: { auto fp8_quantizer_cpp = static_cast(quantizer_cpp.get()); std::tie(unquantized_out_nvte, unquantized_out, amax_buf) = @@ -423,41 +426,36 @@ std::vector rmsnorm_fwd(const py::handle &input, const py::handle &w } } - // Launch normalization kernel - auto launch_norm = [&]() { - // Query workspace size - TensorWrapper workspace; - NVTE_SCOPED_GIL_RELEASE({ - nvte_rmsnorm_fwd(input_nvte.data(), weight_nvte.data(), eps, kernel_out_nvte->data(), - rsigma_nvte.data(), workspace.data(), - at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin, - zero_centered_gamma, at::cuda::getCurrentCUDAStream()); - }); + // Query workspace size + TensorWrapper workspace; + NVTE_SCOPED_GIL_RELEASE({ + nvte_rmsnorm_fwd(input_nvte.data(), weight_nvte.data(), eps, kernel_out_nvte->data(), + rsigma_nvte.data(), workspace.data(), + at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin, + zero_centered_gamma, at::cuda::getCurrentCUDAStream()); + }); - // Allocate workspace - auto workspace_data = allocateSpace(workspace.shape(), workspace.dtype()); - workspace = makeTransformerEngineTensor(workspace_data.data_ptr(), workspace.shape(), - workspace.dtype()); + // Allocate workspace + auto workspace_data = allocateSpace(workspace.shape(), workspace.dtype()); + workspace = + makeTransformerEngineTensor(workspace_data.data_ptr(), workspace.shape(), workspace.dtype()); - // Launch kernel - NVTE_SCOPED_GIL_RELEASE({ - nvte_rmsnorm_fwd(input_nvte.data(), weight_nvte.data(), eps, kernel_out_nvte->data(), - rsigma_nvte.data(), workspace.data(), - at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin, - zero_centered_gamma, at::cuda::getCurrentCUDAStream()); - }); - }; - if (impl == Impl::FUSED_NORM_QUANT_UNSWIZZLED) { - quantize_with_compact_scales(out_nvte, false, launch_norm); - } else { - launch_norm(); - } + // Launch kernel + NVTE_SCOPED_GIL_RELEASE({ + nvte_rmsnorm_fwd(input_nvte.data(), weight_nvte.data(), eps, kernel_out_nvte->data(), + rsigma_nvte.data(), workspace.data(), + at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin, + zero_centered_gamma, at::cuda::getCurrentCUDAStream()); + }); // Quantize output if needed switch (impl) { case Impl::UNFUSED: { quantizer_cpp->quantize(unquantized_out_nvte, out_nvte); } break; + case Impl::FUSED_NORM_QUANT_UNSWIZZLED: { + restore_and_swizzle_scales(out_nvte, redirected_scales); + } break; case Impl::FUSED_NORM_AMAX_FP8: { auto fp8_quantizer_cpp = static_cast(quantizer_cpp.get()); fp8_quantizer_cpp->quantize_with_amax(unquantized_out_nvte, out_nvte, amax_buf); diff --git a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp index 99bb809847f..da26affa732 100644 --- a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp +++ b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp @@ -4,8 +4,6 @@ * See LICENSE for license information. ************************************************************************/ -#include -#include #include #include #include @@ -68,12 +66,6 @@ TensorWrapper scale_view(const TensorWrapper &tensor, bool rowwise, void *scales return view; } -// Compact scratch scales of an output and its own GEMM-swizzled scale buffers -struct RedirectedScales { - at::Tensor scratch[2]; // rowwise, columnwise - void *swizzled[2] = {nullptr, nullptr}; -}; - void redirect_scales(TensorWrapper &output, bool rowwise, at::Tensor scratch, RedirectedScales &redirected, bool preserve_scales) { void *swizzled = get_scales(output, rowwise).data_ptr; @@ -101,8 +93,7 @@ void *restore_scales(TensorWrapper &output, bool rowwise, const RedirectedScales } // namespace -void quantize_with_compact_scales(TensorWrapper &output, bool preserve_scales, - const std::function &quantize) { +RedirectedScales redirect_scales_to_scratch(TensorWrapper &output, bool preserve_scales) { NVTE_CHECK(output.get_with_gemm_swizzled_scales(), "Expected a GEMM-ready output"); RedirectedScales redirected; for (bool rowwise : {true, false}) { @@ -112,12 +103,17 @@ void quantize_with_compact_scales(TensorWrapper &output, bool preserve_scales, preserve_scales); } output.set_with_gemm_swizzled_scales(false); - quantize(); + redirected.active = true; + return redirected; +} + +void restore_and_swizzle_scales(TensorWrapper &output, const RedirectedScales &redirected) { + if (!redirected.active) return; output.set_with_gemm_swizzled_scales(true); for (bool rowwise : {true, false}) { - void *compact_scales = restore_scales(output, rowwise, redirected); - if (compact_scales == nullptr || output.numel() == 0) continue; - auto input_scales = scale_view(output, rowwise, compact_scales, false); + void *compact = restore_scales(output, rowwise, redirected); + if (compact == nullptr || output.numel() == 0) continue; + auto input_scales = scale_view(output, rowwise, compact, false); auto output_scales = scale_view(output, rowwise, get_scales(output, rowwise).data_ptr, true); NVTE_SCOPED_GIL_RELEASE({ nvte_swizzle_scaling_factors(input_scales.data(), output_scales.data(), @@ -126,21 +122,14 @@ void quantize_with_compact_scales(TensorWrapper &output, bool preserve_scales, } } -void quantize_with_compact_scales(std::vector &outputs, - const std::vector &compact_scales, - const std::function &quantize) { +std::vector redirect_scales_to_scratch(std::vector &outputs, + const std::vector &compact_scales) { NVTE_CHECK(outputs.size() == compact_scales.size(), "Mismatched output layouts"); - if (std::none_of(compact_scales.begin(), compact_scales.end(), [](bool b) { return b; })) { - quantize(); - return; - } + std::vector redirected(outputs.size()); for (size_t i = 0; i < outputs.size(); ++i) { NVTE_CHECK(!compact_scales[i] || outputs[i].get_with_gemm_swizzled_scales(), "Expected a GEMM-ready output"); } - - // Redirect scales to scratch buffers, allocated together per usage - std::vector redirected(outputs.size()); for (bool rowwise : {true, false}) { std::vector> shapes; std::vector indices; @@ -160,14 +149,18 @@ void quantize_with_compact_scales(std::vector &outputs, } } for (size_t i = 0; i < outputs.size(); ++i) { - if (compact_scales[i]) outputs[i].set_with_gemm_swizzled_scales(false); + if (!compact_scales[i]) continue; + outputs[i].set_with_gemm_swizzled_scales(false); + redirected[i].active = true; } + return redirected; +} - quantize(); - - // Restore the outputs and swizzle all compact scales with one kernel per usage +void restore_and_swizzle_scales(std::vector &outputs, + const std::vector &redirected) { + NVTE_CHECK(outputs.size() == redirected.size(), "Mismatched quantization outputs"); for (size_t i = 0; i < outputs.size(); ++i) { - if (compact_scales[i]) outputs[i].set_with_gemm_swizzled_scales(true); + if (redirected[i].active) outputs[i].set_with_gemm_swizzled_scales(true); } for (bool rowwise : {true, false}) { std::vector input_views, output_views; diff --git a/transformer_engine/pytorch/csrc/quantizer.cpp b/transformer_engine/pytorch/csrc/quantizer.cpp index 3dc1bee8e65..6cad9c287b6 100644 --- a/transformer_engine/pytorch/csrc/quantizer.cpp +++ b/transformer_engine/pytorch/csrc/quantizer.cpp @@ -2572,11 +2572,10 @@ void NVFP4Quantizer::quantize_with_rht_unfused_helper( void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& out, const std::optional& noop_flag, bool compute_amax) { + RedirectedScales redirected_scales; if (input.numel() != 0 && out.get_with_gemm_swizzled_scales() && !emits_gemm_swizzled_scales(convertShape(input.shape()))) { - quantize_with_compact_scales(out, noop_flag.has_value(), - [&] { quantize_impl(input, out, noop_flag, compute_amax); }); - return; + redirected_scales = redirect_scales_to_scratch(out, noop_flag.has_value()); } auto reduce_amaxes = [&]() { @@ -2805,6 +2804,7 @@ void NVFP4Quantizer::quantize_impl(const TensorWrapper& input, TensorWrapper& ou } else { NVTE_SCOPED_GIL_RELEASE({ nvte_quantize_v2(input.data(), out.data(), quant_config, stream); }); } + restore_and_swizzle_scales(out, redirected_scales); } void NVFP4Quantizer::quantize(const TensorWrapper& input, TensorWrapper& out, diff --git a/transformer_engine/pytorch/csrc/util.h b/transformer_engine/pytorch/csrc/util.h index f3dcf841295..bd690d00a7c 100644 --- a/transformer_engine/pytorch/csrc/util.h +++ b/transformer_engine/pytorch/csrc/util.h @@ -9,7 +9,6 @@ #include -#include #include #include #include @@ -37,18 +36,31 @@ std::optional multi_tensor_swizzle_scales_for_gemm(std::vector multi_tensor_swizzle_scales_for_gemm_unchecked( std::vector& tensors, bool rowwise_usage, bool columnwise_usage); -/*! \brief Run quantize on a GEMM-swizzled output with compact scratch scales, then swizzle them. +/*! \brief Compact scratch scales of an output and its own GEMM-swizzled scale buffers. */ +struct RedirectedScales { + bool active = false; + at::Tensor scratch[2]; // rowwise, columnwise + void* swizzled[2] = {nullptr, nullptr}; // rowwise, columnwise +}; + +/*! \brief Point a GEMM-swizzled output's scales to compact scratch buffers. * - * For kernels that only emit compact scales. preserve_scales keeps the previous scales - * if quantize is a GPU no-op. + * For kernels that only emit compact scales, which can then write directly into the output. + * Must be followed by restore_and_swizzle_scales. preserve_scales keeps the previous scales + * if the kernel is a GPU no-op. */ -void quantize_with_compact_scales(TensorWrapper& output, bool preserve_scales, - const std::function& quantize); +RedirectedScales redirect_scales_to_scratch(TensorWrapper& output, bool preserve_scales = false); + +/*! \brief Restore the output's scale buffers and swizzle the compact scratch scales into them. */ +void restore_and_swizzle_scales(TensorWrapper& output, const RedirectedScales& redirected); + +/*! \brief Multi-tensor redirect_scales_to_scratch for the outputs where compact_scales is set. */ +std::vector redirect_scales_to_scratch(std::vector& outputs, + const std::vector& compact_scales); -/*! \brief Same for the outputs where compact_scales is set, with one swizzle kernel per usage. */ -void quantize_with_compact_scales(std::vector& outputs, - const std::vector& compact_scales, - const std::function& quantize); +/*! \brief Multi-tensor restore_and_swizzle_scales with one swizzle kernel per usage. */ +void restore_and_swizzle_scales(std::vector& outputs, + const std::vector& redirected); using SwizzledGroupedScales = std::pair, std::optional>; From c6b2538bbc7afe85f774632f738cba2868fcac8c Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 5 Nov 2026 20:20:56 +0100 Subject: [PATCH 25/31] Drop NVFP4 shape-gate tests for GEMM-ready outputs Co-Authored-By: Claude Opus 5.5 Signed-off-by: Pawel Gadzinski --- .../test_nvfp4_2d_quantize_swizzle_fusion.py | 38 ++---------- .../test_nvfp4_rht_quantize_swizzle_fusion.py | 58 ++----------------- 2 files changed, 10 insertions(+), 86 deletions(-) diff --git a/tests/pytorch/nvfp4/test_nvfp4_2d_quantize_swizzle_fusion.py b/tests/pytorch/nvfp4/test_nvfp4_2d_quantize_swizzle_fusion.py index cfc279f886e..1edbbe5825a 100644 --- a/tests/pytorch/nvfp4/test_nvfp4_2d_quantize_swizzle_fusion.py +++ b/tests/pytorch/nvfp4/test_nvfp4_2d_quantize_swizzle_fusion.py @@ -180,43 +180,13 @@ def test_nvfp4_2d_quantize_swizzle_fusion( ) -@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) -@pytest.mark.parametrize( - "M, N, expected_swizzled", - [ - # Eligible: both dims % 128 == 0 -> 2D quantize kernel bakes swizzled SF. - (128, 128, True), - (256, 512, True), - # Ineligible (cols % 128 != 0) -> kernel emits compact SF; create_tensor - # reports swizzled=False (the fused kernel did not bake it in). - (128, 192, False), - # Ineligible (rows % 128 != 0). - (64, 128, False), - ], -) -def test_nvfp4_2d_swizzle_fusion_shape_gate(M: int, N: int, expected_swizzled: bool) -> None: - """``create_tensor`` must report in-kernel swizzle only for 128-aligned shapes. - - ``make_empty`` runs ``create_tensor`` only (no quantize kernel, no - post-quantize ``inplace_swizzle_scale_for_gemm`` fallback), so its - ``_with_gemm_swizzled_scales`` observes the in-kernel gate in isolation. - """ - quantizer = _make_quantizer(return_rowwise=True, return_transpose=True) - quantizer.optimize_for_gemm = True - out = quantizer.make_empty([M, N], dtype=torch.bfloat16) - assert out._with_gemm_swizzled_scales is expected_swizzled, ( - f"2D-kernel shape gate expected _with_gemm_swizzled_scales={expected_swizzled} " - f"for shape ({M}, {N}), got {out._with_gemm_swizzled_scales}" - ) - - @pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) @pytest.mark.parametrize( "M, N", [ (128, 128), (256, 512), - # Ineligible shapes still end up swizzled via the post-quantize fallback. + # Ineligible shapes: compact scales are swizzled after quantization. (128, 192), (64, 128), ], @@ -225,9 +195,9 @@ def test_nvfp4_2d_swizzle_fusion_end_to_end_swizzled(M: int, N: int) -> None: """End-to-end quantize must never crash and must always yield swizzled SF. Eligible (128-aligned) shapes get swizzled SF directly from the 2D quantize - kernel; ineligible shapes are gated off the in-kernel path and get swizzled - by the post-quantize ``inplace_swizzle_scale_for_gemm`` fallback. Either way - the final ``_with_gemm_swizzled_scales`` must be True. + kernel; for ineligible shapes the kernel writes compact SF to scratch buffers, + which are then swizzled into the output. Either way the final + ``_with_gemm_swizzled_scales`` must be True. """ quantizer = _make_quantizer(return_rowwise=True, return_transpose=True) quantizer.optimize_for_gemm = True diff --git a/tests/pytorch/nvfp4/test_nvfp4_rht_quantize_swizzle_fusion.py b/tests/pytorch/nvfp4/test_nvfp4_rht_quantize_swizzle_fusion.py index 67dc0b47b88..1486d3195f4 100644 --- a/tests/pytorch/nvfp4/test_nvfp4_rht_quantize_swizzle_fusion.py +++ b/tests/pytorch/nvfp4/test_nvfp4_rht_quantize_swizzle_fusion.py @@ -206,52 +206,6 @@ def _make_swizzle_fusion_quantizer() -> NVFP4Quantizer: return quantizer -@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) -@pytest.mark.parametrize( - "M, N, expected_swizzled", - [ - # Eligible: rows%64==0 AND cols%128==0 -> the fused RHT cast-fusion - # kernel can bake the GEMM-swizzled SF in directly. - (64, 128, True), - (128, 256, True), - # Ineligible (cols%128 != 0) -> the fused kernel is gated off, so - # create_tensor reports swizzled=False (the fused kernel did not bake - # it in). This is the case hit in production at irregular shapes like - # (8192, 11328) where 11328%128==64. - (64, 144, False), - (128, 144, False), - # Ineligible (rows%64 != 0) -> same gating. - (48, 128, False), - ], -) -def test_nvfp4_rht_swizzle_fusion_shape_gate(M: int, N: int, expected_swizzled: bool) -> None: - """The fused-kernel shape gate must reflect cast-fusion eligibility. - - Only ``row_cast_col_hadamard_transform_cast_fusion.cu`` can bake - GEMM-swizzled SF directly, and only for eligible shapes (rows%64==0 AND - cols%128==0 on SM 100/110). ``NVFP4Quantizer::create_tensor`` carries this - gate in ``_with_gemm_swizzled_scales``: it must be True only when the fused - kernel is selectable, and False on ineligible shapes so dispatch routes - them to the unfused path (which is what keeps the defense-in-depth - ``NVTE_CHECK`` in ``quantize_with_rht_unfused_helper`` from hard-aborting - on irregular production shapes such as ``(8192, 11328)``). - - ``make_empty`` runs ``create_tensor`` only -- no quantize kernel, and in - particular no post-quantize ``inplace_swizzle_scale_for_gemm`` fallback -- - so its ``_with_gemm_swizzled_scales`` observes the fused-kernel gate in - isolation. (End-to-end ``quantizer(x)`` differs: the fallback then swizzles - ineligible shapes too and flips the flag back to True; see - ``test_nvfp4_rht_swizzle_fusion_end_to_end_swizzled``.) - """ - quantizer = _make_swizzle_fusion_quantizer() - out = quantizer.make_empty([M, N], dtype=torch.bfloat16) - assert out._with_gemm_swizzled_scales is expected_swizzled, ( - f"Fused-kernel shape gate expected _with_gemm_swizzled_scales={expected_swizzled} " - f"for shape ({M}, {N}) with optimize_for_gemm=True + with_rht=True, " - f"got {out._with_gemm_swizzled_scales}" - ) - - @pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) @pytest.mark.parametrize( "M, N", @@ -260,8 +214,8 @@ def test_nvfp4_rht_swizzle_fusion_shape_gate(M: int, N: int, expected_swizzled: (64, 128), (128, 256), # Ineligible shapes (cols%128 != 0 / rows%64 != 0): the fused kernel is - # gated off, so they take the unfused quantize path and rely on the - # post-quantize swizzle fallback. (8192, 11328)-class production shapes. + # gated off, so they take the unfused quantize path and their compact SF + # are swizzled afterwards. (8192, 11328)-class production shapes. (64, 144), (128, 144), (48, 128), @@ -275,15 +229,15 @@ def test_nvfp4_rht_swizzle_fusion_end_to_end_swizzled(M: int, N: int) -> None: swizzle pass. Eligible shapes get this from the fused cast-fusion kernel in one pass; ineligible shapes are gated off the fused kernel (avoiding the ``NVTE_CHECK`` hard-abort in ``quantize_with_rht_unfused_helper``) and - instead get swizzled by the post-quantize ``inplace_swizzle_scale_for_gemm`` - fallback added in mainline #3076. Either way ``quantizer(x)`` must not raise - and the final ``_with_gemm_swizzled_scales`` must be True. + instead write compact SF to scratch buffers, which are then swizzled into the + output. Either way ``quantizer(x)`` must not raise and the final + ``_with_gemm_swizzled_scales`` must be True. """ quantizer = _make_swizzle_fusion_quantizer() x = torch.randn((M, N), dtype=torch.bfloat16, device="cuda") # Must not raise on ineligible shapes (gate keeps them off the fused - # kernel; the unfused path + swizzle fallback run cleanly). + # kernel; the unfused path + swizzle run cleanly). result = quantizer(x) assert result._with_gemm_swizzled_scales is True, ( "End-to-end quantize expected _with_gemm_swizzled_scales=True for shape " From 2818e63c9e6af5f7f5b87ddb9048df634ce8b046 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 12 Nov 2026 16:52:09 +0100 Subject: [PATCH 26/31] [PyTorch] Reduce activation compile test combinations Reduce 137 cases to 73 while retaining each activation and representative layout, cache, autocast, dynamic-shape, quantization, and CUDA Graph coverage. Exercise cached FP8 input in fused GELU bias backward and aligned NVFP4 RHT output shapes. Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index e9e230b125c..5c1b0963eb2 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -3378,19 +3378,23 @@ def run(x): "SwiGLU", ], ) +def test_te_ops_activation_compile(activation): + _check_te_ops_activation_compile(activation, dtype=torch.bfloat16) + + +@pytest.mark.parametrize("activation", ["GELU", "GEGLU", "SwiGLU"]) @pytest.mark.parametrize( "dtype,layout,cache_input", [ (torch.float32, "contiguous", False), (torch.float16, "contiguous", False), - (torch.bfloat16, "contiguous", False), (torch.bfloat16, "strided", False), (torch.bfloat16, "fp8", False), (torch.bfloat16, "contiguous", True), (torch.bfloat16, "fp8", True), ], ) -def test_te_ops_activation_compile(activation, dtype, layout, cache_input): +def test_te_ops_activation_variants_compile(activation, dtype, layout, cache_input): _check_te_ops_activation_compile( activation, dtype=dtype, layout=layout, cache_input=cache_input ) @@ -3413,10 +3417,19 @@ def test_te_ops_swiglu_dynamic_compile(cache_input, interleave_size): ) -@pytest.mark.parametrize("activation", ["GELU", "ReLU", "GEGLU", "SwiGLU"]) @pytest.mark.parametrize("quantization", [None, "fp8", "nvfp4", "nvfp4_no_rht"]) @pytest.mark.parametrize( - "training,mode", [(False, "default"), (True, "default"), (True, "reduce-overhead")] + "activation,training,mode", + [ + ("GELU", True, "default"), + ("ReLU", True, "default"), + ("GEGLU", True, "default"), + ("SwiGLU", True, "default"), + ("GELU", False, "default"), + ("SwiGLU", False, "default"), + ("GELU", True, "reduce-overhead"), + ("SwiGLU", True, "reduce-overhead"), + ], ) def test_te_ops_activation_mlp_compile(activation, quantization, training, mode, monkeypatch): nvfp4 = quantization in ("nvfp4", "nvfp4_no_rht") @@ -3431,10 +3444,12 @@ def test_te_ops_activation_mlp_compile(activation, quantization, training, mode, monkeypatch.setattr(torch.backends.cuda.matmul, "allow_tf32", False) in_features, hidden = (128, 128) if nvfp4 else (32, 64) gated = activation in ("GEGLU", "SwiGLU") + cache_input = training and activation == "GELU" and quantization == "fp8" + rows = 64 if activation == "GELU" and quantization == "nvfp4" else 32 model = ( te.ops.Sequential( te.ops.Linear(in_features, hidden * (2 if gated else 1), device="cuda", dtype=dtype), - getattr(te.ops, activation)(), + getattr(te.ops, activation)(cache_quantized_input=cache_input), te.ops.Linear(hidden, in_features, device="cuda", dtype=dtype), ) .train(training) @@ -3466,7 +3481,7 @@ def run(x, module=model): eager_optimizer = torch.optim.SGD(eager_model.parameters(), lr=0.01) for step in range(4 if training else 1): torch.compiler.cudagraph_mark_step_begin() - shape = (2, 16, in_features) if training else (32, in_features) + shape = (2, rows // 2, in_features) if training else (rows, in_features) x = torch.randn(shape, device="cuda", dtype=dtype, requires_grad=training) if training and step == 0: # Keep the independent reference reductions well-conditioned. From 3fc3d47faaa69f1bdc6f37d71ea2c295d826bbe9 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 12 Nov 2026 17:15:17 +0100 Subject: [PATCH 27/31] [PyTorch] Compile SiTUGLU through shared activation ops Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 44 ++++- .../pytorch/ops/basic/swiglu.py | 163 ++++++------------ 2 files changed, 83 insertions(+), 124 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 5c1b0963eb2..0dc37835ca8 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -3275,10 +3275,14 @@ def run(x, module=model): ) -def _ops_activation_reference(name, x): +def _ops_activation_reference(name, x, *, beta1=4.0, beta2=25.0, interleave_size=None): gate = None - if name in ("GLU", "GEGLU", "QGEGLU", "ReGLU", "SReGLU", "SwiGLU"): - x, gate = x.chunk(2, dim=-1) + if name in ("GLU", "GEGLU", "QGEGLU", "ReGLU", "SReGLU", "SwiGLU", "SiTUGLU"): + if interleave_size is None: + x, gate = x.chunk(2, dim=-1) + else: + x, gate = x.reshape(*x.shape[:-1], -1, 2, interleave_size).unbind(-2) + x, gate = x.flatten(-2), gate.flatten(-2) if name in ("GELU", "GEGLU"): x = torch.nn.functional.gelu(x, approximate="tanh") elif name in ("QGELU", "QGEGLU"): @@ -3289,6 +3293,9 @@ def _ops_activation_reference(name, x): x = x.relu().square() elif name in ("SiLU", "SwiGLU"): x = torch.nn.functional.silu(x) + elif name == "SiTUGLU": + x = beta1 * torch.tanh(x / beta1) * x.sigmoid() + gate = beta2 * torch.tanh(gate / beta2) elif name == "GLU": x = x.sigmoid() else: @@ -3306,11 +3313,13 @@ def _check_te_ops_activation_compile( training=True, interleave_size=None, dynamic=False, + activation_kwargs=None, ): if (layout == "fp8" or cache_input) and not fp8_available: pytest.skip(reason_for_no_fp8) torch._dynamo.reset() - kwargs = {"cache_quantized_input": cache_input} + activation_kwargs = activation_kwargs or {} + kwargs = {"cache_quantized_input": cache_input, **activation_kwargs} if interleave_size is not None: kwargs["glu_interleave_size"] = interleave_size model = te.ops.Sequential(getattr(te.ops, activation)(**kwargs)) @@ -3343,7 +3352,11 @@ def run(x): with torch.set_grad_enabled(training): actual = compiled(x) expected = ( - run(ref_x) if eager_reference else _ops_activation_reference(activation, ref_x) + run(ref_x) + if eager_reference + else _ops_activation_reference( + activation, ref_x, interleave_size=interleave_size, **activation_kwargs + ) ) torch.testing.assert_close(actual.to(expected.dtype), expected, **tols) if training: @@ -3376,6 +3389,7 @@ def run(x): "SReGLU", "SiLU", "SwiGLU", + "SiTUGLU", ], ) def test_te_ops_activation_compile(activation): @@ -3400,7 +3414,7 @@ def test_te_ops_activation_variants_compile(activation, dtype, layout, cache_inp ) -@pytest.mark.parametrize("activation", ["GELU", "SwiGLU"]) +@pytest.mark.parametrize("activation", ["GELU", "SwiGLU", "SiTUGLU"]) @pytest.mark.parametrize("cache_input", [False, True]) @pytest.mark.parametrize("training", [False, True]) def test_te_ops_activation_autocast_compile(activation, cache_input, training): @@ -3409,11 +3423,22 @@ def test_te_ops_activation_autocast_compile(activation, cache_input, training): ) +@pytest.mark.parametrize("activation", ["SwiGLU", "SiTUGLU"]) @pytest.mark.parametrize("cache_input", [False, True]) @pytest.mark.parametrize("interleave_size", [None, 4]) -def test_te_ops_swiglu_dynamic_compile(cache_input, interleave_size): +def test_te_ops_swiglu_dynamic_compile(activation, cache_input, interleave_size): + _check_te_ops_activation_compile( + activation, cache_input=cache_input, interleave_size=interleave_size, dynamic=True + ) + + +@pytest.mark.parametrize("beta1,beta2", [(0.75, 1.5), (3.0, 12.0)]) +def test_te_ops_situglu_parameters_compile(beta1, beta2): _check_te_ops_activation_compile( - "SwiGLU", cache_input=cache_input, interleave_size=interleave_size, dynamic=True + "SiTUGLU", + cache_input=True, + interleave_size=4, + activation_kwargs={"beta1": beta1, "beta2": beta2}, ) @@ -3425,6 +3450,7 @@ def test_te_ops_swiglu_dynamic_compile(cache_input, interleave_size): ("ReLU", True, "default"), ("GEGLU", True, "default"), ("SwiGLU", True, "default"), + ("SiTUGLU", True, "default"), ("GELU", False, "default"), ("SwiGLU", False, "default"), ("GELU", True, "reduce-overhead"), @@ -3443,7 +3469,7 @@ def test_te_ops_activation_mlp_compile(activation, quantization, training, mode, counters.clear() monkeypatch.setattr(torch.backends.cuda.matmul, "allow_tf32", False) in_features, hidden = (128, 128) if nvfp4 else (32, 64) - gated = activation in ("GEGLU", "SwiGLU") + gated = activation in ("GEGLU", "SwiGLU", "SiTUGLU") cache_input = training and activation == "GELU" and quantization == "fp8" rows = 64 if activation == "GELU" and quantization == "nvfp4" else 32 model = ( diff --git a/transformer_engine/pytorch/ops/basic/swiglu.py b/transformer_engine/pytorch/ops/basic/swiglu.py index 60f61465468..df6e968cf97 100644 --- a/transformer_engine/pytorch/ops/basic/swiglu.py +++ b/transformer_engine/pytorch/ops/basic/swiglu.py @@ -45,6 +45,22 @@ class SwiGLUBwdArgs(ActivationBwdArgs): glu_interleave_size: Optional[int] +@dataclass(slots=True) +class SiTUGLUFwdArgs(SwiGLUFwdArgs): + """SwiGLU inputs with gate and up-branch soft caps.""" + + beta1: float + beta2: float + + +@dataclass(slots=True) +class SiTUGLUBwdArgs(SwiGLUBwdArgs): + """SwiGLU gradients with gate and up-branch soft caps.""" + + beta1: float + beta2: float + + def _interleave_glu(tensor: torch.Tensor, size: int, *, reverse: bool = False) -> torch.Tensor: shape = tensor.shape blocks = shape[-1] // (2 * size) @@ -116,12 +132,12 @@ def __init__( _output_halves_last_dim = True @staticmethod - def _activation_forward_impl(*args, **kwargs) -> torch.Tensor: - return tex.swiglu(*args, **kwargs) + def _activation_forward_impl(input_, quantizer, **_kwargs) -> torch.Tensor: + return tex.swiglu(input_, quantizer) @staticmethod - def _activation_backward_impl(*args, **kwargs) -> torch.Tensor: - return tex.dswiglu(*args, **kwargs) + def _activation_backward_impl(grad_output, input_, quantizer, **_kwargs) -> torch.Tensor: + return tex.dswiglu(grad_output, input_, quantizer) def pack_forward_args(self, *args, **kwargs) -> SwiGLUFwdArgs: args = super().pack_forward_args(*args, **kwargs) @@ -136,7 +152,7 @@ def forward_compute(cls, args: SwiGLUFwdArgs, *, in_custom_op: bool = False): swiglu_in = x if args.glu_interleave_size is not None: swiglu_in = _interleave_glu(x, args.glu_interleave_size, reverse=True) - out = cls._activation_forward_impl(swiglu_in, args.output_quantizer) + out = cls._activation_forward_impl(swiglu_in, args.output_quantizer, args=args) if args.input_quantizer is not None: x = args.input_quantizer(x) save_input = args.requires_grad and (not in_custom_op or args.input_quantizer is not None) @@ -157,7 +173,7 @@ def backward_compute(cls, args: SwiGLUBwdArgs, *, in_custom_op: bool = False): if args.glu_interleave_size is not None: x = _interleave_glu(x, args.glu_interleave_size, reverse=True) quantizer = None - dx = cls._activation_backward_impl(dy, x, quantizer) + dx = cls._activation_backward_impl(dy, x, quantizer, args=args) if args.glu_interleave_size is not None: dx = _interleave_glu(dx, args.glu_interleave_size) if not in_custom_op: @@ -197,8 +213,8 @@ class SiTUGLU(SwiGLU): Block-interleaved GLU layout, as in :class:`SwiGLU`. """ - fwd_args_type = None - bwd_args_type = None + fwd_args_type = SiTUGLUFwdArgs + bwd_args_type = SiTUGLUBwdArgs def __init__( self, @@ -219,123 +235,40 @@ def __init__( if not math.isfinite(self.beta2) or self.beta2 <= 0.0: raise ValueError(f"beta2 must be finite and positive, got {self.beta2}") - def _tex_swiglu_forward( - self, + @staticmethod + def _activation_forward_impl( input_: torch.Tensor, quantizer: Optional[Quantizer], + *, + args: SiTUGLUFwdArgs, ) -> torch.Tensor: - return tex.situglu(input_, quantizer, self.beta1, self.beta2) + return tex.situglu(input_, quantizer, args.beta1, args.beta2) - def _tex_swiglu_backward( - self, + @staticmethod + def _activation_backward_impl( grad_output: torch.Tensor, input_: torch.Tensor, quantizer: Optional[Quantizer], + *, + args: SiTUGLUBwdArgs, ) -> torch.Tensor: - return tex.dsituglu(grad_output, input_, quantizer, self.beta1, self.beta2) - - def op_forward( - self, - ctx: OperationContext, - input_: torch.Tensor, - prev_op_grad_output_quantizer: Optional[Quantizer], - next_op_input_quantizer: Optional[Quantizer], - ) -> torch.Tensor: - - # Compute dtype - dtype: torch.dtype = maybe_autocast_dtype(default_dtype=input_.dtype) - if dtype not in (torch.float32, torch.float16, torch.bfloat16): - raise RuntimeError(f"Unsupported dtype ({dtype})") - - # Check input tensor - input_ = maybe_dequantize(input_.contiguous(), dtype) - - # Remove interleaving if needed - swiglu_in = input_ - if self.glu_interleave_size is not None: - shape = swiglu_in.size() - swiglu_in = swiglu_in.reshape( - -1, - shape[-1] // (2 * self.glu_interleave_size), - 2, - self.glu_interleave_size, - ) - swiglu_in = swiglu_in.transpose(1, 2).contiguous() - swiglu_in = swiglu_in.view(shape) - - # Launch kernel - out = self._tex_swiglu_forward(swiglu_in, next_op_input_quantizer) - - # Quantize input to FP8 before caching if needed - if self.cache_quantized_input: - input_quantizer = Float8CurrentScalingQuantizer( - DType.kFloat8E4M3, - input_.device, - ) - input_quantizer.set_usage(rowwise=True, columnwise=False) - input_ = input_quantizer(input_) - - # Save state for backward pass - if ctx.requires_grad: - if is_cpu_offload_enabled(): - mark_activation_offload(input_) - ctx.save_for_backward(input_) - ctx.dtype = dtype - ctx.prev_op_grad_output_quantizer = prev_op_grad_output_quantizer - - return out - - def op_backward( - self, - ctx: OperationContext, - grad_output: torch.Tensor, - ) -> tuple[torch.Tensor, tuple[()]]: - - # Saved tensors from forward pass - (input_,) = ctx.saved_tensors - - # Make sure tensors have correct dtypes - x = maybe_dequantize(input_.contiguous(), ctx.dtype) - dy = maybe_dequantize(grad_output.contiguous(), ctx.dtype) - - # Remove interleaving if needed - swiglu_in = x - if self.glu_interleave_size is not None: - shape = swiglu_in.size() - swiglu_in = swiglu_in.reshape( - -1, - shape[-1] // (2 * self.glu_interleave_size), - 2, - self.glu_interleave_size, - ) - swiglu_in = swiglu_in.transpose(1, 2).contiguous() - swiglu_in = swiglu_in.view(shape) - - # Quantizer for grad input - quantizer = ctx.prev_op_grad_output_quantizer - if self.glu_interleave_size is not None: - quantizer = None - - # Launch kernel - grad_swiglu_in = self._tex_swiglu_backward(dy, swiglu_in, quantizer) - - # Apply interleaving if needed - dx = grad_swiglu_in - if self.glu_interleave_size is not None: - shape = dx.size() - dx = dx.reshape( - -1, - 2, - shape[-1] // (2 * self.glu_interleave_size), - self.glu_interleave_size, - ) - dx = dx.transpose(1, 2).contiguous() - dx = dx.view(shape) + return tex.dsituglu(grad_output, input_, quantizer, args.beta1, args.beta2) - # Clear input tensor if possible - clear_tensor_data(input_) + def pack_forward_args(self, *args, **kwargs) -> SiTUGLUFwdArgs: + args = super().pack_forward_args(*args, **kwargs) + return SiTUGLUFwdArgs( + **{field.name: getattr(args, field.name) for field in fields(args)}, + beta1=self.beta1, + beta2=self.beta2, + ) - return dx, () + def pack_backward_args(self, *args, **kwargs) -> SiTUGLUBwdArgs: + args = super().pack_backward_args(*args, **kwargs) + return SiTUGLUBwdArgs( + **{field.name: getattr(args, field.name) for field in fields(args)}, + beta1=self.beta1, + beta2=self.beta2, + ) class ClampedSwiGLU(BasicOperation): From 2bcf2cf0e3834a9b5c38ed8cc93ae693cc60969d Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 12 Nov 2026 17:51:16 +0100 Subject: [PATCH 28/31] [PyTorch] Avoid rebuilding activation argument containers Signed-off-by: Pawel Gadzinski --- .../pytorch/ops/basic/activation.py | 4 +- .../pytorch/ops/basic/swiglu.py | 42 ++++++++----------- 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/transformer_engine/pytorch/ops/basic/activation.py b/transformer_engine/pytorch/ops/basic/activation.py index f380647f744..e6e6e5e50fd 100644 --- a/transformer_engine/pytorch/ops/basic/activation.py +++ b/transformer_engine/pytorch/ops/basic/activation.py @@ -141,7 +141,7 @@ def pack_forward_args( input_quantizer = Float8CurrentScalingQuantizer(DType.kFloat8E4M3, input_.device) input_quantizer.set_usage(rowwise=True, columnwise=False) input_quantizer.internal = True - return ActivationFwdArgs( + return self.fwd_args_type( input_, dtype, next_op_input_quantizer, @@ -191,7 +191,7 @@ def pack_backward_args( self, basic_op_ctxs, grad_output, **unused # pylint: disable=unused-argument ) -> ActivationBwdArgs: ctx = basic_op_ctxs[0] - return ActivationBwdArgs( + return self.bwd_args_type( grad_output, ctx.saved_tensors[0], ctx.dtype, ctx.prev_op_grad_output_quantizer ) diff --git a/transformer_engine/pytorch/ops/basic/swiglu.py b/transformer_engine/pytorch/ops/basic/swiglu.py index df6e968cf97..66244c3c134 100644 --- a/transformer_engine/pytorch/ops/basic/swiglu.py +++ b/transformer_engine/pytorch/ops/basic/swiglu.py @@ -7,7 +7,7 @@ from __future__ import annotations from collections.abc import Iterable, Sequence import math -from dataclasses import dataclass, fields +from dataclasses import dataclass from typing import Any, Optional import torch @@ -35,30 +35,30 @@ class SwiGLUFwdArgs(ActivationFwdArgs): """Activation configuration with the GLU input layout.""" - glu_interleave_size: Optional[int] + glu_interleave_size: Optional[int] = None @dataclass(slots=True) class SwiGLUBwdArgs(ActivationBwdArgs): """Activation gradient with the GLU input layout.""" - glu_interleave_size: Optional[int] + glu_interleave_size: Optional[int] = None @dataclass(slots=True) class SiTUGLUFwdArgs(SwiGLUFwdArgs): """SwiGLU inputs with gate and up-branch soft caps.""" - beta1: float - beta2: float + beta1: float = 4.0 + beta2: float = 25.0 @dataclass(slots=True) class SiTUGLUBwdArgs(SwiGLUBwdArgs): """SwiGLU gradients with gate and up-branch soft caps.""" - beta1: float - beta2: float + beta1: float = 4.0 + beta2: float = 25.0 def _interleave_glu(tensor: torch.Tensor, size: int, *, reverse: bool = False) -> torch.Tensor: @@ -141,10 +141,8 @@ def _activation_backward_impl(grad_output, input_, quantizer, **_kwargs) -> torc def pack_forward_args(self, *args, **kwargs) -> SwiGLUFwdArgs: args = super().pack_forward_args(*args, **kwargs) - return SwiGLUFwdArgs( - **{field.name: getattr(args, field.name) for field in fields(args)}, - glu_interleave_size=self.glu_interleave_size, - ) + args.glu_interleave_size = self.glu_interleave_size + return args @classmethod def forward_compute(cls, args: SwiGLUFwdArgs, *, in_custom_op: bool = False): @@ -160,10 +158,8 @@ def forward_compute(cls, args: SwiGLUFwdArgs, *, in_custom_op: bool = False): def pack_backward_args(self, *args, **kwargs) -> SwiGLUBwdArgs: args = super().pack_backward_args(*args, **kwargs) - return SwiGLUBwdArgs( - **{field.name: getattr(args, field.name) for field in fields(args)}, - glu_interleave_size=self.glu_interleave_size, - ) + args.glu_interleave_size = self.glu_interleave_size + return args @classmethod def backward_compute(cls, args: SwiGLUBwdArgs, *, in_custom_op: bool = False): @@ -256,19 +252,15 @@ def _activation_backward_impl( def pack_forward_args(self, *args, **kwargs) -> SiTUGLUFwdArgs: args = super().pack_forward_args(*args, **kwargs) - return SiTUGLUFwdArgs( - **{field.name: getattr(args, field.name) for field in fields(args)}, - beta1=self.beta1, - beta2=self.beta2, - ) + args.beta1 = self.beta1 + args.beta2 = self.beta2 + return args def pack_backward_args(self, *args, **kwargs) -> SiTUGLUBwdArgs: args = super().pack_backward_args(*args, **kwargs) - return SiTUGLUBwdArgs( - **{field.name: getattr(args, field.name) for field in fields(args)}, - beta1=self.beta1, - beta2=self.beta2, - ) + args.beta1 = self.beta1 + args.beta2 = self.beta2 + return args class ClampedSwiGLU(BasicOperation): From c3fc4998605ece7ebc9909469fbf93726d3d2905 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 12 Nov 2026 18:06:59 +0100 Subject: [PATCH 29/31] [PyTorch] Compile ClampedSwiGLU through shared activation ops Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 36 +++- .../pytorch/ops/basic/swiglu.py | 180 ++++++------------ 2 files changed, 84 insertions(+), 132 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 0dc37835ca8..64abdc53b14 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -3275,9 +3275,19 @@ def run(x, module=model): ) -def _ops_activation_reference(name, x, *, beta1=4.0, beta2=25.0, interleave_size=None): +def _ops_activation_reference( + name, + x, + *, + beta1=4.0, + beta2=25.0, + limit=7.0, + alpha=1.702, + glu_linear_offset=1.0, + interleave_size=None, +): gate = None - if name in ("GLU", "GEGLU", "QGEGLU", "ReGLU", "SReGLU", "SwiGLU", "SiTUGLU"): + if name in ("GLU", "GEGLU", "QGEGLU", "ReGLU", "SReGLU", "SwiGLU", "SiTUGLU", "ClampedSwiGLU"): if interleave_size is None: x, gate = x.chunk(2, dim=-1) else: @@ -3296,6 +3306,10 @@ def _ops_activation_reference(name, x, *, beta1=4.0, beta2=25.0, interleave_size elif name == "SiTUGLU": x = beta1 * torch.tanh(x / beta1) * x.sigmoid() gate = beta2 * torch.tanh(gate / beta2) + elif name == "ClampedSwiGLU": + x = x.clamp(max=limit) + x = x * torch.sigmoid(alpha * x) + gate = gate.clamp(min=-limit, max=limit) + glu_linear_offset elif name == "GLU": x = x.sigmoid() else: @@ -3390,6 +3404,7 @@ def run(x): "SiLU", "SwiGLU", "SiTUGLU", + "ClampedSwiGLU", ], ) def test_te_ops_activation_compile(activation): @@ -3414,7 +3429,7 @@ def test_te_ops_activation_variants_compile(activation, dtype, layout, cache_inp ) -@pytest.mark.parametrize("activation", ["GELU", "SwiGLU", "SiTUGLU"]) +@pytest.mark.parametrize("activation", ["GELU", "SwiGLU", "SiTUGLU", "ClampedSwiGLU"]) @pytest.mark.parametrize("cache_input", [False, True]) @pytest.mark.parametrize("training", [False, True]) def test_te_ops_activation_autocast_compile(activation, cache_input, training): @@ -3423,7 +3438,7 @@ def test_te_ops_activation_autocast_compile(activation, cache_input, training): ) -@pytest.mark.parametrize("activation", ["SwiGLU", "SiTUGLU"]) +@pytest.mark.parametrize("activation", ["SwiGLU", "SiTUGLU", "ClampedSwiGLU"]) @pytest.mark.parametrize("cache_input", [False, True]) @pytest.mark.parametrize("interleave_size", [None, 4]) def test_te_ops_swiglu_dynamic_compile(activation, cache_input, interleave_size): @@ -3442,6 +3457,16 @@ def test_te_ops_situglu_parameters_compile(beta1, beta2): ) +@pytest.mark.parametrize("limit,alpha,glu_linear_offset", [(0.75, 1.3, 0.0), (1.25, 2.0, 0.5)]) +def test_te_ops_clamped_swiglu_parameters_compile(limit, alpha, glu_linear_offset): + _check_te_ops_activation_compile( + "ClampedSwiGLU", + cache_input=True, + interleave_size=4, + activation_kwargs={"limit": limit, "alpha": alpha, "glu_linear_offset": glu_linear_offset}, + ) + + @pytest.mark.parametrize("quantization", [None, "fp8", "nvfp4", "nvfp4_no_rht"]) @pytest.mark.parametrize( "activation,training,mode", @@ -3451,6 +3476,7 @@ def test_te_ops_situglu_parameters_compile(beta1, beta2): ("GEGLU", True, "default"), ("SwiGLU", True, "default"), ("SiTUGLU", True, "default"), + ("ClampedSwiGLU", True, "default"), ("GELU", False, "default"), ("SwiGLU", False, "default"), ("GELU", True, "reduce-overhead"), @@ -3469,7 +3495,7 @@ def test_te_ops_activation_mlp_compile(activation, quantization, training, mode, counters.clear() monkeypatch.setattr(torch.backends.cuda.matmul, "allow_tf32", False) in_features, hidden = (128, 128) if nvfp4 else (32, 64) - gated = activation in ("GEGLU", "SwiGLU", "SiTUGLU") + gated = activation in ("GEGLU", "SwiGLU", "SiTUGLU", "ClampedSwiGLU") cache_input = training and activation == "GELU" and quantization == "fp8" rows = 64 if activation == "GELU" and quantization == "nvfp4" else 32 model = ( diff --git a/transformer_engine/pytorch/ops/basic/swiglu.py b/transformer_engine/pytorch/ops/basic/swiglu.py index 66244c3c134..6c939496491 100644 --- a/transformer_engine/pytorch/ops/basic/swiglu.py +++ b/transformer_engine/pytorch/ops/basic/swiglu.py @@ -13,9 +13,8 @@ import torch import transformer_engine_torch as tex -from ...constants import DType from ...cpu_offload import is_cpu_offload_enabled, mark_activation_offload -from ...tensor import Float8CurrentScalingQuantizer, Quantizer +from ...tensor import Quantizer from ...utils import clear_tensor_data from ..op import BasicOperation, OperationContext from .._common import maybe_autocast_dtype, maybe_dequantize @@ -61,6 +60,24 @@ class SiTUGLUBwdArgs(SwiGLUBwdArgs): beta2: float = 25.0 +@dataclass(slots=True) +class ClampedSwiGLUFwdArgs(SwiGLUFwdArgs): + """SwiGLU inputs with clipping and gate parameters.""" + + limit: float = 7.0 + alpha: float = 1.702 + glu_linear_offset: float = 1.0 + + +@dataclass(slots=True) +class ClampedSwiGLUBwdArgs(SwiGLUBwdArgs): + """SwiGLU gradients with clipping and gate parameters.""" + + limit: float = 7.0 + alpha: float = 1.702 + glu_linear_offset: float = 1.0 + + def _interleave_glu(tensor: torch.Tensor, size: int, *, reverse: bool = False) -> torch.Tensor: shape = tensor.shape blocks = shape[-1] // (2 * size) @@ -263,7 +280,7 @@ def pack_backward_args(self, *args, **kwargs) -> SiTUGLUBwdArgs: return args -class ClampedSwiGLU(BasicOperation): +class ClampedSwiGLU(SwiGLU): r"""GPT-OSS Implementation based on `GPT-OSS `__. @@ -296,6 +313,9 @@ class ClampedSwiGLU(BasicOperation): """ + fwd_args_type = ClampedSwiGLUFwdArgs + bwd_args_type = ClampedSwiGLUBwdArgs + def __init__( self, *, @@ -305,142 +325,48 @@ def __init__( cache_quantized_input: bool = False, glu_interleave_size: Optional[int] = None, ): - super().__init__() + super().__init__( + cache_quantized_input=cache_quantized_input, + glu_interleave_size=glu_interleave_size, + ) self.limit: float = limit self.alpha: float = alpha self.glu_linear_offset: float = glu_linear_offset - self.cache_quantized_input: bool = cache_quantized_input - self.glu_interleave_size: Optional[int] = glu_interleave_size - def _tex_clamped_swiglu_forward( - self, - swiglu_in: torch.Tensor, - next_op_input_quantizer: Optional[Quantizer], + @staticmethod + def _activation_forward_impl( + input_: torch.Tensor, + quantizer: Optional[Quantizer], + *, + args: ClampedSwiGLUFwdArgs, ) -> torch.Tensor: - """Call :func:`tex.clamped_swiglu` with this op's ``limit`` / ``alpha`` / ``glu_linear_offset``.""" - return tex.clamped_swiglu( - swiglu_in, - next_op_input_quantizer, - self.limit, - self.alpha, - self.glu_linear_offset, - ) + return tex.clamped_swiglu(input_, quantizer, args.limit, args.alpha, args.glu_linear_offset) - def _tex_clamped_dswiglu( - self, - dy: torch.Tensor, - swiglu_in: torch.Tensor, + @staticmethod + def _activation_backward_impl( + grad_output: torch.Tensor, + input_: torch.Tensor, quantizer: Optional[Quantizer], + *, + args: ClampedSwiGLUBwdArgs, ) -> torch.Tensor: - """Call :func:`tex.clamped_dswiglu` with this op's ``limit`` / ``alpha`` / ``glu_linear_offset``.""" return tex.clamped_dswiglu( - dy, - swiglu_in, - quantizer, - self.limit, - self.alpha, - self.glu_linear_offset, + grad_output, input_, quantizer, args.limit, args.alpha, args.glu_linear_offset ) - def op_forward( - self, - ctx: OperationContext, - input_: torch.Tensor, - prev_op_grad_output_quantizer: Optional[Quantizer], - next_op_input_quantizer: Optional[Quantizer], - ) -> torch.Tensor: - - # Compute dtype - dtype: torch.dtype = maybe_autocast_dtype(default_dtype=input_.dtype) - if dtype not in (torch.float32, torch.float16, torch.bfloat16): - raise RuntimeError(f"Unsupported dtype ({dtype})") - - # Check input tensor - x = maybe_dequantize(input_.contiguous(), dtype) - - # Remove interleaving if needed - swiglu_in = x - if self.glu_interleave_size is not None: - shape = swiglu_in.size() - swiglu_in = swiglu_in.reshape( - -1, - shape[-1] // (2 * self.glu_interleave_size), - 2, - self.glu_interleave_size, - ) - swiglu_in = swiglu_in.transpose(1, 2).contiguous() - swiglu_in = swiglu_in.view(shape) - - # Launch kernel - out = self._tex_clamped_swiglu_forward(swiglu_in, next_op_input_quantizer) - - # Quantize input to FP8 before caching if needed - if self.cache_quantized_input: - input_quantizer = Float8CurrentScalingQuantizer(DType.kFloat8E4M3, x.device) - input_quantizer.set_usage(rowwise=True, columnwise=False) - x = input_quantizer(x) - - # Save state for backward pass - if ctx.requires_grad: - if is_cpu_offload_enabled(): - mark_activation_offload(x) - ctx.save_for_backward(x) - ctx.dtype = dtype - ctx.prev_op_grad_output_quantizer = prev_op_grad_output_quantizer - - return out - - def op_backward( - self, - ctx: OperationContext, - grad_output: torch.Tensor, - ) -> tuple[torch.Tensor, tuple[()]]: - - # Saved tensors from forward pass - (input_,) = ctx.saved_tensors - - # Make sure tensors have correct dtypes - x = maybe_dequantize(input_.contiguous(), ctx.dtype) - dy = maybe_dequantize(grad_output.contiguous(), ctx.dtype) - - # Remove interleaving if needed - swiglu_in = x - if self.glu_interleave_size is not None: - shape = swiglu_in.size() - swiglu_in = swiglu_in.reshape( - -1, - shape[-1] // (2 * self.glu_interleave_size), - 2, - self.glu_interleave_size, - ) - swiglu_in = swiglu_in.transpose(1, 2).contiguous() - swiglu_in = swiglu_in.view(shape) - - # Quantizer for grad input - quantizer = ctx.prev_op_grad_output_quantizer - if self.glu_interleave_size is not None: - quantizer = None - - # Launch kernel - grad_swiglu_in = self._tex_clamped_dswiglu(dy, swiglu_in, quantizer) - - # Apply interleaving if needed - dx = grad_swiglu_in - if self.glu_interleave_size is not None: - shape = dx.size() - dx = dx.reshape( - -1, - 2, - shape[-1] // (2 * self.glu_interleave_size), - self.glu_interleave_size, - ) - dx = dx.transpose(1, 2).contiguous() - dx = dx.view(shape) - - # Clear input tensor if possible - clear_tensor_data(input_) + def pack_forward_args(self, *args, **kwargs) -> ClampedSwiGLUFwdArgs: + args = super().pack_forward_args(*args, **kwargs) + args.limit = self.limit + args.alpha = self.alpha + args.glu_linear_offset = self.glu_linear_offset + return args - return dx, () + def pack_backward_args(self, *args, **kwargs) -> ClampedSwiGLUBwdArgs: + args = super().pack_backward_args(*args, **kwargs) + args.limit = self.limit + args.alpha = self.alpha + args.glu_linear_offset = self.glu_linear_offset + return args class _ScaledGLU(BasicOperation): From 2ebb2203ca2180dd75d7a66e2e786bf24206ceb7 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Thu, 12 Nov 2026 20:36:17 +0100 Subject: [PATCH 30/31] [PyTorch] Preserve strides and prepared activation inputs under compile Signed-off-by: Pawel Gadzinski --- .../pytorch/dynamo/tensor_spec.py | 36 ++++++++++++++++--- .../pytorch/ops/basic/activation.py | 21 +++++++---- transformer_engine/pytorch/ops/basic/bias.py | 12 +++++-- .../pytorch/ops/basic/swiglu.py | 7 ++-- 4 files changed, 61 insertions(+), 15 deletions(-) diff --git a/transformer_engine/pytorch/dynamo/tensor_spec.py b/transformer_engine/pytorch/dynamo/tensor_spec.py index f023dd081db..fbf72b48b92 100644 --- a/transformer_engine/pytorch/dynamo/tensor_spec.py +++ b/transformer_engine/pytorch/dynamo/tensor_spec.py @@ -18,7 +18,7 @@ class TensorSpec: """A data-free description of a tensor or quantized tensor. - Captures ``shape`` / ``dtype`` and, for quantized tensors, the + Captures ``shape`` / ``dtype`` / ``strides`` and, for quantized tensors, the (value-opaque) ``quantizer`` -- enough to rebuild a tensor without holding storage. The common abstraction over plain ``torch.Tensor``, ``QuantizedTensorStorage`` and ``QuantizedTensor``, used for custom-op fake @@ -31,6 +31,7 @@ class TensorSpec: requires_grad: bool = False device: Optional[torch.device] = field(default=None) with_gemm_swizzled_scales: Optional[bool] = None + strides: Optional[Tuple[int, ...]] = None def __post_init__(self) -> None: # Own a private copy of the quantizer so usage changes (update_usage) @@ -45,6 +46,22 @@ def is_quantized(self) -> bool: """Whether this spec describes a quantized tensor.""" return self.quantizer is not None + def stride(self) -> Tuple[int, ...]: + """Explicit strides, or contiguous strides for a fresh output.""" + return self.strides if self.strides is not None else make_contiguous_strides_for(self.shape) + + def is_contiguous(self) -> bool: + """Check contiguous layout using only shape and strides.""" + if self.strides is None or any(size == 0 for size in self.shape): + return True + expected = 1 + for size, stride in zip(reversed(self.shape), reversed(self.strides)): + if size != 1: + if stride != expected: + return False + expected *= size + return True + def update_usage( self, *, @@ -111,7 +128,11 @@ def create_inner_tensors(self) -> List[torch.Tensor]: """ device = self.device if self.device is not None else torch.device(te_device_type()) if self.quantizer is None: - return [torch.empty(tuple(self.shape), dtype=self.dtype, device=device)] + return [ + torch.empty_strided( + tuple(self.shape), self.stride(), dtype=self.dtype, device=device + ) + ] inner = self.quantizer.alloc_tensors(tuple(self.shape), device=device) return [inner[name] for name in self.inner_names()] @@ -141,8 +162,9 @@ def create_tensor(self) -> torch.Tensor: """ if self.quantizer is None: device = self.device if self.device is not None else torch.device(te_device_type()) - return torch.empty( + return torch.empty_strided( tuple(self.shape), + self.stride(), dtype=self.dtype, device=device, requires_grad=self.requires_grad, @@ -161,11 +183,17 @@ def to_tensor_spec(tensor: Any) -> TensorSpec: dtype = getattr(tensor, "dtype", None) if dtype is None: dtype = getattr(tensor, "_dtype", None) + quantizer = getattr(tensor, "_quantizer", None) return TensorSpec( shape=tuple(tensor.shape), dtype=dtype, - quantizer=getattr(tensor, "_quantizer", None), + quantizer=quantizer, requires_grad=requires_grad, device=tensor.device, with_gemm_swizzled_scales=getattr(tensor, "_with_gemm_swizzled_scales", None), + strides=( + tuple(tensor.stride()) + if quantizer is None and isinstance(tensor, torch.Tensor) + else None + ), ) diff --git a/transformer_engine/pytorch/ops/basic/activation.py b/transformer_engine/pytorch/ops/basic/activation.py index e6e6e5e50fd..03564cddf9c 100644 --- a/transformer_engine/pytorch/ops/basic/activation.py +++ b/transformer_engine/pytorch/ops/basic/activation.py @@ -21,7 +21,7 @@ from ...tensor.float8_tensor import Float8CurrentScalingQuantizer, Quantizer from ...utils import _compile_safe_warn, clear_tensor_data from ..op import BasicOperation, OperationContext -from .._common import maybe_autocast_dtype, maybe_dequantize +from .._common import is_quantized_tensor, maybe_autocast_dtype, maybe_dequantize __all__ = [ "GELU", @@ -150,15 +150,24 @@ def pack_forward_args( basic_op_ctxs[0].requires_grad, ) + @staticmethod + def _save_prepared_input(args: ActivationFwdArgs) -> bool: + return args.requires_grad and ( + args.input_quantizer is not None + or is_quantized_tensor(args.input_) + or args.input_.dtype != args.dtype + or not args.input_.is_contiguous() + ) + @classmethod - def forward_compute(cls, args: ActivationFwdArgs, *, in_custom_op: bool = False): + def forward_compute( + cls, args: ActivationFwdArgs, *, in_custom_op: bool = False + ): # pylint: disable=unused-argument x = maybe_dequantize(args.input_, args.dtype).contiguous() y = cls._activation_forward_impl(x, args.output_quantizer) if args.input_quantizer is not None: x = args.input_quantizer(x) - # Custom ops return only fresh saved tensors; setup retains the original input. - save_input = args.requires_grad and (not in_custom_op or args.input_quantizer is not None) - return y, [()], (x,) if save_input else () + return y, [()], (x,) if cls._save_prepared_input(args) else () @classmethod def forward_compute_fake(cls, args: ActivationFwdArgs): @@ -170,7 +179,7 @@ def forward_compute_fake(cls, args: ActivationFwdArgs): gated=cls._output_halves_last_dim, ) saved = () - if args.requires_grad and args.input_quantizer is not None: + if cls._save_prepared_input(args): saved = ( TensorSpec( shape=x.shape, dtype=args.dtype, device=x.device, quantizer=args.input_quantizer diff --git a/transformer_engine/pytorch/ops/basic/bias.py b/transformer_engine/pytorch/ops/basic/bias.py index c5a9dda4f6c..c129e2c1cd0 100644 --- a/transformer_engine/pytorch/ops/basic/bias.py +++ b/transformer_engine/pytorch/ops/basic/bias.py @@ -157,11 +157,19 @@ def forward_compute( @classmethod def forward_compute_fake(cls, args: BiasFwdArgs): + input_ = torch.empty_strided( + args.input_.shape, args.input_.stride(), dtype=args.input_.dtype, device="meta" + ) + bias = torch.empty_strided( + args.bias.shape, args.bias.stride(), dtype=args.bias.dtype, device="meta" + ) + out = input_ + bias return ( TensorSpec( - shape=args.input_.shape, - dtype=torch.promote_types(args.input_.dtype, args.bias.dtype), + shape=tuple(out.shape), + dtype=out.dtype, device=args.input_.device, + strides=tuple(out.stride()), ), [()], (), diff --git a/transformer_engine/pytorch/ops/basic/swiglu.py b/transformer_engine/pytorch/ops/basic/swiglu.py index 6c939496491..a135c432159 100644 --- a/transformer_engine/pytorch/ops/basic/swiglu.py +++ b/transformer_engine/pytorch/ops/basic/swiglu.py @@ -162,7 +162,9 @@ def pack_forward_args(self, *args, **kwargs) -> SwiGLUFwdArgs: return args @classmethod - def forward_compute(cls, args: SwiGLUFwdArgs, *, in_custom_op: bool = False): + def forward_compute( + cls, args: SwiGLUFwdArgs, *, in_custom_op: bool = False + ): # pylint: disable=unused-argument x = maybe_dequantize(args.input_, args.dtype).contiguous() swiglu_in = x if args.glu_interleave_size is not None: @@ -170,8 +172,7 @@ def forward_compute(cls, args: SwiGLUFwdArgs, *, in_custom_op: bool = False): out = cls._activation_forward_impl(swiglu_in, args.output_quantizer, args=args) if args.input_quantizer is not None: x = args.input_quantizer(x) - save_input = args.requires_grad and (not in_custom_op or args.input_quantizer is not None) - return out, [()], (x,) if save_input else () + return out, [()], (x,) if cls._save_prepared_input(args) else () def pack_backward_args(self, *args, **kwargs) -> SwiGLUBwdArgs: args = super().pack_backward_args(*args, **kwargs) From 120d744e3d9120e98699c734aa2a7f45d4537546 Mon Sep 17 00:00:00 2001 From: Pawel Gadzinski Date: Fri, 13 Nov 2026 00:25:03 +0100 Subject: [PATCH 31/31] [PyTorch] Restrict compiled activation inputs to contiguous tensors Signed-off-by: Pawel Gadzinski --- tests/pytorch/test_torch_compile.py | 12 +++++-- .../pytorch/dynamo/tensor_spec.py | 36 +++---------------- .../pytorch/ops/basic/activation.py | 21 ++++++----- transformer_engine/pytorch/ops/basic/bias.py | 20 +++++------ .../pytorch/ops/basic/swiglu.py | 2 +- 5 files changed, 35 insertions(+), 56 deletions(-) diff --git a/tests/pytorch/test_torch_compile.py b/tests/pytorch/test_torch_compile.py index 64abdc53b14..9a7543fce0e 100644 --- a/tests/pytorch/test_torch_compile.py +++ b/tests/pytorch/test_torch_compile.py @@ -3364,7 +3364,14 @@ def run(x): else (x.dequantize() if layout == "fp8" else x).detach().double() ).requires_grad_(training) with torch.set_grad_enabled(training): - actual = compiled(x) + if layout == "strided": + with pytest.raises( + Exception, match="requires contiguous inputs under torch.compile" + ): + compiled(x) + actual = run(x) + else: + actual = compiled(x) expected = ( run(ref_x) if eager_reference @@ -3386,7 +3393,8 @@ def run(x): graph_count = len(graphs) elif step > 1: assert len(graphs) == graph_count - _assert_custom_ops(graphs, activation.lower(), backward=training) + if layout != "strided": + _assert_custom_ops(graphs, activation.lower(), backward=training) @pytest.mark.parametrize( diff --git a/transformer_engine/pytorch/dynamo/tensor_spec.py b/transformer_engine/pytorch/dynamo/tensor_spec.py index fbf72b48b92..f023dd081db 100644 --- a/transformer_engine/pytorch/dynamo/tensor_spec.py +++ b/transformer_engine/pytorch/dynamo/tensor_spec.py @@ -18,7 +18,7 @@ class TensorSpec: """A data-free description of a tensor or quantized tensor. - Captures ``shape`` / ``dtype`` / ``strides`` and, for quantized tensors, the + Captures ``shape`` / ``dtype`` and, for quantized tensors, the (value-opaque) ``quantizer`` -- enough to rebuild a tensor without holding storage. The common abstraction over plain ``torch.Tensor``, ``QuantizedTensorStorage`` and ``QuantizedTensor``, used for custom-op fake @@ -31,7 +31,6 @@ class TensorSpec: requires_grad: bool = False device: Optional[torch.device] = field(default=None) with_gemm_swizzled_scales: Optional[bool] = None - strides: Optional[Tuple[int, ...]] = None def __post_init__(self) -> None: # Own a private copy of the quantizer so usage changes (update_usage) @@ -46,22 +45,6 @@ def is_quantized(self) -> bool: """Whether this spec describes a quantized tensor.""" return self.quantizer is not None - def stride(self) -> Tuple[int, ...]: - """Explicit strides, or contiguous strides for a fresh output.""" - return self.strides if self.strides is not None else make_contiguous_strides_for(self.shape) - - def is_contiguous(self) -> bool: - """Check contiguous layout using only shape and strides.""" - if self.strides is None or any(size == 0 for size in self.shape): - return True - expected = 1 - for size, stride in zip(reversed(self.shape), reversed(self.strides)): - if size != 1: - if stride != expected: - return False - expected *= size - return True - def update_usage( self, *, @@ -128,11 +111,7 @@ def create_inner_tensors(self) -> List[torch.Tensor]: """ device = self.device if self.device is not None else torch.device(te_device_type()) if self.quantizer is None: - return [ - torch.empty_strided( - tuple(self.shape), self.stride(), dtype=self.dtype, device=device - ) - ] + return [torch.empty(tuple(self.shape), dtype=self.dtype, device=device)] inner = self.quantizer.alloc_tensors(tuple(self.shape), device=device) return [inner[name] for name in self.inner_names()] @@ -162,9 +141,8 @@ def create_tensor(self) -> torch.Tensor: """ if self.quantizer is None: device = self.device if self.device is not None else torch.device(te_device_type()) - return torch.empty_strided( + return torch.empty( tuple(self.shape), - self.stride(), dtype=self.dtype, device=device, requires_grad=self.requires_grad, @@ -183,17 +161,11 @@ def to_tensor_spec(tensor: Any) -> TensorSpec: dtype = getattr(tensor, "dtype", None) if dtype is None: dtype = getattr(tensor, "_dtype", None) - quantizer = getattr(tensor, "_quantizer", None) return TensorSpec( shape=tuple(tensor.shape), dtype=dtype, - quantizer=quantizer, + quantizer=getattr(tensor, "_quantizer", None), requires_grad=requires_grad, device=tensor.device, with_gemm_swizzled_scales=getattr(tensor, "_with_gemm_swizzled_scales", None), - strides=( - tuple(tensor.stride()) - if quantizer is None and isinstance(tensor, torch.Tensor) - else None - ), ) diff --git a/transformer_engine/pytorch/ops/basic/activation.py b/transformer_engine/pytorch/ops/basic/activation.py index 03564cddf9c..09511ef0768 100644 --- a/transformer_engine/pytorch/ops/basic/activation.py +++ b/transformer_engine/pytorch/ops/basic/activation.py @@ -133,6 +133,12 @@ def pack_forward_args( ) -> ActivationFwdArgs: if basic_op_kwargs[0]: raise ValueError("Activation forward does not expect keyword arguments") + if ( + torch.compiler.is_compiling() + and not is_quantized_tensor(input_) + and not input_.is_contiguous() + ): + raise RuntimeError("Activation forward requires contiguous inputs under torch.compile") dtype = maybe_autocast_dtype(default_dtype=input_.dtype) if dtype not in (torch.float32, torch.float16, torch.bfloat16): raise RuntimeError(f"Unsupported dtype ({dtype})") @@ -150,15 +156,6 @@ def pack_forward_args( basic_op_ctxs[0].requires_grad, ) - @staticmethod - def _save_prepared_input(args: ActivationFwdArgs) -> bool: - return args.requires_grad and ( - args.input_quantizer is not None - or is_quantized_tensor(args.input_) - or args.input_.dtype != args.dtype - or not args.input_.is_contiguous() - ) - @classmethod def forward_compute( cls, args: ActivationFwdArgs, *, in_custom_op: bool = False @@ -167,7 +164,7 @@ def forward_compute( y = cls._activation_forward_impl(x, args.output_quantizer) if args.input_quantizer is not None: x = args.input_quantizer(x) - return y, [()], (x,) if cls._save_prepared_input(args) else () + return y, [()], (x,) if args.requires_grad and x is not args.input_ else () @classmethod def forward_compute_fake(cls, args: ActivationFwdArgs): @@ -179,7 +176,9 @@ def forward_compute_fake(cls, args: ActivationFwdArgs): gated=cls._output_halves_last_dim, ) saved = () - if cls._save_prepared_input(args): + if args.requires_grad and ( + args.input_quantizer is not None or is_quantized_tensor(x) or x.dtype != args.dtype + ): saved = ( TensorSpec( shape=x.shape, dtype=args.dtype, device=x.device, quantizer=args.input_quantizer diff --git a/transformer_engine/pytorch/ops/basic/bias.py b/transformer_engine/pytorch/ops/basic/bias.py index c129e2c1cd0..7d012c8eed9 100644 --- a/transformer_engine/pytorch/ops/basic/bias.py +++ b/transformer_engine/pytorch/ops/basic/bias.py @@ -15,6 +15,7 @@ from ...dynamo import TensorOrQuantized, TensorSpec from ...dynamo.fake import fake_bgrad from ...quantization import FP8GlobalStateManager +from .._common import is_quantized_tensor from ..op import BasicOperation from ...utils import canonicalize_device, canonicalize_dtype from ...tensor import Quantizer @@ -143,6 +144,13 @@ def pack_forward_args( ) -> BiasFwdArgs: if basic_op_kwargs[0]: raise ValueError("Bias forward does not expect keyword arguments") + if torch.compiler.is_compiling() and ( + (not is_quantized_tensor(input_) and not input_.is_contiguous()) + or not self.bias.is_contiguous() + ): + raise RuntimeError( + "Bias forward requires contiguous inputs and bias under torch.compile" + ) quantizer = prev_op_grad_output_quantizer if FP8GlobalStateManager.is_fp8_enabled(): if FP8GlobalStateManager.get_fp8_recipe().backward_override is not None: @@ -157,19 +165,11 @@ def forward_compute( @classmethod def forward_compute_fake(cls, args: BiasFwdArgs): - input_ = torch.empty_strided( - args.input_.shape, args.input_.stride(), dtype=args.input_.dtype, device="meta" - ) - bias = torch.empty_strided( - args.bias.shape, args.bias.stride(), dtype=args.bias.dtype, device="meta" - ) - out = input_ + bias return ( TensorSpec( - shape=tuple(out.shape), - dtype=out.dtype, + shape=args.input_.shape, + dtype=torch.promote_types(args.input_.dtype, args.bias.dtype), device=args.input_.device, - strides=tuple(out.stride()), ), [()], (), diff --git a/transformer_engine/pytorch/ops/basic/swiglu.py b/transformer_engine/pytorch/ops/basic/swiglu.py index a135c432159..1162b2d31f1 100644 --- a/transformer_engine/pytorch/ops/basic/swiglu.py +++ b/transformer_engine/pytorch/ops/basic/swiglu.py @@ -172,7 +172,7 @@ def forward_compute( out = cls._activation_forward_impl(swiglu_in, args.output_quantizer, args=args) if args.input_quantizer is not None: x = args.input_quantizer(x) - return out, [()], (x,) if cls._save_prepared_input(args) else () + return out, [()], (x,) if args.requires_grad and x is not args.input_ else () def pack_backward_args(self, *args, **kwargs) -> SwiGLUBwdArgs: args = super().pack_backward_args(*args, **kwargs)