Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
217 changes: 175 additions & 42 deletions tests/pytorch/test_torch_compile.py

Large diffs are not rendered by default.

16 changes: 16 additions & 0 deletions transformer_engine/pytorch/csrc/common.h
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,8 @@ class MXFP8Quantizer : public Quantizer {

class NVFP4Quantizer : public Quantizer {
public:
using ShapeAlignment = std::optional<std::pair<size_t, size_t>>;

// amax reduction for low precision FP4 AG
bool with_amax_reduction;
c10::intrusive_ptr<dist_group_type> amax_reduction_group;
Expand Down Expand Up @@ -422,7 +424,21 @@ class NVFP4Quantizer : public Quantizer {
*/
static bool is_eligible_for_2d_swizzle_fusion(const std::vector<size_t>& shape);

// Flattened row/column alignment for directly emitted GEMM-swizzled scales, or no such path.
static ShapeAlignment gemm_swizzled_scales_alignment(bool optimize_for_gemm, bool with_rht,
bool with_2d_quantization,
bool row_scaled_nvfp4, bool nvfp4_use_4over6,
int sm_arch,
std::optional<bool> disable_rht_cast_fusion);

private:
static ShapeAlignment rht_cast_fusion_alignment(int sm_arch,
std::optional<bool> disable_rht_cast_fusion,
bool for_grouped_kernel = false);
static ShapeAlignment get_2d_swizzle_fusion_alignment(int sm_arch);
static bool meets_alignment(const std::vector<size_t>& shape, const ShapeAlignment& alignment);
bool emits_gemm_swizzled_scales(const std::vector<size_t>& shape) const;

void quantize_with_rht_unfused_helper(const TensorWrapper& input, TensorWrapper& out,
TensorWrapper& rht_output_t_cpp,
QuantizationConfigWrapper& quant_config,
Expand Down
5 changes: 5 additions & 0 deletions transformer_engine/pytorch/csrc/extensions/pybind.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,11 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
py::arg("output") = py::none(), py::arg("noop") = py::none());
m.def("dequantize", &transformer_engine::pytorch::dequantize, "Dequantize", py::arg("input"),
py::arg("otype"));
m.def("nvfp4_gemm_swizzled_scales_alignment",
&transformer_engine::pytorch::NVFP4Quantizer::gemm_swizzled_scales_alignment,
py::arg("optimize_for_gemm"), py::arg("with_rht"), py::arg("with_2d_quantization"),
py::arg("row_scaled_nvfp4"), py::arg("nvfp4_use_4over6"), py::arg("sm_arch"),
py::arg("disable_rht_cast_fusion"));
m.def("create_empty_quantized_tensor",
&transformer_engine::pytorch::create_empty_quantized_tensor,
"Create an empty quantized tensor", py::arg("quantizer"), py::arg("shape"),
Expand Down
86 changes: 54 additions & 32 deletions transformer_engine/pytorch/csrc/quantizer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1934,50 +1934,72 @@ void NVFP4Quantizer::set_quantization_params(TensorWrapper* tensor) const {
columnwise_data.shape);
}

bool NVFP4Quantizer::is_eligible_for_rht_cast_fusion(const std::vector<size_t>& shape,
bool for_grouped_kernel) {
if (transformer_engine::getenv<bool>("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION")) {
return false;
}
const auto [rows, cols] = get_2d_dims(shape);
const size_t row_align = for_grouped_kernel ? 128 : 64;
const int sm_arch = transformer_engine::cuda::sm_arch();
NVFP4Quantizer::ShapeAlignment NVFP4Quantizer::rht_cast_fusion_alignment(
int sm_arch, std::optional<bool> disable_rht_cast_fusion, bool for_grouped_kernel) {
const bool disabled =
disable_rht_cast_fusion.has_value()
? *disable_rht_cast_fusion
: transformer_engine::getenv<bool>("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION");
const bool supported_arch = (sm_arch >= 100 && sm_arch <= 110) ||
(!for_grouped_kernel && (sm_arch == 120 || sm_arch == 121));
return rows % row_align == 0 && cols % 128 == 0 && supported_arch;
if (disabled || !supported_arch) {
return std::nullopt;
}
return std::pair<size_t, size_t>{for_grouped_kernel ? 128 : 64, 128};
}

bool NVFP4Quantizer::is_eligible_for_2d_swizzle_fusion(const std::vector<size_t>& shape) {
const auto [rows, cols] = get_2d_dims(shape);
return rows % 128 == 0 && cols % 128 == 0 && transformer_engine::cuda::sm_arch() >= 100 &&
transformer_engine::cuda::sm_arch() <= 110;
NVFP4Quantizer::ShapeAlignment NVFP4Quantizer::get_2d_swizzle_fusion_alignment(int sm_arch) {
if (sm_arch < 100 || sm_arch > 110) {
return std::nullopt;
}
return std::pair<size_t, size_t>{128, 128};
}

namespace {

// Whether this quantizer should emit GEMM-swizzled scale factors directly from the
// quantize kernel (RHT cast-fusion path or the plain 2D quantize path), avoiding the
// standalone post-quantize swizzle pass.
bool nvfp4_emits_gemm_swizzled_scales(const NVFP4Quantizer& q, const std::vector<size_t>& shape) {
if (!q.optimize_for_gemm) {
bool NVFP4Quantizer::meets_alignment(const std::vector<size_t>& shape,
const ShapeAlignment& alignment) {
if (!alignment) {
return false;
}
if (q.with_rht) {
const int sm_arch = transformer_engine::cuda::sm_arch();
const auto [rows, cols] = get_2d_dims(shape);
return rows % alignment->first == 0 && cols % alignment->second == 0;
}

NVFP4Quantizer::ShapeAlignment NVFP4Quantizer::gemm_swizzled_scales_alignment(
bool optimize_for_gemm, bool with_rht, bool with_2d_quantization, bool row_scaled_nvfp4,
bool nvfp4_use_4over6, int sm_arch, std::optional<bool> disable_rht_cast_fusion) {
if (!optimize_for_gemm) {
return std::nullopt;
}
if (with_rht) {
// The SM12x no-TMEM RHT path reuses the compact-scale 1D TMA quantizer.
// Keep the existing post-quantize swizzle until that path gains an in-kernel swizzle epilogue.
if (sm_arch == 120 || sm_arch == 121) {
return false;
return std::nullopt;
}
return NVFP4Quantizer::is_eligible_for_rht_cast_fusion(shape);
return rht_cast_fusion_alignment(sm_arch, disable_rht_cast_fusion);
}
// Plain (non-RHT) 2D quantize kernel can bake the swizzled layout for aligned shapes.
return q.with_2d_quantization && !q.row_scaled_nvfp4 &&
q.nvfp4_4over6_mode == kNVTENVFP44Over6Disabled &&
NVFP4Quantizer::is_eligible_for_2d_swizzle_fusion(shape);
if (with_2d_quantization && !row_scaled_nvfp4 && !nvfp4_use_4over6) {
return get_2d_swizzle_fusion_alignment(sm_arch);
}
return std::nullopt;
}

} // namespace
bool NVFP4Quantizer::is_eligible_for_rht_cast_fusion(const std::vector<size_t>& shape,
bool for_grouped_kernel) {
return meets_alignment(shape, rht_cast_fusion_alignment(transformer_engine::cuda::sm_arch(),
std::nullopt, for_grouped_kernel));
}

bool NVFP4Quantizer::is_eligible_for_2d_swizzle_fusion(const std::vector<size_t>& shape) {
return meets_alignment(shape,
get_2d_swizzle_fusion_alignment(transformer_engine::cuda::sm_arch()));
}

bool NVFP4Quantizer::emits_gemm_swizzled_scales(const std::vector<size_t>& shape) const {
return meets_alignment(shape, gemm_swizzled_scales_alignment(
optimize_for_gemm, with_rht, with_2d_quantization,
row_scaled_nvfp4, nvfp4_4over6_mode != kNVTENVFP44Over6Disabled,
transformer_engine::cuda::sm_arch(), std::nullopt));
}

std::pair<TensorWrapper, py::object> NVFP4Quantizer::create_tensor(
const std::vector<size_t>& shape, DType dtype, std::optional<at::Device> device_opt,
Expand All @@ -1992,7 +2014,7 @@ std::pair<TensorWrapper, py::object> NVFP4Quantizer::create_tensor(
// Swizzled SF is emitted directly by the RHT cast-fusion kernel or by the plain
// 2D quantize kernel (for aligned shapes); other quantize paths reject it and go
// through the standalone post-quantize swizzle pass instead.
const bool with_gemm_swizzled_scales = nvfp4_emits_gemm_swizzled_scales(*this, shape);
const bool with_gemm_swizzled_scales = emits_gemm_swizzled_scales(shape);
NVTE_CHECK(flat_first_dim % NVFP4_BLOCK_SIZE == 0, "First dim for NVFP4 must be divisible by ",
NVFP4_BLOCK_SIZE, " (got shape=", shape, ")");
NVTE_CHECK(flat_last_dim % NVFP4_BLOCK_SIZE == 0,
Expand Down Expand Up @@ -2344,7 +2366,7 @@ std::pair<TensorWrapper, py::object> NVFP4Quantizer::convert_and_update_tensor(
// Swizzled SF is emitted directly by the RHT cast-fusion kernel or by the plain
// 2D quantize kernel (for aligned shapes); other quantize paths reject it and go
// through the standalone post-quantize swizzle pass instead.
const bool with_gemm_swizzled_scales = nvfp4_emits_gemm_swizzled_scales(*this, shape);
const bool with_gemm_swizzled_scales = emits_gemm_swizzled_scales(shape);

const bool row_scaled_nvfp4 = this->row_scaled_nvfp4;
const bool disable_second_level_scale = this->disable_second_level_scale;
Expand Down
94 changes: 94 additions & 0 deletions transformer_engine/pytorch/dynamo/fake.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# See LICENSE for license information.

"""Data-free descriptions of quantized operation outputs."""

from __future__ import annotations
import math
import os
from typing import Optional

import torch
import transformer_engine_torch as tex

from .tensor_spec import TensorSpec
from ..quantized_tensor import Quantizer
from ..tensor.nvfp4_tensor import NVFP4Quantizer
from ..utils import get_device_compute_capability


@torch.compiler.assume_constant_result
def _nvfp4_gemm_swizzled_scales_alignment(
optimize_for_gemm: bool,
with_rht: bool,
with_2d_quantization: bool,
row_scaled_nvfp4: bool,
nvfp4_use_4over6: bool,
sm_arch: int,
disable_rht_cast_fusion: bool,
) -> Optional[tuple[int, int]]:
return tex.nvfp4_gemm_swizzled_scales_alignment(
optimize_for_gemm,
with_rht,
with_2d_quantization,
row_scaled_nvfp4,
nvfp4_use_4over6,
sm_arch,
disable_rht_cast_fusion,
)


def _make_fused_quantization_output_spec(
shape: tuple[int, ...],
dtype: torch.dtype,
device: Optional[torch.device],
quantizer: Optional[Quantizer],
) -> TensorSpec:
"""Describe a native output without a separate post-quantize swizzle."""
spec = TensorSpec(shape=tuple(shape), dtype=dtype, device=device, quantizer=quantizer)
quantizer = spec.quantizer
if not isinstance(quantizer, NVFP4Quantizer):
return spec
major, minor = get_device_compute_capability(device)
# Direct environ access lets Dynamo guard changes to the flag.
disable_rht_cast_fusion = bool(
int(os.environ.get("NVTE_NVFP4_DISABLE_RHT_CAST_FUSION", "0") or "0")
)
alignment = _nvfp4_gemm_swizzled_scales_alignment(
quantizer.optimize_for_gemm,
quantizer.with_rht,
quantizer.with_2d_quantization,
quantizer.row_scaled_nvfp4,
quantizer.nvfp4_use_4over6,
major * 10 + minor,
disable_rht_cast_fusion,
)
# Keep divisibility checks visible to Dynamo without specializing the dimensions.
rows, cols = math.prod(shape[:-1]), shape[-1]
if alignment is None:
spec.with_gemm_swizzled_scales = False
else:
row_alignment, col_alignment = alignment
spec.with_gemm_swizzled_scales = bool(
rows % row_alignment == 0 and cols % col_alignment == 0
)
return spec


def fake_bgrad(
grad_output: TensorSpec,
quantizer: Optional[Quantizer] = None,
) -> tuple[TensorSpec, TensorSpec]:
"""Describe bias reduction and optional gradient quantization, in native order."""
db = TensorSpec(
shape=(grad_output.shape[-1],), dtype=grad_output.dtype, device=grad_output.device
)
dx = (
grad_output
if quantizer is None
else _make_fused_quantization_output_spec(
grad_output.shape, grad_output.dtype, grad_output.device, quantizer
)
)
return db, dx
7 changes: 6 additions & 1 deletion transformer_engine/pytorch/dynamo/tensor_spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ class TensorSpec:
quantizer: Optional[Any] = None
requires_grad: bool = False
device: Optional[torch.device] = field(default=None)
with_gemm_swizzled_scales: Optional[bool] = None

def __post_init__(self) -> None:
# Own a private copy of the quantizer so usage changes (update_usage)
Expand Down Expand Up @@ -94,9 +95,12 @@ def create_metadata(self) -> Dict[str, Any]:
"dtype": self.dtype,
"requires_grad": self.requires_grad,
}
return self.quantizer.create_metadata(
meta = self.quantizer.create_metadata(
tuple(self.shape), dtype=self.dtype, requires_grad=self.requires_grad
)
if self.with_gemm_swizzled_scales is not None:
meta["nontensor_kwargs"]["with_gemm_swizzled_scales"] = self.with_gemm_swizzled_scales
return meta

def create_inner_tensors(self) -> List[torch.Tensor]:
"""Materialize the flat inner tensors (in :meth:`inner_names` order).
Expand Down Expand Up @@ -162,4 +166,5 @@ def to_tensor_spec(tensor: Any) -> TensorSpec:
quantizer=getattr(tensor, "_quantizer", None),
requires_grad=requires_grad,
device=tensor.device,
with_gemm_swizzled_scales=getattr(tensor, "_with_gemm_swizzled_scales", None),
)
14 changes: 5 additions & 9 deletions transformer_engine/pytorch/ops/basic/bias.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

import transformer_engine_torch as tex
from ...dynamo import TensorOrQuantized, TensorSpec
from ...dynamo.fake import fake_bgrad
from ...quantization import FP8GlobalStateManager
from ..op import BasicOperation
from ...utils import canonicalize_device, canonicalize_dtype
Expand Down Expand Up @@ -194,13 +195,8 @@ def backward_compute(
@classmethod
def backward_compute_fake(cls, args: BiasBwdArgs):
dy = args.grad_output
dx = None
if len(dy.shape) > 1 and args.grad_input_quantizer is not None:
dx = TensorSpec(
shape=dy.shape,
dtype=dy.dtype,
device=dy.device,
quantizer=args.grad_input_quantizer,
)
db = TensorSpec(shape=(dy.shape[-1],), dtype=dy.dtype, device=dy.device)
quantizer = args.grad_input_quantizer if len(dy.shape) > 1 else None
db, dx = fake_bgrad(dy, quantizer)
if quantizer is None:
dx = None
return dx, [(db,)], [()]
31 changes: 7 additions & 24 deletions transformer_engine/pytorch/tensor/nvfp4_tensor.py
Original file line number Diff line number Diff line change
Expand Up @@ -330,29 +330,11 @@ def get_scale_shape(self, shape: Iterable[int], columnwise: bool) -> Tuple[int,

@staticmethod
def get_columnwise_shape(shape: Iterable[int]) -> Tuple[int, ...]:
"""Calculate the shape of a tensor after columnwise quantization.

For NVFP4 columnwise quantization, it's performing 16x1 quantization block scaling.

Parameters
----------
shape : Iterable[int]
Original shape of the tensor

Returns
-------
Tuple[int, ...]
New shape with dimensions rearranged for columnwise layout.
For a shape (d1, d2, ..., dn), returns (dn, d1, d2, ..., dn-1).
Returns empty tuple for empty input shape.
"""
if len(shape) == 0:
return tuple()
# and then after AG, a reorganize kernel will be called to restore the shape
colwise_shape = [shape[-1]]
for i in range(len(shape) - 1):
colwise_shape.append(shape[i])
return tuple(colwise_shape)
"""Return the flattened columnwise shape before FP4 packing."""
shape = tuple(shape)
if not shape:
return ()
return (shape[-1], math.prod(shape[:-1]))

@staticmethod
def convert_shape_for_fp4(shape: Iterable[int]) -> Tuple[int, ...]:
Expand Down Expand Up @@ -418,7 +400,8 @@ def inner_tensor_specs(
amax_rowwise_shape = (math.prod(shape[:-1]),) if self.row_scaled_nvfp4 else (1,)
specs["_amax_rowwise"] = (amax_rowwise_shape, torch.float32)
if self.columnwise_usage:
specs["_amax_columnwise"] = ((1,), torch.float32)
amax_columnwise_shape = (shape[-1],) if self.row_scaled_nvfp4 else (1,)
specs["_amax_columnwise"] = (amax_columnwise_shape, torch.float32)
return specs


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,9 @@ def size(self, dim: Optional[int] = None) -> Union[torch.Size, int]:
byte_shape = list(self._rowwise_data.size())
shape = byte_shape[:-1] + [byte_shape[-1] * 2]
elif self._columnwise_data is not None:
warnings.warn("Attempting to get shape of NVFP4 tensor with only column-wise data.")
# Dynamo queries columnwise-only fake shapes; warning emission breaks fullgraph tracing.
if not torch.compiler.is_compiling():
warnings.warn("Attempting to get shape of NVFP4 tensor with only column-wise data.")
byte_shape = list(self._columnwise_data.size())
shape = byte_shape[1:-1] + [byte_shape[-1] * 2, byte_shape[0]]
if shape is None:
Expand Down
Loading