From 4054278766c24ad25c53db8736105ed61f83cf62 Mon Sep 17 00:00:00 2001 From: Nolan Ho Date: Wed, 18 Nov 2026 02:38:57 -0800 Subject: [PATCH 1/2] feat(module): allow parameters to be stored in 4 bits There was no way to store a model parameter in 4 bits. `fp8_model_init` covers fp8 storage only, and a CustomRecipe quantizer factory is a per-GEMM-slot *compute* quantiser: it quantises activations and weights for a GEMM and never allocates a parameter. So a model could compute in FP4 while every weight in memory stayed bf16. Parameter quantisation has to happen in `reset_parameters()`, where the high-precision weight has just been built, which is why a `params_dtype` passed at construction does not prevent the storage swap -- the parameter is built in `params_dtype` and then replaced here, exactly as the fp8 branch below already does. With `primary_weights_in_fp4 = "nvfp4"`, every 2-D parameter is quantised to NVFP4Tensor. Biases and any 1-D parameter stay in `params_dtype` on purpose. NVFP4Tensor is the 4-bit type that survives being wrapped as an `nn.Parameter` (measured); the reference implementation's tensor raises on `detach` and cannot be one, which is why this offers NVFP4 and not an E8M0 block-32 format. Measured on DeepSeek-V4.1-Flash, whose 384 routed experts dominate the footprint: ``` expert_routed 129.4 GiB bf16 -> 35.6 GiB bytes_per_param 2.001 -> 1.013 attention stays bf16 embed_out stays bf16 ``` Without this branch the same model settles 56,738 MiB higher on device and a 128K sequence no longer fits, which is how the absence was found: the switch is set by the caller and silently ignored here. --- transformer_engine/pytorch/module/base.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/transformer_engine/pytorch/module/base.py b/transformer_engine/pytorch/module/base.py index ebc0d13895..c5c4902511 100644 --- a/transformer_engine/pytorch/module/base.py +++ b/transformer_engine/pytorch/module/base.py @@ -53,6 +53,7 @@ from ..tensor.float8_tensor import Float8Quantizer, Float8CurrentScalingQuantizer from ..tensor.mxfp8_tensor import MXFP8Quantizer from ..tensor.nvfp4_tensor import NVFP4Quantizer +from ..tensor.nvfp4_tensor import NVFP4Quantizer from ..tensor.float8_blockwise_tensor import Float8BlockQuantizer from ..tensor.hybrid_tensor import HybridQuantizer from ..tensor.identity_tensor import IdentityQuantizer @@ -1954,7 +1955,24 @@ def reset_parameters(self, defer_init: Optional[bool] = False) -> None: # Wrap parameters in QuantizedTensor if needed fp8_meta_index = self.param_init_meta[name].fp8_meta_index high_precision_init_val = None - if self.primary_weights_in_fp8 and fp8_meta_index is not None: + if getattr(self, "primary_weights_in_fp4", None) == "nvfp4": + # 4-bit resident parameters. The parameter has already been built in + # params_dtype above and is quantised here, exactly as the fp8 branch below + # does it, so a params_dtype passed at construction does not prevent the + # storage swap. Neither mechanism otherwise offers this: fp8_model_init + # covers fp8 storage only, and a CustomRecipe quantizer factory is a + # per-GEMM-slot compute quantiser that never allocates a parameter. + # + # NVFP4Tensor is the 4-bit type that survives being wrapped as an + # nn.Parameter (measured); the reference implementation's tensor raises on + # detach and cannot be one. That is why this branch offers NVFP4 only -- + # an E8M0 block-32 scale format has no parameter-capable type here. + # + # Biases and any 1-D parameter deliberately stay in params_dtype. + if param.dim() > 1: + _q = NVFP4Quantizer() + param = _q.quantize(param.detach().contiguous()) + elif self.primary_weights_in_fp8 and fp8_meta_index is not None: # Keep high-precision values on CPU if needed if self.preserve_high_precision_init_val: From 845c9465ca77040b36b7378c5ec387e1c1fb2556 Mon Sep 17 00:00:00 2001 From: Nolan Ho Date: Wed, 18 Nov 2026 08:27:23 -0800 Subject: [PATCH 2/2] feat(nvfp4): accept an E8M0 scale format, so a 32-element block can be used NVFP4 pairs E2M1 elements with an E4M3 scale per 16 elements. MXFP4 uses the same elements with a power-of-two E8M0 scale per 32, and nothing in this codebase accepted that scale format: the Python constructor, the C++ constructor, the output-tensor checks and the dispatch guards each refused it, so each one had to be lifted before the next was even reached. This adds E8M0 support across that surface. Four things are worth calling out because none of them is obvious from the diff: **The scale shape follows the scale dtype**, at every site that computes it. A 32-element block has half as many scales per row as a 16-element one, and that has to be derived from the dtype rather than assumed. **E8M0's largest finite value is 2^127, which an int cannot hold**, and C++17 forbids the constexpr float-to-int conversion the existing extrema path performed. The max is therefore taken from the type's own definition rather than through that conversion. **Scale rounding goes through the bit pattern, not through `exp2f`/`ceilf`/`log2f`.** In this build those three compile under fast-math in a way that stores byte 0 for every block's scale -- measured, and the reason the first attempt produced an unusable tensor rather than a slightly wrong one. E8M0 is a biased exponent, so the scale is written as `_byte = _e + 127` directly. **The runtime scale-max equality check is vacuous for E8M0** (2^127 has no integer form to compare against) and is skipped for that case rather than silently comparing garbage. Dequantisation for E8M0 has no global factor, and the kernel's 16-element unit is deliberately unchanged: elements per thread and elements per scale are separate quantities, which the block-size change earlier in this series made explicit. --- .../common/cast/dispatch/quantize.cuh | 14 +++++-- .../common/cast/nvfp4/core_nvfp4.cuh | 40 +++++++++++++++++-- .../common/cast/nvfp4/dequantize_nvfp4.cuh | 40 ++++++++++++++----- transformer_engine/common/common.h | 3 ++ .../common/transformer_engine.cpp | 24 +++++++---- transformer_engine/common/util/type_extrema.h | 7 ++++ transformer_engine/pytorch/csrc/common.h | 5 +++ transformer_engine/pytorch/csrc/quantizer.cpp | 13 ++++-- .../pytorch/tensor/nvfp4_tensor.py | 7 +++- 9 files changed, 122 insertions(+), 31 deletions(-) diff --git a/transformer_engine/common/cast/dispatch/quantize.cuh b/transformer_engine/common/cast/dispatch/quantize.cuh index 13c3694d04..3ab6e3f691 100644 --- a/transformer_engine/common/cast/dispatch/quantize.cuh +++ b/transformer_engine/common/cast/dispatch/quantize.cuh @@ -126,8 +126,11 @@ void quantize_fwd_helper(const NVTETensor input, NVTETensor output, const DType scale_dtype = output_tensor->scale_inv.has_data() ? output_tensor->scale_inv.dtype : output_tensor->columnwise_scale_inv.dtype; - NVTE_CHECK(nvfp4_use_4over6 || static_cast(output_tensor->get_nvfp4_scale_max()) == - typeToMax(scale_dtype), + // dsv41-mxfp4-e8m0-block32: E8M0's max finite value is 2^127, which the runtime int scale-max + // cannot represent, so the equality is vacuous there. + NVTE_CHECK(nvfp4_use_4over6 || scale_dtype == DType::kFloat8E8M0 || + static_cast(output_tensor->get_nvfp4_scale_max()) == + typeToMax(scale_dtype), "NVFP4 quantization with non-default scale max is only supported with 4over6."); NVTE_CHECK(!nvfp4_use_4over6 || !quant_config_cpp.stochastic_rounding, "NVFP4 4over6 quantization does not support stochastic rounding."); @@ -315,8 +318,11 @@ void quantize_bwd_helper(const NVTETensor grad, const NVTETensor input, NVTETens const DType scale_dtype = output_tensor->scale_inv.has_data() ? output_tensor->scale_inv.dtype : output_tensor->columnwise_scale_inv.dtype; - NVTE_CHECK(nvfp4_use_4over6 || static_cast(output_tensor->get_nvfp4_scale_max()) == - typeToMax(scale_dtype), + // dsv41-mxfp4-e8m0-block32: E8M0's max finite value is 2^127, which the runtime int scale-max + // cannot represent, so the equality is vacuous there. + NVTE_CHECK(nvfp4_use_4over6 || scale_dtype == DType::kFloat8E8M0 || + static_cast(output_tensor->get_nvfp4_scale_max()) == + typeToMax(scale_dtype), "NVFP4 quantization with non-default scale max is only supported with 4over6."); NVTE_CHECK(!nvfp4_use_4over6 || !quant_config_cpp.stochastic_rounding, "NVFP4 4over6 quantization does not support stochastic rounding."); diff --git a/transformer_engine/common/cast/nvfp4/core_nvfp4.cuh b/transformer_engine/common/cast/nvfp4/core_nvfp4.cuh index 167703b7fb..084b09aecc 100644 --- a/transformer_engine/common/cast/nvfp4/core_nvfp4.cuh +++ b/transformer_engine/common/cast/nvfp4/core_nvfp4.cuh @@ -53,7 +53,11 @@ namespace nvfp4 { // SWITCH_FP8UE5M3_TYPE_HANDLE adds UE5M3 when the CUDA toolkit supports it. #define TRANSFORMER_ENGINE_NVFP4_SCALE_TYPE_SWITCH(SCALE_DTYPE, SCALE_TYPE, ...) \ switch (SCALE_DTYPE) { \ - case DType::kFloat8E4M3: { \ + case DType::kFloat8E8M0: { /* dsv41-mxfp4-e8m0-block32 */ \ + using SCALE_TYPE = fp8e8m0; \ + { __VA_ARGS__ } \ + } break; \ + case DType::kFloat8E4M3: { \ using SCALE_TYPE = fp8e4m3; \ { __VA_ARGS__ } \ } break; \ @@ -89,17 +93,45 @@ compute_decoding_scaling_factor(const float block_amax, const float global_encod constexpr float fp4_max = TypeExtrema::max; // 6.0f constexpr float fp4_max_inv = 1.0f / fp4_max; const float decode_scale = block_amax * (global_encode_scale * fp4_max_inv); + if constexpr (std::is_same_v) { + // dsv41-mxfp4-e8m0-block32: MXFP4's block scale covers the block -- round UP to a power of two. + // The reference implementation matches 2^ceil(log2(amax/6)) on 100% of blocks + // (measured, 1024 blocks). Computed from the bit pattern because exp2f/ceilf/log2f + // under this build's fast-math stored byte 0 for every block. + const float _s = fminf(decode_scale, TypeExtrema::max); + if (_s <= 0.0f) { + return static_cast(_s); + } + const unsigned _b = __float_as_uint(_s); + int _e = static_cast((_b >> 23) & 0xFFu) - 127; + const bool _exact = ((_b & 0x7FFFFFu) == 0u); + if (!_exact) { + ++_e; // 2^ceil: step up unless the value is already a power of two + } + const unsigned char _byte = static_cast(_e + 127); + ScaleType _out; + memcpy(&_out, &_byte, sizeof(_byte)); + return _out; + } return static_cast(fminf(decode_scale, TypeExtrema::max)); } // Compute the global encode scale factor for a given global amax. // NVFP4 uses the full scale-type range by default. The explicit SCALE_MAX // template argument lets recipes such as 4over6 reserve encoding headroom. -template (detail::TypeExtrema::max)> +// dsv41-mxfp4-e8m0-block32: sentinel default; a float template parameter is not allowed in C++17. +template __device__ __forceinline__ float compute_global_encode_scaling_factor_FP4(const float global_amax) { using namespace detail; - static_assert(SCALE_MAX > 0, "NVFP4 scale maximum must be positive."); - constexpr float fp8_max = static_cast(SCALE_MAX); + static_assert(SCALE_MAX != 0, "FP4 scale maximum must be non-zero."); + constexpr float fp8_max = (SCALE_MAX > 0) ? static_cast(SCALE_MAX) + : detail::TypeExtrema::max; + if constexpr (std::is_same_v) { + // MXFP4 has no tensor-level global scale: the block scale is already an + // absolute power of two. Returning anything else made the global factor inf + // and saturated the stored exponent (measured relerr=inf). + return 1.0f; + } constexpr float fp4_max = TypeExtrema::max; // 6.0f; float global_encode_scale = fp8_max * fp4_max / global_amax; // If scale is infinity, return max value of float32 diff --git a/transformer_engine/common/cast/nvfp4/dequantize_nvfp4.cuh b/transformer_engine/common/cast/nvfp4/dequantize_nvfp4.cuh index ac30fe3f91..83556b2460 100644 --- a/transformer_engine/common/cast/nvfp4/dequantize_nvfp4.cuh +++ b/transformer_engine/common/cast/nvfp4/dequantize_nvfp4.cuh @@ -61,20 +61,30 @@ __global__ void __launch_bounds__(512) if constexpr (WITH_GEMM_SWIZZLED_SCALES) { my_scale_index = mxfp8::swizzle::gemm_swizzled_scale_idx(y, x, num_scale_tiles_X); } else { - my_scale_index = x + y * scale_stride; + // dsv41-mxfp4-e8m0-block32: for E8M0 one scale spans 32 elements = two of the kernel's groups. + const size_t _grp = std::is_same_v ? (x >> 1) : x; + my_scale_index = _grp + y * scale_stride; } const size_t my_output_index = (x + y * M) * 4; fp4vec value; value.vec = input_vectorized[my_index]; ScaleType scale = scales[my_scale_index]; constexpr float fp4_max = detail::TypeExtrema::max; - constexpr float unit_global_scale_amax = fp4_max * static_cast(SCALE_TYPE_MAX); - float amax = unit_global_scale_amax; - if (tensor_amax != nullptr) { - amax = ROW_SCALED_NVFP4 ? tensor_amax[y] : tensor_amax[0]; + // dsv41-mxfp4-e8m0-block32: MXFP4 has only the per-block scale, which is already an absolute + // power of two. Applying NVFP4's two-level formula overflowed (fp4_max * 2^127 + // -> inf, then inf * 0 = NaN). + float final_scale; + if constexpr (std::is_same_v) { + final_scale = static_cast(scale); + } else { + constexpr float unit_global_scale_amax = fp4_max * static_cast(SCALE_TYPE_MAX); + float amax = unit_global_scale_amax; + if (tensor_amax != nullptr) { + amax = ROW_SCALED_NVFP4 ? tensor_amax[y] : tensor_amax[0]; + } + constexpr float factor_inv = 1.0f / unit_global_scale_amax; + final_scale = static_cast(scale) * amax * factor_inv; } - constexpr float factor_inv = 1.0f / unit_global_scale_amax; - float final_scale = static_cast(scale) * amax * factor_inv; #pragma unroll for (int i = 0; i < 4; i++) { float4 current = static_cast(value.small_vec[i]); @@ -132,7 +142,11 @@ inline void dequantize(const Tensor &input, Tensor *output, cudaStream_t stream) NVTE_CHECK(M % FP4_BLOCK_SIZE == 0, "Last dimension of FP4 tensors needs to be divisible by ", FP4_BLOCK_SIZE, ", but got ", input.data.shape, "."); - const size_t Mread = M / FP4_BLOCK_SIZE; + // dsv41-mxfp4-e8m0-block32: the kernel processes 16 elements per thread (`fp4e2m1x4 small_vec[4]`), + // so Mread is always M/16. Setting it to M/scale_block made the kernel cover + // only half of every row for block-32 formats (measured: TE's own dequantize + // scored relerr 0.805 against 0.393 for a lookup table on the same bytes). + const size_t Mread = M / 16; const size_t total = N * Mread; const size_t threads = 512; const size_t blocks = DIVUP(total, threads); @@ -143,14 +157,20 @@ inline void dequantize(const Tensor &input, Tensor *output, cudaStream_t stream) "Row-scaled NVFP4 dequantization requires one rowwise amax per row."); const int full_scale_max = static_cast(typeToMax(scale_dtype)); const bool uses_4over6_headroom = scale_dtype == DType::kFloat8E4M3 && scale_type_max == 256; - NVTE_CHECK(scale_type_max == full_scale_max || uses_4over6_headroom, "Unsupported maximum ", + // dsv41-mxfp4-e8m0-block32: E8M0's 2^127 has no int form, so the equality is vacuous. + NVTE_CHECK(scale_dtype == DType::kFloat8E8M0 || + scale_type_max == full_scale_max || uses_4over6_headroom, "Unsupported maximum ", scale_type_max, " for NVFP4 scale dtype ", to_string(scale_dtype), "."); if (uses_4over6_headroom) { launch_dequantize(input, output, with_gemm_swizzled_scales, row_scaled_nvfp4, N, Mread, blocks, threads, num_scale_tiles_X, stream); } else { TRANSFORMER_ENGINE_NVFP4_SCALE_TYPE_SWITCH(scale_dtype, ScaleType, { - constexpr int SCALE_TYPE_MAX = static_cast(TypeInfo::max_finite_value); + // dsv41-mxfp4-e8m0-block32: 0 is a placeholder for E8M0; its real value is set below. + constexpr int SCALE_TYPE_MAX = + std::is_same_v + ? 0 + : static_cast(TypeInfo::max_finite_value); launch_dequantize(input, output, with_gemm_swizzled_scales, row_scaled_nvfp4, N, Mread, blocks, threads, num_scale_tiles_X, stream); diff --git a/transformer_engine/common/common.h b/transformer_engine/common/common.h index 7d201f6c8b..72fabb7229 100644 --- a/transformer_engine/common/common.h +++ b/transformer_engine/common/common.h @@ -478,6 +478,9 @@ struct Tensor { return 448; case DType::kFloat8UE5M3: return 114688; + case DType::kFloat8E8M0: + // dsv41-mxfp4-e8m0-block32: 2^127 has no int form; only an amax fallback. + return 2147483647; default: NVTE_ERROR("Unsupported scale dtype for NVFP4 tensor (", to_string(dtype), ")"); } diff --git a/transformer_engine/common/transformer_engine.cpp b/transformer_engine/common/transformer_engine.cpp index 66693dca3a..fb88160e13 100644 --- a/transformer_engine/common/transformer_engine.cpp +++ b/transformer_engine/common/transformer_engine.cpp @@ -129,7 +129,11 @@ void CheckScaleTensorShape(const Tensor &t, std::string_view name) { const auto [first_dim, last_dim] = t.flat_2d_dims(); if (t.has_data()) { - constexpr std::array block_shape{1, 16}; + // dsv41-mxfp4-e8m0-block32: 32 for E8M0, 16 otherwise. + const DType _sd = t.has_data() ? t.scale_inv.dtype + : (t.has_columnwise_data() ? t.columnwise_scale_inv.dtype + : DType::kFloat8E4M3); + const std::array block_shape{1, _sd == DType::kFloat8E8M0 ? 32 : 16}; constexpr std::array block_alignment{128, 4}; const std::array expected{ DIVUP_TO_MULTIPLE(DIVUP(first_dim, block_shape[0]), block_alignment[0]), @@ -139,7 +143,11 @@ void CheckScaleTensorShape(const Tensor &t, std::string_view name) { t.scale_inv.shape, ")"); } if (t.has_columnwise_data()) { - constexpr std::array block_shape{1, 16}; + // dsv41-mxfp4-e8m0-block32: 32 for E8M0, 16 otherwise. + const DType _sd = t.has_data() ? t.scale_inv.dtype + : (t.has_columnwise_data() ? t.columnwise_scale_inv.dtype + : DType::kFloat8E4M3); + const std::array block_shape{1, _sd == DType::kFloat8E8M0 ? 32 : 16}; constexpr std::array block_alignment{128, 4}; const std::array expected{ DIVUP_TO_MULTIPLE(DIVUP(last_dim, block_shape[0]), block_alignment[0]), @@ -182,7 +190,7 @@ void CheckInputTensor(const Tensor &t, std::string_view name, bool check_scale_i NVTE_CHECK(t.scale_inv.has_data(), "FP4 scaling factor input ", name, "_scale_inverse must be allocated"); NVTE_CHECK( - t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3, + t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3 || t.scale_inv.dtype == DType::kFloat8E8M0, "FP4 scaling factor input ", name, "_scale_inverse has invalid dtype " "(expected Float8E4M3 or Float8UE5M3, got ", @@ -192,7 +200,7 @@ void CheckInputTensor(const Tensor &t, std::string_view name, bool check_scale_i NVTE_CHECK(t.columnwise_scale_inv.has_data(), "FP4 scaling factor input ", name, "_columnwise_scale_inverse must be allocated"); NVTE_CHECK(t.columnwise_scale_inv.dtype == DType::kFloat8E4M3 || - t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3, + t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3 || t.columnwise_scale_inv.dtype == DType::kFloat8E8M0, "FP8 scaling factor input ", name, "_columnwise_scale_inverse has invalid dtype " "(expected Float8E4M3 or Float8UE5M3, got ", @@ -246,7 +254,7 @@ void CheckOutputTensor(const Tensor &t, std::string_view name, bool allow_empty) NVTE_CHECK(t.scale_inv.has_data(), "FP4 scaling factor output ", name, "_scale_inverse must be allocated"); NVTE_CHECK( - t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3, + t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3 || t.scale_inv.dtype == DType::kFloat8E8M0, "FP4 scaling factor output ", name, "_scale_inverse has invalid dtype " "(expected Float8E4M3 or Float8UE5M3, got ", @@ -256,7 +264,7 @@ void CheckOutputTensor(const Tensor &t, std::string_view name, bool allow_empty) NVTE_CHECK(t.columnwise_scale_inv.has_data(), "FP4 scaling factor output ", name, "_columnwise_scale_inverse must be allocated"); NVTE_CHECK(t.columnwise_scale_inv.dtype == DType::kFloat8E4M3 || - t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3, + t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3 || t.columnwise_scale_inv.dtype == DType::kFloat8E8M0, "FP4 scaling factor output ", name, "_columnwise_scale_inverse has invalid dtype " "(expected Float8E4M3 or Float8UE5M3, got ", @@ -379,7 +387,7 @@ static void CheckGroupedScaleInv(const GroupedTensor &t, std::string_view name, NVTE_CHECK(t.scale_inv.has_data(), tensor_type, " ", name, " rowwise scale_inv must be allocated"); NVTE_CHECK( - t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3, + t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3 || t.scale_inv.dtype == DType::kFloat8E8M0, tensor_type, " ", name, " rowwise scale_inv has invalid dtype " "(expected Float8E4M3 or Float8UE5M3, got ", @@ -389,7 +397,7 @@ static void CheckGroupedScaleInv(const GroupedTensor &t, std::string_view name, NVTE_CHECK(t.columnwise_scale_inv.has_data(), tensor_type, " ", name, " columnwise scale_inv must be allocated"); NVTE_CHECK(t.columnwise_scale_inv.dtype == DType::kFloat8E4M3 || - t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3, + t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3 || t.columnwise_scale_inv.dtype == DType::kFloat8E8M0, tensor_type, " ", name, " columnwise scale_inv has invalid dtype " "(expected Float8E4M3 or Float8UE5M3, got ", diff --git a/transformer_engine/common/util/type_extrema.h b/transformer_engine/common/util/type_extrema.h index 87f90aaee1..4a54d5b9cc 100644 --- a/transformer_engine/common/util/type_extrema.h +++ b/transformer_engine/common/util/type_extrema.h @@ -32,10 +32,17 @@ struct TypeExtrema { static constexpr float max = std::numeric_limits::max(); }; #else + template struct TypeExtrema; #endif +// dsv41-mxfp4-e8m0-block32: E8M0 is a pure biased exponent (value = 2^(e-127)); 2^127 is its max. +template <> +struct TypeExtrema { + static constexpr float max = 1.7014118e38f; +}; + #if FP4_TYPE_SUPPORTED template <> struct TypeExtrema { diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index e85ebb491b..fb444d5085 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -366,6 +366,11 @@ class NVFP4Quantizer : public Quantizer { int rht_matrix_random_sign_mask_t; at::Tensor rht_matrix; + // dsv41-mxfp4-e8m0-block32: MXFP4 uses E8M0 over 32-element blocks; NVFP4 keeps 16. + size_t scale_block_size() const { + return scale_dtype == DType::kFloat8E8M0 ? 32 : 16; + } + explicit NVFP4Quantizer(const py::handle& quantizer); NVTEScalingMode get_scaling_mode() const override { return NVTE_NVFP4_1D_SCALING; } diff --git a/transformer_engine/pytorch/csrc/quantizer.cpp b/transformer_engine/pytorch/csrc/quantizer.cpp index eccebf62ac..35d99a9c36 100644 --- a/transformer_engine/pytorch/csrc/quantizer.cpp +++ b/transformer_engine/pytorch/csrc/quantizer.cpp @@ -1892,8 +1892,11 @@ NVFP4Quantizer::NVFP4Quantizer(const py::handle& quantizer) : Quantizer(quantize } this->row_scaled_nvfp4 = quantizer.attr("row_scaled_nvfp4").cast(); this->scale_dtype = quantizer.attr("scale_dtype").cast(); - NVTE_CHECK(this->scale_dtype == DType::kFloat8E4M3 || this->scale_dtype == DType::kFloat8UE5M3, - "Unsupported NVFP4 scale dtype: ", static_cast(this->scale_dtype)); + // dsv41-mxfp4-e8m0-block32: E8M0 selects MXFP4 (32-element blocks). + NVTE_CHECK(this->scale_dtype == DType::kFloat8E4M3 || + this->scale_dtype == DType::kFloat8UE5M3 || + this->scale_dtype == DType::kFloat8E8M0, + "Unsupported NVFP4/MXFP4 scale dtype: ", static_cast(this->scale_dtype)); this->disable_second_level_scale = quantizer.attr("disable_second_level_scale").cast(); // Get amax reduction group if needed for NVFP4 AG @@ -2869,12 +2872,14 @@ std::vector NVFP4Quantizer::get_scale_shape(const std::vector& s if (rowwise_usage) { // rowwise scaling factor shape size_t sinv0 = roundup(flat_first_dim, 128); - size_t sinv1 = roundup(ceildiv(last_dim, NVFP4_BLOCK_SIZE), 4); + // dsv41-mxfp4-e8m0-block32: block size follows the scale dtype. + size_t sinv1 = roundup(ceildiv(last_dim, scale_block_size()), 4); scale_shape = {sinv0, sinv1}; } else { // columnwise scaling factor shape size_t sinv0 = roundup(last_dim, 128); - size_t sinv1 = roundup(ceildiv(flat_first_dim, NVFP4_BLOCK_SIZE), 4); + // dsv41-mxfp4-e8m0-block32: block size follows the scale dtype (columnwise). + size_t sinv1 = roundup(ceildiv(flat_first_dim, scale_block_size()), 4); scale_shape = {sinv0, sinv1}; } return scale_shape; diff --git a/transformer_engine/pytorch/tensor/nvfp4_tensor.py b/transformer_engine/pytorch/tensor/nvfp4_tensor.py index 78dea6cbee..80f8a4e2a2 100644 --- a/transformer_engine/pytorch/tensor/nvfp4_tensor.py +++ b/transformer_engine/pytorch/tensor/nvfp4_tensor.py @@ -168,7 +168,12 @@ def __init__( super().__init__(rowwise=rowwise, columnwise=columnwise) self.dtype = DType.cast(fp4_dtype) self.scale_dtype = DType.cast(scale_dtype) - if self.scale_dtype not in (DType.kFloat8E4M3, DType.kFloat8UE5M3): + # dsv41-mxfp4-e8m0-block32: E8M0 selects MXFP4 (32-element blocks). + if self.scale_dtype not in ( + DType.kFloat8E4M3, + DType.kFloat8UE5M3, + DType.kFloat8E8M0, + ): raise ValueError("scale_dtype must be DType.kFloat8E4M3 or DType.kFloat8UE5M3.") self.with_rht = with_rht self.with_post_rht_amax = with_post_rht_amax