Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions transformer_engine/common/cast/dispatch/quantize.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,11 @@ void quantize_fwd_helper(const NVTETensor input, NVTETensor output,
const DType scale_dtype = output_tensor->scale_inv.has_data()
? output_tensor->scale_inv.dtype
: output_tensor->columnwise_scale_inv.dtype;
NVTE_CHECK(nvfp4_use_4over6 || static_cast<float>(output_tensor->get_nvfp4_scale_max()) ==
typeToMax(scale_dtype),
// dsv41-mxfp4-e8m0-block32: E8M0's max finite value is 2^127, which the runtime int scale-max
// cannot represent, so the equality is vacuous there.
NVTE_CHECK(nvfp4_use_4over6 || scale_dtype == DType::kFloat8E8M0 ||
static_cast<float>(output_tensor->get_nvfp4_scale_max()) ==
typeToMax(scale_dtype),
"NVFP4 quantization with non-default scale max is only supported with 4over6.");
NVTE_CHECK(!nvfp4_use_4over6 || !quant_config_cpp.stochastic_rounding,
"NVFP4 4over6 quantization does not support stochastic rounding.");
Expand Down Expand Up @@ -315,8 +318,11 @@ void quantize_bwd_helper(const NVTETensor grad, const NVTETensor input, NVTETens
const DType scale_dtype = output_tensor->scale_inv.has_data()
? output_tensor->scale_inv.dtype
: output_tensor->columnwise_scale_inv.dtype;
NVTE_CHECK(nvfp4_use_4over6 || static_cast<float>(output_tensor->get_nvfp4_scale_max()) ==
typeToMax(scale_dtype),
// dsv41-mxfp4-e8m0-block32: E8M0's max finite value is 2^127, which the runtime int scale-max
// cannot represent, so the equality is vacuous there.
NVTE_CHECK(nvfp4_use_4over6 || scale_dtype == DType::kFloat8E8M0 ||
static_cast<float>(output_tensor->get_nvfp4_scale_max()) ==
typeToMax(scale_dtype),
"NVFP4 quantization with non-default scale max is only supported with 4over6.");
NVTE_CHECK(!nvfp4_use_4over6 || !quant_config_cpp.stochastic_rounding,
"NVFP4 4over6 quantization does not support stochastic rounding.");
Expand Down
40 changes: 36 additions & 4 deletions transformer_engine/common/cast/nvfp4/core_nvfp4.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,11 @@ namespace nvfp4 {
// SWITCH_FP8UE5M3_TYPE_HANDLE adds UE5M3 when the CUDA toolkit supports it.
#define TRANSFORMER_ENGINE_NVFP4_SCALE_TYPE_SWITCH(SCALE_DTYPE, SCALE_TYPE, ...) \
switch (SCALE_DTYPE) { \
case DType::kFloat8E4M3: { \
case DType::kFloat8E8M0: { /* dsv41-mxfp4-e8m0-block32 */ \
using SCALE_TYPE = fp8e8m0; \
{ __VA_ARGS__ } \
} break; \
case DType::kFloat8E4M3: { \
using SCALE_TYPE = fp8e4m3; \
{ __VA_ARGS__ } \
} break; \
Expand Down Expand Up @@ -89,17 +93,45 @@ compute_decoding_scaling_factor(const float block_amax, const float global_encod
constexpr float fp4_max = TypeExtrema<fp4e2m1>::max; // 6.0f
constexpr float fp4_max_inv = 1.0f / fp4_max;
const float decode_scale = block_amax * (global_encode_scale * fp4_max_inv);
if constexpr (std::is_same_v<ScaleType, fp8e8m0>) {
// dsv41-mxfp4-e8m0-block32: MXFP4's block scale covers the block -- round UP to a power of two.
// The reference implementation matches 2^ceil(log2(amax/6)) on 100% of blocks
// (measured, 1024 blocks). Computed from the bit pattern because exp2f/ceilf/log2f
// under this build's fast-math stored byte 0 for every block.
const float _s = fminf(decode_scale, TypeExtrema<float>::max);
if (_s <= 0.0f) {
return static_cast<ScaleType>(_s);
}
const unsigned _b = __float_as_uint(_s);
int _e = static_cast<int>((_b >> 23) & 0xFFu) - 127;
const bool _exact = ((_b & 0x7FFFFFu) == 0u);
if (!_exact) {
++_e; // 2^ceil: step up unless the value is already a power of two
}
const unsigned char _byte = static_cast<unsigned char>(_e + 127);
ScaleType _out;
memcpy(&_out, &_byte, sizeof(_byte));
return _out;
}
return static_cast<ScaleType>(fminf(decode_scale, TypeExtrema<float>::max));
}

// Compute the global encode scale factor for a given global amax.
// NVFP4 uses the full scale-type range by default. The explicit SCALE_MAX
// template argument lets recipes such as 4over6 reserve encoding headroom.
template <typename ScaleType, int SCALE_MAX = static_cast<int>(detail::TypeExtrema<ScaleType>::max)>
// dsv41-mxfp4-e8m0-block32: sentinel default; a float template parameter is not allowed in C++17.
template <typename ScaleType, int SCALE_MAX = -1>
__device__ __forceinline__ float compute_global_encode_scaling_factor_FP4(const float global_amax) {
using namespace detail;
static_assert(SCALE_MAX > 0, "NVFP4 scale maximum must be positive.");
constexpr float fp8_max = static_cast<float>(SCALE_MAX);
static_assert(SCALE_MAX != 0, "FP4 scale maximum must be non-zero.");
constexpr float fp8_max = (SCALE_MAX > 0) ? static_cast<float>(SCALE_MAX)
: detail::TypeExtrema<ScaleType>::max;
if constexpr (std::is_same_v<ScaleType, fp8e8m0>) {
// MXFP4 has no tensor-level global scale: the block scale is already an
// absolute power of two. Returning anything else made the global factor inf
// and saturated the stored exponent (measured relerr=inf).
return 1.0f;
}
constexpr float fp4_max = TypeExtrema<fp4e2m1>::max; // 6.0f;
float global_encode_scale = fp8_max * fp4_max / global_amax;
// If scale is infinity, return max value of float32
Expand Down
40 changes: 30 additions & 10 deletions transformer_engine/common/cast/nvfp4/dequantize_nvfp4.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -61,20 +61,30 @@ __global__ void __launch_bounds__(512)
if constexpr (WITH_GEMM_SWIZZLED_SCALES) {
my_scale_index = mxfp8::swizzle::gemm_swizzled_scale_idx(y, x, num_scale_tiles_X);
} else {
my_scale_index = x + y * scale_stride;
// dsv41-mxfp4-e8m0-block32: for E8M0 one scale spans 32 elements = two of the kernel's groups.
const size_t _grp = std::is_same_v<ScaleType, fp8e8m0> ? (x >> 1) : x;
my_scale_index = _grp + y * scale_stride;
}
const size_t my_output_index = (x + y * M) * 4;
fp4vec value;
value.vec = input_vectorized[my_index];
ScaleType scale = scales[my_scale_index];
constexpr float fp4_max = detail::TypeExtrema<fp4e2m1>::max;
constexpr float unit_global_scale_amax = fp4_max * static_cast<float>(SCALE_TYPE_MAX);
float amax = unit_global_scale_amax;
if (tensor_amax != nullptr) {
amax = ROW_SCALED_NVFP4 ? tensor_amax[y] : tensor_amax[0];
// dsv41-mxfp4-e8m0-block32: MXFP4 has only the per-block scale, which is already an absolute
// power of two. Applying NVFP4's two-level formula overflowed (fp4_max * 2^127
// -> inf, then inf * 0 = NaN).
float final_scale;
if constexpr (std::is_same_v<ScaleType, fp8e8m0>) {
final_scale = static_cast<float>(scale);
} else {
constexpr float unit_global_scale_amax = fp4_max * static_cast<float>(SCALE_TYPE_MAX);
float amax = unit_global_scale_amax;
if (tensor_amax != nullptr) {
amax = ROW_SCALED_NVFP4 ? tensor_amax[y] : tensor_amax[0];
}
constexpr float factor_inv = 1.0f / unit_global_scale_amax;
final_scale = static_cast<float>(scale) * amax * factor_inv;
}
constexpr float factor_inv = 1.0f / unit_global_scale_amax;
float final_scale = static_cast<float>(scale) * amax * factor_inv;
#pragma unroll
for (int i = 0; i < 4; i++) {
float4 current = static_cast<float4>(value.small_vec[i]);
Expand Down Expand Up @@ -132,7 +142,11 @@ inline void dequantize(const Tensor &input, Tensor *output, cudaStream_t stream)
NVTE_CHECK(M % FP4_BLOCK_SIZE == 0, "Last dimension of FP4 tensors needs to be divisible by ",
FP4_BLOCK_SIZE, ", but got ", input.data.shape, ".");

const size_t Mread = M / FP4_BLOCK_SIZE;
// dsv41-mxfp4-e8m0-block32: the kernel processes 16 elements per thread (`fp4e2m1x4 small_vec[4]`),
// so Mread is always M/16. Setting it to M/scale_block made the kernel cover
// only half of every row for block-32 formats (measured: TE's own dequantize
// scored relerr 0.805 against 0.393 for a lookup table on the same bytes).
const size_t Mread = M / 16;
const size_t total = N * Mread;
const size_t threads = 512;
const size_t blocks = DIVUP(total, threads);
Expand All @@ -143,14 +157,20 @@ inline void dequantize(const Tensor &input, Tensor *output, cudaStream_t stream)
"Row-scaled NVFP4 dequantization requires one rowwise amax per row.");
const int full_scale_max = static_cast<int>(typeToMax(scale_dtype));
const bool uses_4over6_headroom = scale_dtype == DType::kFloat8E4M3 && scale_type_max == 256;
NVTE_CHECK(scale_type_max == full_scale_max || uses_4over6_headroom, "Unsupported maximum ",
// dsv41-mxfp4-e8m0-block32: E8M0's 2^127 has no int form, so the equality is vacuous.
NVTE_CHECK(scale_dtype == DType::kFloat8E8M0 ||
scale_type_max == full_scale_max || uses_4over6_headroom, "Unsupported maximum ",
scale_type_max, " for NVFP4 scale dtype ", to_string(scale_dtype), ".");
if (uses_4over6_headroom) {
launch_dequantize<fp8e4m3, 256>(input, output, with_gemm_swizzled_scales, row_scaled_nvfp4, N,
Mread, blocks, threads, num_scale_tiles_X, stream);
} else {
TRANSFORMER_ENGINE_NVFP4_SCALE_TYPE_SWITCH(scale_dtype, ScaleType, {
constexpr int SCALE_TYPE_MAX = static_cast<int>(TypeInfo<ScaleType>::max_finite_value);
// dsv41-mxfp4-e8m0-block32: 0 is a placeholder for E8M0; its real value is set below.
constexpr int SCALE_TYPE_MAX =
std::is_same_v<ScaleType, fp8e8m0>
? 0
: static_cast<int>(TypeInfo<ScaleType>::max_finite_value);
launch_dequantize<ScaleType, SCALE_TYPE_MAX>(input, output, with_gemm_swizzled_scales,
row_scaled_nvfp4, N, Mread, blocks, threads,
num_scale_tiles_X, stream);
Expand Down
3 changes: 3 additions & 0 deletions transformer_engine/common/common.h
Original file line number Diff line number Diff line change
Expand Up @@ -478,6 +478,9 @@ struct Tensor {
return 448;
case DType::kFloat8UE5M3:
return 114688;
case DType::kFloat8E8M0:
// dsv41-mxfp4-e8m0-block32: 2^127 has no int form; only an amax fallback.
return 2147483647;
default:
NVTE_ERROR("Unsupported scale dtype for NVFP4 tensor (", to_string(dtype), ")");
}
Expand Down
24 changes: 16 additions & 8 deletions transformer_engine/common/transformer_engine.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,11 @@ void CheckScaleTensorShape(const Tensor &t, std::string_view name) {
const auto [first_dim, last_dim] = t.flat_2d_dims();

if (t.has_data()) {
constexpr std::array<size_t, 2> block_shape{1, 16};
// dsv41-mxfp4-e8m0-block32: 32 for E8M0, 16 otherwise.
const DType _sd = t.has_data() ? t.scale_inv.dtype
: (t.has_columnwise_data() ? t.columnwise_scale_inv.dtype
: DType::kFloat8E4M3);
const std::array<size_t, 2> block_shape{1, _sd == DType::kFloat8E8M0 ? 32 : 16};
constexpr std::array<size_t, 2> block_alignment{128, 4};
const std::array<size_t, 2> expected{
DIVUP_TO_MULTIPLE(DIVUP(first_dim, block_shape[0]), block_alignment[0]),
Expand All @@ -139,7 +143,11 @@ void CheckScaleTensorShape(const Tensor &t, std::string_view name) {
t.scale_inv.shape, ")");
}
if (t.has_columnwise_data()) {
constexpr std::array<size_t, 2> block_shape{1, 16};
// dsv41-mxfp4-e8m0-block32: 32 for E8M0, 16 otherwise.
const DType _sd = t.has_data() ? t.scale_inv.dtype
: (t.has_columnwise_data() ? t.columnwise_scale_inv.dtype
: DType::kFloat8E4M3);
const std::array<size_t, 2> block_shape{1, _sd == DType::kFloat8E8M0 ? 32 : 16};
constexpr std::array<size_t, 2> block_alignment{128, 4};
const std::array<size_t, 2> expected{
DIVUP_TO_MULTIPLE(DIVUP(last_dim, block_shape[0]), block_alignment[0]),
Expand Down Expand Up @@ -182,7 +190,7 @@ void CheckInputTensor(const Tensor &t, std::string_view name, bool check_scale_i
NVTE_CHECK(t.scale_inv.has_data(), "FP4 scaling factor input ", name,
"_scale_inverse must be allocated");
NVTE_CHECK(
t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3,
t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3 || t.scale_inv.dtype == DType::kFloat8E8M0,
"FP4 scaling factor input ", name,
"_scale_inverse has invalid dtype "
"(expected Float8E4M3 or Float8UE5M3, got ",
Expand All @@ -192,7 +200,7 @@ void CheckInputTensor(const Tensor &t, std::string_view name, bool check_scale_i
NVTE_CHECK(t.columnwise_scale_inv.has_data(), "FP4 scaling factor input ", name,
"_columnwise_scale_inverse must be allocated");
NVTE_CHECK(t.columnwise_scale_inv.dtype == DType::kFloat8E4M3 ||
t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3,
t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3 || t.columnwise_scale_inv.dtype == DType::kFloat8E8M0,
"FP8 scaling factor input ", name,
"_columnwise_scale_inverse has invalid dtype "
"(expected Float8E4M3 or Float8UE5M3, got ",
Expand Down Expand Up @@ -246,7 +254,7 @@ void CheckOutputTensor(const Tensor &t, std::string_view name, bool allow_empty)
NVTE_CHECK(t.scale_inv.has_data(), "FP4 scaling factor output ", name,
"_scale_inverse must be allocated");
NVTE_CHECK(
t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3,
t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3 || t.scale_inv.dtype == DType::kFloat8E8M0,
"FP4 scaling factor output ", name,
"_scale_inverse has invalid dtype "
"(expected Float8E4M3 or Float8UE5M3, got ",
Expand All @@ -256,7 +264,7 @@ void CheckOutputTensor(const Tensor &t, std::string_view name, bool allow_empty)
NVTE_CHECK(t.columnwise_scale_inv.has_data(), "FP4 scaling factor output ", name,
"_columnwise_scale_inverse must be allocated");
NVTE_CHECK(t.columnwise_scale_inv.dtype == DType::kFloat8E4M3 ||
t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3,
t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3 || t.columnwise_scale_inv.dtype == DType::kFloat8E8M0,
"FP4 scaling factor output ", name,
"_columnwise_scale_inverse has invalid dtype "
"(expected Float8E4M3 or Float8UE5M3, got ",
Expand Down Expand Up @@ -379,7 +387,7 @@ static void CheckGroupedScaleInv(const GroupedTensor &t, std::string_view name,
NVTE_CHECK(t.scale_inv.has_data(), tensor_type, " ", name,
" rowwise scale_inv must be allocated");
NVTE_CHECK(
t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3,
t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3 || t.scale_inv.dtype == DType::kFloat8E8M0,
tensor_type, " ", name,
" rowwise scale_inv has invalid dtype "
"(expected Float8E4M3 or Float8UE5M3, got ",
Expand All @@ -389,7 +397,7 @@ static void CheckGroupedScaleInv(const GroupedTensor &t, std::string_view name,
NVTE_CHECK(t.columnwise_scale_inv.has_data(), tensor_type, " ", name,
" columnwise scale_inv must be allocated");
NVTE_CHECK(t.columnwise_scale_inv.dtype == DType::kFloat8E4M3 ||
t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3,
t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3 || t.columnwise_scale_inv.dtype == DType::kFloat8E8M0,
tensor_type, " ", name,
" columnwise scale_inv has invalid dtype "
"(expected Float8E4M3 or Float8UE5M3, got ",
Expand Down
7 changes: 7 additions & 0 deletions transformer_engine/common/util/type_extrema.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,17 @@ struct TypeExtrema {
static constexpr float max = std::numeric_limits<T>::max();
};
#else

template <typename T>
struct TypeExtrema;
#endif

// dsv41-mxfp4-e8m0-block32: E8M0 is a pure biased exponent (value = 2^(e-127)); 2^127 is its max.
template <>
struct TypeExtrema<fp8e8m0> {
static constexpr float max = 1.7014118e38f;
};

#if FP4_TYPE_SUPPORTED
template <>
struct TypeExtrema<fp4e2m1> {
Expand Down
5 changes: 5 additions & 0 deletions transformer_engine/pytorch/csrc/common.h
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,11 @@ class NVFP4Quantizer : public Quantizer {
int rht_matrix_random_sign_mask_t;
at::Tensor rht_matrix;

// dsv41-mxfp4-e8m0-block32: MXFP4 uses E8M0 over 32-element blocks; NVFP4 keeps 16.
size_t scale_block_size() const {
return scale_dtype == DType::kFloat8E8M0 ? 32 : 16;
}

explicit NVFP4Quantizer(const py::handle& quantizer);

NVTEScalingMode get_scaling_mode() const override { return NVTE_NVFP4_1D_SCALING; }
Expand Down
13 changes: 9 additions & 4 deletions transformer_engine/pytorch/csrc/quantizer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1892,8 +1892,11 @@ NVFP4Quantizer::NVFP4Quantizer(const py::handle& quantizer) : Quantizer(quantize
}
this->row_scaled_nvfp4 = quantizer.attr("row_scaled_nvfp4").cast<bool>();
this->scale_dtype = quantizer.attr("scale_dtype").cast<DType>();
NVTE_CHECK(this->scale_dtype == DType::kFloat8E4M3 || this->scale_dtype == DType::kFloat8UE5M3,
"Unsupported NVFP4 scale dtype: ", static_cast<int>(this->scale_dtype));
// dsv41-mxfp4-e8m0-block32: E8M0 selects MXFP4 (32-element blocks).
NVTE_CHECK(this->scale_dtype == DType::kFloat8E4M3 ||
this->scale_dtype == DType::kFloat8UE5M3 ||
this->scale_dtype == DType::kFloat8E8M0,
"Unsupported NVFP4/MXFP4 scale dtype: ", static_cast<int>(this->scale_dtype));
this->disable_second_level_scale = quantizer.attr("disable_second_level_scale").cast<bool>();

// Get amax reduction group if needed for NVFP4 AG
Expand Down Expand Up @@ -2869,12 +2872,14 @@ std::vector<size_t> NVFP4Quantizer::get_scale_shape(const std::vector<size_t>& s
if (rowwise_usage) {
// rowwise scaling factor shape
size_t sinv0 = roundup(flat_first_dim, 128);
size_t sinv1 = roundup(ceildiv(last_dim, NVFP4_BLOCK_SIZE), 4);
// dsv41-mxfp4-e8m0-block32: block size follows the scale dtype.
size_t sinv1 = roundup(ceildiv(last_dim, scale_block_size()), 4);
scale_shape = {sinv0, sinv1};
} else {
// columnwise scaling factor shape
size_t sinv0 = roundup(last_dim, 128);
size_t sinv1 = roundup(ceildiv(flat_first_dim, NVFP4_BLOCK_SIZE), 4);
// dsv41-mxfp4-e8m0-block32: block size follows the scale dtype (columnwise).
size_t sinv1 = roundup(ceildiv(flat_first_dim, scale_block_size()), 4);
scale_shape = {sinv0, sinv1};
}
return scale_shape;
Expand Down
Loading
Loading