diff --git a/transformer_engine/common/cast/dispatch/quantize.cuh b/transformer_engine/common/cast/dispatch/quantize.cuh index 13c3694d04..3ab6e3f691 100644 --- a/transformer_engine/common/cast/dispatch/quantize.cuh +++ b/transformer_engine/common/cast/dispatch/quantize.cuh @@ -126,8 +126,11 @@ void quantize_fwd_helper(const NVTETensor input, NVTETensor output, const DType scale_dtype = output_tensor->scale_inv.has_data() ? output_tensor->scale_inv.dtype : output_tensor->columnwise_scale_inv.dtype; - NVTE_CHECK(nvfp4_use_4over6 || static_cast(output_tensor->get_nvfp4_scale_max()) == - typeToMax(scale_dtype), + // dsv41-mxfp4-e8m0-block32: E8M0's max finite value is 2^127, which the runtime int scale-max + // cannot represent, so the equality is vacuous there. + NVTE_CHECK(nvfp4_use_4over6 || scale_dtype == DType::kFloat8E8M0 || + static_cast(output_tensor->get_nvfp4_scale_max()) == + typeToMax(scale_dtype), "NVFP4 quantization with non-default scale max is only supported with 4over6."); NVTE_CHECK(!nvfp4_use_4over6 || !quant_config_cpp.stochastic_rounding, "NVFP4 4over6 quantization does not support stochastic rounding."); @@ -315,8 +318,11 @@ void quantize_bwd_helper(const NVTETensor grad, const NVTETensor input, NVTETens const DType scale_dtype = output_tensor->scale_inv.has_data() ? output_tensor->scale_inv.dtype : output_tensor->columnwise_scale_inv.dtype; - NVTE_CHECK(nvfp4_use_4over6 || static_cast(output_tensor->get_nvfp4_scale_max()) == - typeToMax(scale_dtype), + // dsv41-mxfp4-e8m0-block32: E8M0's max finite value is 2^127, which the runtime int scale-max + // cannot represent, so the equality is vacuous there. + NVTE_CHECK(nvfp4_use_4over6 || scale_dtype == DType::kFloat8E8M0 || + static_cast(output_tensor->get_nvfp4_scale_max()) == + typeToMax(scale_dtype), "NVFP4 quantization with non-default scale max is only supported with 4over6."); NVTE_CHECK(!nvfp4_use_4over6 || !quant_config_cpp.stochastic_rounding, "NVFP4 4over6 quantization does not support stochastic rounding."); diff --git a/transformer_engine/common/cast/nvfp4/core_nvfp4.cuh b/transformer_engine/common/cast/nvfp4/core_nvfp4.cuh index 167703b7fb..084b09aecc 100644 --- a/transformer_engine/common/cast/nvfp4/core_nvfp4.cuh +++ b/transformer_engine/common/cast/nvfp4/core_nvfp4.cuh @@ -53,7 +53,11 @@ namespace nvfp4 { // SWITCH_FP8UE5M3_TYPE_HANDLE adds UE5M3 when the CUDA toolkit supports it. #define TRANSFORMER_ENGINE_NVFP4_SCALE_TYPE_SWITCH(SCALE_DTYPE, SCALE_TYPE, ...) \ switch (SCALE_DTYPE) { \ - case DType::kFloat8E4M3: { \ + case DType::kFloat8E8M0: { /* dsv41-mxfp4-e8m0-block32 */ \ + using SCALE_TYPE = fp8e8m0; \ + { __VA_ARGS__ } \ + } break; \ + case DType::kFloat8E4M3: { \ using SCALE_TYPE = fp8e4m3; \ { __VA_ARGS__ } \ } break; \ @@ -89,17 +93,45 @@ compute_decoding_scaling_factor(const float block_amax, const float global_encod constexpr float fp4_max = TypeExtrema::max; // 6.0f constexpr float fp4_max_inv = 1.0f / fp4_max; const float decode_scale = block_amax * (global_encode_scale * fp4_max_inv); + if constexpr (std::is_same_v) { + // dsv41-mxfp4-e8m0-block32: MXFP4's block scale covers the block -- round UP to a power of two. + // The reference implementation matches 2^ceil(log2(amax/6)) on 100% of blocks + // (measured, 1024 blocks). Computed from the bit pattern because exp2f/ceilf/log2f + // under this build's fast-math stored byte 0 for every block. + const float _s = fminf(decode_scale, TypeExtrema::max); + if (_s <= 0.0f) { + return static_cast(_s); + } + const unsigned _b = __float_as_uint(_s); + int _e = static_cast((_b >> 23) & 0xFFu) - 127; + const bool _exact = ((_b & 0x7FFFFFu) == 0u); + if (!_exact) { + ++_e; // 2^ceil: step up unless the value is already a power of two + } + const unsigned char _byte = static_cast(_e + 127); + ScaleType _out; + memcpy(&_out, &_byte, sizeof(_byte)); + return _out; + } return static_cast(fminf(decode_scale, TypeExtrema::max)); } // Compute the global encode scale factor for a given global amax. // NVFP4 uses the full scale-type range by default. The explicit SCALE_MAX // template argument lets recipes such as 4over6 reserve encoding headroom. -template (detail::TypeExtrema::max)> +// dsv41-mxfp4-e8m0-block32: sentinel default; a float template parameter is not allowed in C++17. +template __device__ __forceinline__ float compute_global_encode_scaling_factor_FP4(const float global_amax) { using namespace detail; - static_assert(SCALE_MAX > 0, "NVFP4 scale maximum must be positive."); - constexpr float fp8_max = static_cast(SCALE_MAX); + static_assert(SCALE_MAX != 0, "FP4 scale maximum must be non-zero."); + constexpr float fp8_max = (SCALE_MAX > 0) ? static_cast(SCALE_MAX) + : detail::TypeExtrema::max; + if constexpr (std::is_same_v) { + // MXFP4 has no tensor-level global scale: the block scale is already an + // absolute power of two. Returning anything else made the global factor inf + // and saturated the stored exponent (measured relerr=inf). + return 1.0f; + } constexpr float fp4_max = TypeExtrema::max; // 6.0f; float global_encode_scale = fp8_max * fp4_max / global_amax; // If scale is infinity, return max value of float32 diff --git a/transformer_engine/common/cast/nvfp4/dequantize_nvfp4.cuh b/transformer_engine/common/cast/nvfp4/dequantize_nvfp4.cuh index ac30fe3f91..83556b2460 100644 --- a/transformer_engine/common/cast/nvfp4/dequantize_nvfp4.cuh +++ b/transformer_engine/common/cast/nvfp4/dequantize_nvfp4.cuh @@ -61,20 +61,30 @@ __global__ void __launch_bounds__(512) if constexpr (WITH_GEMM_SWIZZLED_SCALES) { my_scale_index = mxfp8::swizzle::gemm_swizzled_scale_idx(y, x, num_scale_tiles_X); } else { - my_scale_index = x + y * scale_stride; + // dsv41-mxfp4-e8m0-block32: for E8M0 one scale spans 32 elements = two of the kernel's groups. + const size_t _grp = std::is_same_v ? (x >> 1) : x; + my_scale_index = _grp + y * scale_stride; } const size_t my_output_index = (x + y * M) * 4; fp4vec value; value.vec = input_vectorized[my_index]; ScaleType scale = scales[my_scale_index]; constexpr float fp4_max = detail::TypeExtrema::max; - constexpr float unit_global_scale_amax = fp4_max * static_cast(SCALE_TYPE_MAX); - float amax = unit_global_scale_amax; - if (tensor_amax != nullptr) { - amax = ROW_SCALED_NVFP4 ? tensor_amax[y] : tensor_amax[0]; + // dsv41-mxfp4-e8m0-block32: MXFP4 has only the per-block scale, which is already an absolute + // power of two. Applying NVFP4's two-level formula overflowed (fp4_max * 2^127 + // -> inf, then inf * 0 = NaN). + float final_scale; + if constexpr (std::is_same_v) { + final_scale = static_cast(scale); + } else { + constexpr float unit_global_scale_amax = fp4_max * static_cast(SCALE_TYPE_MAX); + float amax = unit_global_scale_amax; + if (tensor_amax != nullptr) { + amax = ROW_SCALED_NVFP4 ? tensor_amax[y] : tensor_amax[0]; + } + constexpr float factor_inv = 1.0f / unit_global_scale_amax; + final_scale = static_cast(scale) * amax * factor_inv; } - constexpr float factor_inv = 1.0f / unit_global_scale_amax; - float final_scale = static_cast(scale) * amax * factor_inv; #pragma unroll for (int i = 0; i < 4; i++) { float4 current = static_cast(value.small_vec[i]); @@ -132,7 +142,11 @@ inline void dequantize(const Tensor &input, Tensor *output, cudaStream_t stream) NVTE_CHECK(M % FP4_BLOCK_SIZE == 0, "Last dimension of FP4 tensors needs to be divisible by ", FP4_BLOCK_SIZE, ", but got ", input.data.shape, "."); - const size_t Mread = M / FP4_BLOCK_SIZE; + // dsv41-mxfp4-e8m0-block32: the kernel processes 16 elements per thread (`fp4e2m1x4 small_vec[4]`), + // so Mread is always M/16. Setting it to M/scale_block made the kernel cover + // only half of every row for block-32 formats (measured: TE's own dequantize + // scored relerr 0.805 against 0.393 for a lookup table on the same bytes). + const size_t Mread = M / 16; const size_t total = N * Mread; const size_t threads = 512; const size_t blocks = DIVUP(total, threads); @@ -143,14 +157,20 @@ inline void dequantize(const Tensor &input, Tensor *output, cudaStream_t stream) "Row-scaled NVFP4 dequantization requires one rowwise amax per row."); const int full_scale_max = static_cast(typeToMax(scale_dtype)); const bool uses_4over6_headroom = scale_dtype == DType::kFloat8E4M3 && scale_type_max == 256; - NVTE_CHECK(scale_type_max == full_scale_max || uses_4over6_headroom, "Unsupported maximum ", + // dsv41-mxfp4-e8m0-block32: E8M0's 2^127 has no int form, so the equality is vacuous. + NVTE_CHECK(scale_dtype == DType::kFloat8E8M0 || + scale_type_max == full_scale_max || uses_4over6_headroom, "Unsupported maximum ", scale_type_max, " for NVFP4 scale dtype ", to_string(scale_dtype), "."); if (uses_4over6_headroom) { launch_dequantize(input, output, with_gemm_swizzled_scales, row_scaled_nvfp4, N, Mread, blocks, threads, num_scale_tiles_X, stream); } else { TRANSFORMER_ENGINE_NVFP4_SCALE_TYPE_SWITCH(scale_dtype, ScaleType, { - constexpr int SCALE_TYPE_MAX = static_cast(TypeInfo::max_finite_value); + // dsv41-mxfp4-e8m0-block32: 0 is a placeholder for E8M0; its real value is set below. + constexpr int SCALE_TYPE_MAX = + std::is_same_v + ? 0 + : static_cast(TypeInfo::max_finite_value); launch_dequantize(input, output, with_gemm_swizzled_scales, row_scaled_nvfp4, N, Mread, blocks, threads, num_scale_tiles_X, stream); diff --git a/transformer_engine/common/common.h b/transformer_engine/common/common.h index 7d201f6c8b..72fabb7229 100644 --- a/transformer_engine/common/common.h +++ b/transformer_engine/common/common.h @@ -478,6 +478,9 @@ struct Tensor { return 448; case DType::kFloat8UE5M3: return 114688; + case DType::kFloat8E8M0: + // dsv41-mxfp4-e8m0-block32: 2^127 has no int form; only an amax fallback. + return 2147483647; default: NVTE_ERROR("Unsupported scale dtype for NVFP4 tensor (", to_string(dtype), ")"); } diff --git a/transformer_engine/common/transformer_engine.cpp b/transformer_engine/common/transformer_engine.cpp index 66693dca3a..fb88160e13 100644 --- a/transformer_engine/common/transformer_engine.cpp +++ b/transformer_engine/common/transformer_engine.cpp @@ -129,7 +129,11 @@ void CheckScaleTensorShape(const Tensor &t, std::string_view name) { const auto [first_dim, last_dim] = t.flat_2d_dims(); if (t.has_data()) { - constexpr std::array block_shape{1, 16}; + // dsv41-mxfp4-e8m0-block32: 32 for E8M0, 16 otherwise. + const DType _sd = t.has_data() ? t.scale_inv.dtype + : (t.has_columnwise_data() ? t.columnwise_scale_inv.dtype + : DType::kFloat8E4M3); + const std::array block_shape{1, _sd == DType::kFloat8E8M0 ? 32 : 16}; constexpr std::array block_alignment{128, 4}; const std::array expected{ DIVUP_TO_MULTIPLE(DIVUP(first_dim, block_shape[0]), block_alignment[0]), @@ -139,7 +143,11 @@ void CheckScaleTensorShape(const Tensor &t, std::string_view name) { t.scale_inv.shape, ")"); } if (t.has_columnwise_data()) { - constexpr std::array block_shape{1, 16}; + // dsv41-mxfp4-e8m0-block32: 32 for E8M0, 16 otherwise. + const DType _sd = t.has_data() ? t.scale_inv.dtype + : (t.has_columnwise_data() ? t.columnwise_scale_inv.dtype + : DType::kFloat8E4M3); + const std::array block_shape{1, _sd == DType::kFloat8E8M0 ? 32 : 16}; constexpr std::array block_alignment{128, 4}; const std::array expected{ DIVUP_TO_MULTIPLE(DIVUP(last_dim, block_shape[0]), block_alignment[0]), @@ -182,7 +190,7 @@ void CheckInputTensor(const Tensor &t, std::string_view name, bool check_scale_i NVTE_CHECK(t.scale_inv.has_data(), "FP4 scaling factor input ", name, "_scale_inverse must be allocated"); NVTE_CHECK( - t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3, + t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3 || t.scale_inv.dtype == DType::kFloat8E8M0, "FP4 scaling factor input ", name, "_scale_inverse has invalid dtype " "(expected Float8E4M3 or Float8UE5M3, got ", @@ -192,7 +200,7 @@ void CheckInputTensor(const Tensor &t, std::string_view name, bool check_scale_i NVTE_CHECK(t.columnwise_scale_inv.has_data(), "FP4 scaling factor input ", name, "_columnwise_scale_inverse must be allocated"); NVTE_CHECK(t.columnwise_scale_inv.dtype == DType::kFloat8E4M3 || - t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3, + t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3 || t.columnwise_scale_inv.dtype == DType::kFloat8E8M0, "FP8 scaling factor input ", name, "_columnwise_scale_inverse has invalid dtype " "(expected Float8E4M3 or Float8UE5M3, got ", @@ -246,7 +254,7 @@ void CheckOutputTensor(const Tensor &t, std::string_view name, bool allow_empty) NVTE_CHECK(t.scale_inv.has_data(), "FP4 scaling factor output ", name, "_scale_inverse must be allocated"); NVTE_CHECK( - t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3, + t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3 || t.scale_inv.dtype == DType::kFloat8E8M0, "FP4 scaling factor output ", name, "_scale_inverse has invalid dtype " "(expected Float8E4M3 or Float8UE5M3, got ", @@ -256,7 +264,7 @@ void CheckOutputTensor(const Tensor &t, std::string_view name, bool allow_empty) NVTE_CHECK(t.columnwise_scale_inv.has_data(), "FP4 scaling factor output ", name, "_columnwise_scale_inverse must be allocated"); NVTE_CHECK(t.columnwise_scale_inv.dtype == DType::kFloat8E4M3 || - t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3, + t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3 || t.columnwise_scale_inv.dtype == DType::kFloat8E8M0, "FP4 scaling factor output ", name, "_columnwise_scale_inverse has invalid dtype " "(expected Float8E4M3 or Float8UE5M3, got ", @@ -379,7 +387,7 @@ static void CheckGroupedScaleInv(const GroupedTensor &t, std::string_view name, NVTE_CHECK(t.scale_inv.has_data(), tensor_type, " ", name, " rowwise scale_inv must be allocated"); NVTE_CHECK( - t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3, + t.scale_inv.dtype == DType::kFloat8E4M3 || t.scale_inv.dtype == DType::kFloat8UE5M3 || t.scale_inv.dtype == DType::kFloat8E8M0, tensor_type, " ", name, " rowwise scale_inv has invalid dtype " "(expected Float8E4M3 or Float8UE5M3, got ", @@ -389,7 +397,7 @@ static void CheckGroupedScaleInv(const GroupedTensor &t, std::string_view name, NVTE_CHECK(t.columnwise_scale_inv.has_data(), tensor_type, " ", name, " columnwise scale_inv must be allocated"); NVTE_CHECK(t.columnwise_scale_inv.dtype == DType::kFloat8E4M3 || - t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3, + t.columnwise_scale_inv.dtype == DType::kFloat8UE5M3 || t.columnwise_scale_inv.dtype == DType::kFloat8E8M0, tensor_type, " ", name, " columnwise scale_inv has invalid dtype " "(expected Float8E4M3 or Float8UE5M3, got ", diff --git a/transformer_engine/common/util/type_extrema.h b/transformer_engine/common/util/type_extrema.h index 87f90aaee1..4a54d5b9cc 100644 --- a/transformer_engine/common/util/type_extrema.h +++ b/transformer_engine/common/util/type_extrema.h @@ -32,10 +32,17 @@ struct TypeExtrema { static constexpr float max = std::numeric_limits::max(); }; #else + template struct TypeExtrema; #endif +// dsv41-mxfp4-e8m0-block32: E8M0 is a pure biased exponent (value = 2^(e-127)); 2^127 is its max. +template <> +struct TypeExtrema { + static constexpr float max = 1.7014118e38f; +}; + #if FP4_TYPE_SUPPORTED template <> struct TypeExtrema { diff --git a/transformer_engine/pytorch/csrc/common.h b/transformer_engine/pytorch/csrc/common.h index e85ebb491b..fb444d5085 100644 --- a/transformer_engine/pytorch/csrc/common.h +++ b/transformer_engine/pytorch/csrc/common.h @@ -366,6 +366,11 @@ class NVFP4Quantizer : public Quantizer { int rht_matrix_random_sign_mask_t; at::Tensor rht_matrix; + // dsv41-mxfp4-e8m0-block32: MXFP4 uses E8M0 over 32-element blocks; NVFP4 keeps 16. + size_t scale_block_size() const { + return scale_dtype == DType::kFloat8E8M0 ? 32 : 16; + } + explicit NVFP4Quantizer(const py::handle& quantizer); NVTEScalingMode get_scaling_mode() const override { return NVTE_NVFP4_1D_SCALING; } diff --git a/transformer_engine/pytorch/csrc/quantizer.cpp b/transformer_engine/pytorch/csrc/quantizer.cpp index eccebf62ac..35d99a9c36 100644 --- a/transformer_engine/pytorch/csrc/quantizer.cpp +++ b/transformer_engine/pytorch/csrc/quantizer.cpp @@ -1892,8 +1892,11 @@ NVFP4Quantizer::NVFP4Quantizer(const py::handle& quantizer) : Quantizer(quantize } this->row_scaled_nvfp4 = quantizer.attr("row_scaled_nvfp4").cast(); this->scale_dtype = quantizer.attr("scale_dtype").cast(); - NVTE_CHECK(this->scale_dtype == DType::kFloat8E4M3 || this->scale_dtype == DType::kFloat8UE5M3, - "Unsupported NVFP4 scale dtype: ", static_cast(this->scale_dtype)); + // dsv41-mxfp4-e8m0-block32: E8M0 selects MXFP4 (32-element blocks). + NVTE_CHECK(this->scale_dtype == DType::kFloat8E4M3 || + this->scale_dtype == DType::kFloat8UE5M3 || + this->scale_dtype == DType::kFloat8E8M0, + "Unsupported NVFP4/MXFP4 scale dtype: ", static_cast(this->scale_dtype)); this->disable_second_level_scale = quantizer.attr("disable_second_level_scale").cast(); // Get amax reduction group if needed for NVFP4 AG @@ -2869,12 +2872,14 @@ std::vector NVFP4Quantizer::get_scale_shape(const std::vector& s if (rowwise_usage) { // rowwise scaling factor shape size_t sinv0 = roundup(flat_first_dim, 128); - size_t sinv1 = roundup(ceildiv(last_dim, NVFP4_BLOCK_SIZE), 4); + // dsv41-mxfp4-e8m0-block32: block size follows the scale dtype. + size_t sinv1 = roundup(ceildiv(last_dim, scale_block_size()), 4); scale_shape = {sinv0, sinv1}; } else { // columnwise scaling factor shape size_t sinv0 = roundup(last_dim, 128); - size_t sinv1 = roundup(ceildiv(flat_first_dim, NVFP4_BLOCK_SIZE), 4); + // dsv41-mxfp4-e8m0-block32: block size follows the scale dtype (columnwise). + size_t sinv1 = roundup(ceildiv(flat_first_dim, scale_block_size()), 4); scale_shape = {sinv0, sinv1}; } return scale_shape; diff --git a/transformer_engine/pytorch/module/base.py b/transformer_engine/pytorch/module/base.py index ebc0d13895..c5c4902511 100644 --- a/transformer_engine/pytorch/module/base.py +++ b/transformer_engine/pytorch/module/base.py @@ -53,6 +53,7 @@ from ..tensor.float8_tensor import Float8Quantizer, Float8CurrentScalingQuantizer from ..tensor.mxfp8_tensor import MXFP8Quantizer from ..tensor.nvfp4_tensor import NVFP4Quantizer +from ..tensor.nvfp4_tensor import NVFP4Quantizer from ..tensor.float8_blockwise_tensor import Float8BlockQuantizer from ..tensor.hybrid_tensor import HybridQuantizer from ..tensor.identity_tensor import IdentityQuantizer @@ -1954,7 +1955,24 @@ def reset_parameters(self, defer_init: Optional[bool] = False) -> None: # Wrap parameters in QuantizedTensor if needed fp8_meta_index = self.param_init_meta[name].fp8_meta_index high_precision_init_val = None - if self.primary_weights_in_fp8 and fp8_meta_index is not None: + if getattr(self, "primary_weights_in_fp4", None) == "nvfp4": + # 4-bit resident parameters. The parameter has already been built in + # params_dtype above and is quantised here, exactly as the fp8 branch below + # does it, so a params_dtype passed at construction does not prevent the + # storage swap. Neither mechanism otherwise offers this: fp8_model_init + # covers fp8 storage only, and a CustomRecipe quantizer factory is a + # per-GEMM-slot compute quantiser that never allocates a parameter. + # + # NVFP4Tensor is the 4-bit type that survives being wrapped as an + # nn.Parameter (measured); the reference implementation's tensor raises on + # detach and cannot be one. That is why this branch offers NVFP4 only -- + # an E8M0 block-32 scale format has no parameter-capable type here. + # + # Biases and any 1-D parameter deliberately stay in params_dtype. + if param.dim() > 1: + _q = NVFP4Quantizer() + param = _q.quantize(param.detach().contiguous()) + elif self.primary_weights_in_fp8 and fp8_meta_index is not None: # Keep high-precision values on CPU if needed if self.preserve_high_precision_init_val: diff --git a/transformer_engine/pytorch/tensor/nvfp4_tensor.py b/transformer_engine/pytorch/tensor/nvfp4_tensor.py index 78dea6cbee..80f8a4e2a2 100644 --- a/transformer_engine/pytorch/tensor/nvfp4_tensor.py +++ b/transformer_engine/pytorch/tensor/nvfp4_tensor.py @@ -168,7 +168,12 @@ def __init__( super().__init__(rowwise=rowwise, columnwise=columnwise) self.dtype = DType.cast(fp4_dtype) self.scale_dtype = DType.cast(scale_dtype) - if self.scale_dtype not in (DType.kFloat8E4M3, DType.kFloat8UE5M3): + # dsv41-mxfp4-e8m0-block32: E8M0 selects MXFP4 (32-element blocks). + if self.scale_dtype not in ( + DType.kFloat8E4M3, + DType.kFloat8UE5M3, + DType.kFloat8E8M0, + ): raise ValueError("scale_dtype must be DType.kFloat8E4M3 or DType.kFloat8UE5M3.") self.with_rht = with_rht self.with_post_rht_amax = with_post_rht_amax