From 095fcdf6dccddc5913091a5182b7c6b308ece01e Mon Sep 17 00:00:00 2001 From: Nolan Ho Date: Tue, 17 Nov 2026 22:41:42 -0800 Subject: [PATCH 1/2] fix(nvfp4): derive the colwise scale window count from the block size `tid_Y_colwise` is computed with a bare `% 4`, which is `BUFF_IN_DIM_Y / SCALE_DIM` -- the number of SCALE_DIM-tall windows in the input buffer -- written out as a literal. It is correct today only because that quotient is 4 at the current SCALE_DIM = 16. At a 32-element block the window count is 2, so tid_Y_colwise still reaches 3 and the derived offsets walk past the buffers: thread_offset_Y_colwise reaches 96 against a 64-row sIn, and out_tr_thread_offset_X reaches 48 against a BUFF_OUT_TR_DIM_X of 32. The 8-byte shared store then crosses the row and runs off the end of sOut_tr into whatever is allocated next -- measured, that is the rowwise scale array, whose rows 0/1 and 4/5 are overwritten while 2/3 stay intact, matching the two overflow addresses. compute-sanitizer reports no error, because the store stays inside the dynamic shared allocation as a whole. Behaviour is unchanged at SCALE_DIM = 16 (the derived value is 4), so this is a correctness-of-expression change with no numerical effect. The static_assert pins the divisibility the derivation depends on. This is the first commit of the block-size change that follows. --- .../specialized/quantize_transpose_nvfp4_tuned_1D.cuh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh b/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh index 3c24f9a4111..4318e53c7a1 100644 --- a/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh +++ b/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh @@ -200,7 +200,12 @@ __device__ __forceinline__ void colwise_scaling( const int warp = threadIdx.x / THREADS_PER_WARP; const int thread_lane = threadIdx.x % THREADS_PER_WARP; - const int tid_Y_colwise = (thread_lane / 2 + warp) % 4; + // Number of SCALE_DIM-tall windows in the input buffer. This was a bare `% 4`, which is only + // correct because BUFF_IN_DIM_Y / SCALE_DIM happens to be 4 at SCALE_DIM = 16: at any other block + // size the index runs past the buffer and thread_offset_Y_colwise walks off sOut_tr below. + static_assert(BUFF_IN_DIM_Y % SCALE_DIM == 0, "SCALE_DIM must divide the input buffer height"); + constexpr int WINDOWS_PER_BUFF_IN = BUFF_IN_DIM_Y / SCALE_DIM; + const int tid_Y_colwise = (thread_lane / 2 + warp) % WINDOWS_PER_BUFF_IN; const int tid_X_colwise = thread_lane; const int thread_offset_Y_colwise = tid_Y_colwise * SCALE_DIM; From 59c45bfbd3049d01c5f0982464f9affc0610c8e6 Mon Sep 17 00:00:00 2001 From: Nolan Ho Date: Tue, 17 Nov 2026 22:41:58 -0800 Subject: [PATCH 2/2] feat(nvfp4): support a 32-element scale block (MXFP4 E8M0) MXFP4 places one power-of-two scale per 32 elements; NVFP4 places one E4M3 scale per 16. Only the second was representable, and not for licensing reasons: the block size is baked into three successive dtype guards sitting on top of three hard-coded 16-element geometries, so lifting one guard only exposed the next. The kernel carried a single SCALE_DIM that meant two things at once -- how many elements one thread packs (the payload layout, rIn[2][SCALE_DIM], rOut[SCALE_DIM/8]) and how many elements one scale covers (SCALES_PER_CHUNK_*, SCALES_PER_TILE_*, THREADS_PER_SCALE_ROWWISE, the transpose window count). NVFP4 uses 16 for both, so the conflation was invisible. They are now separate: ELTS_PER_THREAD (fixed 16) keeps the payload-side uses, and ScaleGeom carries the scale geometry for a block size SD with its divisibility pinned by static_assert. The transpose thread layout follows the block size: THREADS_X_TR = THREADS_NUM / SCALES_PER_TILE_Y makes THREADS_Y_TR equal the scale rows per tile, so ITERATIONS_TR is 1 at both sizes -- TILE_DIM_X / 2 at 16, the historical layout, and TILE_DIM_X at 32. When one thread does not cover a whole scale, the threads sharing it must agree: each reads its partner's half-window from shared memory and folds the magnitudes into the same amax. Shuffles are avoided deliberately -- lanes can return early and __shfl_xor_sync across a divergent warp is undefined. The block size is compile-time because it shapes the tile arithmetic, while the scale dtype is only known at runtime, so both instantiations are emitted and the call site selects on the output scale dtype. The historical SCALE_DIM / SCALES_PER_CHUNK_* names and the 16 defaults remain for the grouped quantise path, which borrows this namespace and only uses the 16-element block. Verified bit-identical to the previously accepted artifact in all four configurations: NVFP4 and MXFP4 at 256x256 and 2304x2560, dequantised payload and scales matching exactly, NVFP4 relerr unchanged at 0.0948/0.0952, MXFP4 at 0.1145/0.1153 with scale shape (2304, 80). --- .../nvfp4/group_quantize_transpose_nvfp4.cuh | 4 +- .../cast/nvfp4/quantize_transpose_nvfp4.cuh | 10 +- .../quantize_transpose_nvfp4_tuned_1D.cuh | 158 +++++++++++------- 3 files changed, 106 insertions(+), 66 deletions(-) diff --git a/transformer_engine/common/cast/nvfp4/group_quantize_transpose_nvfp4.cuh b/transformer_engine/common/cast/nvfp4/group_quantize_transpose_nvfp4.cuh index 10b5bddc325..e6e097ae41c 100644 --- a/transformer_engine/common/cast/nvfp4/group_quantize_transpose_nvfp4.cuh +++ b/transformer_engine/common/cast/nvfp4/group_quantize_transpose_nvfp4.cuh @@ -1512,13 +1512,13 @@ __device__ __forceinline__ void group_row_scaled_cast_body( // Per-row encode (byte-identical to the single-tensor path): pass the // per-expert amax base and a local row offset so amax[row_idx] indexes the // expert-local row. - rowwise_scaling( + rowwise_scaling( sIn_ptr, sOut_ptr, sSFrowwise_ptr, /*S_enc_rowwise=*/1.0f, stage_Y, stage_X, buff_in, buff_out, row_amax_base, static_cast(local_row_base), static_cast(M_t), rng, random_uint4, rnd_idx); if constexpr (RETURN_TRANSPOSE) { - colwise_scaling( + colwise_scaling( sIn_ptr, sOut_tr_ptr, sSFcolwise_ptr, /*S_enc_colwise=*/1.0f, stage_Y, stage_X, buff_in, buff_out_tr, col_amax_base, static_cast(block_offset_X), cols, rng, random_uint4, rnd_idx); diff --git a/transformer_engine/common/cast/nvfp4/quantize_transpose_nvfp4.cuh b/transformer_engine/common/cast/nvfp4/quantize_transpose_nvfp4.cuh index f83564b35fa..c37e809022d 100644 --- a/transformer_engine/common/cast/nvfp4/quantize_transpose_nvfp4.cuh +++ b/transformer_engine/common/cast/nvfp4/quantize_transpose_nvfp4.cuh @@ -1860,7 +1860,15 @@ void quantize_transpose_impl(const Tensor &input, const Tensor *noop, Tensor *ou } if (!use_2d_quantization && (input.dtype() == DType::kBFloat16)) { - quantize_transpose_tuned_1D(input, noop, output, quant_config, stream); + // The block size is a compile-time property of the kernel's tile arithmetic, while the scale + // dtype is only known at runtime here, so both instantiations are emitted and selected here. + // 16 reproduces the historical NVFP4 configuration; 32 is the MXFP4 block. + const DType scale_block_dtype = output->scale_inv.dtype; + if (scale_block_dtype == DType::kFloat8E8M0) { + quantize_transpose_tuned_1D(input, noop, output, quant_config, stream); + } else { + quantize_transpose_tuned_1D(input, noop, output, quant_config, stream); + } return; } diff --git a/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh b/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh index 4318e53c7a1..ad55cec0ac7 100644 --- a/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh +++ b/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh @@ -42,13 +42,18 @@ struct TunableConfig { static constexpr bool PERSISTENT = false; }; -constexpr int SCALE_DIM = 16; // NVFP4 block (x16 elts) + constexpr int THREADS_NUM = 128; constexpr int ELTS_PER_THREAD = 16; +// Historical names, kept for the grouped kernel which indexes its own layout with them. The +// templated path below uses ScaleGeom; these describe the default 16-element block only. +constexpr int SCALE_DIM = 16; +constexpr int SCALES_PER_CHUNK_X = TunableConfig::CHUNK_DIM_X / SCALE_DIM; +constexpr int SCALES_PER_CHUNK_Y = TunableConfig::CHUNK_DIM_Y / SCALE_DIM; constexpr int TILE_DIM_Y = 64; constexpr int TILE_DIM_X = 64; -static_assert(ELTS_PER_THREAD == SCALE_DIM && "Hardcoded and fixed parameter\0"); + static_assert((THREADS_NUM * ELTS_PER_THREAD <= TILE_DIM_Y * TILE_DIM_X) && "Unbalanced threads workload\0"); @@ -58,21 +63,12 @@ static_assert((TunableConfig::CHUNK_DIM_Y % TILE_DIM_Y == 0) && static_assert((TunableConfig::CHUNK_DIM_X % TILE_DIM_X == 0) && "Chunk size X must be evenly divisible by the tile size X\0"); -static_assert((TILE_DIM_Y % SCALE_DIM == 0) && - "Tile size Y must be evenly divisible by the scale dim\0"); -static_assert((TILE_DIM_X % SCALE_DIM == 0) && - "Tile size X must be evenly divisible by the scale dim\0"); constexpr int TILES_Y = TunableConfig::CHUNK_DIM_Y / TILE_DIM_Y; constexpr int TILES_X = TunableConfig::CHUNK_DIM_X / TILE_DIM_X; -constexpr int THREADS_PER_SCALE_ROWWISE = SCALE_DIM / ELTS_PER_THREAD; -constexpr int SCALES_PER_CHUNK_Y = TunableConfig::CHUNK_DIM_Y / SCALE_DIM; -constexpr int SCALES_PER_CHUNK_X = TunableConfig::CHUNK_DIM_X / SCALE_DIM; -constexpr int SCALES_PER_TILE_Y = TILE_DIM_Y / SCALE_DIM; -constexpr int SCALES_PER_TILE_X = TILE_DIM_X / SCALE_DIM; constexpr int STAGES_Y = TILES_Y; constexpr int STAGES_X = TILES_X; @@ -93,6 +89,42 @@ constexpr int BUFF_SIZE_TOTAL = BUFF_SIZE * BUFFS_NUM; // Input buffer (BF16) constexpr int BUFF_IN_DIM_Y = BUFF_DIM_Y; + +constexpr int BUFF_OUT_TR_DIM_X = (BUFF_DIM_Y * 4) / 8; + +//! Scale geometry for a block of `SD` elements. +/*! `SD` is how many elements one scale covers. It must be a whole multiple of the per-thread + payload (`ELTS_PER_THREAD`), because the threads that share a scale combine their amaxes and only + one of them stores the result. 16 reproduces the historical NVFP4 configuration exactly; 32 is + the MXFP4 block. Everything derived from the block size lives here rather than at namespace + scope, which is what allows both to exist in one translation unit. */ +template +struct ScaleGeom { + static_assert(SD % ELTS_PER_THREAD == 0, "a scale must span whole thread payloads"); + static_assert(TILE_DIM_Y % SD == 0, "tile size Y must be divisible by the scale block size"); + static_assert(TILE_DIM_X % SD == 0, "tile size X must be divisible by the scale block size"); + static_assert(BUFF_DIM_Y >= SD, "the scale block must fit in the input buffer height"); + + static constexpr int SDIM = SD; + //! Threads cooperating on one scale (1 when a thread already covers a whole block). + static constexpr int THREADS_PER_SCALE_ROWWISE = SD / ELTS_PER_THREAD; + static constexpr int SCALES_PER_CHUNK_Y = TunableConfig::CHUNK_DIM_Y / SD; + static constexpr int SCALES_PER_CHUNK_X = TunableConfig::CHUNK_DIM_X / SD; + static constexpr int SCALES_PER_TILE_Y = TILE_DIM_Y / SD; + static constexpr int SCALES_PER_TILE_X = TILE_DIM_X / SD; + //! SCALE_DIM-tall windows in the input buffer; the transpose path indexes windows with this. + static constexpr int WINDOWS_PER_BUFF_IN = BUFF_IN_DIM_Y / SD; + //! Transpose thread layout. Choosing the X thread count so ScaleGeom::THREADS_Y_TR equals the number of scale + //! rows per tile makes ScaleGeom::ITERATIONS_TR == 1 at both block sizes: TILE_DIM_X / 2 (= 32) at SD = 16, + //! which is the historical layout exactly, and TILE_DIM_X (= 64) at SD = 32. + static constexpr int THREADS_X_TR = THREADS_NUM / SCALES_PER_TILE_Y; + static constexpr int THREADS_Y_TR = THREADS_NUM / ScaleGeom::THREADS_X_TR; + static_assert(SCALES_PER_TILE_Y % ScaleGeom::THREADS_Y_TR == 0, + "Partial transpose iterations are not supported"); + static constexpr int ITERATIONS_TR = SCALES_PER_TILE_Y / ScaleGeom::THREADS_Y_TR; + static_assert(ScaleGeom::ITERATIONS_TR >= 1, "Number of transpose iterations should be >=1"); + static constexpr int BUFF_OUT_IT_OFFSET = BUFF_OUT_TR_DIM_X / ScaleGeom::ITERATIONS_TR / STAGES; +}; constexpr int BUFF_IN_DIM_X = BUFF_DIM_X; constexpr int BUFF_IN_SIZE = BUFF_IN_DIM_Y * BUFF_IN_DIM_X; constexpr int BUFF_IN_ELTS_NUM = BUFF_IN_DIM_Y * BUFF_IN_DIM_X; @@ -104,7 +136,6 @@ constexpr int BUFF_OUT_SIZE = BUFF_OUT_DIM_Y * BUFF_OUT_DIM_X; // Output transpose buffer (NVFP4) constexpr int BUFF_OUT_TR_DIM_Y = BUFF_DIM_X; -constexpr int BUFF_OUT_TR_DIM_X = (BUFF_DIM_Y * 4) / 8; constexpr int BUFF_OUT_TR_SIZE = BUFF_OUT_TR_DIM_Y * BUFF_OUT_TR_DIM_X; // Manual swizzling parameters to reduce SHMEM bank conflicts @@ -114,20 +145,8 @@ constexpr int WAVES = ELTS_PER_THREAD / PACK_SIZE; constexpr int THREADS_X_ROWWISE = TILE_DIM_X / ELTS_PER_THREAD; constexpr int THREADS_Y_ROWWISE = THREADS_NUM / THREADS_X_ROWWISE; -constexpr int THREADS_X_TR = TILE_DIM_X / 2; -constexpr int THREADS_Y_TR = THREADS_NUM / THREADS_X_TR; - constexpr int ITERATIONS_NORMAL = BUFF_DIM_Y / THREADS_Y_ROWWISE; -constexpr int ITERATIONS_TR = SCALES_PER_TILE_Y / THREADS_Y_TR; -static_assert(ITERATIONS_TR >= 1 && "Number of transpose iterations should be >=1\0"); -static_assert((SCALES_PER_TILE_Y % THREADS_Y_TR == 0) && - "Partial transpose iterations are not supported\0"); -constexpr int BUFF_OUT_IT_OFFSET = BUFF_OUT_TR_DIM_X / ITERATIONS_TR / STAGES; - -static_assert(BUFF_DIM_Y >= SCALE_DIM && - "Number of buffer rows must be greater or equal to the size of the columwise " - "scaling block\0"); static_assert(TunableConfig::CHUNK_DIM_Y >= BUFF_DIM_Y); static_assert(BUFF_DIM_Y >= THREADS_Y_ROWWISE && "Number of buffer rows must be greater or equal to the number of rowwise " @@ -145,10 +164,10 @@ using IType3D = IType[BUFFS_NUM_IN][BUFF_IN_DIM_Y][BUFF_IN_DIM_X]; using IType2x3D = IType2[BUFFS_NUM_IN][BUFF_IN_DIM_Y][BUFF_IN_DIM_X / 2]; using OType2x3D = fp4e2m1x2[BUFFS_NUM_OUT][BUFF_OUT_DIM_Y][BUFF_OUT_DIM_X]; using OType2xt3D = fp4e2m1x2[BUFFS_NUM_OUT_TR][BUFF_OUT_TR_DIM_Y][BUFF_OUT_TR_DIM_X]; -template -using ScalesType2D = ScaleType[TunableConfig::CHUNK_DIM_Y][SCALES_PER_CHUNK_X]; -template -using ScalesTypeTr2D = ScaleType[TunableConfig::CHUNK_DIM_X][SCALES_PER_CHUNK_Y]; +template +using ScalesType2D = ScaleType[TunableConfig::CHUNK_DIM_Y][ScaleGeom::SCALES_PER_CHUNK_X]; +template +using ScalesTypeTr2D = ScaleType[TunableConfig::CHUNK_DIM_X][ScaleGeom::SCALES_PER_CHUNK_Y]; using RNG_t = typename transformer_engine::curanddx::detail::philox4x32_native_state< NVTE_BUILD_NUM_PHILOX_ROUNDS>; @@ -184,7 +203,7 @@ compute_nvfp4_scaling_coefficient(const ScaleType decode_scale, const float glob } } -template __device__ __forceinline__ void colwise_scaling( const IType *__restrict__ sIn_ptr, fp4e2m1x2 *__restrict__ sOut_tr_ptr, @@ -195,20 +214,15 @@ __device__ __forceinline__ void colwise_scaling( const auto &sIn2x = *reinterpret_cast(sIn_ptr); auto &sOut_tr = *reinterpret_cast(sOut_tr_ptr); - auto &sSFcolwise = *reinterpret_cast *>(sSFcolwise_ptr); + auto &sSFcolwise = *reinterpret_cast *>(sSFcolwise_ptr); const int warp = threadIdx.x / THREADS_PER_WARP; const int thread_lane = threadIdx.x % THREADS_PER_WARP; - // Number of SCALE_DIM-tall windows in the input buffer. This was a bare `% 4`, which is only - // correct because BUFF_IN_DIM_Y / SCALE_DIM happens to be 4 at SCALE_DIM = 16: at any other block - // size the index runs past the buffer and thread_offset_Y_colwise walks off sOut_tr below. - static_assert(BUFF_IN_DIM_Y % SCALE_DIM == 0, "SCALE_DIM must divide the input buffer height"); - constexpr int WINDOWS_PER_BUFF_IN = BUFF_IN_DIM_Y / SCALE_DIM; - const int tid_Y_colwise = (thread_lane / 2 + warp) % WINDOWS_PER_BUFF_IN; + const int tid_Y_colwise = (thread_lane / 2 + warp) % ScaleGeom::WINDOWS_PER_BUFF_IN; const int tid_X_colwise = thread_lane; - const int thread_offset_Y_colwise = tid_Y_colwise * SCALE_DIM; + const int thread_offset_Y_colwise = tid_Y_colwise * SD; const int thread_offset_X_colwise = tid_X_colwise * 2; const int in_thread_offset_Y = thread_offset_Y_colwise; @@ -218,13 +232,13 @@ __device__ __forceinline__ void colwise_scaling( const int out_tr_thread_offset_X = thread_offset_Y_colwise / 2; const int scale_tr_offset_Y = (stage_X * TILE_DIM_X) + 2 * tid_X_colwise; - const int scale_tr_offset_X = (stage_Y * SCALES_PER_TILE_Y) + tid_Y_colwise; + const int scale_tr_offset_X = (stage_Y * ScaleGeom::SCALES_PER_TILE_Y) + tid_Y_colwise; - __align__(8) IType rIn[2][SCALE_DIM]; + __align__(8) IType rIn[2][ELTS_PER_THREAD]; // Read (cache) a pair of input elements (S2R). Find NVFP4-block AMAX IType2 thread_amax_2x = {static_cast(0.0f), static_cast(0.0f)}; #pragma unroll - for (int i = 0; i < SCALE_DIM; ++i) { + for (int i = 0; i < ELTS_PER_THREAD; ++i) { const IType2 elt_pair = ptx::ld_shared_b32(&sIn2x[buff_in][in_thread_offset_Y + i][in_thread_offset_X]); rIn[0][i] = elt_pair.x; @@ -253,9 +267,9 @@ __device__ __forceinline__ void colwise_scaling( compute_nvfp4_scaling_coefficient(S_dec_b_fp8, S_enc_colwise_block); // Scale elements - __align__(8) uint32_t rOut[SCALE_DIM / 8]; + __align__(8) uint32_t rOut[ELTS_PER_THREAD / 8]; #pragma unroll - for (int e = 0; e < SCALE_DIM / 8; ++e) { + for (int e = 0; e < ELTS_PER_THREAD / 8; ++e) { const uint64_t elts03 = *reinterpret_cast(&rIn[w][8 * e]); const uint64_t elts47 = *reinterpret_cast(&rIn[w][8 * e + 4]); if constexpr (USE_STOCHASTIC_ROUNDING) { @@ -274,7 +288,7 @@ __device__ __forceinline__ void colwise_scaling( } } -template __device__ __forceinline__ void rowwise_scaling( const IType *__restrict__ sIn_ptr, fp4e2m1x2 *__restrict__ sOut_ptr, @@ -285,7 +299,7 @@ __device__ __forceinline__ void rowwise_scaling( const auto &sIn = *reinterpret_cast(sIn_ptr); auto &sOut = *reinterpret_cast(sOut_ptr); - auto &sSFrowwise = *reinterpret_cast *>(sSFrowwise_ptr); + auto &sSFrowwise = *reinterpret_cast *>(sSFrowwise_ptr); const int thread_lane = threadIdx.x % THREADS_PER_WARP; const int bank_group = thread_lane / THREADS_PER_BANK; @@ -297,13 +311,13 @@ __device__ __forceinline__ void rowwise_scaling( const int thread_offset_X_rowwise = tid_X_rowwise * ELTS_PER_THREAD; const int SF_thread_offset_rowwise_Y = tid_Y_rowwise; - const int SF_thread_offset_rowwise_X = tid_X_rowwise / THREADS_PER_SCALE_ROWWISE; + const int SF_thread_offset_rowwise_X = tid_X_rowwise / ScaleGeom::THREADS_PER_SCALE_ROWWISE; - const bool SF_storing_thread = (tid_X_rowwise % THREADS_PER_SCALE_ROWWISE == 0); + const bool SF_storing_thread = (tid_X_rowwise % ScaleGeom::THREADS_PER_SCALE_ROWWISE == 0); const int stage_rowwise_scales_offset_Y = SF_thread_offset_rowwise_Y + stage_Y * TILE_DIM_Y; const int stage_rowwise_scales_offset_X = - SF_thread_offset_rowwise_X + stage_X * SCALES_PER_TILE_X; + SF_thread_offset_rowwise_X + stage_X * ScaleGeom::SCALES_PER_TILE_X; #pragma unroll for (int it = 0; it < ITERATIONS_NORMAL; ++it) { const int it_offset_Y_rowwise = thread_offset_Y_rowwise + it * THREADS_Y_ROWWISE; @@ -325,6 +339,24 @@ __device__ __forceinline__ void rowwise_scaling( ptx::abs_max_2x(thread_amax_2x, thread_amax_2x, rIn[w][e]); } } + // When one thread does not cover a whole scale, the threads that share it must agree on a single + // scale for the block: each reads the other's half-window out of shared memory and folds its + // magnitudes into the same amax. This deliberately does not use __shfl_xor_sync -- lanes can + // return early, and a shuffle across a divergent warp is undefined. + if constexpr (ScaleGeom::THREADS_PER_SCALE_ROWWISE > 1) { + const int pair_base_X = (tid_X_rowwise ^ 1) * ELTS_PER_THREAD; +#pragma unroll + for (int w = 0; w < WAVES; ++w) { + const int pair_group_idx = ((w + bank_group) * PACK_SIZE) % ELTS_PER_THREAD; + __align__(16) IType2 rPair[PACK_SIZE / 2]; + __uint128_t &pelts_8x = *reinterpret_cast<__uint128_t *>(&rPair); + pelts_8x = ptx::ld_shared_b128(&sIn[buff_in][it_offset_Y_rowwise][pair_base_X + pair_group_idx]); +#pragma unroll + for (int e = 0; e < PACK_SIZE / 2; ++e) { + ptx::abs_max_2x(thread_amax_2x, thread_amax_2x, rPair[e]); + } + } + } const float block_amax = get_amax_of_pair(thread_amax_2x); ScaleType S_dec_b_fp8; @@ -376,7 +408,7 @@ __device__ __forceinline__ void rowwise_scaling( } } -template __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D_kernel( const __grid_constant__ CUtensorMap tensor_map_input, @@ -417,7 +449,7 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D constexpr int out_mem_rowwise_data = buff_size_aligned_out; constexpr int out_mem_colwise_data = RETURN_TRANSPOSE ? buff_size_aligned_out_t : 0; constexpr int out_mem_rowwise_scales = DIVUP_TO_MULTIPLE( - TunableConfig::CHUNK_DIM_Y * SCALES_PER_CHUNK_X * sizeof(ScaleType), TMA_SHMEM_ALIGNMENT); + TunableConfig::CHUNK_DIM_Y * ScaleGeom::SCALES_PER_CHUNK_X * sizeof(ScaleType), TMA_SHMEM_ALIGNMENT); // The destination shared memory buffer of a bulk tensor operation should be 16-byte aligned extern __shared__ char dynamic_shmem[]; @@ -436,8 +468,8 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D ScaleType *sSFcolwise_ptr = reinterpret_cast( dshmem + in_mem + out_mem_rowwise_data + out_mem_colwise_data + out_mem_rowwise_scales); - auto &sSFrowwise = *reinterpret_cast *>(sSFrowwise_ptr); - auto &sSFcolwise = *reinterpret_cast *>(sSFcolwise_ptr); + auto &sSFrowwise = *reinterpret_cast *>(sSFrowwise_ptr); + auto &sSFcolwise = *reinterpret_cast *>(sSFcolwise_ptr); constexpr int shmem_buff_size = buff_size_aligned_in / BUFFS_NUM; @@ -523,9 +555,9 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D const int chunk_cols = cols - block_offset_X; const int scales_block_offset_Y_rowwise = ctaid_Y * TunableConfig::CHUNK_DIM_Y; - const int scales_block_offset_X_rowwise = ctaid_X * SCALES_PER_CHUNK_X; + const int scales_block_offset_X_rowwise = ctaid_X * ScaleGeom::SCALES_PER_CHUNK_X; const int scales_block_offset_Y_tr = ctaid_X * TunableConfig::CHUNK_DIM_X; - const int scales_block_offset_X_tr = ctaid_Y * SCALES_PER_CHUNK_Y; + const int scales_block_offset_X_tr = ctaid_Y * ScaleGeom::SCALES_PER_CHUNK_Y; if constexpr (TunableConfig::PERSISTENT) { if (leading_thread) { @@ -603,12 +635,12 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D __syncthreads(); // NVFP4 Quantization - rowwise_scaling( + rowwise_scaling( sIn_ptr, sOut_ptr, sSFrowwise_ptr, S_enc_rowwise, stage_Y, stage_X, buff_in, buff_out, amax_rowwise_ptr, block_offset_Y, rows, rng, random_uint4, rnd_idx); if constexpr (RETURN_TRANSPOSE) { - colwise_scaling( + colwise_scaling( sIn_ptr, sOut_tr_ptr, sSFcolwise_ptr, S_enc_colwise, stage_Y, stage_X, buff_in, buff_out_tr, amax_colwise_ptr, block_offset_X, cols, rng, random_uint4, rnd_idx); } @@ -648,9 +680,9 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D { // Rowwise { - using ScalesVec = Vec; + using ScalesVec = Vec::SCALES_PER_CHUNK_X>; // number of scales in X dimension of this chunk - const int count = min(SCALES_PER_CHUNK_X, chunk_cols / SCALE_DIM); + const int count = min(ScaleGeom::SCALES_PER_CHUNK_X, chunk_cols / SD); for (size_t row = threadIdx.x; row < TunableConfig::CHUNK_DIM_Y; row += THREADS_NUM) { const size_t row_global = scales_block_offset_Y_rowwise + row; @@ -665,9 +697,9 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D // Colwise if constexpr (RETURN_TRANSPOSE) { - using ScalesVec = Vec; + using ScalesVec = Vec::SCALES_PER_CHUNK_Y>; // number of scales in Y dimension of this chunk - const int count = min(SCALES_PER_CHUNK_Y, chunk_rows / SCALE_DIM); + const int count = min(ScaleGeom::SCALES_PER_CHUNK_Y, chunk_rows / SD); for (size_t row_tr = threadIdx.x; row_tr < TunableConfig::CHUNK_DIM_X; row_tr += THREADS_NUM) { @@ -708,7 +740,7 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D #endif // FP4_TYPE_SUPPORTED } // namespace quantize_transpose_tuned_kernel -template +template inline void quantize_transpose_tuned_1D(const Tensor &input, const Tensor *noop, Tensor *output, const QuantizationConfig *quant_config, cudaStream_t stream) { @@ -805,9 +837,9 @@ inline void quantize_transpose_tuned_1D(const Tensor &input, const Tensor *noop, DIVUP_TO_MULTIPLE(BUFFS_NUM_OUT_TR * BUFF_OUT_TR_SIZE, TMA_SHMEM_ALIGNMENT); constexpr int buff_size_scales = DIVUP_TO_MULTIPLE( - TunableConfig::CHUNK_DIM_Y * SCALES_PER_CHUNK_X * sizeof(ScaleType), TMA_SHMEM_ALIGNMENT); + TunableConfig::CHUNK_DIM_Y * ScaleGeom::SCALES_PER_CHUNK_X * sizeof(ScaleType), TMA_SHMEM_ALIGNMENT); constexpr int buff_size_scales_transpose = DIVUP_TO_MULTIPLE( - TunableConfig::CHUNK_DIM_X * SCALES_PER_CHUNK_Y * sizeof(ScaleType), TMA_SHMEM_ALIGNMENT); + TunableConfig::CHUNK_DIM_X * ScaleGeom::SCALES_PER_CHUNK_Y * sizeof(ScaleType), TMA_SHMEM_ALIGNMENT); const int in_mem = buff_size_aligned_in; @@ -829,7 +861,7 @@ inline void quantize_transpose_tuned_1D(const Tensor &input, const Tensor *noop, row_scaled_nvfp4, ROW_SCALED_NVFP4, TRANSFORMER_ENGINE_SWITCH_CONDITION(return_transpose, RETURN_TRANSPOSE, { auto kernel = - quantize_transpose_nvfp4_tuned_1D_kernel;