diff --git a/transformer_engine/common/cast/nvfp4/group_quantize_transpose_nvfp4.cuh b/transformer_engine/common/cast/nvfp4/group_quantize_transpose_nvfp4.cuh index 10b5bddc325..e6e097ae41c 100644 --- a/transformer_engine/common/cast/nvfp4/group_quantize_transpose_nvfp4.cuh +++ b/transformer_engine/common/cast/nvfp4/group_quantize_transpose_nvfp4.cuh @@ -1512,13 +1512,13 @@ __device__ __forceinline__ void group_row_scaled_cast_body( // Per-row encode (byte-identical to the single-tensor path): pass the // per-expert amax base and a local row offset so amax[row_idx] indexes the // expert-local row. - rowwise_scaling( + rowwise_scaling( sIn_ptr, sOut_ptr, sSFrowwise_ptr, /*S_enc_rowwise=*/1.0f, stage_Y, stage_X, buff_in, buff_out, row_amax_base, static_cast(local_row_base), static_cast(M_t), rng, random_uint4, rnd_idx); if constexpr (RETURN_TRANSPOSE) { - colwise_scaling( + colwise_scaling( sIn_ptr, sOut_tr_ptr, sSFcolwise_ptr, /*S_enc_colwise=*/1.0f, stage_Y, stage_X, buff_in, buff_out_tr, col_amax_base, static_cast(block_offset_X), cols, rng, random_uint4, rnd_idx); diff --git a/transformer_engine/common/cast/nvfp4/quantize_transpose_nvfp4.cuh b/transformer_engine/common/cast/nvfp4/quantize_transpose_nvfp4.cuh index f83564b35fa..c37e809022d 100644 --- a/transformer_engine/common/cast/nvfp4/quantize_transpose_nvfp4.cuh +++ b/transformer_engine/common/cast/nvfp4/quantize_transpose_nvfp4.cuh @@ -1860,7 +1860,15 @@ void quantize_transpose_impl(const Tensor &input, const Tensor *noop, Tensor *ou } if (!use_2d_quantization && (input.dtype() == DType::kBFloat16)) { - quantize_transpose_tuned_1D(input, noop, output, quant_config, stream); + // The block size is a compile-time property of the kernel's tile arithmetic, while the scale + // dtype is only known at runtime here, so both instantiations are emitted and selected here. + // 16 reproduces the historical NVFP4 configuration; 32 is the MXFP4 block. + const DType scale_block_dtype = output->scale_inv.dtype; + if (scale_block_dtype == DType::kFloat8E8M0) { + quantize_transpose_tuned_1D(input, noop, output, quant_config, stream); + } else { + quantize_transpose_tuned_1D(input, noop, output, quant_config, stream); + } return; } diff --git a/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh b/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh index 3c24f9a4111..ad55cec0ac7 100644 --- a/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh +++ b/transformer_engine/common/cast/nvfp4/specialized/quantize_transpose_nvfp4_tuned_1D.cuh @@ -42,13 +42,18 @@ struct TunableConfig { static constexpr bool PERSISTENT = false; }; -constexpr int SCALE_DIM = 16; // NVFP4 block (x16 elts) + constexpr int THREADS_NUM = 128; constexpr int ELTS_PER_THREAD = 16; +// Historical names, kept for the grouped kernel which indexes its own layout with them. The +// templated path below uses ScaleGeom; these describe the default 16-element block only. +constexpr int SCALE_DIM = 16; +constexpr int SCALES_PER_CHUNK_X = TunableConfig::CHUNK_DIM_X / SCALE_DIM; +constexpr int SCALES_PER_CHUNK_Y = TunableConfig::CHUNK_DIM_Y / SCALE_DIM; constexpr int TILE_DIM_Y = 64; constexpr int TILE_DIM_X = 64; -static_assert(ELTS_PER_THREAD == SCALE_DIM && "Hardcoded and fixed parameter\0"); + static_assert((THREADS_NUM * ELTS_PER_THREAD <= TILE_DIM_Y * TILE_DIM_X) && "Unbalanced threads workload\0"); @@ -58,21 +63,12 @@ static_assert((TunableConfig::CHUNK_DIM_Y % TILE_DIM_Y == 0) && static_assert((TunableConfig::CHUNK_DIM_X % TILE_DIM_X == 0) && "Chunk size X must be evenly divisible by the tile size X\0"); -static_assert((TILE_DIM_Y % SCALE_DIM == 0) && - "Tile size Y must be evenly divisible by the scale dim\0"); -static_assert((TILE_DIM_X % SCALE_DIM == 0) && - "Tile size X must be evenly divisible by the scale dim\0"); constexpr int TILES_Y = TunableConfig::CHUNK_DIM_Y / TILE_DIM_Y; constexpr int TILES_X = TunableConfig::CHUNK_DIM_X / TILE_DIM_X; -constexpr int THREADS_PER_SCALE_ROWWISE = SCALE_DIM / ELTS_PER_THREAD; -constexpr int SCALES_PER_CHUNK_Y = TunableConfig::CHUNK_DIM_Y / SCALE_DIM; -constexpr int SCALES_PER_CHUNK_X = TunableConfig::CHUNK_DIM_X / SCALE_DIM; -constexpr int SCALES_PER_TILE_Y = TILE_DIM_Y / SCALE_DIM; -constexpr int SCALES_PER_TILE_X = TILE_DIM_X / SCALE_DIM; constexpr int STAGES_Y = TILES_Y; constexpr int STAGES_X = TILES_X; @@ -93,6 +89,42 @@ constexpr int BUFF_SIZE_TOTAL = BUFF_SIZE * BUFFS_NUM; // Input buffer (BF16) constexpr int BUFF_IN_DIM_Y = BUFF_DIM_Y; + +constexpr int BUFF_OUT_TR_DIM_X = (BUFF_DIM_Y * 4) / 8; + +//! Scale geometry for a block of `SD` elements. +/*! `SD` is how many elements one scale covers. It must be a whole multiple of the per-thread + payload (`ELTS_PER_THREAD`), because the threads that share a scale combine their amaxes and only + one of them stores the result. 16 reproduces the historical NVFP4 configuration exactly; 32 is + the MXFP4 block. Everything derived from the block size lives here rather than at namespace + scope, which is what allows both to exist in one translation unit. */ +template +struct ScaleGeom { + static_assert(SD % ELTS_PER_THREAD == 0, "a scale must span whole thread payloads"); + static_assert(TILE_DIM_Y % SD == 0, "tile size Y must be divisible by the scale block size"); + static_assert(TILE_DIM_X % SD == 0, "tile size X must be divisible by the scale block size"); + static_assert(BUFF_DIM_Y >= SD, "the scale block must fit in the input buffer height"); + + static constexpr int SDIM = SD; + //! Threads cooperating on one scale (1 when a thread already covers a whole block). + static constexpr int THREADS_PER_SCALE_ROWWISE = SD / ELTS_PER_THREAD; + static constexpr int SCALES_PER_CHUNK_Y = TunableConfig::CHUNK_DIM_Y / SD; + static constexpr int SCALES_PER_CHUNK_X = TunableConfig::CHUNK_DIM_X / SD; + static constexpr int SCALES_PER_TILE_Y = TILE_DIM_Y / SD; + static constexpr int SCALES_PER_TILE_X = TILE_DIM_X / SD; + //! SCALE_DIM-tall windows in the input buffer; the transpose path indexes windows with this. + static constexpr int WINDOWS_PER_BUFF_IN = BUFF_IN_DIM_Y / SD; + //! Transpose thread layout. Choosing the X thread count so ScaleGeom::THREADS_Y_TR equals the number of scale + //! rows per tile makes ScaleGeom::ITERATIONS_TR == 1 at both block sizes: TILE_DIM_X / 2 (= 32) at SD = 16, + //! which is the historical layout exactly, and TILE_DIM_X (= 64) at SD = 32. + static constexpr int THREADS_X_TR = THREADS_NUM / SCALES_PER_TILE_Y; + static constexpr int THREADS_Y_TR = THREADS_NUM / ScaleGeom::THREADS_X_TR; + static_assert(SCALES_PER_TILE_Y % ScaleGeom::THREADS_Y_TR == 0, + "Partial transpose iterations are not supported"); + static constexpr int ITERATIONS_TR = SCALES_PER_TILE_Y / ScaleGeom::THREADS_Y_TR; + static_assert(ScaleGeom::ITERATIONS_TR >= 1, "Number of transpose iterations should be >=1"); + static constexpr int BUFF_OUT_IT_OFFSET = BUFF_OUT_TR_DIM_X / ScaleGeom::ITERATIONS_TR / STAGES; +}; constexpr int BUFF_IN_DIM_X = BUFF_DIM_X; constexpr int BUFF_IN_SIZE = BUFF_IN_DIM_Y * BUFF_IN_DIM_X; constexpr int BUFF_IN_ELTS_NUM = BUFF_IN_DIM_Y * BUFF_IN_DIM_X; @@ -104,7 +136,6 @@ constexpr int BUFF_OUT_SIZE = BUFF_OUT_DIM_Y * BUFF_OUT_DIM_X; // Output transpose buffer (NVFP4) constexpr int BUFF_OUT_TR_DIM_Y = BUFF_DIM_X; -constexpr int BUFF_OUT_TR_DIM_X = (BUFF_DIM_Y * 4) / 8; constexpr int BUFF_OUT_TR_SIZE = BUFF_OUT_TR_DIM_Y * BUFF_OUT_TR_DIM_X; // Manual swizzling parameters to reduce SHMEM bank conflicts @@ -114,20 +145,8 @@ constexpr int WAVES = ELTS_PER_THREAD / PACK_SIZE; constexpr int THREADS_X_ROWWISE = TILE_DIM_X / ELTS_PER_THREAD; constexpr int THREADS_Y_ROWWISE = THREADS_NUM / THREADS_X_ROWWISE; -constexpr int THREADS_X_TR = TILE_DIM_X / 2; -constexpr int THREADS_Y_TR = THREADS_NUM / THREADS_X_TR; - constexpr int ITERATIONS_NORMAL = BUFF_DIM_Y / THREADS_Y_ROWWISE; -constexpr int ITERATIONS_TR = SCALES_PER_TILE_Y / THREADS_Y_TR; -static_assert(ITERATIONS_TR >= 1 && "Number of transpose iterations should be >=1\0"); -static_assert((SCALES_PER_TILE_Y % THREADS_Y_TR == 0) && - "Partial transpose iterations are not supported\0"); -constexpr int BUFF_OUT_IT_OFFSET = BUFF_OUT_TR_DIM_X / ITERATIONS_TR / STAGES; - -static_assert(BUFF_DIM_Y >= SCALE_DIM && - "Number of buffer rows must be greater or equal to the size of the columwise " - "scaling block\0"); static_assert(TunableConfig::CHUNK_DIM_Y >= BUFF_DIM_Y); static_assert(BUFF_DIM_Y >= THREADS_Y_ROWWISE && "Number of buffer rows must be greater or equal to the number of rowwise " @@ -145,10 +164,10 @@ using IType3D = IType[BUFFS_NUM_IN][BUFF_IN_DIM_Y][BUFF_IN_DIM_X]; using IType2x3D = IType2[BUFFS_NUM_IN][BUFF_IN_DIM_Y][BUFF_IN_DIM_X / 2]; using OType2x3D = fp4e2m1x2[BUFFS_NUM_OUT][BUFF_OUT_DIM_Y][BUFF_OUT_DIM_X]; using OType2xt3D = fp4e2m1x2[BUFFS_NUM_OUT_TR][BUFF_OUT_TR_DIM_Y][BUFF_OUT_TR_DIM_X]; -template -using ScalesType2D = ScaleType[TunableConfig::CHUNK_DIM_Y][SCALES_PER_CHUNK_X]; -template -using ScalesTypeTr2D = ScaleType[TunableConfig::CHUNK_DIM_X][SCALES_PER_CHUNK_Y]; +template +using ScalesType2D = ScaleType[TunableConfig::CHUNK_DIM_Y][ScaleGeom::SCALES_PER_CHUNK_X]; +template +using ScalesTypeTr2D = ScaleType[TunableConfig::CHUNK_DIM_X][ScaleGeom::SCALES_PER_CHUNK_Y]; using RNG_t = typename transformer_engine::curanddx::detail::philox4x32_native_state< NVTE_BUILD_NUM_PHILOX_ROUNDS>; @@ -184,7 +203,7 @@ compute_nvfp4_scaling_coefficient(const ScaleType decode_scale, const float glob } } -template __device__ __forceinline__ void colwise_scaling( const IType *__restrict__ sIn_ptr, fp4e2m1x2 *__restrict__ sOut_tr_ptr, @@ -195,15 +214,15 @@ __device__ __forceinline__ void colwise_scaling( const auto &sIn2x = *reinterpret_cast(sIn_ptr); auto &sOut_tr = *reinterpret_cast(sOut_tr_ptr); - auto &sSFcolwise = *reinterpret_cast *>(sSFcolwise_ptr); + auto &sSFcolwise = *reinterpret_cast *>(sSFcolwise_ptr); const int warp = threadIdx.x / THREADS_PER_WARP; const int thread_lane = threadIdx.x % THREADS_PER_WARP; - const int tid_Y_colwise = (thread_lane / 2 + warp) % 4; + const int tid_Y_colwise = (thread_lane / 2 + warp) % ScaleGeom::WINDOWS_PER_BUFF_IN; const int tid_X_colwise = thread_lane; - const int thread_offset_Y_colwise = tid_Y_colwise * SCALE_DIM; + const int thread_offset_Y_colwise = tid_Y_colwise * SD; const int thread_offset_X_colwise = tid_X_colwise * 2; const int in_thread_offset_Y = thread_offset_Y_colwise; @@ -213,13 +232,13 @@ __device__ __forceinline__ void colwise_scaling( const int out_tr_thread_offset_X = thread_offset_Y_colwise / 2; const int scale_tr_offset_Y = (stage_X * TILE_DIM_X) + 2 * tid_X_colwise; - const int scale_tr_offset_X = (stage_Y * SCALES_PER_TILE_Y) + tid_Y_colwise; + const int scale_tr_offset_X = (stage_Y * ScaleGeom::SCALES_PER_TILE_Y) + tid_Y_colwise; - __align__(8) IType rIn[2][SCALE_DIM]; + __align__(8) IType rIn[2][ELTS_PER_THREAD]; // Read (cache) a pair of input elements (S2R). Find NVFP4-block AMAX IType2 thread_amax_2x = {static_cast(0.0f), static_cast(0.0f)}; #pragma unroll - for (int i = 0; i < SCALE_DIM; ++i) { + for (int i = 0; i < ELTS_PER_THREAD; ++i) { const IType2 elt_pair = ptx::ld_shared_b32(&sIn2x[buff_in][in_thread_offset_Y + i][in_thread_offset_X]); rIn[0][i] = elt_pair.x; @@ -248,9 +267,9 @@ __device__ __forceinline__ void colwise_scaling( compute_nvfp4_scaling_coefficient(S_dec_b_fp8, S_enc_colwise_block); // Scale elements - __align__(8) uint32_t rOut[SCALE_DIM / 8]; + __align__(8) uint32_t rOut[ELTS_PER_THREAD / 8]; #pragma unroll - for (int e = 0; e < SCALE_DIM / 8; ++e) { + for (int e = 0; e < ELTS_PER_THREAD / 8; ++e) { const uint64_t elts03 = *reinterpret_cast(&rIn[w][8 * e]); const uint64_t elts47 = *reinterpret_cast(&rIn[w][8 * e + 4]); if constexpr (USE_STOCHASTIC_ROUNDING) { @@ -269,7 +288,7 @@ __device__ __forceinline__ void colwise_scaling( } } -template __device__ __forceinline__ void rowwise_scaling( const IType *__restrict__ sIn_ptr, fp4e2m1x2 *__restrict__ sOut_ptr, @@ -280,7 +299,7 @@ __device__ __forceinline__ void rowwise_scaling( const auto &sIn = *reinterpret_cast(sIn_ptr); auto &sOut = *reinterpret_cast(sOut_ptr); - auto &sSFrowwise = *reinterpret_cast *>(sSFrowwise_ptr); + auto &sSFrowwise = *reinterpret_cast *>(sSFrowwise_ptr); const int thread_lane = threadIdx.x % THREADS_PER_WARP; const int bank_group = thread_lane / THREADS_PER_BANK; @@ -292,13 +311,13 @@ __device__ __forceinline__ void rowwise_scaling( const int thread_offset_X_rowwise = tid_X_rowwise * ELTS_PER_THREAD; const int SF_thread_offset_rowwise_Y = tid_Y_rowwise; - const int SF_thread_offset_rowwise_X = tid_X_rowwise / THREADS_PER_SCALE_ROWWISE; + const int SF_thread_offset_rowwise_X = tid_X_rowwise / ScaleGeom::THREADS_PER_SCALE_ROWWISE; - const bool SF_storing_thread = (tid_X_rowwise % THREADS_PER_SCALE_ROWWISE == 0); + const bool SF_storing_thread = (tid_X_rowwise % ScaleGeom::THREADS_PER_SCALE_ROWWISE == 0); const int stage_rowwise_scales_offset_Y = SF_thread_offset_rowwise_Y + stage_Y * TILE_DIM_Y; const int stage_rowwise_scales_offset_X = - SF_thread_offset_rowwise_X + stage_X * SCALES_PER_TILE_X; + SF_thread_offset_rowwise_X + stage_X * ScaleGeom::SCALES_PER_TILE_X; #pragma unroll for (int it = 0; it < ITERATIONS_NORMAL; ++it) { const int it_offset_Y_rowwise = thread_offset_Y_rowwise + it * THREADS_Y_ROWWISE; @@ -320,6 +339,24 @@ __device__ __forceinline__ void rowwise_scaling( ptx::abs_max_2x(thread_amax_2x, thread_amax_2x, rIn[w][e]); } } + // When one thread does not cover a whole scale, the threads that share it must agree on a single + // scale for the block: each reads the other's half-window out of shared memory and folds its + // magnitudes into the same amax. This deliberately does not use __shfl_xor_sync -- lanes can + // return early, and a shuffle across a divergent warp is undefined. + if constexpr (ScaleGeom::THREADS_PER_SCALE_ROWWISE > 1) { + const int pair_base_X = (tid_X_rowwise ^ 1) * ELTS_PER_THREAD; +#pragma unroll + for (int w = 0; w < WAVES; ++w) { + const int pair_group_idx = ((w + bank_group) * PACK_SIZE) % ELTS_PER_THREAD; + __align__(16) IType2 rPair[PACK_SIZE / 2]; + __uint128_t &pelts_8x = *reinterpret_cast<__uint128_t *>(&rPair); + pelts_8x = ptx::ld_shared_b128(&sIn[buff_in][it_offset_Y_rowwise][pair_base_X + pair_group_idx]); +#pragma unroll + for (int e = 0; e < PACK_SIZE / 2; ++e) { + ptx::abs_max_2x(thread_amax_2x, thread_amax_2x, rPair[e]); + } + } + } const float block_amax = get_amax_of_pair(thread_amax_2x); ScaleType S_dec_b_fp8; @@ -371,7 +408,7 @@ __device__ __forceinline__ void rowwise_scaling( } } -template __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D_kernel( const __grid_constant__ CUtensorMap tensor_map_input, @@ -412,7 +449,7 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D constexpr int out_mem_rowwise_data = buff_size_aligned_out; constexpr int out_mem_colwise_data = RETURN_TRANSPOSE ? buff_size_aligned_out_t : 0; constexpr int out_mem_rowwise_scales = DIVUP_TO_MULTIPLE( - TunableConfig::CHUNK_DIM_Y * SCALES_PER_CHUNK_X * sizeof(ScaleType), TMA_SHMEM_ALIGNMENT); + TunableConfig::CHUNK_DIM_Y * ScaleGeom::SCALES_PER_CHUNK_X * sizeof(ScaleType), TMA_SHMEM_ALIGNMENT); // The destination shared memory buffer of a bulk tensor operation should be 16-byte aligned extern __shared__ char dynamic_shmem[]; @@ -431,8 +468,8 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D ScaleType *sSFcolwise_ptr = reinterpret_cast( dshmem + in_mem + out_mem_rowwise_data + out_mem_colwise_data + out_mem_rowwise_scales); - auto &sSFrowwise = *reinterpret_cast *>(sSFrowwise_ptr); - auto &sSFcolwise = *reinterpret_cast *>(sSFcolwise_ptr); + auto &sSFrowwise = *reinterpret_cast *>(sSFrowwise_ptr); + auto &sSFcolwise = *reinterpret_cast *>(sSFcolwise_ptr); constexpr int shmem_buff_size = buff_size_aligned_in / BUFFS_NUM; @@ -518,9 +555,9 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D const int chunk_cols = cols - block_offset_X; const int scales_block_offset_Y_rowwise = ctaid_Y * TunableConfig::CHUNK_DIM_Y; - const int scales_block_offset_X_rowwise = ctaid_X * SCALES_PER_CHUNK_X; + const int scales_block_offset_X_rowwise = ctaid_X * ScaleGeom::SCALES_PER_CHUNK_X; const int scales_block_offset_Y_tr = ctaid_X * TunableConfig::CHUNK_DIM_X; - const int scales_block_offset_X_tr = ctaid_Y * SCALES_PER_CHUNK_Y; + const int scales_block_offset_X_tr = ctaid_Y * ScaleGeom::SCALES_PER_CHUNK_Y; if constexpr (TunableConfig::PERSISTENT) { if (leading_thread) { @@ -598,12 +635,12 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D __syncthreads(); // NVFP4 Quantization - rowwise_scaling( + rowwise_scaling( sIn_ptr, sOut_ptr, sSFrowwise_ptr, S_enc_rowwise, stage_Y, stage_X, buff_in, buff_out, amax_rowwise_ptr, block_offset_Y, rows, rng, random_uint4, rnd_idx); if constexpr (RETURN_TRANSPOSE) { - colwise_scaling( + colwise_scaling( sIn_ptr, sOut_tr_ptr, sSFcolwise_ptr, S_enc_colwise, stage_Y, stage_X, buff_in, buff_out_tr, amax_colwise_ptr, block_offset_X, cols, rng, random_uint4, rnd_idx); } @@ -643,9 +680,9 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D { // Rowwise { - using ScalesVec = Vec; + using ScalesVec = Vec::SCALES_PER_CHUNK_X>; // number of scales in X dimension of this chunk - const int count = min(SCALES_PER_CHUNK_X, chunk_cols / SCALE_DIM); + const int count = min(ScaleGeom::SCALES_PER_CHUNK_X, chunk_cols / SD); for (size_t row = threadIdx.x; row < TunableConfig::CHUNK_DIM_Y; row += THREADS_NUM) { const size_t row_global = scales_block_offset_Y_rowwise + row; @@ -660,9 +697,9 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D // Colwise if constexpr (RETURN_TRANSPOSE) { - using ScalesVec = Vec; + using ScalesVec = Vec::SCALES_PER_CHUNK_Y>; // number of scales in Y dimension of this chunk - const int count = min(SCALES_PER_CHUNK_Y, chunk_rows / SCALE_DIM); + const int count = min(ScaleGeom::SCALES_PER_CHUNK_Y, chunk_rows / SD); for (size_t row_tr = threadIdx.x; row_tr < TunableConfig::CHUNK_DIM_X; row_tr += THREADS_NUM) { @@ -703,7 +740,7 @@ __global__ void __launch_bounds__(THREADS_NUM) quantize_transpose_nvfp4_tuned_1D #endif // FP4_TYPE_SUPPORTED } // namespace quantize_transpose_tuned_kernel -template +template inline void quantize_transpose_tuned_1D(const Tensor &input, const Tensor *noop, Tensor *output, const QuantizationConfig *quant_config, cudaStream_t stream) { @@ -800,9 +837,9 @@ inline void quantize_transpose_tuned_1D(const Tensor &input, const Tensor *noop, DIVUP_TO_MULTIPLE(BUFFS_NUM_OUT_TR * BUFF_OUT_TR_SIZE, TMA_SHMEM_ALIGNMENT); constexpr int buff_size_scales = DIVUP_TO_MULTIPLE( - TunableConfig::CHUNK_DIM_Y * SCALES_PER_CHUNK_X * sizeof(ScaleType), TMA_SHMEM_ALIGNMENT); + TunableConfig::CHUNK_DIM_Y * ScaleGeom::SCALES_PER_CHUNK_X * sizeof(ScaleType), TMA_SHMEM_ALIGNMENT); constexpr int buff_size_scales_transpose = DIVUP_TO_MULTIPLE( - TunableConfig::CHUNK_DIM_X * SCALES_PER_CHUNK_Y * sizeof(ScaleType), TMA_SHMEM_ALIGNMENT); + TunableConfig::CHUNK_DIM_X * ScaleGeom::SCALES_PER_CHUNK_Y * sizeof(ScaleType), TMA_SHMEM_ALIGNMENT); const int in_mem = buff_size_aligned_in; @@ -824,7 +861,7 @@ inline void quantize_transpose_tuned_1D(const Tensor &input, const Tensor *noop, row_scaled_nvfp4, ROW_SCALED_NVFP4, TRANSFORMER_ENGINE_SWITCH_CONDITION(return_transpose, RETURN_TRANSPOSE, { auto kernel = - quantize_transpose_nvfp4_tuned_1D_kernel;