diff --git a/tests/cpp/operator/test_swizzle.cu b/tests/cpp/operator/test_swizzle.cu index 4ebb23fdc5..5c3d33665d 100644 --- a/tests/cpp/operator/test_swizzle.cu +++ b/tests/cpp/operator/test_swizzle.cu @@ -224,6 +224,181 @@ static void zero_scale_inv_padding(uint8_t *buf, set_scale_inv_padding(buf, padded_rows, padded_cols, orig_rows, orig_cols, 0); } +class SwizzleUnpaddedOuterDimTestSuite + : public ::testing::TestWithParam< + std::tuple, NVTEScalingMode, bool>> { + protected: + void Run(size_t num_tensors, bool unchecked = false, bool mixed_padding = false) { + using namespace test; + const auto& [shape, scaling_mode, rowwise] = GetParam(); + const auto [rows, cols] = shape; + SCOPED_TRACE(::testing::Message() << "num_tensors=" << num_tensors); + const bool nvfp4 = scaling_mode == NVTE_NVFP4_1D_SCALING; + const bool row_major_scales = rowwise || nvfp4; + const size_t block_size = nvfp4 ? 16 : 32; + const size_t outer_dim = + nvfp4 ? (rowwise ? rows : cols) : (rowwise ? rows : divide_round_up(rows, block_size)); + const size_t inner_dim = nvfp4 ? divide_round_up(rowwise ? cols : rows, block_size) + : (rowwise ? divide_round_up(cols, block_size) : cols); + const size_t padded_outer = round_up_to_nearest_multiple(outer_dim, row_major_scales ? 128 : 4); + const size_t padded_inner = round_up_to_nearest_multiple(inner_dim, row_major_scales ? 4 : 128); + ASSERT_LT(outer_dim, padded_outer); + const std::vector output_scale_shape{padded_outer, padded_inner}; + const std::vector data_shape = + nvfp4 && !rowwise ? std::vector{cols, rows} : std::vector{rows, cols}; + const DType data_dtype = nvfp4 ? DType::kFloat4E2M1 : DType::kFloat8E4M3; + const DType scale_dtype = nvfp4 ? DType::kFloat8E4M3 : DType::kFloat8E8M0; + const size_t output_size = padded_outer * padded_inner; + + std::vector> input_buffers, output_buffers; + std::vector inputs, outputs; + std::vector input_handles, output_handles; + std::vector> references; + inputs.reserve(num_tensors); + outputs.reserve(num_tensors); + for (size_t tensor_index = 0; tensor_index < num_tensors; ++tensor_index) { + const bool unpadded = !mixed_padding || tensor_index % 2 == 0; + const std::vector input_scale_shape{unpadded ? outer_dim : padded_outer, padded_inner}; + const size_t input_size = input_scale_shape[0] * padded_inner; + std::vector input_scales(input_size, unpadded ? 0xFF : 0); + std::vector padded_scales(output_size, 0); + for (size_t outer = 0; outer < outer_dim; ++outer) { + for (size_t inner = 0; inner < inner_dim; ++inner) { + const uint8_t value = 1 + (37 * outer + 13 * inner + 17 * tensor_index) % 254; + input_scales[outer * padded_inner + inner] = value; + padded_scales[outer * padded_inner + inner] = value; + } + } + references.emplace_back(output_size); + if (row_major_scales) { + compute_ref_swizzle<128, 4, true>(padded_scales.data(), references.back().data(), + padded_outer, padded_inner); + } else { + compute_ref_swizzle<128, 4, false>(padded_scales.data(), references.back().data(), + padded_inner, padded_outer); + } + + input_buffers.push_back(cuda_alloc(input_size)); + output_buffers.push_back(cuda_alloc(output_size)); + NVTE_CHECK_CUDA(cudaMemcpy(input_buffers.back().get(), input_scales.data(), input_size, + cudaMemcpyHostToDevice)); + NVTE_CHECK_CUDA(cudaMemset(output_buffers.back().get(), 0xA5, output_size)); + inputs.emplace_back(scaling_mode); + outputs.emplace_back(scaling_mode); + if (rowwise) { + inputs.back().set_rowwise_data(nullptr, data_dtype, data_shape); + inputs.back().set_rowwise_scale_inv(input_buffers.back().get(), scale_dtype, + input_scale_shape); + outputs.back().set_rowwise_data(nullptr, data_dtype, data_shape); + outputs.back().set_rowwise_scale_inv(output_buffers.back().get(), scale_dtype, + output_scale_shape); + } else { + inputs.back().set_columnwise_data(nullptr, data_dtype, data_shape); + inputs.back().set_columnwise_scale_inv(input_buffers.back().get(), scale_dtype, + input_scale_shape); + outputs.back().set_columnwise_data(nullptr, data_dtype, data_shape); + outputs.back().set_columnwise_scale_inv(output_buffers.back().get(), scale_dtype, + output_scale_shape); + } + outputs.back().set_with_gemm_swizzled_scales(true); + input_handles.push_back(inputs.back().data()); + output_handles.push_back(outputs.back().data()); + } + + const auto launch = [&](cudaStream_t stream) { + if (num_tensors == 1) { + nvte_swizzle_scaling_factors(input_handles.front(), output_handles.front(), stream); + } else if (unchecked) { + nvte_multi_tensor_swizzle_scaling_factors_unchecked( + input_handles.data(), output_handles.data(), num_tensors, stream); + } else { + nvte_multi_tensor_swizzle_scaling_factors(input_handles.data(), output_handles.data(), + num_tensors, stream); + } + }; + launch(0); + NVTE_CHECK_CUDA(cudaDeviceSynchronize()); + if (mixed_padding) { + cudaStream_t stream; + NVTE_CHECK_CUDA(cudaStreamCreate(&stream)); + NVTE_CHECK_CUDA(cudaStreamBeginCapture(stream, cudaStreamCaptureModeRelaxed)); + launch(stream); + cudaGraph_t graph; + NVTE_CHECK_CUDA(cudaStreamEndCapture(stream, &graph)); + size_t num_nodes = 0; + NVTE_CHECK_CUDA(cudaGraphGetNodes(graph, nullptr, &num_nodes)); + std::vector nodes(num_nodes); + NVTE_CHECK_CUDA(cudaGraphGetNodes(graph, nodes.data(), &num_nodes)); + size_t num_kernels = 0; + for (const auto node : nodes) { + cudaGraphNodeType node_type; + NVTE_CHECK_CUDA(cudaGraphNodeGetType(node, &node_type)); + num_kernels += node_type == cudaGraphNodeTypeKernel; + } + NVTE_CHECK_CUDA(cudaGraphDestroy(graph)); + NVTE_CHECK_CUDA(cudaStreamDestroy(stream)); + const size_t num_padded = num_tensors / 2; + const size_t num_unpadded = num_tensors - num_padded; + EXPECT_EQ(num_kernels, divide_round_up(num_padded, size_t{64}) + + divide_round_up(num_unpadded, size_t{64})); + } + for (size_t tensor_index = 0; tensor_index < num_tensors; ++tensor_index) { + std::vector actual(output_size); + NVTE_CHECK_CUDA(cudaMemcpy(actual.data(), output_buffers[tensor_index].get(), output_size, + cudaMemcpyDeviceToHost)); + compareResults("unpadded_outer_dim_" + std::to_string(tensor_index), actual.data(), + references[tensor_index].data(), output_size); + } + } +}; + +TEST_P(SwizzleUnpaddedOuterDimTestSuite, SingleTensor) { Run(1); } + +TEST_P(SwizzleUnpaddedOuterDimTestSuite, MultiTensor) { + Run(4); + Run(65); +} + +TEST_P(SwizzleUnpaddedOuterDimTestSuite, MultiTensorUnchecked) { + Run(4, true); + Run(65, true); +} + +INSTANTIATE_TEST_SUITE_P( + OperatorTest, SwizzleUnpaddedOuterDimTestSuite, + ::testing::Combine( + ::testing::ValuesIn(std::vector>{ + {32, 32}, {160, 96}, {4063, 96}, {4192, 2016}, {160, 8096}, {8096, 160}}), + ::testing::Values(NVTE_MXFP8_1D_SCALING, NVTE_NVFP4_1D_SCALING), ::testing::Bool()), + [](const testing::TestParamInfo& info) { + const auto& shape = std::get<0>(info.param); + const auto scaling_mode = std::get<1>(info.param); + const auto rowwise = std::get<2>(info.param); + return "M" + std::to_string(shape.first) + "_K" + std::to_string(shape.second) + + (scaling_mode == NVTE_NVFP4_1D_SCALING ? "_NVFP4" : "_MXFP8") + + (rowwise ? "_rowwise" : "_columnwise"); + }); + +class SwizzleMixedPaddingTestSuite : public SwizzleUnpaddedOuterDimTestSuite {}; + +TEST_P(SwizzleMixedPaddingTestSuite, MultiTensor) { + for (const size_t num_tensors : {64, 65, 129, 130}) { + Run(num_tensors, false, true); + } +} + +TEST_P(SwizzleMixedPaddingTestSuite, MultiTensorUnchecked) { + for (const size_t num_tensors : {64, 65, 129, 130}) { + Run(num_tensors, true, true); + } +} + +INSTANTIATE_TEST_SUITE_P( + OperatorTest, SwizzleMixedPaddingTestSuite, + ::testing::Combine(::testing::Values(std::pair{160, 96}), + ::testing::Values(NVTE_MXFP8_1D_SCALING, NVTE_NVFP4_1D_SCALING), + ::testing::Bool())); + void performTestSwizzleNVFP4(const size_t M, const size_t K, bool rowwise) { using namespace test; constexpr size_t BLOCK_SIZE = 16; diff --git a/tests/pytorch/mxfp8/test_swizzle_unpadded_scales.py b/tests/pytorch/mxfp8/test_swizzle_unpadded_scales.py new file mode 100644 index 0000000000..cbd113e047 --- /dev/null +++ b/tests/pytorch/mxfp8/test_swizzle_unpadded_scales.py @@ -0,0 +1,87 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# See LICENSE for license information. + +"""Scale swizzling directly pads compact MXFP8 scale buffers.""" + +import pytest +import torch +from mxfp8_utils import swizzle_mxfp8_scale + +import transformer_engine.pytorch as te +import transformer_engine_torch as tex +from transformer_engine.pytorch.tensor.mxfp8_tensor import MXFP8Quantizer, MXFP8Tensor + + +def _round_up(value, multiple): + return (value + multiple - 1) // multiple * multiple + + +def _scale_shape(rows, cols, rowwise, compact): + if rowwise: + return ( + rows if compact else _round_up(rows, 128), + _round_up((cols + 31) // 32, 4), + ) + return ( + (rows + 31) // 32 if compact else _round_up((rows + 31) // 32, 4), + _round_up(cols, 128), + ) + + +def _make_tensor_and_reference(shape, rowwise): + rows, cols = shape + compact_shape = _scale_shape(rows, cols, rowwise, compact=True) + logical_shape = (rows, (cols + 31) // 32) if rowwise else ((rows + 31) // 32, cols) + scales = torch.zeros(compact_shape, dtype=torch.uint8, device="cuda") + scales[: logical_shape[0], : logical_shape[1]] = torch.randint( + 1, 256, logical_shape, dtype=torch.uint8, device="cuda" + ) + + padded_rows, padded_cols = _round_up(rows, 128), _round_up(cols, 128) + padded_shape = _scale_shape(padded_rows, padded_cols, rowwise, compact=False) + padded = torch.zeros(padded_shape, dtype=torch.uint8, device="cuda") + padded[: compact_shape[0], : compact_shape[1]].copy_(scales) + reference = swizzle_mxfp8_scale(padded_rows, padded_cols, padded, columnwise=not rowwise) + + tensor = MXFP8Tensor( + shape=shape, + dtype=torch.bfloat16, + device="cuda", + rowwise_data=torch.empty(shape, dtype=torch.uint8, device="cuda") if rowwise else None, + columnwise_data=None if rowwise else torch.empty(shape, dtype=torch.uint8, device="cuda"), + rowwise_scale_inv=scales if rowwise else None, + columnwise_scale_inv=None if rowwise else scales, + fp8_dtype=te.DType.kFloat8E4M3, + quantizer=MXFP8Quantizer(te.DType.kFloat8E4M3, rowwise=rowwise, columnwise=not rowwise), + with_gemm_swizzled_scales=False, + ) + return tensor, reference + + +@pytest.mark.skipif(not te.is_mxfp8_available(), reason="MXFP8 requires Blackwell") +@pytest.mark.parametrize("shape", [(32, 32), (160, 96), (288, 224), (4096, 4128)]) +@pytest.mark.parametrize("rowwise", [False, True]) +def test_compact_scale_swizzle(shape, rowwise): + """Single-tensor swizzle pads compact scales and zero-fills the missing outer extent.""" + tensor, reference = _make_tensor_and_reference(shape, rowwise) + tex.swizzle_scales_for_gemm_(tensor) + actual = tensor._rowwise_scale_inv if rowwise else tensor._columnwise_scale_inv + torch.testing.assert_close(actual, reference, rtol=0, atol=0) + assert tensor._with_gemm_swizzled_scales + + +@pytest.mark.skipif(not te.is_mxfp8_available(), reason="MXFP8 requires Blackwell") +@pytest.mark.parametrize("rowwise", [False, True]) +@pytest.mark.parametrize("num_tensors", [3, 64, 130]) +def test_multi_tensor_compact_scale_swizzle(rowwise, num_tensors): + """Multi-tensor swizzle handles compact and already outer-aligned inputs in one batch.""" + shapes = ((160, 96), (256, 128), (288, 224)) + tensors_and_references = [ + _make_tensor_and_reference(shapes[index % len(shapes)], rowwise) + for index in range(num_tensors) + ] + tensors = [item[0] for item in tensors_and_references] + tex.multi_tensor_swizzle_scales_for_gemm_(tensors, rowwise, not rowwise) + for tensor, reference in tensors_and_references: + actual = tensor._rowwise_scale_inv if rowwise else tensor._columnwise_scale_inv + torch.testing.assert_close(actual, reference, rtol=0, atol=0) diff --git a/transformer_engine/common/include/transformer_engine/swizzle.h b/transformer_engine/common/include/transformer_engine/swizzle.h index c7f72bcbdf..013b679c39 100644 --- a/transformer_engine/common/include/transformer_engine/swizzle.h +++ b/transformer_engine/common/include/transformer_engine/swizzle.h @@ -25,7 +25,10 @@ extern "C" { * * Requirements: * - scale_inv is stored in row-major. - * - scale_inv size is padded to 128x4 for row-scale and 4x128 for col-scale. + * - Output scale_inv size is padded to 128x4 for row-scale and 4x128 for col-scale. + * - MXFP8 and NVFP4 input scale_inv may omit padding on the outer dimension, while retaining + * contiguous-dimension padding so each scale row is 4-byte aligned. Missing outer-dimension + * padding is filled with zeros during swizzling. * - Input and output scale_inv buffer pointers used by this operation must be 16-byte aligned. * - data is quantized along K-dimension, i.e. 1D-scaling block lies along the K-dimension. */ @@ -73,7 +76,10 @@ void nvte_pack_mxfp8_columnwise_scales_for_batched_gemm(const NVTETensor input, * * Requirements: * - scale_inv is stored in row-major. - * - scale_inv size is padded to 128x4 for row-scale and 4x128 for col-scale. + * - Output scale_inv size is padded to 128x4 for row-scale and 4x128 for col-scale. + * - MXFP8 and NVFP4 input scale_inv may omit padding on the outer dimension, while retaining + * contiguous-dimension padding so each scale row is 4-byte aligned. Missing outer-dimension + * padding is filled with zeros during swizzling. * - Input and output scale_inv buffer pointers used by this operation must be 16-byte aligned. * - data is quantized along K-dimension, i.e. 1D-scaling block lies along the K-dimension. */ diff --git a/transformer_engine/common/swizzle/swizzle.cu b/transformer_engine/common/swizzle/swizzle.cu index ced4ccfd65..a90252b031 100644 --- a/transformer_engine/common/swizzle/swizzle.cu +++ b/transformer_engine/common/swizzle/swizzle.cu @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -783,10 +784,41 @@ __global__ void __launch_bounds__(TB_DIM* TB_DIM) padding_m); } +// True when unswizzled scales omit padding on the outer dimension only. The contiguous dimension +// remains padded so each scale row is 4-byte aligned. +bool has_unpadded_outer_scale_dim(const Tensor& input, bool rowwise) { + if (input.with_gemm_swizzled_scales) return false; + + const auto& scales = rowwise ? input.scale_inv : input.columnwise_scale_inv; + if (!scales.has_data()) return false; + const auto& shape = scales.shape; + if (shape.size() != 2) return false; + + const auto [rows, cols] = input.flat_2d_dims(); + size_t outer_dim, inner_dim; + size_t outer_alignment, inner_alignment; + if (input.scaling_mode == NVTE_MXFP8_1D_SCALING) { + outer_dim = rowwise ? rows : DIVUP(rows, size_t{MXFP8_BLOCK_SIZE}); + inner_dim = rowwise ? DIVUP(cols, size_t{MXFP8_BLOCK_SIZE}) : cols; + outer_alignment = rowwise ? kSwizzleTileDimM : kScalesPerWord; + inner_alignment = rowwise ? kScalesPerWord : kSwizzleTileDimM; + } else if (input.scaling_mode == NVTE_NVFP4_1D_SCALING) { + outer_dim = rowwise ? rows : cols; + inner_dim = DIVUP(rowwise ? cols : rows, size_t{NVFP4_BLOCK_SIZE}); + outer_alignment = kSwizzleTileDimM; + inner_alignment = kScalesPerWord; + } else { + return false; + } + return shape[0] == outer_dim && shape[1] == DIVUP_TO_MULTIPLE(inner_dim, inner_alignment) && + outer_dim != DIVUP_TO_MULTIPLE(outer_dim, outer_alignment); +} + // Optimized scale swizzle for narrow scale widths. "Rowwise full width" means that a block stages // the entire padded K dimension for one or more 128-row tiles. "Columnwise chunk" means that a // block stages only 32 (or 128 for large grids) K rows for one 128-column M tile. Both paths use -// coalesced 16-byte loads and stores through shared memory. +// coalesced 16-byte loads and stores through shared memory, with scalar 4-byte loads for any +// rowwise input tail shorter than 16 bytes. // The 4 scales packed in `word` (the first one in column first_column), with those in columns // >= num_columns set to zero. __device__ __forceinline__ uint32_t zero_padding_columns(uint32_t word, int first_column, @@ -813,6 +845,7 @@ inline uint32_t division_magic(int divisor) { // input and of the output. Shared-memory rows are padded to an odd number of words, so the column // gathers are free of bank conflicts. Requires num_tiles * 128 * padded_k / 4 < 2^16 (guaranteed // by the row-wise shared-memory limit = 30 KB). +template __device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict__ input, uint8_t* __restrict__ output, int padded_k, uint32_t words_per_row_magic, int original_m, @@ -820,9 +853,16 @@ __device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict_ uint32_t* smem) { const int words_per_row = padded_k / kScalesPerWord; const int smem_row_words = words_per_row | 1; - const int num_vectors = num_tiles * kVectorsPerMSpan * padded_k; const size_t chunk_offset = static_cast(first_tile) * kSwizzleTileDimM * padded_k; - const int4* input_vectors = reinterpret_cast(input + chunk_offset); + const uint32_t* input_words = reinterpret_cast(input + chunk_offset); + const int4* input_vectors = reinterpret_cast(input_words); + const int valid_rows = + kUnpaddedOuterDim + ? max(0, min(num_tiles * kSwizzleTileDimM, original_m - first_tile * kSwizzleTileDimM)) + : num_tiles * kSwizzleTileDimM; + const int valid_words = valid_rows * words_per_row; + const int num_vectors = + kUnpaddedOuterDim ? valid_words / kScalesPerWord : num_tiles * kVectorsPerMSpan * padded_k; for (int i = threadIdx.x; i < num_vectors; i += kRowwiseThreadsPerBlock) { const int4 vector = __ldg(input_vectors + i); const uint32_t words[kScalesPerWord] = { @@ -835,6 +875,14 @@ __device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict_ smem[row * smem_row_words + (word_index - row * words_per_row)] = words[w]; } } + if constexpr (kUnpaddedOuterDim) { + for (int word_index = kScalesPerWord * num_vectors + threadIdx.x; word_index < valid_words; + word_index += kRowwiseThreadsPerBlock) { + const uint32_t row = divide_by_magic(word_index, words_per_row_magic); + smem[row * smem_row_words + (word_index - row * words_per_row)] = + __ldg(input_words + word_index); + } + } __syncthreads(); // An output tile holds padded_k / 4 blocks of 512 bytes, i.e. 32 vectors each; vector r of a @@ -852,8 +900,15 @@ __device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict_ #pragma unroll for (int group = 0; group < kScalesPerWord; group++) { const int row = row_in_group + kMPositionsPerSwizzleGroup * group; - uint32_t word = smem[(tile_first_row + row) * smem_row_words + k_block]; - if (global_first_row + row >= original_m) word = 0u; + uint32_t word; + if constexpr (kUnpaddedOuterDim) { + word = global_first_row + row < original_m + ? smem[(tile_first_row + row) * smem_row_words + k_block] + : 0u; + } else { + word = smem[(tile_first_row + row) * smem_row_words + k_block]; + if (global_first_row + row >= original_m) word = 0u; + } words[group] = zero_padding_columns(word, k_block * kScalesPerWord, original_k); } output_vectors[v] = make_int4(static_cast(words[0]), static_cast(words[1]), @@ -865,15 +920,17 @@ __device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict_ // Column-wise: tile m_tile (128 columns), scale rows [first_k, first_k + kChunkRows) clipped to // padded_k. Each input row segment is 128 contiguous bytes; each output block of 4 scale rows is // 512 contiguous bytes. -template +template __device__ __forceinline__ void swizzle_columnwise_chunk(const uint8_t* __restrict__ input, uint8_t* __restrict__ output, int padded_m, int padded_k, int original_m, int original_k, int m_tile, int first_k, int4* smem) { const int num_rows = min(kChunkRows, padded_k - first_k); // multiple of kScalesPerWord - const int num_vectors = num_rows * kVectorsPerMSpan; - for (int i = threadIdx.x; i < num_vectors; i += kColumnwiseThreadsPerBlock) { + const int num_input_vectors = + (kUnpaddedOuterDim ? max(0, min(num_rows, original_k - first_k)) : num_rows) * + kVectorsPerMSpan; + for (int i = threadIdx.x; i < num_input_vectors; i += kColumnwiseThreadsPerBlock) { const int row = i / kVectorsPerMSpan; const int vector_in_row = i % kVectorsPerMSpan; smem[i] = @@ -886,6 +943,7 @@ __device__ __forceinline__ void swizzle_columnwise_chunk(const uint8_t* __restri const uint8_t* smem_bytes = reinterpret_cast(smem); const size_t first_output_block = static_cast(m_tile) * (padded_k / kScalesPerWord) + (first_k / kScalesPerWord); + const int num_vectors = num_rows * kVectorsPerMSpan; for (int v = threadIdx.x; v < num_vectors; v += kColumnwiseThreadsPerBlock) { const int k_block = v / kMPositionsPerSwizzleGroup; const int column_in_group = v % kMPositionsPerSwizzleGroup; @@ -912,21 +970,23 @@ __device__ __forceinline__ void swizzle_columnwise_chunk(const uint8_t* __restri } } +template __global__ void __launch_bounds__(kRowwiseThreadsPerBlock) swizzle_rowwise_full_width_kernel(const uint8_t* __restrict__ input, uint8_t* __restrict__ output, int padded_k, uint32_t words_per_row_magic, int original_m, int original_k) { extern __shared__ uint32_t rowwise_smem[]; - swizzle_rowwise_tiles(input, output, padded_k, words_per_row_magic, original_m, original_k, - blockIdx.x, 1, rowwise_smem); + swizzle_rowwise_tiles(input, output, padded_k, words_per_row_magic, original_m, + original_k, blockIdx.x, 1, rowwise_smem); } +template __global__ void __launch_bounds__(kColumnwiseThreadsPerBlock) swizzle_columnwise_chunk_kernel(const uint8_t* __restrict__ input, uint8_t* __restrict__ output, int padded_m, int padded_k, int original_m, int original_k) { __shared__ int4 columnwise_smem[kColumnwiseSmallChunkRows * kVectorsPerMSpan]; - swizzle_columnwise_chunk( + swizzle_columnwise_chunk( input, output, padded_m, padded_k, original_m, original_k, blockIdx.x, blockIdx.y * kColumnwiseSmallChunkRows, columnwise_smem); } @@ -938,6 +998,7 @@ struct SwizzleTensorDescriptor { const uint8_t* input; uint8_t* output; int padded_m, padded_k, original_m, original_k; + bool input_has_unpadded_outer_dim; }; struct MultiTensorSwizzleArgs { @@ -954,6 +1015,7 @@ __device__ __forceinline__ int tensor_of_block(const MultiTensorSwizzleArgs& arg return tensor; } +template __global__ void __launch_bounds__(kRowwiseThreadsPerBlock) multi_tensor_swizzle_rowwise_full_width_kernel(const MultiTensorSwizzleArgs args) { extern __shared__ uint32_t rowwise_multi_smem[]; @@ -962,12 +1024,13 @@ __global__ void __launch_bounds__(kRowwiseThreadsPerBlock) const int first_tile = (blockIdx.x - args.first_block[t]) * row_tiles_per_block; const int num_tiles = min(row_tiles_per_block, args.tensors[t].padded_m / kSwizzleTileDimM - first_tile); - swizzle_rowwise_tiles(args.tensors[t].input, args.tensors[t].output, args.tensors[t].padded_k, - args.words_per_row_magic[t], args.tensors[t].original_m, - args.tensors[t].original_k, first_tile, num_tiles, rowwise_multi_smem); + swizzle_rowwise_tiles(args.tensors[t].input, args.tensors[t].output, + args.tensors[t].padded_k, args.words_per_row_magic[t], + args.tensors[t].original_m, args.tensors[t].original_k, + first_tile, num_tiles, rowwise_multi_smem); } -template +template __global__ void __launch_bounds__(kColumnwiseThreadsPerBlock) multi_tensor_swizzle_columnwise_chunk_kernel(const MultiTensorSwizzleArgs args) { __shared__ int4 columnwise_multi_smem[kChunkRows * kVectorsPerMSpan]; @@ -976,10 +1039,10 @@ __global__ void __launch_bounds__(kColumnwiseThreadsPerBlock) const int num_m_tiles = args.tensors[t].padded_m / kSwizzleTileDimM; const int m_tile = block_in_tensor % num_m_tiles; const int first_k = (block_in_tensor / num_m_tiles) * kChunkRows; - swizzle_columnwise_chunk(args.tensors[t].input, args.tensors[t].output, - args.tensors[t].padded_m, args.tensors[t].padded_k, - args.tensors[t].original_m, args.tensors[t].original_k, - m_tile, first_k, columnwise_multi_smem); + swizzle_columnwise_chunk( + args.tensors[t].input, args.tensors[t].output, args.tensors[t].padded_m, + args.tensors[t].padded_k, args.tensors[t].original_m, args.tensors[t].original_k, m_tile, + first_k, columnwise_multi_smem); } inline bool is_16_byte_aligned(const void* ptr) { @@ -1006,6 +1069,7 @@ inline bool should_use_rowwise_full_width_swizzle(int padded_k, int64_t total_ro // Launches the rowwise full-width kernel, where each block stages the entire padded K dimension, // for up to kMaxTensorsPerLaunch tensors. The row tiles per block are the smallest power of two // that keeps the grid within kRowwiseMaxBlocks, subject to the shared-memory limit. +template inline void launch_multi_tensor_rowwise_full_width_swizzle(const SwizzleTensorDescriptor* tensors, int num_tensors, cudaStream_t stream) { MultiTensorSwizzleArgs args{}; @@ -1037,15 +1101,17 @@ inline void launch_multi_tensor_rowwise_full_width_swizzle(const SwizzleTensorDe } const int grid = args.first_block[num_tensors]; if (grid == 0) return; - multi_tensor_swizzle_rowwise_full_width_kernel<<< - grid, kRowwiseThreadsPerBlock, - row_tiles_per_block * kSwizzleTileDimM * max_smem_row_words * kBytesPerWord, stream>>>(args); + multi_tensor_swizzle_rowwise_full_width_kernel + <<>>( + args); NVTE_CHECK_CUDA(cudaGetLastError()); } // Launches the columnwise chunk kernel, where each block stages part of K for one 128-column M // tile, for up to kMaxTensorsPerLaunch tensors. It uses 128-row K chunks once 32-row chunks would // exceed kColumnwiseMaxBlocksWith32RowChunks blocks. +template inline void launch_multi_tensor_columnwise_chunk_swizzle(const SwizzleTensorDescriptor* tensors, int num_tensors, cudaStream_t stream) { MultiTensorSwizzleArgs args{}; @@ -1071,10 +1137,10 @@ inline void launch_multi_tensor_columnwise_chunk_swizzle(const SwizzleTensorDesc const int grid = args.first_block[num_tensors]; if (grid == 0) return; if (chunk_rows == kColumnwiseLargeChunkRows) { - multi_tensor_swizzle_columnwise_chunk_kernel + multi_tensor_swizzle_columnwise_chunk_kernel <<>>(args); } else { - multi_tensor_swizzle_columnwise_chunk_kernel + multi_tensor_swizzle_columnwise_chunk_kernel <<>>(args); } NVTE_CHECK_CUDA(cudaGetLastError()); @@ -1085,9 +1151,15 @@ inline void launch_rowwise_full_width_swizzle(const SwizzleTensorDescriptor& ten cudaStream_t stream) { const int grid = tensor.padded_m / kSwizzleTileDimM; const int smem = kSwizzleTileDimM * ((tensor.padded_k / kScalesPerWord) | 1) * kBytesPerWord; - swizzle_rowwise_full_width_kernel<<>>( - tensor.input, tensor.output, tensor.padded_k, - division_magic(tensor.padded_k / kScalesPerWord), tensor.original_m, tensor.original_k); + if (tensor.input_has_unpadded_outer_dim) { + swizzle_rowwise_full_width_kernel<<>>( + tensor.input, tensor.output, tensor.padded_k, + division_magic(tensor.padded_k / kScalesPerWord), tensor.original_m, tensor.original_k); + } else { + swizzle_rowwise_full_width_kernel<<>>( + tensor.input, tensor.output, tensor.padded_k, + division_magic(tensor.padded_k / kScalesPerWord), tensor.original_m, tensor.original_k); + } NVTE_CHECK_CUDA(cudaGetLastError()); } @@ -1095,9 +1167,15 @@ inline void launch_rowwise_full_width_swizzle(const SwizzleTensorDescriptor& ten inline void launch_columnwise_chunk_swizzle(const SwizzleTensorDescriptor& tensor, cudaStream_t stream) { dim3 grid(tensor.padded_m / kSwizzleTileDimM, DIVUP(tensor.padded_k, kColumnwiseSmallChunkRows)); - swizzle_columnwise_chunk_kernel<<>>( - tensor.input, tensor.output, tensor.padded_m, tensor.padded_k, tensor.original_m, - tensor.original_k); + if (tensor.input_has_unpadded_outer_dim) { + swizzle_columnwise_chunk_kernel<<>>( + tensor.input, tensor.output, tensor.padded_m, tensor.padded_k, tensor.original_m, + tensor.original_k); + } else { + swizzle_columnwise_chunk_kernel<<>>( + tensor.input, tensor.output, tensor.padded_m, tensor.padded_k, tensor.original_m, + tensor.original_k); + } NVTE_CHECK_CUDA(cudaGetLastError()); } @@ -1109,8 +1187,17 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s NVTE_CHECK(scaling_mode == NVTE_MXFP8_1D_SCALING || scaling_mode == NVTE_NVFP4_1D_SCALING, "Input tensor has invalid scaling mode (", to_string(input->scaling_mode), ")."); - CheckInputTensor(*input, "scaling_factor_input"); + const bool has_rowwise_scale_inv = input->scale_inv.has_data(); + const bool has_columnwise_scale_inv = input->columnwise_scale_inv.has_data(); + NVTE_CHECK(!has_rowwise_scale_inv || !has_columnwise_scale_inv, + "Input tensor has both row-wise and column-wise scaling factors"); + const bool input_has_unpadded_outer_dim = + has_unpadded_outer_scale_dim(*input, has_rowwise_scale_inv); + CheckInputTensor(*input, "scaling_factor_input", !input_has_unpadded_outer_dim); CheckInputTensor(*output, "scaling_factor_output"); + NVTE_CHECK( + output->scaling_mode == scaling_mode && output->flat_2d_dims() == input->flat_2d_dims(), + "Input and output tensor layouts must match."); NVTE_CHECK(!input->with_gemm_swizzled_scales, "Expected input tensor with scales in compact format."); NVTE_CHECK(output->with_gemm_swizzled_scales, @@ -1129,44 +1216,41 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s } // Check if scaling factors are non-trivial - const bool has_rowwise_scale_inv = input->scale_inv.has_data(); - const bool has_columnwise_scale_inv = input->columnwise_scale_inv.has_data(); - NVTE_CHECK(!has_rowwise_scale_inv || !has_columnwise_scale_inv, - "Input tensor has both row-wise and column-wise scaling factors"); if (!has_rowwise_scale_inv && !has_columnwise_scale_inv) { return; } - // Deduce tensor dims + // Deduce padded swizzle dimensions from the output. A compact input may omit outer-dimension + // padding, while the output always has the full GEMM layout. int m{0}, k{0}; switch (scaling_mode) { case NVTE_MXFP8_1D_SCALING: { if (has_rowwise_scale_inv) { - NVTE_CHECK(input->scale_inv.shape.size() == 2, - "Expected 2D scaling factors, got shape=", input->scale_inv.shape, "."); - m = input->scale_inv.shape[0]; - k = input->scale_inv.shape[1]; + NVTE_CHECK(output->scale_inv.shape.size() == 2, + "Expected 2D scaling factors, got shape=", output->scale_inv.shape, "."); + m = output->scale_inv.shape[0]; + k = output->scale_inv.shape[1]; } else if (has_columnwise_scale_inv) { - NVTE_CHECK(input->columnwise_scale_inv.shape.size() == 2, - "Expected 2D scaling factors, got shape=", input->columnwise_scale_inv.shape, + NVTE_CHECK(output->columnwise_scale_inv.shape.size() == 2, + "Expected 2D scaling factors, got shape=", output->columnwise_scale_inv.shape, "."); - m = input->columnwise_scale_inv.shape[1]; - k = input->columnwise_scale_inv.shape[0]; + m = output->columnwise_scale_inv.shape[1]; + k = output->columnwise_scale_inv.shape[0]; } break; } case NVTE_NVFP4_1D_SCALING: { if (has_rowwise_scale_inv) { - NVTE_CHECK(input->scale_inv.shape.size() == 2, - "Expected 2D scaling factors, got shape=", input->scale_inv.shape, "."); - m = input->scale_inv.shape[0]; - k = input->scale_inv.shape[1]; + NVTE_CHECK(output->scale_inv.shape.size() == 2, + "Expected 2D scaling factors, got shape=", output->scale_inv.shape, "."); + m = output->scale_inv.shape[0]; + k = output->scale_inv.shape[1]; } else if (has_columnwise_scale_inv) { - NVTE_CHECK(input->columnwise_scale_inv.shape.size() == 2, - "Expected 2D scaling factors, got shape=", input->columnwise_scale_inv.shape, + NVTE_CHECK(output->columnwise_scale_inv.shape.size() == 2, + "Expected 2D scaling factors, got shape=", output->columnwise_scale_inv.shape, "."); - m = input->columnwise_scale_inv.shape[0]; - k = input->columnwise_scale_inv.shape[1]; + m = output->columnwise_scale_inv.shape[0]; + k = output->columnwise_scale_inv.shape[1]; } break; } @@ -1234,7 +1318,7 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s switch (scaling_mode) { case NVTE_MXFP8_1D_SCALING: { original_M = input->flat_first_dim(); - original_K = input->flat_last_dim() / MXFP8_BLOCK_SIZE; + original_K = DIVUP(input->flat_last_dim(), static_cast(MXFP8_BLOCK_SIZE)); input_scale_inv_ptr = input->scale_inv.dptr; output_scale_inv_ptr = output->scale_inv.dptr; break; @@ -1242,12 +1326,12 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s case NVTE_NVFP4_1D_SCALING: { if (has_rowwise_scale_inv) { original_M = input->flat_first_dim(); - original_K = input->flat_last_dim() / NVFP4_BLOCK_SIZE; + original_K = DIVUP(input->flat_last_dim(), static_cast(NVFP4_BLOCK_SIZE)); input_scale_inv_ptr = input->scale_inv.dptr; output_scale_inv_ptr = output->scale_inv.dptr; } else if (has_columnwise_scale_inv) { original_M = input->flat_last_dim(); - original_K = input->flat_first_dim() / NVFP4_BLOCK_SIZE; + original_K = DIVUP(input->flat_first_dim(), static_cast(NVFP4_BLOCK_SIZE)); input_scale_inv_ptr = input->columnwise_scale_inv.dptr; output_scale_inv_ptr = output->columnwise_scale_inv.dptr; } @@ -1266,11 +1350,16 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s m, k, original_M, - original_K}; + original_K, + input_has_unpadded_outer_dim}; if (should_use_rowwise_full_width_swizzle(descriptor.padded_k, descriptor.padded_m / SF_TILE_DIM_M)) { if (m / SF_TILE_DIM_M > kRowwiseMaxBlocks) { - launch_multi_tensor_rowwise_full_width_swizzle(&descriptor, 1, stream); + if (input_has_unpadded_outer_dim) { + launch_multi_tensor_rowwise_full_width_swizzle(&descriptor, 1, stream); + } else { + launch_multi_tensor_rowwise_full_width_swizzle(&descriptor, 1, stream); + } } else { launch_rowwise_full_width_swizzle(descriptor, stream); } @@ -1315,7 +1404,7 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s // Perform column-wise swizzle if (columnwise_swizzle) { const int original_M = input->flat_last_dim(); - const int original_K = input->flat_first_dim() / MXFP8_BLOCK_SIZE; + const int original_K = DIVUP(input->flat_first_dim(), static_cast(MXFP8_BLOCK_SIZE)); const SwizzleTensorDescriptor descriptor{ static_cast(input->columnwise_scale_inv.dptr), @@ -1323,11 +1412,16 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s m, k, original_M, - original_K}; + original_K, + input_has_unpadded_outer_dim}; const int64_t grid_size = static_cast(m / SF_TILE_DIM_M) * DIVUP(k, kColumnwiseSmallChunkRows); if (grid_size > kColumnwiseMaxBlocksWith32RowChunks) { - launch_multi_tensor_columnwise_chunk_swizzle(&descriptor, 1, stream); + if (input_has_unpadded_outer_dim) { + launch_multi_tensor_columnwise_chunk_swizzle(&descriptor, 1, stream); + } else { + launch_multi_tensor_columnwise_chunk_swizzle(&descriptor, 1, stream); + } } else { launch_columnwise_chunk_swizzle(descriptor, stream); } @@ -1474,7 +1568,7 @@ void launch_multi_tensor_unswizzle_scaling_factors(MultiSwizzleArgs& kernel_args namespace { // Multi-tensor swizzle using rowwise full-width or columnwise chunk kernels with aligned int4 -// (16-byte) loads and stores. +// (16-byte) stores and loads, except for scalar 4-byte loads for a partial rowwise input vector. void int4_vectorized_multi_tensor_swizzle(const std::vector& input, std::vector& output, bool rowwise, cudaStream_t stream) { @@ -1488,34 +1582,61 @@ void int4_vectorized_multi_tensor_swizzle(const std::vector& input, const auto& out_scales = selected_rowwise ? output[i]->scale_inv : output[i]->columnwise_scale_inv; NVTE_CHECK(in_scales.shape.size() == 2, "Expected two-dimensional scaling factors."); + NVTE_CHECK(out_scales.shape.size() == 2, "Expected two-dimensional output scaling factors."); const int padded_m = - static_cast((rowwise || nvfp4) ? in_scales.shape[0] : in_scales.shape[1]); + static_cast((rowwise || nvfp4) ? out_scales.shape[0] : out_scales.shape[1]); const int padded_k = - static_cast((rowwise || nvfp4) ? in_scales.shape[1] : in_scales.shape[0]); + static_cast((rowwise || nvfp4) ? out_scales.shape[1] : out_scales.shape[0]); NVTE_CHECK(padded_m > 0 && padded_k > 0 && padded_m % kSwizzleTileDimM == 0 && padded_k % kScalesPerWord == 0, "Invalid scaling-factor shape (", padded_m, ", ", padded_k, ")."); - NVTE_CHECK(in_scales.numel() == static_cast(padded_m) * padded_k && - out_scales.numel() == static_cast(padded_m) * padded_k, - "Input and output scaling-factor shapes must match."); + NVTE_CHECK(out_scales.numel() == static_cast(padded_m) * padded_k, + "Output scaling factors must use the padded GEMM layout."); const auto [first_dim, last_dim] = input[i]->flat_2d_dims(); const int block_size = nvfp4 ? NVFP4_BLOCK_SIZE : MXFP8_BLOCK_SIZE; const int original_m = static_cast(selected_rowwise ? first_dim : last_dim); - const int original_k = static_cast((selected_rowwise ? last_dim : first_dim) / block_size); + const int original_k = static_cast( + DIVUP(selected_rowwise ? last_dim : first_dim, static_cast(block_size))); + const bool input_has_unpadded_outer_dim = + in_scales.numel() != static_cast(padded_m) * padded_k; + NVTE_CHECK( + !input_has_unpadded_outer_dim || has_unpadded_outer_scale_dim(*input[i], selected_rowwise), + "Input scaling factors must either use the padded GEMM layout or omit only " + "outer-dimension padding."); tensors[i] = {static_cast(in_scales.dptr), static_cast(out_scales.dptr), padded_m, padded_k, original_m, - original_k}; - } - for (size_t first = 0; first < num_tensors; first += kMaxTensorsPerLaunch) { - const int count = static_cast(std::min(kMaxTensorsPerLaunch, num_tensors - first)); + original_k, + input_has_unpadded_outer_dim}; + } + std::partition(tensors.begin(), tensors.end(), [](const SwizzleTensorDescriptor& tensor) { + return tensor.input_has_unpadded_outer_dim; + }); + for (size_t first = 0; first < num_tensors;) { + size_t end = first + 1; + while (end < num_tensors && end - first < kMaxTensorsPerLaunch && + tensors[end].input_has_unpadded_outer_dim == + tensors[first].input_has_unpadded_outer_dim) { + ++end; + } + const int count = static_cast(end - first); if (rowwise) { - launch_multi_tensor_rowwise_full_width_swizzle(tensors.data() + first, count, stream); + if (tensors[first].input_has_unpadded_outer_dim) { + launch_multi_tensor_rowwise_full_width_swizzle(tensors.data() + first, count, stream); + } else { + launch_multi_tensor_rowwise_full_width_swizzle(tensors.data() + first, count, + stream); + } } else { - launch_multi_tensor_columnwise_chunk_swizzle(tensors.data() + first, count, stream); + if (tensors[first].input_has_unpadded_outer_dim) { + launch_multi_tensor_columnwise_chunk_swizzle(tensors.data() + first, count, stream); + } else { + launch_multi_tensor_columnwise_chunk_swizzle(tensors.data() + first, count, stream); + } } + first = end; } } @@ -1542,8 +1663,9 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, // We don't allow empty tensors. They should be filtered out before calling this function. NVTE_CHECK(input[i]->numel() != 0, "Tensor input[", i, "] is empty."); - CheckInputTensor(*input[i], "scaling_factor_input", check_scale_inv_shapes); - CheckInputTensor(*output[i], "scaling_factor_output", check_scale_inv_shapes); + NVTE_CHECK(output[i]->scaling_mode == scaling_mode && + output[i]->flat_2d_dims() == input[i]->flat_2d_dims(), + "Input and output tensor ", i, " layouts must match."); all_has_data = all_has_data && input[i]->scale_inv.has_data(); all_has_columnwise_data = (all_has_columnwise_data && input[i]->columnwise_scale_inv.has_data()); @@ -1561,13 +1683,21 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, for (size_t i = 0; i < num_tensors; i++) { const bool use_rowwise_buffer = input[i]->scale_inv.has_data(); const bool selected_rowwise = rowwise_swizzle && use_rowwise_buffer; + const bool input_has_unpadded_outer_dim = + has_unpadded_outer_scale_dim(*input[i], selected_rowwise); + CheckInputTensor(*input[i], "scaling_factor_input", + check_scale_inv_shapes && !input_has_unpadded_outer_dim); + CheckInputTensor(*output[i], "scaling_factor_output", check_scale_inv_shapes); const auto& in_scales = selected_rowwise ? input[i]->scale_inv : input[i]->columnwise_scale_inv; + const auto& out_scales = + selected_rowwise ? output[i]->scale_inv : output[i]->columnwise_scale_inv; check_scale_inv_alignment(input[i], "Input", selected_rowwise); check_scale_inv_alignment(output[i], "Output", selected_rowwise); if (rowwise_swizzle) { - NVTE_CHECK(in_scales.shape.size() == 2, "Expected two-dimensional scaling factors."); - max_rowwise_padded_k = std::max(max_rowwise_padded_k, static_cast(in_scales.shape[1])); - total_rowwise_tiles += static_cast(in_scales.shape[0]) / kSwizzleTileDimM; + NVTE_CHECK(in_scales.shape.size() == 2 && out_scales.shape.size() == 2, + "Expected two-dimensional scaling factors."); + max_rowwise_padded_k = std::max(max_rowwise_padded_k, static_cast(out_scales.shape[1])); + total_rowwise_tiles += static_cast(out_scales.shape[0]) / kSwizzleTileDimM; } } if (rowwise_swizzle) { @@ -1602,12 +1732,12 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, int m, k; if (all_has_data) { - m = input[i]->scale_inv.shape[0]; - k = input[i]->scale_inv.shape[1]; + m = output[i]->scale_inv.shape[0]; + k = output[i]->scale_inv.shape[1]; } else { NVTE_CHECK(all_nvfp4, "When doing rowwise swizzle with rowwise data, it has to be NVFP4"); - m = input[i]->columnwise_scale_inv.shape[0]; - k = input[i]->columnwise_scale_inv.shape[1]; + m = output[i]->columnwise_scale_inv.shape[0]; + k = output[i]->columnwise_scale_inv.shape[1]; } NVTE_CHECK(m % SF_TILE_DIM_M == 0, "Input should be padded in M/N dimension!"); @@ -1647,12 +1777,12 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, kernel_args.input_list[pos] = const_cast(input[i]->scale_inv.dptr); kernel_args.output_list[pos] = output[i]->scale_inv.dptr; kernel_args.original_m_list[pos] = first_dim; - kernel_args.original_k_list[pos] = last_dim / block_scale_size; + kernel_args.original_k_list[pos] = DIVUP(last_dim, static_cast(block_scale_size)); } else { kernel_args.input_list[pos] = const_cast(input[i]->columnwise_scale_inv.dptr); kernel_args.output_list[pos] = output[i]->columnwise_scale_inv.dptr; kernel_args.original_m_list[pos] = last_dim; - kernel_args.original_k_list[pos] = first_dim / NVFP4_BLOCK_SIZE; + kernel_args.original_k_list[pos] = DIVUP(first_dim, static_cast(NVFP4_BLOCK_SIZE)); } kernel_args.num_tensors++; } diff --git a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp index 3172549be7..08b38da8a9 100644 --- a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp +++ b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp @@ -45,6 +45,19 @@ bool is_empty_grouped_tensor_param(const GroupedTensorWrapper &input) { return false; } +NVTEShape padded_scale_shape_for_gemm(NVTEShape shape, NVTEScalingMode scaling_mode, bool rowwise) { + if (shape.ndim != 2) return shape; + const bool scales_are_row_major = rowwise || scaling_mode == NVTE_NVFP4_1D_SCALING; + if (scales_are_row_major) { + shape.data[0] = DIVUP_TO_MULTIPLE(shape.data[0], 128); + shape.data[1] = DIVUP_TO_MULTIPLE(shape.data[1], 4); + } else { + shape.data[0] = DIVUP_TO_MULTIPLE(shape.data[0], 4); + shape.data[1] = DIVUP_TO_MULTIPLE(shape.data[1], 128); + } + return shape; +} + } // namespace void set_compact_tensor_data(TensorWrapper &view, const TensorWrapper &tensor, bool rowwise, @@ -100,8 +113,9 @@ std::tuple, std::optional> swizzle_scales_ const auto scales_dtype = static_cast(input_scales_nvte.dtype); // Allocate buffer for swizzled scales (uint8, like all quantizer allocations) - const NVTEShape output_scales_shape = input_scales_shape; - rowwise_scales_pyt = allocateSpace(input_scales_shape, DType::kByte, false); + const NVTEShape output_scales_shape = + padded_scale_shape_for_gemm(input_scales_shape, scaling_mode, /*rowwise=*/true); + rowwise_scales_pyt = allocateSpace(output_scales_shape, DType::kByte, false); void *output_scales_dptr = getDataPtr(*rowwise_scales_pyt); // Initialize TE tensors with scales @@ -133,8 +147,9 @@ std::tuple, std::optional> swizzle_scales_ const auto scales_dtype = static_cast(input_scales_nvte.dtype); // Allocate buffer for swizzled scales (uint8, like all quantizer allocations) - const NVTEShape output_scales_shape = input_scales_shape; - columnwise_scales_pyt = allocateSpace(input_scales_shape, DType::kByte, false); + const NVTEShape output_scales_shape = + padded_scale_shape_for_gemm(input_scales_shape, scaling_mode, /*rowwise=*/false); + columnwise_scales_pyt = allocateSpace(output_scales_shape, DType::kByte, false); void *output_scales_dptr = getDataPtr(*columnwise_scales_pyt); // Initialize TE tensors with scales @@ -208,16 +223,19 @@ std::optional multi_tensor_swizzle_scales_for_gemm_impl( // Determine buffer size needed for swizzled scales std::vector output_scales_offsets; output_scales_offsets.reserve(tensors_needing_swizzle.size()); + std::vector output_scales_shapes; + output_scales_shapes.reserve(tensors_needing_swizzle.size()); size_t output_scales_bytes = 0; for (auto &tensor : tensors_needing_swizzle) { const auto scales_nvte = (rowwise_usage ? tensor->get_rowwise_scale_inv() : tensor->get_columnwise_scale_inv()); - const auto &shape = scales_nvte.shape; + const auto shape = padded_scale_shape_for_gemm(scales_nvte.shape, scaling_mode, rowwise_usage); const auto dtype = static_cast(scales_nvte.dtype); const auto dtype_bits = transformer_engine::pytorch::typeToNumBits(dtype); const auto size = product(shape, 0, shape.ndim); output_scales_bytes = roundup(output_scales_bytes, 16); // align to 16B output_scales_offsets.push_back(output_scales_bytes); + output_scales_shapes.push_back(shape); output_scales_bytes += ceildiv(size * dtype_bits, 8); } @@ -242,8 +260,6 @@ std::optional multi_tensor_swizzle_scales_for_gemm_impl( // Cache output scale dtype/shape per tensor so we can update the // source TensorWrappers without re-reading from the output NVTETensors. std::vector output_scales_dtypes(n_swizzle); - std::vector output_scales_shapes(n_swizzle); - for (size_t i = 0; i < n_swizzle; ++i) { auto &tensor = *tensors_needing_swizzle[i]; const uint8_t swizzled_flag = 1; @@ -255,26 +271,26 @@ std::optional multi_tensor_swizzle_scales_for_gemm_impl( const auto data_dtype = static_cast(data_nvte.dtype); const auto scales_dtype = static_cast(scales_nvte.dtype); output_scales_dtypes[i] = scales_dtype; - output_scales_shapes[i] = scales_nvte.shape; set_param(inputs_nvte[i], kNVTERowwiseData, nullptr, data_dtype, data_nvte.shape); set_param(inputs_nvte[i], kNVTERowwiseScaleInv, scales_nvte.data_ptr, scales_dtype, scales_nvte.shape); set_param(outputs_nvte[i], kNVTERowwiseData, nullptr, data_dtype, data_nvte.shape); set_param(outputs_nvte[i], kNVTERowwiseScaleInv, - output_scales_dptr + output_scales_offsets[i], scales_dtype, scales_nvte.shape); + output_scales_dptr + output_scales_offsets[i], scales_dtype, + output_scales_shapes[i]); } else { const auto data_nvte = tensor.get_columnwise_data(); const auto scales_nvte = tensor.get_columnwise_scale_inv(); const auto data_dtype = static_cast(data_nvte.dtype); const auto scales_dtype = static_cast(scales_nvte.dtype); output_scales_dtypes[i] = scales_dtype; - output_scales_shapes[i] = scales_nvte.shape; set_param(inputs_nvte[i], kNVTEColumnwiseData, nullptr, data_dtype, data_nvte.shape); set_param(inputs_nvte[i], kNVTEColumnwiseScaleInv, scales_nvte.data_ptr, scales_dtype, scales_nvte.shape); set_param(outputs_nvte[i], kNVTEColumnwiseData, nullptr, data_dtype, data_nvte.shape); set_param(outputs_nvte[i], kNVTEColumnwiseScaleInv, - output_scales_dptr + output_scales_offsets[i], scales_dtype, scales_nvte.shape); + output_scales_dptr + output_scales_offsets[i], scales_dtype, + output_scales_shapes[i]); } }