From 8ae90bcc71d2f167ce164e98d02f82e43c8b7cf1 Mon Sep 17 00:00:00 2001 From: Varun Thumbe Date: Fri, 23 Oct 2026 09:19:37 -0700 Subject: [PATCH 1/9] Optimize scale swizzle kernels Use per-tile swizzle kernels for small MXFP8 scale tensors and multi-tensor cases. Require 16-byte alignment for input and output scale buffers selected by the swizzle. Signed-off-by: Varun Thumbe --- docs/envvars.rst | 6 - tests/cpp/operator/test_multi_swizzle.cu | 212 ++++- tests/cpp/operator/test_swizzle.cu | 185 +++- .../include/transformer_engine/swizzle.h | 3 + transformer_engine/common/swizzle/swizzle.cu | 873 +++++------------- 5 files changed, 561 insertions(+), 718 deletions(-) diff --git a/docs/envvars.rst b/docs/envvars.rst index 09e2b462780..5ff430eabea 100644 --- a/docs/envvars.rst +++ b/docs/envvars.rst @@ -351,12 +351,6 @@ Kernel Configuration :Default: ``0`` :Description: Disable NVRTC (CUDA Runtime Compilation) support. When set to ``1``, runtime kernel compilation is disabled. Existing transpose operations select their static fallback automatically. NVFP4 4over6 quantization, fused softmax, and normalization paths require their corresponding ``NVTE_BUILD_LEGACY_STATIC_NVFP4``, ``NVTE_BUILD_LEGACY_STATIC_FUSED_SOFTMAX``, or ``NVTE_BUILD_LEGACY_STATIC_NORM`` CMake option to have been enabled when the library was built; otherwise no static fallback is available. -.. envvar:: NVTE_MXFP8_PER_TILE_SWIZZLE - - :Type: ``int`` (0 or 1) - :Default: ``0`` - :Description: Swizzle MXFP8 scaling factors for GEMM with kernels that give each 128-row tile of scaling factors (row-wise) or each 128-column by 32-row block (column-wise) its own thread block, instead of batching 32 tiles in one block. This lowers the latency of swizzling the scaling factors of tensors with few tiles, such as activations. - .. envvar:: NVTE_USE_CUTLASS_GROUPED_GEMM :Type: ``int`` (0 or 1) diff --git a/tests/cpp/operator/test_multi_swizzle.cu b/tests/cpp/operator/test_multi_swizzle.cu index 4bd1800cea2..69ef1cef99f 100644 --- a/tests/cpp/operator/test_multi_swizzle.cu +++ b/tests/cpp/operator/test_multi_swizzle.cu @@ -51,18 +51,25 @@ void compute_ref_swizzle(const uint8_t *h_input, uint8_t *h_output, } } -static void zero_scale_inv_padding(uint8_t *buf, - size_t padded_rows, size_t padded_cols, - size_t orig_rows, size_t orig_cols) { +static void set_scale_inv_padding(uint8_t *buf, + size_t padded_rows, size_t padded_cols, + size_t orig_rows, size_t orig_cols, + uint8_t value) { for (size_t r = 0; r < padded_rows; ++r) { for (size_t c = 0; c < padded_cols; ++c) { if (r >= orig_rows || c >= orig_cols) { - buf[r * padded_cols + c] = 0; + buf[r * padded_cols + c] = value; } } } } +static void zero_scale_inv_padding(uint8_t *buf, + size_t padded_rows, size_t padded_cols, + size_t orig_rows, size_t orig_cols) { + set_scale_inv_padding(buf, padded_rows, padded_cols, orig_rows, orig_cols, 0); +} + // =================================================================== // Multi-tensor swizzle test // =================================================================== @@ -141,6 +148,159 @@ void performTestMultiTensorSwizzle(const int num_tensors, const size_t M, const } } +void performTestMultiTensorSwizzleNVFP4(const int num_tensors, const size_t M, const size_t K, + bool rowwise) { + using namespace test; + constexpr size_t BLOCK_SIZE = 16; + std::vector> inputs; + std::vector> outputs; + std::vector input_handles; + std::vector output_handles; + + for (int i = 0; i < num_tensors; ++i) { + auto input = std::make_unique("input_" + std::to_string(i), + std::vector{M, K}, DType::kFloat4E2M1, rowwise, + !rowwise, NVTE_NVFP4_1D_SCALING); + auto output = std::make_unique("output_" + std::to_string(i), + std::vector{M, K}, DType::kFloat4E2M1, rowwise, + !rowwise, NVTE_NVFP4_1D_SCALING); + output->set_with_gemm_swizzled_scales(true); + if (rowwise) { + input->fill_uniform_rowwise_scale_inv(); + } else { + input->fill_uniform_columnwise_scale_inv(); + } + const NVTEShape scale_shape = + rowwise ? input->rowwise_scale_inv_shape() : input->columnwise_scale_inv_shape(); + uint8_t *input_scales = rowwise ? input->rowwise_cpu_scale_inv_ptr() + : input->columnwise_cpu_scale_inv_ptr(); + const size_t original_rows = rowwise ? M : K; + const size_t original_cols = + rowwise ? (K + BLOCK_SIZE - 1) / BLOCK_SIZE : (M + BLOCK_SIZE - 1) / BLOCK_SIZE; + set_scale_inv_padding(input_scales, scale_shape.data[0], scale_shape.data[1], original_rows, + original_cols, 0xFF); + input->from_cpu(); + input_handles.push_back(input->data()); + output_handles.push_back(output->data()); + inputs.emplace_back(std::move(input)); + outputs.emplace_back(std::move(output)); + } + + nvte_multi_tensor_swizzle_scaling_factors(input_handles.data(), output_handles.data(), + num_tensors, 0); + NVTE_CHECK_CUDA(cudaDeviceSynchronize()); + for (int i = 0; i < num_tensors; ++i) { + outputs[i]->to_cpu(); + const NVTEShape scale_shape = + rowwise ? inputs[i]->rowwise_scale_inv_shape() : inputs[i]->columnwise_scale_inv_shape(); + const uint8_t *input_scales = rowwise ? inputs[i]->rowwise_cpu_scale_inv_ptr() + : inputs[i]->columnwise_cpu_scale_inv_ptr(); + const uint8_t *actual = rowwise ? outputs[i]->rowwise_cpu_scale_inv_ptr() + : outputs[i]->columnwise_cpu_scale_inv_ptr(); + std::vector reference_input( + input_scales, input_scales + scale_shape.data[0] * scale_shape.data[1]); + const size_t original_rows = rowwise ? M : K; + const size_t original_cols = + rowwise ? (K + BLOCK_SIZE - 1) / BLOCK_SIZE : (M + BLOCK_SIZE - 1) / BLOCK_SIZE; + zero_scale_inv_padding(reference_input.data(), scale_shape.data[0], scale_shape.data[1], + original_rows, original_cols); + std::vector reference(scale_shape.data[0] * scale_shape.data[1]); + compute_ref_swizzle<128, 4, true>(reference_input.data(), reference.data(), scale_shape.data[0], + scale_shape.data[1]); + compareResults("multi_nvfp4_swizzle", actual, reference.data(), reference.size()); + } +} + +TEST(MultiTensorSwizzleTest, MXFP8ColumnwiseBatchIgnoresUnusedRowwiseBuffer) { + using namespace test; + constexpr int num_tensors = 2; + const std::vector shape{256, 128}; + std::vector> inputs; + std::vector> outputs; + std::vector input_handles; + std::vector output_handles; + + for (int i = 0; i < num_tensors; ++i) { + const bool has_rowwise_scales = i == 0; + auto input = std::make_unique("input_" + std::to_string(i), shape, + DType::kFloat8E4M3, has_rowwise_scales, true, + NVTE_MXFP8_1D_SCALING); + auto output = std::make_unique("output_" + std::to_string(i), shape, + DType::kFloat8E4M3, has_rowwise_scales, true, + NVTE_MXFP8_1D_SCALING); + if (has_rowwise_scales) input->fill_uniform_rowwise_scale_inv(); + input->fill_uniform_columnwise_scale_inv(); + output->set_with_gemm_swizzled_scales(true); + input->from_cpu(); + input_handles.push_back(input->data()); + output_handles.push_back(output->data()); + inputs.emplace_back(std::move(input)); + outputs.emplace_back(std::move(output)); + } + + nvte_multi_tensor_swizzle_scaling_factors(input_handles.data(), output_handles.data(), + num_tensors, 0); + NVTE_CHECK_CUDA(cudaDeviceSynchronize()); + for (int i = 0; i < num_tensors; ++i) { + outputs[i]->to_cpu(); + const NVTEShape scale_shape = inputs[i]->columnwise_scale_inv_shape(); + std::vector reference(scale_shape.data[0] * scale_shape.data[1]); + compute_ref_swizzle<128, 4, false>( + inputs[i]->columnwise_cpu_scale_inv_ptr(), reference.data(), scale_shape.data[1], + scale_shape.data[0]); + compareResults("multi_mixed_buffers_columnwise_" + std::to_string(i), + outputs[i]->columnwise_cpu_scale_inv_ptr(), reference.data(), + reference.size()); + } +} + +TEST(MultiTensorSwizzleTest, NVFP4FullWidthChunkLaunchSplitting) { + for (const bool rowwise : {false, true}) { + performTestMultiTensorSwizzleNVFP4(4, 160, 96, rowwise); + performTestMultiTensorSwizzleNVFP4(4, 128, 128, rowwise); + performTestMultiTensorSwizzleNVFP4(65, 128, 128, rowwise); + } +} + +TEST(MultiTensorSwizzleTest, RowwiseMixedSharedMemoryFallback) { + using namespace test; + // The first tensor fits the full-width shared-memory limit, but the second does not. + const std::vector> shapes{{128, 7168}, {128, 8064}}; + std::vector> inputs; + std::vector> outputs; + std::vector input_handles; + std::vector output_handles; + for (size_t i = 0; i < shapes.size(); ++i) { + const auto [M, K] = shapes[i]; + auto input = std::make_unique("input_" + std::to_string(i), + std::vector{M, K}, DType::kFloat8E4M3, true, + false, NVTE_MXFP8_1D_SCALING); + auto output = std::make_unique("output_" + std::to_string(i), + std::vector{M, K}, DType::kFloat8E4M3, true, + false, NVTE_MXFP8_1D_SCALING); + output->set_with_gemm_swizzled_scales(true); + input->fill_uniform_rowwise_scale_inv(); + input->from_cpu(); + input_handles.push_back(input->data()); + output_handles.push_back(output->data()); + inputs.emplace_back(std::move(input)); + outputs.emplace_back(std::move(output)); + } + + nvte_multi_tensor_swizzle_scaling_factors( + input_handles.data(), output_handles.data(), static_cast(inputs.size()), 0); + NVTE_CHECK_CUDA(cudaDeviceSynchronize()); + for (size_t i = 0; i < inputs.size(); ++i) { + outputs[i]->to_cpu(); + const NVTEShape scale_shape = inputs[i]->rowwise_scale_inv_shape(); + std::vector reference(scale_shape.data[0] * scale_shape.data[1]); + compute_ref_swizzle<128, 4, true>(inputs[i]->rowwise_cpu_scale_inv_ptr(), + reference.data(), scale_shape.data[0], scale_shape.data[1]); + compareResults("multi_rowwise_fallback", outputs[i]->rowwise_cpu_scale_inv_ptr(), + reference.data(), reference.size()); + } +} + // =================================================================== // Multi-tensor unswizzle test (uses single-tensor swizzle to prepare) // =================================================================== @@ -300,22 +460,11 @@ void performTestMultiTensorRoundtrip(const int num_tensors, const size_t M, cons // Test suites // =================================================================== -// Sets NVTE_MXFP8_PER_TILE_SWIZZLE while it lives, so that MXFP8 swizzles use the per-tile -// kernels (enable) or the default ones. -class PerTileSwizzle { - public: - explicit PerTileSwizzle(bool enable) { - setenv("NVTE_MXFP8_PER_TILE_SWIZZLE", enable ? "1" : "0", 1); - } - ~PerTileSwizzle() { unsetenv("NVTE_MXFP8_PER_TILE_SWIZZLE"); } -}; - class MultiTensorSwizzleTestSuite - : public ::testing::TestWithParam, bool>> {}; + : public ::testing::TestWithParam> {}; TEST_P(MultiTensorSwizzleTestSuite, TestMultiTensorSwizzle) { - const auto& [num_tensors, M, K, rowwise] = std::get<0>(GetParam()); - const PerTileSwizzle per_tile_swizzle(std::get<1>(GetParam())); + const auto& [num_tensors, M, K, rowwise] = GetParam(); performTestMultiTensorSwizzle(num_tensors, M, K, rowwise); } @@ -343,17 +492,7 @@ TEST_P(MultiTensorRoundtripTestSuite, TestMultiTensorRoundtrip) { namespace { -// Shapes that exercise the narrow_k kernel (rowwise) / narrow_m kernel (colwise): -// Narrow-K fires when ALL tensors have scale num_tiles_k < TB_DIM (32), -// i.e. padded ceil(K/32) < 128. -// Narrow-M fires analogously for colwise when padded K < 4096 -// (since colwise m = K padded to 128, num_tiles_m = m / 128 < 32). -// -// Shapes that bypass narrow and use the regular multi_tensor kernel: -// K >= 4096 makes num_tiles_k >= 32 (rowwise) and num_tiles_m >= 32 (colwise). - std::vector> multi_tensor_test_cases = { - // --- Narrow path cases (K small → narrow_k for row, narrow_m for col) --- // M and K both aligned to 128 {3, 256, 256, true}, {3, 256, 256, false}, @@ -365,7 +504,7 @@ std::vector> multi_tensor_test_cases = { {3, 192, 256, false}, {2, 64, 256, true}, {2, 64, 256, false}, - // Larger narrow K (num_tiles_k = 8, shared mem = 128 KB) + // Larger K {2, 128, 1024, true}, {2, 128, 1024, false}, // K not divisible by 128 @@ -380,13 +519,16 @@ std::vector> multi_tensor_test_cases = { {4, 32, 64, true}, {4, 32, 64, false}, - // --- Non-narrow path cases (K >= 4096 → regular multi_tensor kernel) --- {3, 256, 4096, true}, {3, 256, 4096, false}, {2, 128, 8192, true}, {2, 128, 8192, false}, // Rowwise num_tiles_k = 34 selects vec_load_size = 2 (int2 kernel) {2, 128, 4352, true}, + // Wide row-wise scales fall back for an underfilled grid and use the full-width kernel once + // the aggregate grid is large enough. + {2, 2048, 7168, true}, // 32 aggregate row tiles: regular fallback. + {16, 2048, 7168, true}, // 256 aggregate row tiles: rowwise full-width kernel. // Colwise num_tiles_k = 512 / 32 / 4 = 4 selects vec_load_size = 4 (int4 kernel) {2, 512, 4096, false}, // MXFP8 swizzle: more tensors than one launch takes, several row-wise tiles per CTA, @@ -404,14 +546,12 @@ std::vector> multi_tensor_test_cases = { INSTANTIATE_TEST_SUITE_P( OperatorTest, MultiTensorSwizzleTestSuite, - ::testing::Combine(::testing::ValuesIn(multi_tensor_test_cases), ::testing::Values(false, true)), + ::testing::ValuesIn(multi_tensor_test_cases), [](const testing::TestParamInfo& info) { - const auto& test_case = std::get<0>(info.param); - return "n" + std::to_string(std::get<0>(test_case)) + - "_M" + std::to_string(std::get<1>(test_case)) + - "_K" + std::to_string(std::get<2>(test_case)) + - (std::get<3>(test_case) ? "_row" : "_col") + - (std::get<1>(info.param) ? "_per_tile" : ""); + return "n" + std::to_string(std::get<0>(info.param)) + + "_M" + std::to_string(std::get<1>(info.param)) + + "_K" + std::to_string(std::get<2>(info.param)) + + (std::get<3>(info.param) ? "_row" : "_col"); }); INSTANTIATE_TEST_SUITE_P( diff --git a/tests/cpp/operator/test_swizzle.cu b/tests/cpp/operator/test_swizzle.cu index a442215ae15..7025a4b055d 100644 --- a/tests/cpp/operator/test_swizzle.cu +++ b/tests/cpp/operator/test_swizzle.cu @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -204,18 +205,156 @@ void performTestUnswizzle1D(const size_t M, const size_t K, bool rowwise, bool c // The buffer is stored in leading-dim-major order (row-major for rowwise, // column-major for colwise). `padded_rows x padded_cols` is the full // (padded) shape; `orig_rows` / `orig_cols` are the unpadded extents. -static void zero_scale_inv_padding(uint8_t *buf, - size_t padded_rows, size_t padded_cols, - size_t orig_rows, size_t orig_cols) { +static void set_scale_inv_padding(uint8_t *buf, + size_t padded_rows, size_t padded_cols, + size_t orig_rows, size_t orig_cols, + uint8_t value) { for (size_t r = 0; r < padded_rows; ++r) { for (size_t c = 0; c < padded_cols; ++c) { if (r >= orig_rows || c >= orig_cols) { - buf[r * padded_cols + c] = 0; + buf[r * padded_cols + c] = value; } } } } +static void zero_scale_inv_padding(uint8_t *buf, + size_t padded_rows, size_t padded_cols, + size_t orig_rows, size_t orig_cols) { + set_scale_inv_padding(buf, padded_rows, padded_cols, orig_rows, orig_cols, 0); +} + +void performTestSwizzleNVFP4(const size_t M, const size_t K, bool rowwise) { + using namespace test; + constexpr size_t BLOCK_SIZE = 16; + + const std::vector shape{M, K}; + Tensor input("input", shape, DType::kFloat4E2M1, rowwise, !rowwise, NVTE_NVFP4_1D_SCALING); + Tensor output("output", shape, DType::kFloat4E2M1, rowwise, !rowwise, NVTE_NVFP4_1D_SCALING); + output.set_with_gemm_swizzled_scales(true); + if (rowwise) { + input.fill_uniform_rowwise_scale_inv(); + } else { + input.fill_uniform_columnwise_scale_inv(); + } + + const NVTEShape scale_shape = + rowwise ? input.rowwise_scale_inv_shape() : input.columnwise_scale_inv_shape(); + uint8_t *input_scales = rowwise ? input.rowwise_cpu_scale_inv_ptr() + : input.columnwise_cpu_scale_inv_ptr(); + const size_t original_rows = rowwise ? M : K; + const size_t original_cols = + rowwise ? divide_round_up(K, BLOCK_SIZE) : divide_round_up(M, BLOCK_SIZE); + set_scale_inv_padding(input_scales, scale_shape.data[0], scale_shape.data[1], original_rows, + original_cols, 0xFF); + std::vector reference_input( + input_scales, input_scales + scale_shape.data[0] * scale_shape.data[1]); + zero_scale_inv_padding(reference_input.data(), scale_shape.data[0], scale_shape.data[1], + original_rows, original_cols); + input.from_cpu(); + + std::vector reference(scale_shape.data[0] * scale_shape.data[1]); + compute_ref_swizzle<128, 4, true>(reference_input.data(), reference.data(), scale_shape.data[0], + scale_shape.data[1]); + nvte_swizzle_scaling_factors(input.data(), output.data(), 0); + NVTE_CHECK_CUDA(cudaDeviceSynchronize()); + output.to_cpu(); + + const uint8_t *actual = rowwise ? output.rowwise_cpu_scale_inv_ptr() + : output.columnwise_cpu_scale_inv_ptr(); + compareResults("nvfp4_swizzle", actual, reference.data(), reference.size()); +} + +TEST(SwizzleTest, NVFP4FullWidthChunkAndFallback) { + performTestSwizzleNVFP4(128, 128, true); + performTestSwizzleNVFP4(128, 128, false); + performTestSwizzleNVFP4(160, 96, true); // Padding in both scale dimensions. + performTestSwizzleNVFP4(160, 96, false); // Transposed padding in both scale dimensions. + performTestSwizzleNVFP4(128, 3584, true); // 28.5 KiB, 1 tile: rowwise full-width kernel. + performTestSwizzleNVFP4(2048, 3584, true); // 28.5 KiB, 16 tiles: rowwise full-width kernel. + performTestSwizzleNVFP4(2176, 3584, true); // 28.5 KiB, 17 tiles: regular fallback. + performTestSwizzleNVFP4(128, 4032, true); // 31.5 KiB: regular fallback. + performTestSwizzleNVFP4(128, 4096, true); // 32.5 KiB: regular fallback. + performTestSwizzleNVFP4(3584, 128, false); // 28.5 KiB, 1 tile: transposed full-width kernel. + performTestSwizzleNVFP4(3584, 2048, false); // 28.5 KiB, 16 tiles: transposed full-width kernel. + performTestSwizzleNVFP4(3584, 2176, false); // 28.5 KiB, 17 tiles: regular fallback. + performTestSwizzleNVFP4(4032, 128, false); // 31.5 KiB: regular fallback. + performTestSwizzleNVFP4(4096, 128, false); // 32.5 KiB: regular fallback. +} + +void performTestRejectUnalignedMXFP8Scales(bool rowwise, bool offset_input) { + using namespace test; + constexpr size_t kDataSize = 128 * 128; + constexpr size_t kScaleSize = 128 * 4; + auto input_data = cuda_alloc(kDataSize); + auto output_data = cuda_alloc(kDataSize); + auto input_storage = cuda_alloc(kScaleSize + 4); + auto output_storage = cuda_alloc(kScaleSize + 4); + uint8_t* input_scales = input_storage.get() + (offset_input ? 4 : 0); + uint8_t* output_scales = output_storage.get() + (offset_input ? 0 : 4); + TensorWrapper input(NVTE_MXFP8_1D_SCALING); + TensorWrapper output(NVTE_MXFP8_1D_SCALING); + const std::vector data_shape{128, 128}; + const std::vector scale_shape = rowwise ? std::vector{128, 4} + : std::vector{4, 128}; + if (rowwise) { + input.set_rowwise_data(input_data.get(), DType::kFloat8E4M3, data_shape); + input.set_rowwise_scale_inv(input_scales, DType::kFloat8E8M0, scale_shape); + output.set_rowwise_data(output_data.get(), DType::kFloat8E4M3, data_shape); + output.set_rowwise_scale_inv(output_scales, DType::kFloat8E8M0, scale_shape); + } else { + input.set_columnwise_data(input_data.get(), DType::kFloat8E4M3, data_shape); + input.set_columnwise_scale_inv(input_scales, DType::kFloat8E8M0, scale_shape); + output.set_columnwise_data(output_data.get(), DType::kFloat8E4M3, data_shape); + output.set_columnwise_scale_inv(output_scales, DType::kFloat8E8M0, scale_shape); + } + output.set_with_gemm_swizzled_scales(true); + EXPECT_THROW(nvte_swizzle_scaling_factors(input.data(), output.data(), 0), std::runtime_error); +} + +TEST(SwizzleTest, RejectUnalignedMXFP8ScaleBuffers) { + for (bool rowwise : {true, false}) { + performTestRejectUnalignedMXFP8Scales(rowwise, true); + performTestRejectUnalignedMXFP8Scales(rowwise, false); + } +} + +TEST(SwizzleTest, RejectUnalignedMultiMXFP8ScaleBuffers) { + using namespace test; + constexpr size_t kScaleSize = 128 * 4; + const std::vector data_shape{128, 128}; + const std::vector scale_shape{128, 4}; + for (bool offset_input : {true, false}) { + std::vector> input_data, output_data, input_storage, output_storage; + std::vector input_tensors, output_tensors; + std::vector inputs, outputs; + input_tensors.reserve(2); + output_tensors.reserve(2); + for (int i = 0; i < 2; i++) { + input_data.push_back(cuda_alloc(128 * 128)); + output_data.push_back(cuda_alloc(128 * 128)); + input_storage.push_back(cuda_alloc(kScaleSize + 4)); + output_storage.push_back(cuda_alloc(kScaleSize + 4)); + uint8_t* input_scales = input_storage.back().get() + (i == 0 && offset_input ? 4 : 0); + uint8_t* output_scales = output_storage.back().get() + (i == 0 && !offset_input ? 4 : 0); + input_tensors.emplace_back(NVTE_MXFP8_1D_SCALING); + output_tensors.emplace_back(NVTE_MXFP8_1D_SCALING); + input_tensors.back().set_rowwise_data(input_data.back().get(), DType::kFloat8E4M3, + data_shape); + input_tensors.back().set_rowwise_scale_inv(input_scales, DType::kFloat8E8M0, scale_shape); + output_tensors.back().set_rowwise_data(output_data.back().get(), DType::kFloat8E4M3, + data_shape); + output_tensors.back().set_rowwise_scale_inv(output_scales, DType::kFloat8E8M0, scale_shape); + output_tensors.back().set_with_gemm_swizzled_scales(true); + inputs.push_back(input_tensors.back().data()); + outputs.push_back(output_tensors.back().data()); + } + EXPECT_THROW( + nvte_multi_tensor_swizzle_scaling_factors(inputs.data(), outputs.data(), inputs.size(), 0), + std::runtime_error); + } +} + void performTestGroupedSwizzleMXFP8(const int num_tensors, const size_t M, const size_t K) { using namespace transformer_engine; using namespace test; @@ -308,17 +447,9 @@ void performTestGroupedSwizzleMXFP8(const int num_tensors, const size_t M, const num_tensors * col_numel); } -// Sets NVTE_MXFP8_PER_TILE_SWIZZLE while it lives, so that MXFP8 swizzles use the per-tile -// kernels (enable) or the default ones. -class PerTileSwizzle { - public: - explicit PerTileSwizzle(bool enable) { - setenv("NVTE_MXFP8_PER_TILE_SWIZZLE", enable ? "1" : "0", 1); - } - ~PerTileSwizzle() { unsetenv("NVTE_MXFP8_PER_TILE_SWIZZLE"); } -}; - -class SwizzleTestSuite : public ::testing::TestWithParam, std::pair, bool, bool>> {}; +class SwizzleTestSuite + : public ::testing::TestWithParam< + std::tuple, std::pair, bool>> {}; TEST_P(SwizzleTestSuite, TestSwizzle) { @@ -328,7 +459,6 @@ TEST_P(SwizzleTestSuite, TestSwizzle) { const auto num_tiles = std::get<0>(GetParam()); const auto scaling_mode = std::get<1>(GetParam()); const auto transa = std::get<2>(GetParam()); - const PerTileSwizzle per_tile_swizzle(std::get<3>(GetParam())); performTestSwizzle1D(num_tiles.first, num_tiles.second, scaling_mode.first, scaling_mode.second, @@ -1007,14 +1137,15 @@ std::vector> num_tiles = { {65, 257}, {65, 258}, {65, 259}, - // Additional narrow-path coverage: narrow_k (row) when num_tiles_K < 32, - // narrow_m (col) when num_tiles_M < 32. - {1, 4}, // narrow_k with 4 K-tiles - {1, 8}, // narrow_k with 8 K-tiles - {4, 1}, // narrow_m with 4 M-tiles - {8, 1}, // narrow_m with 8 M-tiles - {31, 1}, // narrow_m at boundary (31 < TB_DIM=32) - {1, 31}, // narrow_k at boundary (31 < TB_DIM=32) + {1, 4}, + {1, 8}, + {4, 1}, + {8, 1}, + {31, 1}, + {1, 31}, + {2, 63}, // Above 30 KiB: regular fallback. + {2, 64}, // Above 32 KiB: regular fallback. + {16, 56}, // 28.5 KiB: rowwise full-width kernel. {1600, 1}, // MXFP8: more row-wise tiles than one CTA per tile allows }; @@ -1051,8 +1182,7 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Combine( ::testing::ValuesIn(num_tiles), ::testing::ValuesIn(scaling_mode), - ::testing::ValuesIn(transa), - ::testing::Values(false, true) + ::testing::ValuesIn(transa) ), [](const testing::TestParamInfo& info) { std::string name = "ntiles" + @@ -1060,8 +1190,7 @@ INSTANTIATE_TEST_SUITE_P( std::to_string(std::get<0>(info.param).second) + "smode" + std::to_string(std::get<1>(info.param).first) + "X"+ std::to_string(std::get<1>(info.param).second) + "trans" + - std::to_string(std::get<2>(info.param)) + - (std::get<3>(info.param) ? "PerTile" : ""); + std::to_string(std::get<2>(info.param)); return name; }); diff --git a/transformer_engine/common/include/transformer_engine/swizzle.h b/transformer_engine/common/include/transformer_engine/swizzle.h index 396093b5439..cffb11a3f8f 100644 --- a/transformer_engine/common/include/transformer_engine/swizzle.h +++ b/transformer_engine/common/include/transformer_engine/swizzle.h @@ -26,6 +26,7 @@ extern "C" { * Requirements: * - scale_inv is stored in row-major. * - scale_inv size is padded to 128x4 for row-scale and 4x128 for col-scale. + * - Input and output scale_inv buffer pointers used by this operation must be 16-byte aligned. * - data is quantized along K-dimension, i.e. 1D-scaling block lies along the K-dimension. */ void nvte_swizzle_scaling_factors(const NVTETensor input, NVTETensor output, cudaStream_t stream); @@ -40,6 +41,7 @@ void nvte_swizzle_scaling_factors(const NVTETensor input, NVTETensor output, cud * Requirements: * - scale_inv is stored in row-major. * - scale_inv size is padded to 128x4 for row-scale and 4x128 for col-scale. + * - Input and output scale_inv buffer pointers used by this operation must be 16-byte aligned. * - data is quantized along K-dimension, i.e. 1D-scaling block lies along the K-dimension. */ void nvte_multi_tensor_swizzle_scaling_factors(const NVTETensor* inputs, NVTETensor* outputs, @@ -50,6 +52,7 @@ void nvte_multi_tensor_swizzle_scaling_factors(const NVTETensor* inputs, NVTETen * * Use this variant when the data and scale_inv tensors intentionally have * different shapes, e.g. when scale_invs have been transposed for attention. + * The 16-byte alignment requirement for scale_inv buffer pointers used by this operation still applies. */ void nvte_multi_tensor_swizzle_scaling_factors_unchecked(const NVTETensor* inputs, NVTETensor* outputs, diff --git a/transformer_engine/common/swizzle/swizzle.cu b/transformer_engine/common/swizzle/swizzle.cu index 19fb8e848ad..af8fe32fe17 100644 --- a/transformer_engine/common/swizzle/swizzle.cu +++ b/transformer_engine/common/swizzle/swizzle.cu @@ -223,17 +223,6 @@ __device__ __forceinline__ void dispatch_swizzle_col_scaling_kernel_impl( } } -template -__global__ void __launch_bounds__(TB_DIM* TB_DIM) - swizzle_col_scaling_kernel(const void* input, void* output, const int M, const int K, - const int original_M, const int original_K) { - const bool padding_m = (blockIdx.y == gridDim.y - 1) && (original_M < M); - const bool padding_k = (blockIdx.x == gridDim.x - 1) && (original_K < K); - dispatch_swizzle_col_scaling_kernel_impl( - input, output, M, K, original_M, original_K, blockIdx.x, blockIdx.y, gridDim.x, gridDim.y, - padding_k, padding_m); -} - template __device__ inline void regs_shuffle(LType* regs_vec) { constexpr int N_TILE_PER_TD = sizeof(LType) / sizeof(int); @@ -394,163 +383,6 @@ __global__ void __launch_bounds__(TB_DIM* TB_DIM) padding_k, padding_m); } -// Narrow-K specialization for row scaling swizzle. -// When K is small (num_tiles_k < TB_DIM), the standard kernel wastes threadIdx.x -// because there aren't enough K-tiles to distribute across threads. -// This kernel repurposes the thread dimensions: threadIdx.x iterates rows within -// an M-tile, threadIdx.y indexes M-tiles within the block, processing TB_DIM -// M-tiles per block with full thread utilization. -template -__device__ void swizzle_row_scaling_narrow_k_kernel_impl(const void* input, void* output, - const int M, const int K, - const int original_M, const int original_K, - const int bid, const int grid_dim) { - constexpr int SF_TILE_SIZE_I32 = SF_TILE_DIM_M * SF_TILE_DIM_K / 4; - const int K_i32 = K / 4; - const int num_tiles_m = M / SF_TILE_DIM_M; - - const int m_tile = bid * blockDim.y + threadIdx.y; - const bool active = (m_tile < num_tiles_m); - - extern __shared__ int4 slm_v4i[]; - const int slm_tile_v4i = K_i32 * (SF_TILE_SIZE_I32 / 4); - - if (active) { - const bool padding_m = (m_tile == num_tiles_m - 1) && (original_M < M); - const bool padding_k = (original_K < K); - - int4* my_slm = slm_v4i + threadIdx.y * slm_tile_v4i; - - for (int k = 0; k < K_i32; k++) { - const int input_base = m_tile * SF_TILE_DIM_M * K_i32 + k; - const int* input_i32 = reinterpret_cast(input) + input_base; - - int regs[N_SF_PER_TD_PER_TILE]; -#pragma unroll - for (int i = 0; i < N_SF_PER_TD_PER_TILE; i++) { - const int row = i * TB_DIM + threadIdx.x; - regs[i] = __ldg(input_i32 + row * K_i32); - if (padding_m || padding_k) { - for (int j = 0; j < 4; j++) { - const int byte_row = m_tile * SF_TILE_DIM_M + row; - const int byte_col = k * 4 + j; - if (byte_row >= original_M || byte_col >= original_K) { - reinterpret_cast(®s[i])[j] = 0; - } - } - } - } - - my_slm[k * (SF_TILE_SIZE_I32 / 4) + threadIdx.x] = *reinterpret_cast(regs); - } - } - - __syncthreads(); - - if (active) { - int4* my_slm = slm_v4i + threadIdx.y * slm_tile_v4i; - int4* out_v4i = - reinterpret_cast(reinterpret_cast(output) + m_tile * SF_TILE_DIM_M * K_i32); - - for (int i = threadIdx.x; i < slm_tile_v4i; i += blockDim.x) { - out_v4i[i] = my_slm[i]; - } - } -} - -template -__global__ void __launch_bounds__(TB_DIM* TB_DIM) - swizzle_row_scaling_narrow_k_kernel(const void* input, void* output, const int M, const int K, - const int original_M, const int original_K) { - swizzle_row_scaling_narrow_k_kernel_impl( - input, output, M, K, original_M, original_K, blockIdx.x, gridDim.x); -} - -// Narrow-M variant of the column scaling swizzle kernel, for when num_tiles_m < TB_DIM. -// Analogous to the narrow-K row kernel: when the M dimension is small, the normal -// col kernel underutilizes threads in the load phase because threadIdx.x covers M -// positions with vectorized loads, leaving many threads idle. This kernel repurposes -// thread dimensions: threadIdx.y indexes K-tiles within the block, threadIdx.x covers -// one int32 column of an M-tile, and M-tiles are iterated serially. -template -__device__ void swizzle_col_scaling_narrow_m_kernel_impl(const void* input, void* output, - const int M, const int K, - const int original_M, const int original_K, - const int bid, const int grid_dim) { - constexpr int SF_TILE_SIZE_I32 = SF_TILE_DIM_M * SF_TILE_DIM_K / 4; - constexpr int SF_TILE_DIM_M_I32 = SF_TILE_DIM_M / 4; - constexpr int SF_TILE_DIM_K_I32 = SF_TILE_DIM_K; - - const int M_i32 = M / 4; - const int K_i32 = K; - const int num_tiles_m = M / SF_TILE_DIM_M; - const int num_tiles_k = K / SF_TILE_DIM_K; - - const int k_tile = bid * blockDim.y + threadIdx.y; - const bool active = (k_tile < num_tiles_k); - const int remaining = num_tiles_k - bid * static_cast(blockDim.y); - const int k_tiles_in_block = remaining <= 0 ? 0 : (remaining < TB_DIM ? remaining : TB_DIM); - - extern __shared__ int slm_narrow_m[]; - - if (active) { - const bool padding_k = (k_tile == num_tiles_k - 1) && (original_K < K); - const int32_t* input_i32 = reinterpret_cast(input); - - for (int m_tile = 0; m_tile < num_tiles_m; m_tile++) { - const bool padding_m = (m_tile == num_tiles_m - 1) && (original_M < M); - - int regs[N_SF_PER_TD_PER_TILE]; -#pragma unroll - for (int i = 0; i < N_SF_PER_TD_PER_TILE; i++) { - const int k_row = k_tile * SF_TILE_DIM_K_I32 + i; - const int m_col = m_tile * SF_TILE_DIM_M_I32 + threadIdx.x; - regs[i] = __ldg(input_i32 + k_row * M_i32 + m_col); - if (padding_m || padding_k) { - for (int j = 0; j < 4; j++) { - if (m_col * 4 + j >= original_M || k_row >= original_K) { - reinterpret_cast(®s[i])[j] = 0; - } - } - } - } - - regs_shuffle_with_bit_shifts(regs); - - int tM = threadIdx.x * N_SF_PER_TD_PER_TILE; - int* slm_tile = - slm_narrow_m + m_tile * TB_DIM * SF_TILE_SIZE_I32 + threadIdx.y * SF_TILE_SIZE_I32; -#pragma unroll - for (int i = 0; i < N_SF_PER_TD_PER_TILE; i++) { - slm_tile[(tM % SF_TILE_DIM_M) / NEW_SF_TILE_DIM_M_I32 + - ((tM + i) % NEW_SF_TILE_DIM_M_I32) * NEW_SF_TILE_DIM_K_I32] = regs[i]; - } - } - } - - __syncthreads(); - - const int linear_id = threadIdx.y * blockDim.x + threadIdx.x; - for (int m_tile = 0; m_tile < num_tiles_m; m_tile++) { - int4* out_v4i = reinterpret_cast(reinterpret_cast(output) + - m_tile * SF_TILE_DIM_M_I32 * K_i32 + - bid * TB_DIM * SF_TILE_SIZE_I32); - int4* slm_v4i = reinterpret_cast(slm_narrow_m + m_tile * TB_DIM * SF_TILE_SIZE_I32); - const int n_v4i = k_tiles_in_block * SF_TILE_SIZE_I32 / 4; - for (int j = linear_id; j < n_v4i; j += blockDim.x * blockDim.y) { - out_v4i[j] = slm_v4i[j]; - } - } -} - -template -__global__ void __launch_bounds__(TB_DIM* TB_DIM) - swizzle_col_scaling_narrow_m_kernel(const void* input, void* output, const int M, const int K, - const int original_M, const int original_K) { - swizzle_col_scaling_narrow_m_kernel_impl( - input, output, M, K, original_M, original_K, blockIdx.x, gridDim.x); -} - template __device__ void unswizzle_row_scaling_kernel_impl(const void* input, void* output, const int M, const int K, const int bid_x, const int bid_y, @@ -867,102 +699,25 @@ __global__ void __launch_bounds__(TB_DIM* TB_DIM) padding_m); } -template -__global__ void __launch_bounds__(TB_DIM* TB_DIM) - multi_tensor_swizzle_col_scaling_kernel(MultiSwizzleArgs kernel_args) { - // Find tensor corresponding to block - const int bid = blockIdx.x; - int tensor_id = 0; - while (kernel_args.block_range[tensor_id + 1] <= bid) { - ++tensor_id; - } - // Get args corresponding to block - const void* input = kernel_args.input_list[tensor_id]; - void* output = kernel_args.output_list[tensor_id]; - const int M = kernel_args.m_list[tensor_id]; - const int K = kernel_args.k_list[tensor_id]; - const int original_M = kernel_args.original_m_list[tensor_id]; - const int original_K = kernel_args.original_k_list[tensor_id]; - - constexpr int N_TILE_PER_TD = sizeof(LType) / sizeof(int); - - // Get block index in grid. Emulate 2D grid. - const int num_tiles_k = K / SF_TILE_DIM_K; - const int num_tiles_m = M / SF_TILE_DIM_M; - const int grid_dim_x = DIVUP(num_tiles_k, TB_DIM); - const int grid_dim_y = DIVUP(num_tiles_m, N_TILE_PER_TD); - const int bid_x = (bid - kernel_args.block_range[tensor_id]) / grid_dim_y; - const int bid_y = (bid - kernel_args.block_range[tensor_id]) % grid_dim_y; - - const bool padding_m = (bid_y == grid_dim_y - 1) && (original_M < M); - const bool padding_k = (bid_x == grid_dim_x - 1) && (original_K < K); - dispatch_swizzle_col_scaling_kernel_impl( - input, output, M, K, original_M, original_K, bid_x, bid_y, grid_dim_x, grid_dim_y, padding_k, - padding_m); -} - -template -__global__ void __launch_bounds__(TB_DIM* TB_DIM) - multi_tensor_swizzle_row_scaling_narrow_k_kernel(MultiSwizzleArgs kernel_args) { - const int bid = blockIdx.x; - int tensor_id = 0; - while (kernel_args.block_range[tensor_id + 1] <= bid) { - ++tensor_id; - } - const void* input = kernel_args.input_list[tensor_id]; - void* output = kernel_args.output_list[tensor_id]; - const int M = kernel_args.m_list[tensor_id]; - const int K = kernel_args.k_list[tensor_id]; - const int original_M = kernel_args.original_m_list[tensor_id]; - const int original_K = kernel_args.original_k_list[tensor_id]; - const int flat_bid = bid - kernel_args.block_range[tensor_id]; - const int num_tiles_m = M / SF_TILE_DIM_M; - const int grid_dim = DIVUP(num_tiles_m, TB_DIM); - - swizzle_row_scaling_narrow_k_kernel_impl( - input, output, M, K, original_M, original_K, flat_bid, grid_dim); -} - -template -__global__ void __launch_bounds__(TB_DIM* TB_DIM) - multi_tensor_swizzle_col_scaling_narrow_m_kernel(MultiSwizzleArgs kernel_args) { - const int bid = blockIdx.x; - int tensor_id = 0; - while (kernel_args.block_range[tensor_id + 1] <= bid) { - ++tensor_id; - } - const void* input = kernel_args.input_list[tensor_id]; - void* output = kernel_args.output_list[tensor_id]; - const int M = kernel_args.m_list[tensor_id]; - const int K = kernel_args.k_list[tensor_id]; - const int original_M = kernel_args.original_m_list[tensor_id]; - const int original_K = kernel_args.original_k_list[tensor_id]; - const int flat_bid = bid - kernel_args.block_range[tensor_id]; - const int num_tiles_k = K / SF_TILE_DIM_K; - const int grid_dim = DIVUP(num_tiles_k, TB_DIM); - - swizzle_col_scaling_narrow_m_kernel_impl( - input, output, M, K, original_M, original_K, flat_bid, grid_dim); -} - -// MXFP8 scale swizzle with one thread block per scale tile, used when -// NVTE_MXFP8_PER_TILE_SWIZZLE=1. The narrow-K / narrow-M kernels above put 32 tiles in one -// 1024-thread block, so e.g. the scales of a 4096-row activation are swizzled by a single block. -// These kernels give every 128-row tile (row-wise) or 128-column x 32-row block (column-wise) its -// own block, which moves a contiguous chunk with coalesced 16-byte loads and stores through shared -// memory. Multi-tensor launches put several tiles in one block only when the grid would otherwise -// be very large. +// Optimized scale swizzle for narrow scale widths. "Rowwise full width" means that a block stages +// the entire padded K dimension for one or more 128-row tiles. "Columnwise chunk" means that a +// block stages only 32 (or 128 for large grids) K rows for one 128-column M tile. Both paths use +// coalesced 16-byte loads and stores through shared memory. constexpr int kRowwiseThreadsPerBlock = 128; constexpr int kColumnwiseThreadsPerBlock = 256; constexpr int kMaxTensorsPerLaunch = 64; -// Row-wise: the largest grid before several tiles share a block. +constexpr int kRowwiseMaxSharedMemoryBytes = 32 * 1024; +// Wide rows use the full-width kernel for tiny launches or once the aggregate grid is large enough +// to occupy the GPU. Narrow rows use it regardless of grid size. +constexpr int kRowwiseNarrowSharedMemoryBytes = 17 * 1024; +constexpr int kRowwiseWideSharedMemoryBytes = 30 * 1024; +constexpr int kRowwiseMaxTinyTiles = 16; +constexpr int kRowwiseMinSaturatingTiles = 128; +// Row-wise: the largest grid before several row tiles share a block. constexpr int kRowwiseMaxBlocks = 1536; // Column-wise: the largest grid with 32-row chunks before switching to 128-row chunks. constexpr int kColumnwiseMaxBlocksWith32RowChunks = 1024; -// Whether MXFP8 swizzles use the per-tile kernels (NVTE_MXFP8_PER_TILE_SWIZZLE=1). -bool use_per_tile_swizzle() { return getenv("NVTE_MXFP8_PER_TILE_SWIZZLE"); } - // The 4 scales packed in `word` (the first one in column first_column), with those in columns // >= num_columns set to zero. __device__ __forceinline__ uint32_t zero_padding_columns(uint32_t word, int first_column, @@ -983,10 +738,10 @@ inline uint32_t division_magic(int divisor) { return divisor <= 1 ? 0u : static_cast(((1ull << 32) + divisor - 1) / divisor); } -// Row-wise: tiles [first_tile, first_tile + num_tiles) of one tensor, a contiguous chunk of the +// Row-wise: tiles [first_tile, first_tile + num_tiles) of one tensor, a consecutive chunk of the // input and of the output. Shared-memory rows are padded to an odd number of words, so the column // gathers are free of bank conflicts. Requires num_tiles * 128 * padded_k / 4 < 2^16 (guaranteed -// by the 48 KB shared-memory limit). +// by the row-wise shared-memory limit = 30 KB). __device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict__ input, uint8_t* __restrict__ output, int padded_k, uint32_t words_per_row_magic, int original_m, @@ -1084,18 +839,18 @@ __device__ __forceinline__ void swizzle_columnwise_chunk(const uint8_t* __restri } __global__ void __launch_bounds__(kRowwiseThreadsPerBlock) - swizzle_rowwise_per_tile_kernel(const uint8_t* __restrict__ input, uint8_t* __restrict__ output, - int padded_k, uint32_t words_per_row_magic, int original_m, - int original_k) { + swizzle_rowwise_full_width_kernel(const uint8_t* __restrict__ input, + uint8_t* __restrict__ output, int padded_k, + uint32_t words_per_row_magic, int original_m, + int original_k) { extern __shared__ uint32_t rowwise_smem[]; swizzle_rowwise_tiles(input, output, padded_k, words_per_row_magic, original_m, original_k, blockIdx.x, 1, rowwise_smem); } __global__ void __launch_bounds__(kColumnwiseThreadsPerBlock) - swizzle_columnwise_per_tile_kernel(const uint8_t* __restrict__ input, - uint8_t* __restrict__ output, int padded_m, int padded_k, - int original_m, int original_k) { + swizzle_columnwise_chunk_kernel(const uint8_t* __restrict__ input, uint8_t* __restrict__ output, + int padded_m, int padded_k, int original_m, int original_k) { __shared__ int4 columnwise_smem[32 * 8]; swizzle_columnwise_chunk<32>(input, output, padded_m, padded_k, original_m, original_k, blockIdx.x, blockIdx.y * 32, columnwise_smem); @@ -1104,33 +859,33 @@ __global__ void __launch_bounds__(kColumnwiseThreadsPerBlock) // The scales of one tensor: padded_m (a multiple of 128) by padded_k (a multiple of 4) in // row-wise terms; for column-wise scales, m runs along the columns of the data. Scales at // m >= original_m or k >= original_k are padding and written as zeros. -struct PerTileSwizzleTensor { +struct SwizzleTensorDescriptor { const uint8_t* input; uint8_t* output; int padded_m, padded_k, original_m, original_k; }; -struct PerTileMultiSwizzleArgs { - PerTileSwizzleTensor tensors[kMaxTensorsPerLaunch]; +struct MultiTensorSwizzleArgs { + SwizzleTensorDescriptor tensors[kMaxTensorsPerLaunch]; int first_block[kMaxTensorsPerLaunch + 1]; uint32_t words_per_row_magic[kMaxTensorsPerLaunch]; // row-wise only - int tiles_per_block; // row-wise only + int row_tiles_per_block; // row-wise only int num_tensors; }; -__device__ __forceinline__ int tensor_of_block(const PerTileMultiSwizzleArgs& args, int block) { +__device__ __forceinline__ int tensor_of_block(const MultiTensorSwizzleArgs& args, int block) { int tensor = 0; while (tensor + 1 < args.num_tensors && args.first_block[tensor + 1] <= block) tensor++; return tensor; } __global__ void __launch_bounds__(kRowwiseThreadsPerBlock) - multi_tensor_swizzle_rowwise_per_tile_kernel(const PerTileMultiSwizzleArgs args) { + multi_tensor_swizzle_rowwise_full_width_kernel(const MultiTensorSwizzleArgs args) { extern __shared__ uint32_t rowwise_multi_smem[]; const int t = tensor_of_block(args, blockIdx.x); - const int tiles_per_block = args.tiles_per_block; - const int first_tile = (blockIdx.x - args.first_block[t]) * tiles_per_block; - const int num_tiles = min(tiles_per_block, args.tensors[t].padded_m / 128 - first_tile); + const int row_tiles_per_block = args.row_tiles_per_block; + const int first_tile = (blockIdx.x - args.first_block[t]) * row_tiles_per_block; + const int num_tiles = min(row_tiles_per_block, args.tensors[t].padded_m / 128 - first_tile); swizzle_rowwise_tiles(args.tensors[t].input, args.tensors[t].output, args.tensors[t].padded_k, args.words_per_row_magic[t], args.tensors[t].original_m, args.tensors[t].original_k, first_tile, num_tiles, rowwise_multi_smem); @@ -1138,7 +893,7 @@ __global__ void __launch_bounds__(kRowwiseThreadsPerBlock) template __global__ void __launch_bounds__(kColumnwiseThreadsPerBlock) - multi_tensor_swizzle_columnwise_per_tile_kernel(const PerTileMultiSwizzleArgs args) { + multi_tensor_swizzle_columnwise_chunk_kernel(const MultiTensorSwizzleArgs args) { __shared__ int4 columnwise_multi_smem[kChunkRows * 8]; const int t = tensor_of_block(args, blockIdx.x); const int block_in_tensor = blockIdx.x - args.first_block[t]; @@ -1155,116 +910,114 @@ inline bool is_16_byte_aligned(const void* ptr) { return (reinterpret_cast(ptr) & 15u) == 0; } -inline bool per_tile_swizzle_supported(const PerTileSwizzleTensor& tensor, bool rowwise) { - if (tensor.padded_m <= 0 || tensor.padded_k <= 0 || tensor.padded_m % 128 != 0 || - tensor.padded_k % 4 != 0) { - return false; +inline void check_scale_inv_alignment(const Tensor* tensor, const char* name, bool rowwise) { + const auto& scales = rowwise ? tensor->scale_inv : tensor->columnwise_scale_inv; + if (scales.has_data()) { + NVTE_CHECK(is_16_byte_aligned(scales.dptr), name, rowwise ? " row-wise" : " column-wise", + " scale_inv buffer must be 16-byte aligned."); } - if (!is_16_byte_aligned(tensor.input) || !is_16_byte_aligned(tensor.output)) return false; - if (rowwise && 128 * ((tensor.padded_k >> 2) | 1) * 4 > 48 * 1024) return false; - return true; } -// One kernel for up to kMaxTensorsPerLaunch tensors, all row-wise or all column-wise. Row-wise: -// the tiles per block are the smallest power of two that keeps the grid within kRowwiseMaxBlocks -// (shared memory permitting). Column-wise: 128-row chunks once 32-row chunks would exceed -// kColumnwiseMaxBlocksWith32RowChunks blocks. Returns false, without launching anything, if any -// tensor is not supported. -inline bool launch_multi_tensor_per_tile_swizzle(const PerTileSwizzleTensor* tensors, - int num_tensors, bool rowwise, - cudaStream_t stream) { - if (num_tensors <= 0 || num_tensors > kMaxTensorsPerLaunch) return false; - PerTileMultiSwizzleArgs args; +inline bool should_use_rowwise_full_width_swizzle(int padded_k, int64_t total_row_tiles) { + const int smem_bytes = 128 * ((padded_k >> 2) | 1) * 4; + if (smem_bytes <= kRowwiseNarrowSharedMemoryBytes) return true; + return smem_bytes <= kRowwiseWideSharedMemoryBytes && + (total_row_tiles <= kRowwiseMaxTinyTiles || total_row_tiles >= kRowwiseMinSaturatingTiles); +} + +// Launches the rowwise full-width kernel, where each block stages the entire padded K dimension, +// for up to kMaxTensorsPerLaunch tensors. The row tiles per block are the smallest power of two +// that keeps the grid within kRowwiseMaxBlocks, subject to the shared-memory limit. +inline void launch_multi_tensor_rowwise_full_width_swizzle(const SwizzleTensorDescriptor* tensors, + int num_tensors, cudaStream_t stream) { + MultiTensorSwizzleArgs args{}; args.num_tensors = num_tensors; args.first_block[0] = 0; - int64_t total_tiles = 0; + int64_t total_row_tiles = 0; int max_smem_row_words = 1; for (int i = 0; i < num_tensors; i++) { - if (!per_tile_swizzle_supported(tensors[i], rowwise)) return false; args.tensors[i] = tensors[i]; args.words_per_row_magic[i] = division_magic(tensors[i].padded_k >> 2); - if (rowwise) { - total_tiles += tensors[i].padded_m / 128; - max_smem_row_words = max(max_smem_row_words, (tensors[i].padded_k >> 2) | 1); - } else { - total_tiles += - static_cast(tensors[i].padded_m / 128) * ((tensors[i].padded_k + 31) / 32); - } + total_row_tiles += tensors[i].padded_m / 128; + max_smem_row_words = max(max_smem_row_words, (tensors[i].padded_k >> 2) | 1); } - int tiles_per_block = 1, chunk_rows = 32; - if (rowwise) { - const int max_tiles_per_block = (48 * 1024) / (128 * max_smem_row_words * 4); - while (total_tiles > static_cast(kRowwiseMaxBlocks) * tiles_per_block && - tiles_per_block * 2 <= max_tiles_per_block) { - tiles_per_block *= 2; - } - } else { - if (total_tiles > kColumnwiseMaxBlocksWith32RowChunks) chunk_rows = 128; + int row_tiles_per_block = 1; + const int max_row_tiles_per_block = kRowwiseMaxSharedMemoryBytes / (128 * max_smem_row_words * 4); + while (total_row_tiles > static_cast(kRowwiseMaxBlocks) * row_tiles_per_block && + row_tiles_per_block * 2 <= max_row_tiles_per_block) { + row_tiles_per_block *= 2; } - args.tiles_per_block = tiles_per_block; + args.row_tiles_per_block = row_tiles_per_block; for (int i = 0; i < num_tensors; i++) { - const PerTileSwizzleTensor& tensor = tensors[i]; - const int blocks = - rowwise ? (tensor.padded_m / 128 + tiles_per_block - 1) / tiles_per_block - : (tensor.padded_m / 128) * ((tensor.padded_k + chunk_rows - 1) / chunk_rows); + const SwizzleTensorDescriptor& tensor = tensors[i]; + const int blocks = (tensor.padded_m / 128 + row_tiles_per_block - 1) / row_tiles_per_block; args.first_block[i + 1] = args.first_block[i] + blocks; } for (int i = num_tensors + 1; i <= kMaxTensorsPerLaunch; i++) { args.first_block[i] = args.first_block[num_tensors]; } const int grid = args.first_block[num_tensors]; - if (grid == 0) return true; - if (rowwise) { - multi_tensor_swizzle_rowwise_per_tile_kernel<<< - grid, kRowwiseThreadsPerBlock, tiles_per_block * 128 * max_smem_row_words * 4, stream>>>( - args); - } else if (chunk_rows == 128) { - multi_tensor_swizzle_columnwise_per_tile_kernel<128> + if (grid == 0) return; + multi_tensor_swizzle_rowwise_full_width_kernel<<< + grid, kRowwiseThreadsPerBlock, row_tiles_per_block * 128 * max_smem_row_words * 4, stream>>>( + args); + NVTE_CHECK_CUDA(cudaGetLastError()); +} + +// Launches the columnwise chunk kernel, where each block stages part of K for one 128-column M +// tile, for up to kMaxTensorsPerLaunch tensors. It uses 128-row K chunks once 32-row chunks would +// exceed kColumnwiseMaxBlocksWith32RowChunks blocks. +inline void launch_multi_tensor_columnwise_chunk_swizzle(const SwizzleTensorDescriptor* tensors, + int num_tensors, cudaStream_t stream) { + MultiTensorSwizzleArgs args{}; + args.num_tensors = num_tensors; + args.first_block[0] = 0; + int64_t total_chunks = 0; + for (int i = 0; i < num_tensors; i++) { + args.tensors[i] = tensors[i]; + total_chunks += + static_cast(tensors[i].padded_m / 128) * ((tensors[i].padded_k + 31) / 32); + } + const int chunk_rows = total_chunks > kColumnwiseMaxBlocksWith32RowChunks ? 128 : 32; + for (int i = 0; i < num_tensors; i++) { + const SwizzleTensorDescriptor& tensor = tensors[i]; + const int blocks = (tensor.padded_m / 128) * ((tensor.padded_k + chunk_rows - 1) / chunk_rows); + args.first_block[i + 1] = args.first_block[i] + blocks; + } + for (int i = num_tensors + 1; i <= kMaxTensorsPerLaunch; i++) { + args.first_block[i] = args.first_block[num_tensors]; + } + const int grid = args.first_block[num_tensors]; + if (grid == 0) return; + if (chunk_rows == 128) { + multi_tensor_swizzle_columnwise_chunk_kernel<128> <<>>(args); } else { - multi_tensor_swizzle_columnwise_per_tile_kernel<32> + multi_tensor_swizzle_columnwise_chunk_kernel<32> <<>>(args); } NVTE_CHECK_CUDA(cudaGetLastError()); - return true; } -inline bool launch_rowwise_per_tile_swizzle(const PerTileSwizzleTensor& tensor, - cudaStream_t stream) { - if (!per_tile_swizzle_supported(tensor, true)) return false; - swizzle_rowwise_per_tile_kernel<<> 2) | 1) * 4, stream>>>( +// Each block stages the full padded K dimension for one 128-row tile. +inline void launch_rowwise_full_width_swizzle(const SwizzleTensorDescriptor& tensor, + cudaStream_t stream) { + const int grid = tensor.padded_m / 128; + const int smem = 128 * ((tensor.padded_k >> 2) | 1) * 4; + swizzle_rowwise_full_width_kernel<<>>( tensor.input, tensor.output, tensor.padded_k, division_magic(tensor.padded_k >> 2), tensor.original_m, tensor.original_k); NVTE_CHECK_CUDA(cudaGetLastError()); - return true; } -inline bool launch_columnwise_per_tile_swizzle(const PerTileSwizzleTensor& tensor, - cudaStream_t stream) { - if (!per_tile_swizzle_supported(tensor, false)) return false; +// Each block stages a 32-row K chunk for one 128-column M tile. +inline void launch_columnwise_chunk_swizzle(const SwizzleTensorDescriptor& tensor, + cudaStream_t stream) { dim3 grid(tensor.padded_m / 128, (tensor.padded_k + 31) / 32); - swizzle_columnwise_per_tile_kernel<<>>( + swizzle_columnwise_chunk_kernel<<>>( tensor.input, tensor.output, tensor.padded_m, tensor.padded_k, tensor.original_m, tensor.original_k); NVTE_CHECK_CUDA(cudaGetLastError()); - return true; -} - -// A single tensor uses the one-tile-per-block kernels unless its grid would be large; several -// tensors go through one multi-tensor launch. -inline bool launch_per_tile_swizzle(const PerTileSwizzleTensor* tensors, int num_tensors, - bool rowwise, cudaStream_t stream) { - const bool grid_too_large = - num_tensors == 1 && (rowwise ? tensors[0].padded_m / 128 > kRowwiseMaxBlocks - : static_cast(tensors[0].padded_m / 128) * - ((tensors[0].padded_k + 31) / 32) > - kColumnwiseMaxBlocksWith32RowChunks); - if (num_tensors == 1 && !grid_too_large) { - return rowwise ? launch_rowwise_per_tile_swizzle(tensors[0], stream) - : launch_columnwise_per_tile_swizzle(tensors[0], stream); - } - return launch_multi_tensor_per_tile_swizzle(tensors, num_tensors, rowwise, stream); } } // namespace @@ -1275,7 +1028,6 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s NVTE_CHECK(scaling_mode == NVTE_MXFP8_1D_SCALING || scaling_mode == NVTE_NVFP4_1D_SCALING, "Input tensor has invalid scaling mode (", to_string(input->scaling_mode), ")."); - // Check tensors CheckInputTensor(*input, "scaling_factor_input"); CheckInputTensor(*output, "scaling_factor_output"); NVTE_CHECK(!input->with_gemm_swizzled_scales, @@ -1341,9 +1093,10 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s NVTE_ERROR("Invalid scaling mode"); } - // Check dims constexpr int SF_TILE_DIM_M = 128; constexpr int SF_TILE_DIM_K = 4; + + // Check dims NVTE_CHECK(m % SF_TILE_DIM_M == 0, "Input should be padded in M/N dimension!"); NVTE_CHECK(k % SF_TILE_DIM_K == 0, "Input should be padded in K dimension!"); @@ -1367,6 +1120,9 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s return; } + check_scale_inv_alignment(input, "Input", has_rowwise_scale_inv); + check_scale_inv_alignment(output, "Output", has_rowwise_scale_inv); + // Choose swizzle implementation bool rowwise_swizzle{false}, columnwise_swizzle{false}; switch (scaling_mode) { @@ -1420,31 +1176,24 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s NVTE_ERROR("Invalid scaling mode"); } - const int narrow_k_slm_size = - TB_DIM * num_tiles_k * SF_TILE_DIM_M * SF_TILE_DIM_K * static_cast(sizeof(int8_t)); - const PerTileSwizzleTensor per_tile_tensor{static_cast(input_scale_inv_ptr), - static_cast(output_scale_inv_ptr), - m, - k, - original_M, - original_K}; - if (scaling_mode == NVTE_MXFP8_1D_SCALING && use_per_tile_swizzle() && - per_tile_swizzle_supported(per_tile_tensor, true)) { - launch_per_tile_swizzle(&per_tile_tensor, 1, true, stream); - } else if (num_tiles_k < TB_DIM && static_cast(narrow_k_slm_size) <= - cuda::max_shared_memory_per_block_optin()) { - // Narrow-K: batch TB_DIM M-tiles per block, fully utilizing all threads. - dim3 num_blocks_narrow(DIVUP(num_tiles_m, TB_DIM)); - NVTE_CHECK_CUDA( - cudaFuncSetAttribute(swizzle_row_scaling_narrow_k_kernel, - cudaFuncAttributeMaxDynamicSharedMemorySize, narrow_k_slm_size)); - swizzle_row_scaling_narrow_k_kernel - <<>>( - input_scale_inv_ptr, output_scale_inv_ptr, m, k, original_M, original_K); + int vec_load_size = (num_tiles_k - 1) % 4 + 1; + /* there is no int3 and misaligned if using int4/int2 */ + if (vec_load_size == 3) vec_load_size = 1; + // Shared by the row-wise dispatch heuristic and optimized kernels. + const SwizzleTensorDescriptor descriptor{static_cast(input_scale_inv_ptr), + static_cast(output_scale_inv_ptr), + m, + k, + original_M, + original_K}; + if (should_use_rowwise_full_width_swizzle(descriptor.padded_k, + descriptor.padded_m / SF_TILE_DIM_M)) { + if (m / SF_TILE_DIM_M > kRowwiseMaxBlocks) { + launch_multi_tensor_rowwise_full_width_swizzle(&descriptor, 1, stream); + } else { + launch_rowwise_full_width_swizzle(descriptor, stream); + } } else { - int vec_load_size = (num_tiles_k - 1) % 4 + 1; - /* there is no int3 and misaligned if using int4/int2 */ - if (vec_load_size == 3) vec_load_size = 1; int n_tiles_in_tb = TB_DIM * vec_load_size; dim3 num_blocks(DIVUP(num_tiles_k, n_tiles_in_tb), num_tiles_m); int slm_size = n_tiles_in_tb * SF_TILE_DIM_M * SF_TILE_DIM_K * sizeof(int8_t); @@ -1487,77 +1236,26 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s const int original_M = input->flat_last_dim(); const int original_K = input->flat_first_dim() / MXFP8_BLOCK_SIZE; - const int narrow_m_slm_size = - TB_DIM * num_tiles_m * SF_TILE_DIM_M * SF_TILE_DIM_K * static_cast(sizeof(int8_t)); - const PerTileSwizzleTensor per_tile_tensor{ + const SwizzleTensorDescriptor descriptor{ static_cast(input->columnwise_scale_inv.dptr), static_cast(output->columnwise_scale_inv.dptr), m, k, original_M, original_K}; - if (use_per_tile_swizzle() && per_tile_swizzle_supported(per_tile_tensor, false)) { - launch_per_tile_swizzle(&per_tile_tensor, 1, false, stream); - } else if (num_tiles_m < TB_DIM && static_cast(narrow_m_slm_size) <= - cuda::max_shared_memory_per_block_optin()) { - // Narrow-M: batch TB_DIM K-tiles per block, fully utilizing all threads. - dim3 num_blocks_narrow(DIVUP(num_tiles_k, TB_DIM)); - NVTE_CHECK_CUDA( - cudaFuncSetAttribute(swizzle_col_scaling_narrow_m_kernel, - cudaFuncAttributeMaxDynamicSharedMemorySize, narrow_m_slm_size)); - swizzle_col_scaling_narrow_m_kernel - <<>>( - input->columnwise_scale_inv.dptr, output->columnwise_scale_inv.dptr, m, k, original_M, - original_K); + const int64_t grid_size = static_cast(m / SF_TILE_DIM_M) * ((k + 31) / 32); + if (grid_size > kColumnwiseMaxBlocksWith32RowChunks) { + launch_multi_tensor_columnwise_chunk_swizzle(&descriptor, 1, stream); } else { - int vec_load_size = (num_tiles_m - 1) % 4 + 1; - if (vec_load_size == 3) vec_load_size = 1; /* no int3 and misaligned if using int4/int2 */ - int n_tiles_in_tb = TB_DIM * vec_load_size; - dim3 num_blocks(DIVUP(num_tiles_k, TB_DIM), DIVUP(num_tiles_m, vec_load_size)); - int slm_size = n_tiles_in_tb * SF_TILE_DIM_M * SF_TILE_DIM_K * sizeof(int8_t); - - switch (vec_load_size) { - case 4: - NVTE_CHECK_CUDA( - cudaFuncSetAttribute(swizzle_col_scaling_kernel, - cudaFuncAttributeMaxDynamicSharedMemorySize, slm_size)); - swizzle_col_scaling_kernel - <<>>(input->columnwise_scale_inv.dptr, - output->columnwise_scale_inv.dptr, m, - k, original_M, original_K); - break; - case 2: - NVTE_CHECK_CUDA( - cudaFuncSetAttribute(swizzle_col_scaling_kernel, - cudaFuncAttributeMaxDynamicSharedMemorySize, slm_size)); - swizzle_col_scaling_kernel - <<>>(input->columnwise_scale_inv.dptr, - output->columnwise_scale_inv.dptr, m, - k, original_M, original_K); - break; - case 1: - NVTE_CHECK_CUDA( - cudaFuncSetAttribute(swizzle_col_scaling_kernel, - cudaFuncAttributeMaxDynamicSharedMemorySize, slm_size)); - swizzle_col_scaling_kernel - <<>>(input->columnwise_scale_inv.dptr, - output->columnwise_scale_inv.dptr, m, - k, original_M, original_K); - break; - default: - NVTE_ERROR("Not valid vec_load_size."); - break; - } + launch_columnwise_chunk_swizzle(descriptor, stream); } NVTE_CHECK_CUDA(cudaGetLastError()); } } template -void launch_multi_tensor_swizzle_scaling_factors(MultiSwizzleArgs& kernel_args, - const int vec_load_size, const bool is_rowwise, - const bool use_narrow_k, const bool use_narrow_m, - cudaStream_t stream) { +void launch_multi_tensor_row_scaling_swizzle(MultiSwizzleArgs& kernel_args, const int vec_load_size, + cudaStream_t stream) { // cudaFuncSetAttribute is a host-synchronous driver call; cache the max shared memory // setting per kernel variant so we only pay the cost when slm_size actually increases. auto set_smem_if_needed = [](auto kernel_fn, int slm, int& cached) { @@ -1570,125 +1268,40 @@ void launch_multi_tensor_swizzle_scaling_factors(MultiSwizzleArgs& kernel_args, dim3 block_size(TB_DIM, TB_DIM); - if (is_rowwise && use_narrow_k) { - // Narrow-K path: each block handles TB_DIM M-tiles with full thread utilization. - // slm_size depends on num_tiles_k, which can vary per tensor — use the max. - int max_num_tiles_k = 0; - for (size_t j = 0; j < kernel_args.num_tensors; j++) { - const int num_tiles_m = kernel_args.m_list[j] / SF_TILE_DIM_M; - const int num_tiles_k = kernel_args.k_list[j] / SF_TILE_DIM_K; - max_num_tiles_k = std::max(max_num_tiles_k, num_tiles_k); - kernel_args.block_range[j + 1] = kernel_args.block_range[j] + DIVUP(num_tiles_m, TB_DIM); - } - int slm_size = TB_DIM * max_num_tiles_k * SF_TILE_DIM_M * SF_TILE_DIM_K * sizeof(int8_t); - const int num_blocks = kernel_args.block_range[kernel_args.num_tensors]; - - static int cached_narrow_k = -1; - set_smem_if_needed( - multi_tensor_swizzle_row_scaling_narrow_k_kernel, slm_size, - cached_narrow_k); - multi_tensor_swizzle_row_scaling_narrow_k_kernel - <<>>(kernel_args); - } else if (!is_rowwise && use_narrow_m) { - // Narrow-M path: each block handles TB_DIM K-tiles with full thread utilization. - // slm_size depends on num_tiles_m, which can vary per tensor — use the max. - int max_num_tiles_m = 0; - for (size_t j = 0; j < kernel_args.num_tensors; j++) { - const int num_tiles_m = kernel_args.m_list[j] / SF_TILE_DIM_M; - const int num_tiles_k = kernel_args.k_list[j] / SF_TILE_DIM_K; - max_num_tiles_m = std::max(max_num_tiles_m, num_tiles_m); - kernel_args.block_range[j + 1] = kernel_args.block_range[j] + DIVUP(num_tiles_k, TB_DIM); - } - int slm_size = TB_DIM * max_num_tiles_m * SF_TILE_DIM_M * SF_TILE_DIM_K * sizeof(int8_t); - const int num_blocks = kernel_args.block_range[kernel_args.num_tensors]; - - static int cached_narrow_m = -1; - set_smem_if_needed( - multi_tensor_swizzle_col_scaling_narrow_m_kernel, slm_size, - cached_narrow_m); - multi_tensor_swizzle_col_scaling_narrow_m_kernel - <<>>(kernel_args); - } else { - int n_tiles_in_tb = TB_DIM * vec_load_size; - int slm_size = n_tiles_in_tb * SF_TILE_DIM_M * SF_TILE_DIM_K * sizeof(int8_t); - /* Calculate number of CUDA blocks needed for each tensor. - * We have to do it here because we have to iterate over all tensors in this batch to - * get the minimum vec_load_size. - */ - for (size_t j = 0; j < kernel_args.num_tensors; j++) { - const int m = kernel_args.m_list[j]; - const int k = kernel_args.k_list[j]; - int num_tiles_m = m / SF_TILE_DIM_M; - int num_tiles_k = k / SF_TILE_DIM_K; - if (is_rowwise) { - kernel_args.block_range[j + 1] = - kernel_args.block_range[j] + DIVUP(num_tiles_k, n_tiles_in_tb) * num_tiles_m; - } else { - kernel_args.block_range[j + 1] = - kernel_args.block_range[j] + - DIVUP(num_tiles_k, TB_DIM) * DIVUP(num_tiles_m, vec_load_size); - } - } - const int num_blocks = kernel_args.block_range[kernel_args.num_tensors]; - - static int cached_row_int4 = -1, cached_row_int2 = -1, cached_row_int1 = -1; - static int cached_col_int4 = -1, cached_col_int2 = -1, cached_col_int1 = -1; - - if (is_rowwise) { - switch (vec_load_size) { - case 4: - set_smem_if_needed( - multi_tensor_swizzle_row_scaling_kernel, slm_size, - cached_row_int4); - multi_tensor_swizzle_row_scaling_kernel - <<>>(kernel_args); - break; - case 2: - set_smem_if_needed( - multi_tensor_swizzle_row_scaling_kernel, slm_size, - cached_row_int2); - multi_tensor_swizzle_row_scaling_kernel - <<>>(kernel_args); - break; - case 1: - set_smem_if_needed( - multi_tensor_swizzle_row_scaling_kernel, slm_size, - cached_row_int1); - multi_tensor_swizzle_row_scaling_kernel - <<>>(kernel_args); - break; - default: - NVTE_ERROR("Not valid vec_load_size."); - break; - } - } else { - switch (vec_load_size) { - case 4: - set_smem_if_needed( - multi_tensor_swizzle_col_scaling_kernel, slm_size, - cached_col_int4); - multi_tensor_swizzle_col_scaling_kernel - <<>>(kernel_args); - break; - case 2: - set_smem_if_needed( - multi_tensor_swizzle_col_scaling_kernel, slm_size, - cached_col_int2); - multi_tensor_swizzle_col_scaling_kernel - <<>>(kernel_args); - break; - case 1: - set_smem_if_needed( - multi_tensor_swizzle_col_scaling_kernel, slm_size, - cached_col_int1); - multi_tensor_swizzle_col_scaling_kernel - <<>>(kernel_args); - break; - default: - NVTE_ERROR("Not valid vec_load_size."); - break; - } - } + int n_tiles_in_tb = TB_DIM * vec_load_size; + int slm_size = n_tiles_in_tb * SF_TILE_DIM_M * SF_TILE_DIM_K * sizeof(int8_t); + for (size_t j = 0; j < kernel_args.num_tensors; j++) { + const int num_tiles_m = kernel_args.m_list[j] / SF_TILE_DIM_M; + const int num_tiles_k = kernel_args.k_list[j] / SF_TILE_DIM_K; + kernel_args.block_range[j + 1] = + kernel_args.block_range[j] + DIVUP(num_tiles_k, n_tiles_in_tb) * num_tiles_m; + } + const int num_blocks = kernel_args.block_range[kernel_args.num_tensors]; + + static int cached_row_int4 = -1, cached_row_int2 = -1, cached_row_int1 = -1; + switch (vec_load_size) { + case 4: + set_smem_if_needed( + multi_tensor_swizzle_row_scaling_kernel, slm_size, + cached_row_int4); + multi_tensor_swizzle_row_scaling_kernel + <<>>(kernel_args); + break; + case 2: + set_smem_if_needed( + multi_tensor_swizzle_row_scaling_kernel, slm_size, + cached_row_int2); + multi_tensor_swizzle_row_scaling_kernel + <<>>(kernel_args); + break; + case 1: + set_smem_if_needed(multi_tensor_swizzle_row_scaling_kernel, + slm_size, cached_row_int1); + multi_tensor_swizzle_row_scaling_kernel + <<>>(kernel_args); + break; + default: + NVTE_ERROR("Not valid vec_load_size."); } NVTE_CHECK_CUDA(cudaGetLastError()); } @@ -1778,36 +1391,49 @@ void launch_multi_tensor_unswizzle_scaling_factors(MultiSwizzleArgs& kernel_args namespace { -// MXFP8 multi-tensor swizzle with the per-tile kernels. Returns false without launching anything -// if any tensor is not supported (the caller then uses the generic kernels, which also report -// shape errors). -bool multi_tensor_per_tile_swizzle(const std::vector& input, std::vector& output, - bool rowwise, cudaStream_t stream) { +// Multi-tensor swizzle using rowwise full-width or columnwise chunk kernels with aligned int4 +// (16-byte) loads and stores. +void int4_vectorized_multi_tensor_swizzle(const std::vector& input, + std::vector& output, bool rowwise, + cudaStream_t stream) { const size_t num_tensors = input.size(); - std::vector tensors(num_tensors); + std::vector tensors(num_tensors); for (size_t i = 0; i < num_tensors; i++) { - if (!is_mxfp8_scaling(input[i]->scaling_mode)) return false; - const auto& in_scales = rowwise ? input[i]->scale_inv : input[i]->columnwise_scale_inv; - const auto& out_scales = rowwise ? output[i]->scale_inv : output[i]->columnwise_scale_inv; - if (in_scales.shape.size() != 2) return false; - const int padded_m = static_cast(rowwise ? in_scales.shape[0] : in_scales.shape[1]); - const int padded_k = static_cast(rowwise ? in_scales.shape[1] : in_scales.shape[0]); - if (padded_m <= 0 || padded_k <= 0 || padded_m % 128 != 0 || padded_k % 4 != 0) return false; - if (out_scales.numel() != static_cast(padded_m) * padded_k) return false; + const bool nvfp4 = is_nvfp4_scaling(input[i]->scaling_mode); + const bool use_rowwise_buffer = input[i]->scale_inv.has_data(); + const bool selected_rowwise = rowwise && use_rowwise_buffer; + const auto& in_scales = selected_rowwise ? input[i]->scale_inv : input[i]->columnwise_scale_inv; + const auto& out_scales = + selected_rowwise ? output[i]->scale_inv : output[i]->columnwise_scale_inv; + NVTE_CHECK(in_scales.shape.size() == 2, "Expected two-dimensional scaling factors."); + const int padded_m = + static_cast((rowwise || nvfp4) ? in_scales.shape[0] : in_scales.shape[1]); + const int padded_k = + static_cast((rowwise || nvfp4) ? in_scales.shape[1] : in_scales.shape[0]); + NVTE_CHECK(padded_m > 0 && padded_k > 0 && padded_m % 128 == 0 && padded_k % 4 == 0, + "Invalid scaling-factor shape (", padded_m, ", ", padded_k, ")."); + NVTE_CHECK(in_scales.numel() == static_cast(padded_m) * padded_k && + out_scales.numel() == static_cast(padded_m) * padded_k, + "Input and output scaling-factor shapes must match."); const auto [first_dim, last_dim] = input[i]->flat_2d_dims(); + const int block_size = nvfp4 ? NVFP4_BLOCK_SIZE : MXFP8_BLOCK_SIZE; + const int original_m = static_cast(selected_rowwise ? first_dim : last_dim); + const int original_k = static_cast((selected_rowwise ? last_dim : first_dim) / block_size); tensors[i] = {static_cast(in_scales.dptr), static_cast(out_scales.dptr), padded_m, padded_k, - static_cast(rowwise ? first_dim : last_dim), - static_cast((rowwise ? last_dim : first_dim) / MXFP8_BLOCK_SIZE)}; - if (!per_tile_swizzle_supported(tensors[i], rowwise)) return false; + original_m, + original_k}; } for (size_t first = 0; first < num_tensors; first += kMaxTensorsPerLaunch) { const int count = static_cast(std::min(kMaxTensorsPerLaunch, num_tensors - first)); - launch_per_tile_swizzle(tensors.data() + first, count, rowwise, stream); + if (rowwise) { + launch_multi_tensor_rowwise_full_width_swizzle(tensors.data() + first, count, stream); + } else { + launch_multi_tensor_columnwise_chunk_swizzle(tensors.data() + first, count, stream); + } } - return true; } } // namespace @@ -1847,11 +1473,30 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, const bool rowwise_swizzle = all_has_data || all_nvfp4; const bool columnwise_swizzle = all_has_columnwise_data && !all_nvfp4; - if (!all_nvfp4 && use_per_tile_swizzle() && - multi_tensor_per_tile_swizzle(input, output, rowwise_swizzle, stream)) { + int max_rowwise_padded_k = 0; + int64_t total_rowwise_tiles = 0; + for (size_t i = 0; i < num_tensors; i++) { + const bool use_rowwise_buffer = input[i]->scale_inv.has_data(); + const bool selected_rowwise = rowwise_swizzle && use_rowwise_buffer; + const auto& in_scales = selected_rowwise ? input[i]->scale_inv : input[i]->columnwise_scale_inv; + check_scale_inv_alignment(input[i], "Input", selected_rowwise); + check_scale_inv_alignment(output[i], "Output", selected_rowwise); + if (rowwise_swizzle) { + NVTE_CHECK(in_scales.shape.size() == 2, "Expected two-dimensional scaling factors."); + max_rowwise_padded_k = std::max(max_rowwise_padded_k, static_cast(in_scales.shape[1])); + total_rowwise_tiles += static_cast(in_scales.shape[0]) / 128; + } + } + if (rowwise_swizzle) { + if (should_use_rowwise_full_width_swizzle(max_rowwise_padded_k, total_rowwise_tiles)) { + int4_vectorized_multi_tensor_swizzle(input, output, true, stream); + return; + } + } else if (columnwise_swizzle) { + int4_vectorized_multi_tensor_swizzle(input, output, false, stream); return; } - + // Fallback to the generic rowwise swizzle kernel. constexpr int SF_TILE_DIM_M = 128; constexpr int SF_TILE_DIM_K = 4; if (rowwise_swizzle) { @@ -1859,18 +1504,16 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, kernel_args.num_tensors = 0; kernel_args.block_range[0] = 0; int vec_load_size = 4; - bool all_narrow_k = true; for (size_t i = 0; i < num_tensors; i++) { //Launch kernel if argument struct is full if (kernel_args.num_tensors == kMaxTensorsPerKernel) { // There is no int3 and misaligned if using int4/int2. if (vec_load_size == 3) vec_load_size = 1; - launch_multi_tensor_swizzle_scaling_factors( - kernel_args, vec_load_size, true, all_narrow_k, false, stream); + launch_multi_tensor_row_scaling_swizzle( + kernel_args, vec_load_size, stream); // Reset the argument struct and vec_load_size kernel_args.num_tensors = 0; vec_load_size = 4; - all_narrow_k = true; } int m, k; @@ -1904,11 +1547,6 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, } int num_tiles_k = k / SF_TILE_DIM_K; - const int narrow_k_slm = - TB_DIM * num_tiles_k * SF_TILE_DIM_M * SF_TILE_DIM_K * static_cast(sizeof(int8_t)); - all_narrow_k = - all_narrow_k && (num_tiles_k < TB_DIM) && - (static_cast(narrow_k_slm) <= cuda::max_shared_memory_per_block_optin()); int vec_load_size_i = (num_tiles_k - 1) % 4 + 1; // We use the minimum vec_load_size across all tensors. // TODO(zhongbo): fix vec_load_size for NVFP4 @@ -1938,69 +1576,8 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, // Launch the remaining tensors // There is no int3 and misaligned if using int4/int2. if (vec_load_size == 3) vec_load_size = 1; - launch_multi_tensor_swizzle_scaling_factors( - kernel_args, vec_load_size, true, all_narrow_k, false, stream); - } - - if (columnwise_swizzle) { - // NVFP4 shouldn't end up here because it only needs rowwise swizzle - NVTE_CHECK(!all_nvfp4, "NVFP4 shouldn't end up here because it only needs rowwise swizzle"); - - MultiSwizzleArgs kernel_args; - kernel_args.num_tensors = 0; - kernel_args.block_range[0] = 0; - int vec_load_size = 4; - bool all_narrow_m = true; - for (size_t i = 0; i < num_tensors; i++) { - //Launch kernel if argument struct is full - if (kernel_args.num_tensors == kMaxTensorsPerKernel) { - // There is no int3 and misaligned if using int4/int2. - if (vec_load_size == 3) vec_load_size = 1; - launch_multi_tensor_swizzle_scaling_factors( - kernel_args, vec_load_size, false, false, all_narrow_m, stream); - // Reset the argument struct and vec_load_size - kernel_args.num_tensors = 0; - vec_load_size = 4; - all_narrow_m = true; - } - const int m = input[i]->columnwise_scale_inv.shape[1]; - const int k = input[i]->columnwise_scale_inv.shape[0]; - - NVTE_CHECK(m % SF_TILE_DIM_M == 0, "Input should be padded in M/N dimension!"); - NVTE_CHECK(k % SF_TILE_DIM_K == 0, "Input should be padded in K dimension!"); - NVTE_CHECK(k > 0, "Input scale inverse should be 2D!"); - NVTE_CHECK(m * k == std::accumulate(output[i]->columnwise_scale_inv.shape.begin(), - output[i]->columnwise_scale_inv.shape.end(), 1, - std::multiplies()), - "Input.columnwise_scale_inv size is not equal to " - "Output.columnwise_scale_inv size!"); - - int num_tiles_m = m / SF_TILE_DIM_M; - int num_tiles_k = k / SF_TILE_DIM_K; - const int narrow_m_slm = - TB_DIM * num_tiles_m * SF_TILE_DIM_M * SF_TILE_DIM_K * static_cast(sizeof(int8_t)); - all_narrow_m = - all_narrow_m && (num_tiles_m < TB_DIM) && - (static_cast(narrow_m_slm) <= cuda::max_shared_memory_per_block_optin()); - int vec_load_size_i = (num_tiles_k - 1) % 4 + 1; - // We use the minimum vec_load_size across all tensors. - vec_load_size = std::min(vec_load_size, vec_load_size_i); - - const int pos = kernel_args.num_tensors; - kernel_args.input_list[pos] = const_cast(input[i]->columnwise_scale_inv.dptr); - kernel_args.output_list[pos] = output[i]->columnwise_scale_inv.dptr; - kernel_args.m_list[pos] = m; - kernel_args.k_list[pos] = k; - const auto [first_dim, last_dim] = input[i]->flat_2d_dims(); - kernel_args.original_m_list[pos] = last_dim; - kernel_args.original_k_list[pos] = first_dim / MXFP8_BLOCK_SIZE; - kernel_args.num_tensors++; - } - // Launch the remaining tensors - // There is no int3 and misaligned if using int4/int2. - if (vec_load_size == 3) vec_load_size = 1; - launch_multi_tensor_swizzle_scaling_factors( - kernel_args, vec_load_size, false, false, all_narrow_m, stream); + launch_multi_tensor_row_scaling_swizzle(kernel_args, + vec_load_size, stream); } } From 34b8a17940f23d53a6a7b304f80f3c00638b3707 Mon Sep 17 00:00:00 2001 From: Varun Thumbe Date: Sun, 25 Oct 2026 20:05:05 -0700 Subject: [PATCH 2/9] Fuse compact scale padding with swizzle Signed-off-by: Varun Thumbe --- .../mxfp8/test_compact_scale_swizzle.py | 90 +++++ .../include/transformer_engine/swizzle.h | 9 +- transformer_engine/common/swizzle/swizzle.cu | 307 +++++++++++++----- .../pytorch/csrc/extensions/swizzle.cpp | 40 ++- 4 files changed, 346 insertions(+), 100 deletions(-) create mode 100644 tests/pytorch/mxfp8/test_compact_scale_swizzle.py diff --git a/tests/pytorch/mxfp8/test_compact_scale_swizzle.py b/tests/pytorch/mxfp8/test_compact_scale_swizzle.py new file mode 100644 index 00000000000..68872d2285d --- /dev/null +++ b/tests/pytorch/mxfp8/test_compact_scale_swizzle.py @@ -0,0 +1,90 @@ +# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# See LICENSE for license information. + +"""Scale swizzling directly pads compact MXFP8 scale buffers.""" + +import pytest +import torch +from mxfp8_utils import swizzle_mxfp8_scale + +import transformer_engine_torch as tex +import transformer_engine.pytorch as te +from transformer_engine.pytorch.tensor.mxfp8_tensor import MXFP8Quantizer, MXFP8Tensor + + +def _round_up(value, multiple): + return (value + multiple - 1) // multiple * multiple + + +def _scale_shape(rows, cols, rowwise, compact): + if rowwise: + return ( + rows if compact else _round_up(rows, 128), + _round_up((cols + 31) // 32, 4), + ) + return ( + (rows + 31) // 32 if compact else _round_up((rows + 31) // 32, 4), + _round_up(cols, 128), + ) + + +def _make_tensor_and_reference(shape, rowwise): + rows, cols = shape + compact_shape = _scale_shape(rows, cols, rowwise, compact=True) + logical_shape = (rows, (cols + 31) // 32) if rowwise else ((rows + 31) // 32, cols) + scales = torch.zeros(compact_shape, dtype=torch.uint8, device="cuda") + scales[: logical_shape[0], : logical_shape[1]] = torch.randint( + 1, 256, logical_shape, dtype=torch.uint8, device="cuda" + ) + + padded_rows, padded_cols = _round_up(rows, 128), _round_up(cols, 128) + padded_shape = _scale_shape(padded_rows, padded_cols, rowwise, compact=False) + padded = torch.zeros(padded_shape, dtype=torch.uint8, device="cuda") + padded[: compact_shape[0], : compact_shape[1]].copy_(scales) + reference = swizzle_mxfp8_scale( + padded_rows, padded_cols, padded, columnwise=not rowwise + ) + + tensor = MXFP8Tensor( + shape=shape, + dtype=torch.bfloat16, + device="cuda", + rowwise_data=torch.empty(shape, dtype=torch.uint8, device="cuda") if rowwise else None, + columnwise_data=None + if rowwise + else torch.empty(shape, dtype=torch.uint8, device="cuda"), + rowwise_scale_inv=scales if rowwise else None, + columnwise_scale_inv=None if rowwise else scales, + fp8_dtype=tex.DType.kFloat8E4M3, + quantizer=MXFP8Quantizer( + tex.DType.kFloat8E4M3, rowwise=rowwise, columnwise=not rowwise + ), + with_gemm_swizzled_scales=False, + ) + return tensor, reference + + +@pytest.mark.skipif(not te.is_mxfp8_available(), reason="MXFP8 requires Blackwell") +@pytest.mark.parametrize("shape", [(32, 32), (160, 96), (288, 224), (4096, 4128)]) +@pytest.mark.parametrize("rowwise", [False, True]) +def test_compact_scale_swizzle(shape, rowwise): + """Single-tensor swizzle pads compact scales and zero-fills the missing outer extent.""" + tensor, reference = _make_tensor_and_reference(shape, rowwise) + tex.swizzle_scales_for_gemm_(tensor) + actual = tensor._rowwise_scale_inv if rowwise else tensor._columnwise_scale_inv + torch.testing.assert_close(actual, reference, rtol=0, atol=0) + assert tensor._with_gemm_swizzled_scales + + +@pytest.mark.skipif(not te.is_mxfp8_available(), reason="MXFP8 requires Blackwell") +@pytest.mark.parametrize("rowwise", [False, True]) +def test_multi_tensor_compact_scale_swizzle(rowwise): + """Multi-tensor swizzle handles compact and already outer-aligned inputs in one batch.""" + tensors_and_references = [ + _make_tensor_and_reference(shape, rowwise) for shape in ((160, 96), (256, 128), (288, 224)) + ] + tensors = [item[0] for item in tensors_and_references] + tex.multi_tensor_swizzle_scales_for_gemm_(tensors, rowwise, not rowwise) + for tensor, reference in tensors_and_references: + actual = tensor._rowwise_scale_inv if rowwise else tensor._columnwise_scale_inv + torch.testing.assert_close(actual, reference, rtol=0, atol=0) diff --git a/transformer_engine/common/include/transformer_engine/swizzle.h b/transformer_engine/common/include/transformer_engine/swizzle.h index 1c54e5f8981..4c76ac680ea 100644 --- a/transformer_engine/common/include/transformer_engine/swizzle.h +++ b/transformer_engine/common/include/transformer_engine/swizzle.h @@ -25,7 +25,10 @@ extern "C" { * * Requirements: * - scale_inv is stored in row-major. - * - scale_inv size is padded to 128x4 for row-scale and 4x128 for col-scale. + * - Output scale_inv size is padded to 128x4 for row-scale and 4x128 for col-scale. + * - MXFP8 and NVFP4 input scale_inv may omit padding on the outer dimension, while retaining + * contiguous-dimension padding so each scale row is 4-byte aligned. Missing outer-dimension + * padding is filled with zeros during swizzling. * - Input and output scale_inv buffer pointers used by this operation must be 16-byte aligned. * - data is quantized along K-dimension, i.e. 1D-scaling block lies along the K-dimension. */ @@ -40,7 +43,9 @@ void nvte_swizzle_scaling_factors(const NVTETensor input, NVTETensor output, cud * * Requirements: * - scale_inv is stored in row-major. - * - scale_inv size is padded to 128x4 for row-scale and 4x128 for col-scale. + * - Output scale_inv size is padded to 128x4 for row-scale and 4x128 for col-scale. + * - Input scale_inv may omit outer-dimension padding under the same conditions as + * nvte_swizzle_scaling_factors. * - Input and output scale_inv buffer pointers used by this operation must be 16-byte aligned. * - data is quantized along K-dimension, i.e. 1D-scaling block lies along the K-dimension. */ diff --git a/transformer_engine/common/swizzle/swizzle.cu b/transformer_engine/common/swizzle/swizzle.cu index af8fe32fe17..ab33b7ec084 100644 --- a/transformer_engine/common/swizzle/swizzle.cu +++ b/transformer_engine/common/swizzle/swizzle.cu @@ -699,6 +699,49 @@ __global__ void __launch_bounds__(TB_DIM* TB_DIM) padding_m); } +// True when unswizzled scales omit padding on the outer dimension only. The contiguous dimension +// remains padded so each scale row is 4-byte aligned. +bool column_padded_compact_scales(const Tensor& input, bool rowwise) { + if (input.with_gemm_swizzled_scales) return false; + + const auto& scales = rowwise ? input.scale_inv : input.columnwise_scale_inv; + if (!scales.has_data()) return false; + const auto& shape = scales.shape; + if (shape.size() != 2) return false; + + const auto [rows, cols] = input.flat_2d_dims(); + if (input.scaling_mode == NVTE_MXFP8_1D_SCALING && rowwise) { + const size_t padded_cols = + round_up_to_multiple(DIVUP(cols, size_t{MXFP8_BLOCK_SIZE}), size_t{4}); + return shape[0] == rows && shape[1] == padded_cols && + rows != round_up_to_multiple(rows, size_t{128}); + } + if (input.scaling_mode == NVTE_MXFP8_1D_SCALING) { + const size_t scale_rows = DIVUP(rows, size_t{MXFP8_BLOCK_SIZE}); + const size_t padded_cols = round_up_to_multiple(cols, size_t{128}); + return shape[0] == scale_rows && shape[1] == padded_cols && + scale_rows != round_up_to_multiple(scale_rows, size_t{4}); + } + if (input.scaling_mode == NVTE_NVFP4_1D_SCALING && rowwise) { + const size_t padded_cols = + round_up_to_multiple(DIVUP(cols, size_t{NVFP4_BLOCK_SIZE}), size_t{4}); + return shape[0] == rows && shape[1] == padded_cols && + rows != round_up_to_multiple(rows, size_t{128}); + } + if (input.scaling_mode == NVTE_NVFP4_1D_SCALING && has_columnwise) { + const size_t padded_cols = + round_up_to_multiple(DIVUP(rows, size_t{NVFP4_BLOCK_SIZE}), size_t{4}); + return shape[0] == cols && shape[1] == padded_cols && + cols != round_up_to_multiple(cols, size_t{128}); + } + return false; +} + +bool column_padded_compact_scales(const Tensor& input) { + return column_padded_compact_scales(input, true) || + column_padded_compact_scales(input, false); +} + // Optimized scale swizzle for narrow scale widths. "Rowwise full width" means that a block stages // the entire padded K dimension for one or more 128-row tiles. "Columnwise chunk" means that a // block stages only 32 (or 128 for large grids) K rows for one 128-column M tile. Both paths use @@ -742,17 +785,21 @@ inline uint32_t division_magic(int divisor) { // input and of the output. Shared-memory rows are padded to an odd number of words, so the column // gathers are free of bank conflicts. Requires num_tiles * 128 * padded_k / 4 < 2^16 (guaranteed // by the row-wise shared-memory limit = 30 KB). -__device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict__ input, - uint8_t* __restrict__ output, int padded_k, - uint32_t words_per_row_magic, int original_m, - int original_k, int first_tile, int num_tiles, - uint32_t* smem) { +template +__device__ __forceinline__ void swizzle_rowwise_tiles( + const uint8_t* __restrict__ input, uint8_t* __restrict__ output, int padded_k, + uint32_t words_per_row_magic, int original_m, int original_k, int first_tile, int num_tiles, + uint32_t* smem) { const int words_per_row = padded_k >> 2; const int smem_row_words = words_per_row | 1; - const int num_vectors = - num_tiles * 8 * padded_k; // 16-byte vectors: num_tiles * 128 * padded_k / 16 const size_t chunk_offset = static_cast(first_tile) * 128 * padded_k; - const int4* input_vectors = reinterpret_cast(input + chunk_offset); + const uint32_t* input_words = reinterpret_cast(input + chunk_offset); + const int4* input_vectors = reinterpret_cast(input_words); + const int valid_rows = + kCompactInput ? max(0, min(num_tiles * 128, original_m - first_tile * 128)) + : num_tiles * 128; + const int valid_words = valid_rows * words_per_row; + const int num_vectors = kCompactInput ? valid_words >> 2 : num_tiles * 8 * padded_k; for (int i = threadIdx.x; i < num_vectors; i += kRowwiseThreadsPerBlock) { const int4 vector = __ldg(input_vectors + i); const uint32_t words[4] = {static_cast(vector.x), static_cast(vector.y), @@ -764,6 +811,14 @@ __device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict_ smem[row * smem_row_words + (word_index - row * words_per_row)] = words[w]; } } + if constexpr (kCompactInput) { + for (int word_index = 4 * num_vectors + threadIdx.x; word_index < valid_words; + word_index += kRowwiseThreadsPerBlock) { + const uint32_t row = divide_by_magic(word_index, words_per_row_magic); + smem[row * smem_row_words + (word_index - row * words_per_row)] = + __ldg(input_words + word_index); + } + } __syncthreads(); // An output tile holds padded_k / 4 blocks of 512 bytes, i.e. 32 vectors each; vector r of a @@ -781,8 +836,15 @@ __device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict_ #pragma unroll for (int group = 0; group < 4; group++) { const int row = row_in_group + 32 * group; - uint32_t word = smem[(tile_first_row + row) * smem_row_words + k_block]; - if (global_first_row + row >= original_m) word = 0u; + uint32_t word; + if constexpr (kCompactInput) { + word = global_first_row + row < original_m + ? smem[(tile_first_row + row) * smem_row_words + k_block] + : 0u; + } else { + word = smem[(tile_first_row + row) * smem_row_words + k_block]; + if (global_first_row + row >= original_m) word = 0u; + } words[group] = zero_padding_columns(word, k_block * 4, original_k); } output_vectors[v] = make_int4(static_cast(words[0]), static_cast(words[1]), @@ -794,15 +856,16 @@ __device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict_ // Column-wise: tile m_tile (128 columns), scale rows [first_k, first_k + kChunkRows) clipped to // padded_k. Each input row segment is 128 contiguous bytes; each output block of 4 scale rows is // 512 contiguous bytes. -template +template __device__ __forceinline__ void swizzle_columnwise_chunk(const uint8_t* __restrict__ input, uint8_t* __restrict__ output, int padded_m, int padded_k, int original_m, int original_k, int m_tile, int first_k, int4* smem) { const int num_rows = min(kChunkRows, padded_k - first_k); // multiple of 4 - const int num_vectors = num_rows * 8; - for (int i = threadIdx.x; i < num_vectors; i += kColumnwiseThreadsPerBlock) { + const int num_input_vectors = + (kCompactInput ? max(0, min(num_rows, original_k - first_k)) : num_rows) * 8; + for (int i = threadIdx.x; i < num_input_vectors; i += kColumnwiseThreadsPerBlock) { const int row = i >> 3; const int vector_in_row = i & 7; smem[i] = __ldg(reinterpret_cast( @@ -813,6 +876,7 @@ __device__ __forceinline__ void swizzle_columnwise_chunk(const uint8_t* __restri const uint8_t* smem_bytes = reinterpret_cast(smem); const size_t first_output_block = static_cast(m_tile) * (padded_k >> 2) + (first_k >> 2); + const int num_vectors = num_rows * 8; for (int v = threadIdx.x; v < num_vectors; v += kColumnwiseThreadsPerBlock) { const int k_block = v >> 5; const int column_in_group = v & 31; @@ -838,22 +902,25 @@ __device__ __forceinline__ void swizzle_columnwise_chunk(const uint8_t* __restri } } +template __global__ void __launch_bounds__(kRowwiseThreadsPerBlock) swizzle_rowwise_full_width_kernel(const uint8_t* __restrict__ input, uint8_t* __restrict__ output, int padded_k, uint32_t words_per_row_magic, int original_m, int original_k) { extern __shared__ uint32_t rowwise_smem[]; - swizzle_rowwise_tiles(input, output, padded_k, words_per_row_magic, original_m, original_k, - blockIdx.x, 1, rowwise_smem); + swizzle_rowwise_tiles(input, output, padded_k, words_per_row_magic, original_m, + original_k, blockIdx.x, 1, rowwise_smem); } +template __global__ void __launch_bounds__(kColumnwiseThreadsPerBlock) swizzle_columnwise_chunk_kernel(const uint8_t* __restrict__ input, uint8_t* __restrict__ output, int padded_m, int padded_k, int original_m, int original_k) { __shared__ int4 columnwise_smem[32 * 8]; - swizzle_columnwise_chunk<32>(input, output, padded_m, padded_k, original_m, original_k, - blockIdx.x, blockIdx.y * 32, columnwise_smem); + swizzle_columnwise_chunk<32, kCompactInput>(input, output, padded_m, padded_k, original_m, + original_k, blockIdx.x, blockIdx.y * 32, + columnwise_smem); } // The scales of one tensor: padded_m (a multiple of 128) by padded_k (a multiple of 4) in @@ -863,6 +930,7 @@ struct SwizzleTensorDescriptor { const uint8_t* input; uint8_t* output; int padded_m, padded_k, original_m, original_k; + bool compact_input; }; struct MultiTensorSwizzleArgs { @@ -879,6 +947,7 @@ __device__ __forceinline__ int tensor_of_block(const MultiTensorSwizzleArgs& arg return tensor; } +template __global__ void __launch_bounds__(kRowwiseThreadsPerBlock) multi_tensor_swizzle_rowwise_full_width_kernel(const MultiTensorSwizzleArgs args) { extern __shared__ uint32_t rowwise_multi_smem[]; @@ -886,12 +955,13 @@ __global__ void __launch_bounds__(kRowwiseThreadsPerBlock) const int row_tiles_per_block = args.row_tiles_per_block; const int first_tile = (blockIdx.x - args.first_block[t]) * row_tiles_per_block; const int num_tiles = min(row_tiles_per_block, args.tensors[t].padded_m / 128 - first_tile); - swizzle_rowwise_tiles(args.tensors[t].input, args.tensors[t].output, args.tensors[t].padded_k, - args.words_per_row_magic[t], args.tensors[t].original_m, - args.tensors[t].original_k, first_tile, num_tiles, rowwise_multi_smem); + swizzle_rowwise_tiles( + args.tensors[t].input, args.tensors[t].output, args.tensors[t].padded_k, + args.words_per_row_magic[t], args.tensors[t].original_m, args.tensors[t].original_k, + first_tile, num_tiles, rowwise_multi_smem); } -template +template __global__ void __launch_bounds__(kColumnwiseThreadsPerBlock) multi_tensor_swizzle_columnwise_chunk_kernel(const MultiTensorSwizzleArgs args) { __shared__ int4 columnwise_multi_smem[kChunkRows * 8]; @@ -900,10 +970,10 @@ __global__ void __launch_bounds__(kColumnwiseThreadsPerBlock) const int num_m_tiles = args.tensors[t].padded_m / 128; const int m_tile = block_in_tensor % num_m_tiles; const int first_k = (block_in_tensor / num_m_tiles) * kChunkRows; - swizzle_columnwise_chunk(args.tensors[t].input, args.tensors[t].output, - args.tensors[t].padded_m, args.tensors[t].padded_k, - args.tensors[t].original_m, args.tensors[t].original_k, - m_tile, first_k, columnwise_multi_smem); + swizzle_columnwise_chunk( + args.tensors[t].input, args.tensors[t].output, args.tensors[t].padded_m, + args.tensors[t].padded_k, args.tensors[t].original_m, args.tensors[t].original_k, m_tile, + first_k, columnwise_multi_smem); } inline bool is_16_byte_aligned(const void* ptr) { @@ -928,6 +998,7 @@ inline bool should_use_rowwise_full_width_swizzle(int padded_k, int64_t total_ro // Launches the rowwise full-width kernel, where each block stages the entire padded K dimension, // for up to kMaxTensorsPerLaunch tensors. The row tiles per block are the smallest power of two // that keeps the grid within kRowwiseMaxBlocks, subject to the shared-memory limit. +template inline void launch_multi_tensor_rowwise_full_width_swizzle(const SwizzleTensorDescriptor* tensors, int num_tensors, cudaStream_t stream) { MultiTensorSwizzleArgs args{}; @@ -958,7 +1029,7 @@ inline void launch_multi_tensor_rowwise_full_width_swizzle(const SwizzleTensorDe } const int grid = args.first_block[num_tensors]; if (grid == 0) return; - multi_tensor_swizzle_rowwise_full_width_kernel<<< + multi_tensor_swizzle_rowwise_full_width_kernel<<< grid, kRowwiseThreadsPerBlock, row_tiles_per_block * 128 * max_smem_row_words * 4, stream>>>( args); NVTE_CHECK_CUDA(cudaGetLastError()); @@ -967,6 +1038,7 @@ inline void launch_multi_tensor_rowwise_full_width_swizzle(const SwizzleTensorDe // Launches the columnwise chunk kernel, where each block stages part of K for one 128-column M // tile, for up to kMaxTensorsPerLaunch tensors. It uses 128-row K chunks once 32-row chunks would // exceed kColumnwiseMaxBlocksWith32RowChunks blocks. +template inline void launch_multi_tensor_columnwise_chunk_swizzle(const SwizzleTensorDescriptor* tensors, int num_tensors, cudaStream_t stream) { MultiTensorSwizzleArgs args{}; @@ -990,10 +1062,10 @@ inline void launch_multi_tensor_columnwise_chunk_swizzle(const SwizzleTensorDesc const int grid = args.first_block[num_tensors]; if (grid == 0) return; if (chunk_rows == 128) { - multi_tensor_swizzle_columnwise_chunk_kernel<128> + multi_tensor_swizzle_columnwise_chunk_kernel<128, kCompactInput> <<>>(args); } else { - multi_tensor_swizzle_columnwise_chunk_kernel<32> + multi_tensor_swizzle_columnwise_chunk_kernel<32, kCompactInput> <<>>(args); } NVTE_CHECK_CUDA(cudaGetLastError()); @@ -1004,9 +1076,15 @@ inline void launch_rowwise_full_width_swizzle(const SwizzleTensorDescriptor& ten cudaStream_t stream) { const int grid = tensor.padded_m / 128; const int smem = 128 * ((tensor.padded_k >> 2) | 1) * 4; - swizzle_rowwise_full_width_kernel<<>>( - tensor.input, tensor.output, tensor.padded_k, division_magic(tensor.padded_k >> 2), - tensor.original_m, tensor.original_k); + if (tensor.compact_input) { + swizzle_rowwise_full_width_kernel<<>>( + tensor.input, tensor.output, tensor.padded_k, division_magic(tensor.padded_k >> 2), + tensor.original_m, tensor.original_k); + } else { + swizzle_rowwise_full_width_kernel<<>>( + tensor.input, tensor.output, tensor.padded_k, division_magic(tensor.padded_k >> 2), + tensor.original_m, tensor.original_k); + } NVTE_CHECK_CUDA(cudaGetLastError()); } @@ -1014,9 +1092,15 @@ inline void launch_rowwise_full_width_swizzle(const SwizzleTensorDescriptor& ten inline void launch_columnwise_chunk_swizzle(const SwizzleTensorDescriptor& tensor, cudaStream_t stream) { dim3 grid(tensor.padded_m / 128, (tensor.padded_k + 31) / 32); - swizzle_columnwise_chunk_kernel<<>>( - tensor.input, tensor.output, tensor.padded_m, tensor.padded_k, tensor.original_m, - tensor.original_k); + if (tensor.compact_input) { + swizzle_columnwise_chunk_kernel<<>>( + tensor.input, tensor.output, tensor.padded_m, tensor.padded_k, tensor.original_m, + tensor.original_k); + } else { + swizzle_columnwise_chunk_kernel<<>>( + tensor.input, tensor.output, tensor.padded_m, tensor.padded_k, tensor.original_m, + tensor.original_k); + } NVTE_CHECK_CUDA(cudaGetLastError()); } @@ -1028,8 +1112,12 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s NVTE_CHECK(scaling_mode == NVTE_MXFP8_1D_SCALING || scaling_mode == NVTE_NVFP4_1D_SCALING, "Input tensor has invalid scaling mode (", to_string(input->scaling_mode), ")."); - CheckInputTensor(*input, "scaling_factor_input"); + const bool compact_input = column_padded_compact_scales(*input); + CheckInputTensor(*input, "scaling_factor_input", !compact_input); CheckInputTensor(*output, "scaling_factor_output"); + NVTE_CHECK(output->scaling_mode == scaling_mode && + output->flat_2d_dims() == input->flat_2d_dims(), + "Input and output tensor layouts must match."); NVTE_CHECK(!input->with_gemm_swizzled_scales, "Expected input tensor with scales in compact format."); NVTE_CHECK(output->with_gemm_swizzled_scales, @@ -1056,36 +1144,37 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s return; } - // Deduce tensor dims + // Deduce padded swizzle dimensions from the output. A compact input may omit outer-dimension + // padding, while the output always has the full GEMM layout. int m{0}, k{0}; switch (scaling_mode) { case NVTE_MXFP8_1D_SCALING: { if (has_rowwise_scale_inv) { - NVTE_CHECK(input->scale_inv.shape.size() == 2, - "Expected 2D scaling factors, got shape=", input->scale_inv.shape, "."); - m = input->scale_inv.shape[0]; - k = input->scale_inv.shape[1]; + NVTE_CHECK(output->scale_inv.shape.size() == 2, + "Expected 2D scaling factors, got shape=", output->scale_inv.shape, "."); + m = output->scale_inv.shape[0]; + k = output->scale_inv.shape[1]; } else if (has_columnwise_scale_inv) { - NVTE_CHECK(input->columnwise_scale_inv.shape.size() == 2, - "Expected 2D scaling factors, got shape=", input->columnwise_scale_inv.shape, + NVTE_CHECK(output->columnwise_scale_inv.shape.size() == 2, + "Expected 2D scaling factors, got shape=", output->columnwise_scale_inv.shape, "."); - m = input->columnwise_scale_inv.shape[1]; - k = input->columnwise_scale_inv.shape[0]; + m = output->columnwise_scale_inv.shape[1]; + k = output->columnwise_scale_inv.shape[0]; } break; } case NVTE_NVFP4_1D_SCALING: { if (has_rowwise_scale_inv) { - NVTE_CHECK(input->scale_inv.shape.size() == 2, - "Expected 2D scaling factors, got shape=", input->scale_inv.shape, "."); - m = input->scale_inv.shape[0]; - k = input->scale_inv.shape[1]; + NVTE_CHECK(output->scale_inv.shape.size() == 2, + "Expected 2D scaling factors, got shape=", output->scale_inv.shape, "."); + m = output->scale_inv.shape[0]; + k = output->scale_inv.shape[1]; } else if (has_columnwise_scale_inv) { - NVTE_CHECK(input->columnwise_scale_inv.shape.size() == 2, - "Expected 2D scaling factors, got shape=", input->columnwise_scale_inv.shape, + NVTE_CHECK(output->columnwise_scale_inv.shape.size() == 2, + "Expected 2D scaling factors, got shape=", output->columnwise_scale_inv.shape, "."); - m = input->columnwise_scale_inv.shape[0]; - k = input->columnwise_scale_inv.shape[1]; + m = output->columnwise_scale_inv.shape[0]; + k = output->columnwise_scale_inv.shape[1]; } break; } @@ -1153,7 +1242,8 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s switch (scaling_mode) { case NVTE_MXFP8_1D_SCALING: { original_M = input->flat_first_dim(); - original_K = input->flat_last_dim() / MXFP8_BLOCK_SIZE; + original_K = + DIVUP(input->flat_last_dim(), static_cast(MXFP8_BLOCK_SIZE)); input_scale_inv_ptr = input->scale_inv.dptr; output_scale_inv_ptr = output->scale_inv.dptr; break; @@ -1161,12 +1251,14 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s case NVTE_NVFP4_1D_SCALING: { if (has_rowwise_scale_inv) { original_M = input->flat_first_dim(); - original_K = input->flat_last_dim() / NVFP4_BLOCK_SIZE; + original_K = + DIVUP(input->flat_last_dim(), static_cast(NVFP4_BLOCK_SIZE)); input_scale_inv_ptr = input->scale_inv.dptr; output_scale_inv_ptr = output->scale_inv.dptr; } else if (has_columnwise_scale_inv) { original_M = input->flat_last_dim(); - original_K = input->flat_first_dim() / NVFP4_BLOCK_SIZE; + original_K = + DIVUP(input->flat_first_dim(), static_cast(NVFP4_BLOCK_SIZE)); input_scale_inv_ptr = input->columnwise_scale_inv.dptr; output_scale_inv_ptr = output->columnwise_scale_inv.dptr; } @@ -1185,11 +1277,16 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s m, k, original_M, - original_K}; + original_K, + compact_input}; if (should_use_rowwise_full_width_swizzle(descriptor.padded_k, descriptor.padded_m / SF_TILE_DIM_M)) { if (m / SF_TILE_DIM_M > kRowwiseMaxBlocks) { - launch_multi_tensor_rowwise_full_width_swizzle(&descriptor, 1, stream); + if (compact_input) { + launch_multi_tensor_rowwise_full_width_swizzle(&descriptor, 1, stream); + } else { + launch_multi_tensor_rowwise_full_width_swizzle(&descriptor, 1, stream); + } } else { launch_rowwise_full_width_swizzle(descriptor, stream); } @@ -1234,7 +1331,8 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s // Perform column-wise swizzle if (columnwise_swizzle) { const int original_M = input->flat_last_dim(); - const int original_K = input->flat_first_dim() / MXFP8_BLOCK_SIZE; + const int original_K = + DIVUP(input->flat_first_dim(), static_cast(MXFP8_BLOCK_SIZE)); const SwizzleTensorDescriptor descriptor{ static_cast(input->columnwise_scale_inv.dptr), @@ -1242,10 +1340,15 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s m, k, original_M, - original_K}; + original_K, + compact_input}; const int64_t grid_size = static_cast(m / SF_TILE_DIM_M) * ((k + 31) / 32); if (grid_size > kColumnwiseMaxBlocksWith32RowChunks) { - launch_multi_tensor_columnwise_chunk_swizzle(&descriptor, 1, stream); + if (compact_input) { + launch_multi_tensor_columnwise_chunk_swizzle(&descriptor, 1, stream); + } else { + launch_multi_tensor_columnwise_chunk_swizzle(&descriptor, 1, stream); + } } else { launch_columnwise_chunk_swizzle(descriptor, stream); } @@ -1393,7 +1496,7 @@ namespace { // Multi-tensor swizzle using rowwise full-width or columnwise chunk kernels with aligned int4 // (16-byte) loads and stores. -void int4_vectorized_multi_tensor_swizzle(const std::vector& input, +bool int4_vectorized_multi_tensor_swizzle(const std::vector& input, std::vector& output, bool rowwise, cudaStream_t stream) { const size_t num_tensors = input.size(); @@ -1406,34 +1509,56 @@ void int4_vectorized_multi_tensor_swizzle(const std::vector& input, const auto& out_scales = selected_rowwise ? output[i]->scale_inv : output[i]->columnwise_scale_inv; NVTE_CHECK(in_scales.shape.size() == 2, "Expected two-dimensional scaling factors."); + NVTE_CHECK(out_scales.shape.size() == 2, "Expected two-dimensional output scaling factors."); const int padded_m = - static_cast((rowwise || nvfp4) ? in_scales.shape[0] : in_scales.shape[1]); + static_cast((rowwise || nvfp4) ? out_scales.shape[0] : out_scales.shape[1]); const int padded_k = - static_cast((rowwise || nvfp4) ? in_scales.shape[1] : in_scales.shape[0]); + static_cast((rowwise || nvfp4) ? out_scales.shape[1] : out_scales.shape[0]); NVTE_CHECK(padded_m > 0 && padded_k > 0 && padded_m % 128 == 0 && padded_k % 4 == 0, "Invalid scaling-factor shape (", padded_m, ", ", padded_k, ")."); - NVTE_CHECK(in_scales.numel() == static_cast(padded_m) * padded_k && - out_scales.numel() == static_cast(padded_m) * padded_k, - "Input and output scaling-factor shapes must match."); + NVTE_CHECK(out_scales.numel() == static_cast(padded_m) * padded_k, + "Output scaling factors must use the padded GEMM layout."); const auto [first_dim, last_dim] = input[i]->flat_2d_dims(); const int block_size = nvfp4 ? NVFP4_BLOCK_SIZE : MXFP8_BLOCK_SIZE; const int original_m = static_cast(selected_rowwise ? first_dim : last_dim); - const int original_k = static_cast((selected_rowwise ? last_dim : first_dim) / block_size); + const int original_k = static_cast( + DIVUP(selected_rowwise ? last_dim : first_dim, static_cast(block_size))); + const bool compact_input = + in_scales.numel() != static_cast(padded_m) * padded_k; + NVTE_CHECK(!compact_input || column_padded_compact_scales(*input[i], selected_rowwise), + "Input scaling factors must either use the padded GEMM layout or omit only " + "outer-dimension padding."); tensors[i] = {static_cast(in_scales.dptr), static_cast(out_scales.dptr), padded_m, padded_k, original_m, - original_k}; - } - for (size_t first = 0; first < num_tensors; first += kMaxTensorsPerLaunch) { - const int count = static_cast(std::min(kMaxTensorsPerLaunch, num_tensors - first)); + original_k, + compact_input}; + } + for (size_t first = 0; first < num_tensors;) { + size_t end = first + 1; + while (end < num_tensors && end - first < kMaxTensorsPerLaunch && + tensors[end].compact_input == tensors[first].compact_input) { + ++end; + } + const int count = static_cast(end - first); if (rowwise) { - launch_multi_tensor_rowwise_full_width_swizzle(tensors.data() + first, count, stream); + if (tensors[first].compact_input) { + launch_multi_tensor_rowwise_full_width_swizzle(tensors.data() + first, count, stream); + } else { + launch_multi_tensor_rowwise_full_width_swizzle(tensors.data() + first, count, stream); + } } else { - launch_multi_tensor_columnwise_chunk_swizzle(tensors.data() + first, count, stream); + if (tensors[first].compact_input) { + launch_multi_tensor_columnwise_chunk_swizzle(tensors.data() + first, count, stream); + } else { + launch_multi_tensor_columnwise_chunk_swizzle(tensors.data() + first, count, stream); + } } + first = end; } + return true; } } // namespace @@ -1459,8 +1584,13 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, // We don't allow empty tensors. They should be filtered out before calling this function. NVTE_CHECK(input[i]->numel() != 0, "Tensor input[", i, "] is empty."); - CheckInputTensor(*input[i], "scaling_factor_input", check_scale_inv_shapes); + const bool compact_input = column_padded_compact_scales(*input[i]); + CheckInputTensor(*input[i], "scaling_factor_input", + check_scale_inv_shapes && !compact_input); CheckInputTensor(*output[i], "scaling_factor_output", check_scale_inv_shapes); + NVTE_CHECK(output[i]->scaling_mode == scaling_mode && + output[i]->flat_2d_dims() == input[i]->flat_2d_dims(), + "Input and output tensor ", i, " layouts must match."); all_has_data = all_has_data && input[i]->scale_inv.has_data(); all_has_columnwise_data = (all_has_columnwise_data && input[i]->columnwise_scale_inv.has_data()); @@ -1479,22 +1609,23 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, const bool use_rowwise_buffer = input[i]->scale_inv.has_data(); const bool selected_rowwise = rowwise_swizzle && use_rowwise_buffer; const auto& in_scales = selected_rowwise ? input[i]->scale_inv : input[i]->columnwise_scale_inv; + const auto& out_scales = + selected_rowwise ? output[i]->scale_inv : output[i]->columnwise_scale_inv; check_scale_inv_alignment(input[i], "Input", selected_rowwise); check_scale_inv_alignment(output[i], "Output", selected_rowwise); if (rowwise_swizzle) { - NVTE_CHECK(in_scales.shape.size() == 2, "Expected two-dimensional scaling factors."); - max_rowwise_padded_k = std::max(max_rowwise_padded_k, static_cast(in_scales.shape[1])); - total_rowwise_tiles += static_cast(in_scales.shape[0]) / 128; + NVTE_CHECK(in_scales.shape.size() == 2 && out_scales.shape.size() == 2, + "Expected two-dimensional scaling factors."); + max_rowwise_padded_k = std::max(max_rowwise_padded_k, static_cast(out_scales.shape[1])); + total_rowwise_tiles += static_cast(out_scales.shape[0]) / 128; } } if (rowwise_swizzle) { if (should_use_rowwise_full_width_swizzle(max_rowwise_padded_k, total_rowwise_tiles)) { - int4_vectorized_multi_tensor_swizzle(input, output, true, stream); - return; + if (int4_vectorized_multi_tensor_swizzle(input, output, true, stream)) return; } } else if (columnwise_swizzle) { - int4_vectorized_multi_tensor_swizzle(input, output, false, stream); - return; + if (int4_vectorized_multi_tensor_swizzle(input, output, false, stream)) return; } // Fallback to the generic rowwise swizzle kernel. constexpr int SF_TILE_DIM_M = 128; @@ -1519,12 +1650,12 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, int m, k; if (all_has_data) { - m = input[i]->scale_inv.shape[0]; - k = input[i]->scale_inv.shape[1]; + m = output[i]->scale_inv.shape[0]; + k = output[i]->scale_inv.shape[1]; } else { NVTE_CHECK(all_nvfp4, "When doing rowwise swizzle with rowwise data, it has to be NVFP4"); - m = input[i]->columnwise_scale_inv.shape[0]; - k = input[i]->columnwise_scale_inv.shape[1]; + m = output[i]->columnwise_scale_inv.shape[0]; + k = output[i]->columnwise_scale_inv.shape[1]; } NVTE_CHECK(m % SF_TILE_DIM_M == 0, "Input should be padded in M/N dimension!"); @@ -1564,12 +1695,14 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, kernel_args.input_list[pos] = const_cast(input[i]->scale_inv.dptr); kernel_args.output_list[pos] = output[i]->scale_inv.dptr; kernel_args.original_m_list[pos] = first_dim; - kernel_args.original_k_list[pos] = last_dim / block_scale_size; + kernel_args.original_k_list[pos] = + DIVUP(last_dim, static_cast(block_scale_size)); } else { kernel_args.input_list[pos] = const_cast(input[i]->columnwise_scale_inv.dptr); kernel_args.output_list[pos] = output[i]->columnwise_scale_inv.dptr; kernel_args.original_m_list[pos] = last_dim; - kernel_args.original_k_list[pos] = first_dim / NVFP4_BLOCK_SIZE; + kernel_args.original_k_list[pos] = + DIVUP(first_dim, static_cast(NVFP4_BLOCK_SIZE)); } kernel_args.num_tensors++; } diff --git a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp index 03eec7d80fd..d0c55534ec3 100644 --- a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp +++ b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp @@ -42,6 +42,20 @@ bool is_empty_grouped_tensor_param(const NVTEBasicTensor &t) { return t.shape.ndim == 1 && t.shape.data[0] == 0; } +NVTEShape padded_scale_shape_for_gemm(NVTEShape shape, NVTEScalingMode scaling_mode, + bool rowwise) { + if (shape.ndim != 2) return shape; + const bool scales_are_row_major = rowwise || scaling_mode == NVTE_NVFP4_1D_SCALING; + if (scales_are_row_major) { + shape.data[0] = ceildiv(shape.data[0], size_t{128}) * 128; + shape.data[1] = ceildiv(shape.data[1], size_t{4}) * 4; + } else { + shape.data[0] = ceildiv(shape.data[0], size_t{4}) * 4; + shape.data[1] = ceildiv(shape.data[1], size_t{128}) * 128; + } + return shape; +} + } // namespace std::tuple, std::optional> swizzle_scales_for_gemm( @@ -78,8 +92,9 @@ std::tuple, std::optional> swizzle_scales_ const auto scales_dtype = static_cast(input_scales_nvte.dtype); // Allocate buffer for swizzled scales (uint8, like all quantizer allocations) - const NVTEShape output_scales_shape = input_scales_shape; - rowwise_scales_pyt = allocateSpace(input_scales_shape, DType::kByte, false); + const NVTEShape output_scales_shape = + padded_scale_shape_for_gemm(input_scales_shape, scaling_mode, /*rowwise=*/true); + rowwise_scales_pyt = allocateSpace(output_scales_shape, DType::kByte, false); void *output_scales_dptr = getDataPtr(*rowwise_scales_pyt); // Initialize TE tensors with scales @@ -111,8 +126,9 @@ std::tuple, std::optional> swizzle_scales_ const auto scales_dtype = static_cast(input_scales_nvte.dtype); // Allocate buffer for swizzled scales (uint8, like all quantizer allocations) - const NVTEShape output_scales_shape = input_scales_shape; - columnwise_scales_pyt = allocateSpace(input_scales_shape, DType::kByte, false); + const NVTEShape output_scales_shape = + padded_scale_shape_for_gemm(input_scales_shape, scaling_mode, /*rowwise=*/false); + columnwise_scales_pyt = allocateSpace(output_scales_shape, DType::kByte, false); void *output_scales_dptr = getDataPtr(*columnwise_scales_pyt); // Initialize TE tensors with scales @@ -186,16 +202,20 @@ std::optional multi_tensor_swizzle_scales_for_gemm_impl( // Determine buffer size needed for swizzled scales std::vector output_scales_offsets; output_scales_offsets.reserve(tensors_needing_swizzle.size()); + std::vector output_scales_shapes; + output_scales_shapes.reserve(tensors_needing_swizzle.size()); size_t output_scales_bytes = 0; for (auto &tensor : tensors_needing_swizzle) { const auto scales_nvte = (rowwise_usage ? tensor->get_rowwise_scale_inv() : tensor->get_columnwise_scale_inv()); - const auto &shape = scales_nvte.shape; + const auto shape = + padded_scale_shape_for_gemm(scales_nvte.shape, scaling_mode, rowwise_usage); const auto dtype = static_cast(scales_nvte.dtype); const auto dtype_bits = transformer_engine::pytorch::typeToNumBits(dtype); const auto size = product(shape, 0, shape.ndim); output_scales_bytes = roundup(output_scales_bytes, 16); // align to 16B output_scales_offsets.push_back(output_scales_bytes); + output_scales_shapes.push_back(shape); output_scales_bytes += ceildiv(size * dtype_bits, 8); } @@ -220,8 +240,6 @@ std::optional multi_tensor_swizzle_scales_for_gemm_impl( // Cache output scale dtype/shape per tensor so we can update the // source TensorWrappers without re-reading from the output NVTETensors. std::vector output_scales_dtypes(n_swizzle); - std::vector output_scales_shapes(n_swizzle); - for (size_t i = 0; i < n_swizzle; ++i) { auto &tensor = *tensors_needing_swizzle[i]; const uint8_t swizzled_flag = 1; @@ -233,26 +251,26 @@ std::optional multi_tensor_swizzle_scales_for_gemm_impl( const auto data_dtype = static_cast(data_nvte.dtype); const auto scales_dtype = static_cast(scales_nvte.dtype); output_scales_dtypes[i] = scales_dtype; - output_scales_shapes[i] = scales_nvte.shape; set_param(inputs_nvte[i], kNVTERowwiseData, nullptr, data_dtype, data_nvte.shape); set_param(inputs_nvte[i], kNVTERowwiseScaleInv, scales_nvte.data_ptr, scales_dtype, scales_nvte.shape); set_param(outputs_nvte[i], kNVTERowwiseData, nullptr, data_dtype, data_nvte.shape); set_param(outputs_nvte[i], kNVTERowwiseScaleInv, - output_scales_dptr + output_scales_offsets[i], scales_dtype, scales_nvte.shape); + output_scales_dptr + output_scales_offsets[i], scales_dtype, + output_scales_shapes[i]); } else { const auto data_nvte = tensor.get_columnwise_data(); const auto scales_nvte = tensor.get_columnwise_scale_inv(); const auto data_dtype = static_cast(data_nvte.dtype); const auto scales_dtype = static_cast(scales_nvte.dtype); output_scales_dtypes[i] = scales_dtype; - output_scales_shapes[i] = scales_nvte.shape; set_param(inputs_nvte[i], kNVTEColumnwiseData, nullptr, data_dtype, data_nvte.shape); set_param(inputs_nvte[i], kNVTEColumnwiseScaleInv, scales_nvte.data_ptr, scales_dtype, scales_nvte.shape); set_param(outputs_nvte[i], kNVTEColumnwiseData, nullptr, data_dtype, data_nvte.shape); set_param(outputs_nvte[i], kNVTEColumnwiseScaleInv, - output_scales_dptr + output_scales_offsets[i], scales_dtype, scales_nvte.shape); + output_scales_dptr + output_scales_offsets[i], scales_dtype, + output_scales_shapes[i]); } } From ccf5241a475f85e7b1d44d52a7a5bb9936d6ee30 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:43:23 +0000 Subject: [PATCH 3/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../mxfp8/test_compact_scale_swizzle.py | 12 +- transformer_engine/common/swizzle/swizzle.cu | 104 ++++++++---------- .../pytorch/csrc/extensions/swizzle.cpp | 6 +- 3 files changed, 49 insertions(+), 73 deletions(-) diff --git a/tests/pytorch/mxfp8/test_compact_scale_swizzle.py b/tests/pytorch/mxfp8/test_compact_scale_swizzle.py index 68872d2285d..8a262894a82 100644 --- a/tests/pytorch/mxfp8/test_compact_scale_swizzle.py +++ b/tests/pytorch/mxfp8/test_compact_scale_swizzle.py @@ -41,24 +41,18 @@ def _make_tensor_and_reference(shape, rowwise): padded_shape = _scale_shape(padded_rows, padded_cols, rowwise, compact=False) padded = torch.zeros(padded_shape, dtype=torch.uint8, device="cuda") padded[: compact_shape[0], : compact_shape[1]].copy_(scales) - reference = swizzle_mxfp8_scale( - padded_rows, padded_cols, padded, columnwise=not rowwise - ) + reference = swizzle_mxfp8_scale(padded_rows, padded_cols, padded, columnwise=not rowwise) tensor = MXFP8Tensor( shape=shape, dtype=torch.bfloat16, device="cuda", rowwise_data=torch.empty(shape, dtype=torch.uint8, device="cuda") if rowwise else None, - columnwise_data=None - if rowwise - else torch.empty(shape, dtype=torch.uint8, device="cuda"), + columnwise_data=None if rowwise else torch.empty(shape, dtype=torch.uint8, device="cuda"), rowwise_scale_inv=scales if rowwise else None, columnwise_scale_inv=None if rowwise else scales, fp8_dtype=tex.DType.kFloat8E4M3, - quantizer=MXFP8Quantizer( - tex.DType.kFloat8E4M3, rowwise=rowwise, columnwise=not rowwise - ), + quantizer=MXFP8Quantizer(tex.DType.kFloat8E4M3, rowwise=rowwise, columnwise=not rowwise), with_gemm_swizzled_scales=False, ) return tensor, reference diff --git a/transformer_engine/common/swizzle/swizzle.cu b/transformer_engine/common/swizzle/swizzle.cu index 53350cbe5ba..39a351f130a 100644 --- a/transformer_engine/common/swizzle/swizzle.cu +++ b/transformer_engine/common/swizzle/swizzle.cu @@ -739,8 +739,7 @@ bool column_padded_compact_scales(const Tensor& input, bool rowwise) { const auto [rows, cols] = input.flat_2d_dims(); if (input.scaling_mode == NVTE_MXFP8_1D_SCALING && rowwise) { - const size_t padded_cols = - DIVUP_TO_MULTIPLE(DIVUP(cols, size_t{MXFP8_BLOCK_SIZE}), size_t{4}); + const size_t padded_cols = DIVUP_TO_MULTIPLE(DIVUP(cols, size_t{MXFP8_BLOCK_SIZE}), size_t{4}); return shape[0] == rows && shape[1] == padded_cols && rows != DIVUP_TO_MULTIPLE(rows, size_t{128}); } @@ -751,14 +750,12 @@ bool column_padded_compact_scales(const Tensor& input, bool rowwise) { scale_rows != DIVUP_TO_MULTIPLE(scale_rows, size_t{4}); } if (input.scaling_mode == NVTE_NVFP4_1D_SCALING && rowwise) { - const size_t padded_cols = - DIVUP_TO_MULTIPLE(DIVUP(cols, size_t{NVFP4_BLOCK_SIZE}), size_t{4}); + const size_t padded_cols = DIVUP_TO_MULTIPLE(DIVUP(cols, size_t{NVFP4_BLOCK_SIZE}), size_t{4}); return shape[0] == rows && shape[1] == padded_cols && rows != DIVUP_TO_MULTIPLE(rows, size_t{128}); } if (input.scaling_mode == NVTE_NVFP4_1D_SCALING && !rowwise) { - const size_t padded_cols = - DIVUP_TO_MULTIPLE(DIVUP(rows, size_t{NVFP4_BLOCK_SIZE}), size_t{4}); + const size_t padded_cols = DIVUP_TO_MULTIPLE(DIVUP(rows, size_t{NVFP4_BLOCK_SIZE}), size_t{4}); return shape[0] == cols && shape[1] == padded_cols && cols != DIVUP_TO_MULTIPLE(cols, size_t{128}); } @@ -766,8 +763,7 @@ bool column_padded_compact_scales(const Tensor& input, bool rowwise) { } bool column_padded_compact_scales(const Tensor& input) { - return column_padded_compact_scales(input, true) || - column_padded_compact_scales(input, false); + return column_padded_compact_scales(input, true) || column_padded_compact_scales(input, false); } // Optimized scale swizzle for narrow scale widths. "Rowwise full width" means that a block stages @@ -801,19 +797,20 @@ inline uint32_t division_magic(int divisor) { // gathers are free of bank conflicts. Requires num_tiles * 128 * padded_k / 4 < 2^16 (guaranteed // by the row-wise shared-memory limit = 30 KB). template -__device__ __forceinline__ void swizzle_rowwise_tiles( - const uint8_t* __restrict__ input, uint8_t* __restrict__ output, int padded_k, - uint32_t words_per_row_magic, int original_m, int original_k, int first_tile, int num_tiles, - uint32_t* smem) { +__device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict__ input, + uint8_t* __restrict__ output, int padded_k, + uint32_t words_per_row_magic, int original_m, + int original_k, int first_tile, int num_tiles, + uint32_t* smem) { const int words_per_row = padded_k / kScalesPerWord; const int smem_row_words = words_per_row | 1; const size_t chunk_offset = static_cast(first_tile) * kSwizzleTileDimM * padded_k; const uint32_t* input_words = reinterpret_cast(input + chunk_offset); const int4* input_vectors = reinterpret_cast(input_words); const int valid_rows = - kCompactInput ? max(0, min(num_tiles * kSwizzleTileDimM, - original_m - first_tile * kSwizzleTileDimM)) - : num_tiles * kSwizzleTileDimM; + kCompactInput + ? max(0, min(num_tiles * kSwizzleTileDimM, original_m - first_tile * kSwizzleTileDimM)) + : num_tiles * kSwizzleTileDimM; const int valid_words = valid_rows * words_per_row; const int num_vectors = kCompactInput ? valid_words / kScalesPerWord : num_tiles * kVectorsPerMSpan * padded_k; @@ -882,15 +879,14 @@ __device__ __forceinline__ void swizzle_columnwise_chunk(const uint8_t* __restri int4* smem) { const int num_rows = min(kChunkRows, padded_k - first_k); // multiple of kScalesPerWord const int num_input_vectors = - (kCompactInput ? max(0, min(num_rows, original_k - first_k)) : num_rows) * - kVectorsPerMSpan; + (kCompactInput ? max(0, min(num_rows, original_k - first_k)) : num_rows) * kVectorsPerMSpan; for (int i = threadIdx.x; i < num_input_vectors; i += kColumnwiseThreadsPerBlock) { const int row = i / kVectorsPerMSpan; const int vector_in_row = i % kVectorsPerMSpan; - smem[i] = __ldg(reinterpret_cast( - input + static_cast(first_k + row) * padded_m + - m_tile * kSwizzleTileDimM) + - vector_in_row); + smem[i] = + __ldg(reinterpret_cast(input + static_cast(first_k + row) * padded_m + + m_tile * kSwizzleTileDimM) + + vector_in_row); } __syncthreads(); @@ -932,7 +928,7 @@ __global__ void __launch_bounds__(kRowwiseThreadsPerBlock) int original_k) { extern __shared__ uint32_t rowwise_smem[]; swizzle_rowwise_tiles(input, output, padded_k, words_per_row_magic, original_m, - original_k, blockIdx.x, 1, rowwise_smem); + original_k, blockIdx.x, 1, rowwise_smem); } template @@ -978,10 +974,10 @@ __global__ void __launch_bounds__(kRowwiseThreadsPerBlock) const int first_tile = (blockIdx.x - args.first_block[t]) * row_tiles_per_block; const int num_tiles = min(row_tiles_per_block, args.tensors[t].padded_m / kSwizzleTileDimM - first_tile); - swizzle_rowwise_tiles( - args.tensors[t].input, args.tensors[t].output, args.tensors[t].padded_k, - args.words_per_row_magic[t], args.tensors[t].original_m, args.tensors[t].original_k, - first_tile, num_tiles, rowwise_multi_smem); + swizzle_rowwise_tiles(args.tensors[t].input, args.tensors[t].output, + args.tensors[t].padded_k, args.words_per_row_magic[t], + args.tensors[t].original_m, args.tensors[t].original_k, + first_tile, num_tiles, rowwise_multi_smem); } template @@ -1035,8 +1031,7 @@ inline void launch_multi_tensor_rowwise_full_width_swizzle(const SwizzleTensorDe args.tensors[i] = tensors[i]; args.words_per_row_magic[i] = division_magic(tensors[i].padded_k / kScalesPerWord); total_row_tiles += tensors[i].padded_m / kSwizzleTileDimM; - max_smem_row_words = - max(max_smem_row_words, (tensors[i].padded_k / kScalesPerWord) | 1); + max_smem_row_words = max(max_smem_row_words, (tensors[i].padded_k / kScalesPerWord) | 1); } int row_tiles_per_block = 1; const int max_row_tiles_per_block = @@ -1056,10 +1051,10 @@ inline void launch_multi_tensor_rowwise_full_width_swizzle(const SwizzleTensorDe } const int grid = args.first_block[num_tensors]; if (grid == 0) return; - multi_tensor_swizzle_rowwise_full_width_kernel<<< - grid, kRowwiseThreadsPerBlock, - row_tiles_per_block * kSwizzleTileDimM * max_smem_row_words * kBytesPerWord, stream>>>( - args); + multi_tensor_swizzle_rowwise_full_width_kernel + <<>>( + args); NVTE_CHECK_CUDA(cudaGetLastError()); } @@ -1105,18 +1100,15 @@ inline void launch_multi_tensor_columnwise_chunk_swizzle(const SwizzleTensorDesc inline void launch_rowwise_full_width_swizzle(const SwizzleTensorDescriptor& tensor, cudaStream_t stream) { const int grid = tensor.padded_m / kSwizzleTileDimM; - const int smem = - kSwizzleTileDimM * ((tensor.padded_k / kScalesPerWord) | 1) * kBytesPerWord; + const int smem = kSwizzleTileDimM * ((tensor.padded_k / kScalesPerWord) | 1) * kBytesPerWord; if (tensor.compact_input) { swizzle_rowwise_full_width_kernel<<>>( tensor.input, tensor.output, tensor.padded_k, - division_magic(tensor.padded_k / kScalesPerWord), - tensor.original_m, tensor.original_k); + division_magic(tensor.padded_k / kScalesPerWord), tensor.original_m, tensor.original_k); } else { swizzle_rowwise_full_width_kernel<<>>( tensor.input, tensor.output, tensor.padded_k, - division_magic(tensor.padded_k / kScalesPerWord), - tensor.original_m, tensor.original_k); + division_magic(tensor.padded_k / kScalesPerWord), tensor.original_m, tensor.original_k); } NVTE_CHECK_CUDA(cudaGetLastError()); } @@ -1124,8 +1116,7 @@ inline void launch_rowwise_full_width_swizzle(const SwizzleTensorDescriptor& ten // Each block stages a 32-row K chunk for one 128-column M tile. inline void launch_columnwise_chunk_swizzle(const SwizzleTensorDescriptor& tensor, cudaStream_t stream) { - dim3 grid(tensor.padded_m / kSwizzleTileDimM, - DIVUP(tensor.padded_k, kColumnwiseSmallChunkRows)); + dim3 grid(tensor.padded_m / kSwizzleTileDimM, DIVUP(tensor.padded_k, kColumnwiseSmallChunkRows)); if (tensor.compact_input) { swizzle_columnwise_chunk_kernel<<>>( tensor.input, tensor.output, tensor.padded_m, tensor.padded_k, tensor.original_m, @@ -1149,9 +1140,9 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s const bool compact_input = column_padded_compact_scales(*input); CheckInputTensor(*input, "scaling_factor_input", !compact_input); CheckInputTensor(*output, "scaling_factor_output"); - NVTE_CHECK(output->scaling_mode == scaling_mode && - output->flat_2d_dims() == input->flat_2d_dims(), - "Input and output tensor layouts must match."); + NVTE_CHECK( + output->scaling_mode == scaling_mode && output->flat_2d_dims() == input->flat_2d_dims(), + "Input and output tensor layouts must match."); NVTE_CHECK(!input->with_gemm_swizzled_scales, "Expected input tensor with scales in compact format."); NVTE_CHECK(output->with_gemm_swizzled_scales, @@ -1276,8 +1267,7 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s switch (scaling_mode) { case NVTE_MXFP8_1D_SCALING: { original_M = input->flat_first_dim(); - original_K = - DIVUP(input->flat_last_dim(), static_cast(MXFP8_BLOCK_SIZE)); + original_K = DIVUP(input->flat_last_dim(), static_cast(MXFP8_BLOCK_SIZE)); input_scale_inv_ptr = input->scale_inv.dptr; output_scale_inv_ptr = output->scale_inv.dptr; break; @@ -1285,14 +1275,12 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s case NVTE_NVFP4_1D_SCALING: { if (has_rowwise_scale_inv) { original_M = input->flat_first_dim(); - original_K = - DIVUP(input->flat_last_dim(), static_cast(NVFP4_BLOCK_SIZE)); + original_K = DIVUP(input->flat_last_dim(), static_cast(NVFP4_BLOCK_SIZE)); input_scale_inv_ptr = input->scale_inv.dptr; output_scale_inv_ptr = output->scale_inv.dptr; } else if (has_columnwise_scale_inv) { original_M = input->flat_last_dim(); - original_K = - DIVUP(input->flat_first_dim(), static_cast(NVFP4_BLOCK_SIZE)); + original_K = DIVUP(input->flat_first_dim(), static_cast(NVFP4_BLOCK_SIZE)); input_scale_inv_ptr = input->columnwise_scale_inv.dptr; output_scale_inv_ptr = output->columnwise_scale_inv.dptr; } @@ -1365,8 +1353,7 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s // Perform column-wise swizzle if (columnwise_swizzle) { const int original_M = input->flat_last_dim(); - const int original_K = - DIVUP(input->flat_first_dim(), static_cast(MXFP8_BLOCK_SIZE)); + const int original_K = DIVUP(input->flat_first_dim(), static_cast(MXFP8_BLOCK_SIZE)); const SwizzleTensorDescriptor descriptor{ static_cast(input->columnwise_scale_inv.dptr), @@ -1558,8 +1545,7 @@ bool int4_vectorized_multi_tensor_swizzle(const std::vector& input, const int original_m = static_cast(selected_rowwise ? first_dim : last_dim); const int original_k = static_cast( DIVUP(selected_rowwise ? last_dim : first_dim, static_cast(block_size))); - const bool compact_input = - in_scales.numel() != static_cast(padded_m) * padded_k; + const bool compact_input = in_scales.numel() != static_cast(padded_m) * padded_k; NVTE_CHECK(!compact_input || column_padded_compact_scales(*input[i], selected_rowwise), "Input scaling factors must either use the padded GEMM layout or omit only " "outer-dimension padding."); @@ -1582,7 +1568,8 @@ bool int4_vectorized_multi_tensor_swizzle(const std::vector& input, if (tensors[first].compact_input) { launch_multi_tensor_rowwise_full_width_swizzle(tensors.data() + first, count, stream); } else { - launch_multi_tensor_rowwise_full_width_swizzle(tensors.data() + first, count, stream); + launch_multi_tensor_rowwise_full_width_swizzle(tensors.data() + first, count, + stream); } } else { if (tensors[first].compact_input) { @@ -1620,8 +1607,7 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, // We don't allow empty tensors. They should be filtered out before calling this function. NVTE_CHECK(input[i]->numel() != 0, "Tensor input[", i, "] is empty."); const bool compact_input = column_padded_compact_scales(*input[i]); - CheckInputTensor(*input[i], "scaling_factor_input", - check_scale_inv_shapes && !compact_input); + CheckInputTensor(*input[i], "scaling_factor_input", check_scale_inv_shapes && !compact_input); CheckInputTensor(*output[i], "scaling_factor_output", check_scale_inv_shapes); NVTE_CHECK(output[i]->scaling_mode == scaling_mode && output[i]->flat_2d_dims() == input[i]->flat_2d_dims(), @@ -1730,14 +1716,12 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, kernel_args.input_list[pos] = const_cast(input[i]->scale_inv.dptr); kernel_args.output_list[pos] = output[i]->scale_inv.dptr; kernel_args.original_m_list[pos] = first_dim; - kernel_args.original_k_list[pos] = - DIVUP(last_dim, static_cast(block_scale_size)); + kernel_args.original_k_list[pos] = DIVUP(last_dim, static_cast(block_scale_size)); } else { kernel_args.input_list[pos] = const_cast(input[i]->columnwise_scale_inv.dptr); kernel_args.output_list[pos] = output[i]->columnwise_scale_inv.dptr; kernel_args.original_m_list[pos] = last_dim; - kernel_args.original_k_list[pos] = - DIVUP(first_dim, static_cast(NVFP4_BLOCK_SIZE)); + kernel_args.original_k_list[pos] = DIVUP(first_dim, static_cast(NVFP4_BLOCK_SIZE)); } kernel_args.num_tensors++; } diff --git a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp index d0c55534ec3..25ebf932363 100644 --- a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp +++ b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp @@ -42,8 +42,7 @@ bool is_empty_grouped_tensor_param(const NVTEBasicTensor &t) { return t.shape.ndim == 1 && t.shape.data[0] == 0; } -NVTEShape padded_scale_shape_for_gemm(NVTEShape shape, NVTEScalingMode scaling_mode, - bool rowwise) { +NVTEShape padded_scale_shape_for_gemm(NVTEShape shape, NVTEScalingMode scaling_mode, bool rowwise) { if (shape.ndim != 2) return shape; const bool scales_are_row_major = rowwise || scaling_mode == NVTE_NVFP4_1D_SCALING; if (scales_are_row_major) { @@ -208,8 +207,7 @@ std::optional multi_tensor_swizzle_scales_for_gemm_impl( for (auto &tensor : tensors_needing_swizzle) { const auto scales_nvte = (rowwise_usage ? tensor->get_rowwise_scale_inv() : tensor->get_columnwise_scale_inv()); - const auto shape = - padded_scale_shape_for_gemm(scales_nvte.shape, scaling_mode, rowwise_usage); + const auto shape = padded_scale_shape_for_gemm(scales_nvte.shape, scaling_mode, rowwise_usage); const auto dtype = static_cast(scales_nvte.dtype); const auto dtype_bits = transformer_engine::pytorch::typeToNumBits(dtype); const auto size = product(shape, 0, shape.ndim); From 9d2b9705f7165d7214176d5f247b9412eefa62e0 Mon Sep 17 00:00:00 2001 From: Varun Thumbe Date: Tue, 17 Nov 2026 21:19:29 -0800 Subject: [PATCH 4/9] Clarify swizzle padding checks and helper contracts Signed-off-by: Varun Thumbe --- transformer_engine/common/swizzle/swizzle.cu | 128 ++++++++++--------- 1 file changed, 67 insertions(+), 61 deletions(-) diff --git a/transformer_engine/common/swizzle/swizzle.cu b/transformer_engine/common/swizzle/swizzle.cu index 39a351f130a..e81293f44fe 100644 --- a/transformer_engine/common/swizzle/swizzle.cu +++ b/transformer_engine/common/swizzle/swizzle.cu @@ -729,7 +729,7 @@ __global__ void __launch_bounds__(TB_DIM* TB_DIM) // True when unswizzled scales omit padding on the outer dimension only. The contiguous dimension // remains padded so each scale row is 4-byte aligned. -bool column_padded_compact_scales(const Tensor& input, bool rowwise) { +bool has_unpadded_outer_scale_dim(const Tensor& input, bool rowwise) { if (input.with_gemm_swizzled_scales) return false; const auto& scales = rowwise ? input.scale_inv : input.columnwise_scale_inv; @@ -762,14 +762,11 @@ bool column_padded_compact_scales(const Tensor& input, bool rowwise) { return false; } -bool column_padded_compact_scales(const Tensor& input) { - return column_padded_compact_scales(input, true) || column_padded_compact_scales(input, false); -} - // Optimized scale swizzle for narrow scale widths. "Rowwise full width" means that a block stages // the entire padded K dimension for one or more 128-row tiles. "Columnwise chunk" means that a // block stages only 32 (or 128 for large grids) K rows for one 128-column M tile. Both paths use -// coalesced 16-byte loads and stores through shared memory. +// coalesced 16-byte loads and stores through shared memory, with scalar 4-byte loads for any +// rowwise input tail shorter than 16 bytes. // The 4 scales packed in `word` (the first one in column first_column), with those in columns // >= num_columns set to zero. __device__ __forceinline__ uint32_t zero_padding_columns(uint32_t word, int first_column, @@ -796,7 +793,7 @@ inline uint32_t division_magic(int divisor) { // input and of the output. Shared-memory rows are padded to an odd number of words, so the column // gathers are free of bank conflicts. Requires num_tiles * 128 * padded_k / 4 < 2^16 (guaranteed // by the row-wise shared-memory limit = 30 KB). -template +template __device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict__ input, uint8_t* __restrict__ output, int padded_k, uint32_t words_per_row_magic, int original_m, @@ -808,12 +805,12 @@ __device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict_ const uint32_t* input_words = reinterpret_cast(input + chunk_offset); const int4* input_vectors = reinterpret_cast(input_words); const int valid_rows = - kCompactInput + kUnpaddedOuterDim ? max(0, min(num_tiles * kSwizzleTileDimM, original_m - first_tile * kSwizzleTileDimM)) : num_tiles * kSwizzleTileDimM; const int valid_words = valid_rows * words_per_row; const int num_vectors = - kCompactInput ? valid_words / kScalesPerWord : num_tiles * kVectorsPerMSpan * padded_k; + kUnpaddedOuterDim ? valid_words / kScalesPerWord : num_tiles * kVectorsPerMSpan * padded_k; for (int i = threadIdx.x; i < num_vectors; i += kRowwiseThreadsPerBlock) { const int4 vector = __ldg(input_vectors + i); const uint32_t words[kScalesPerWord] = { @@ -826,7 +823,7 @@ __device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict_ smem[row * smem_row_words + (word_index - row * words_per_row)] = words[w]; } } - if constexpr (kCompactInput) { + if constexpr (kUnpaddedOuterDim) { for (int word_index = kScalesPerWord * num_vectors + threadIdx.x; word_index < valid_words; word_index += kRowwiseThreadsPerBlock) { const uint32_t row = divide_by_magic(word_index, words_per_row_magic); @@ -852,7 +849,7 @@ __device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict_ for (int group = 0; group < kScalesPerWord; group++) { const int row = row_in_group + kMPositionsPerSwizzleGroup * group; uint32_t word; - if constexpr (kCompactInput) { + if constexpr (kUnpaddedOuterDim) { word = global_first_row + row < original_m ? smem[(tile_first_row + row) * smem_row_words + k_block] : 0u; @@ -871,7 +868,7 @@ __device__ __forceinline__ void swizzle_rowwise_tiles(const uint8_t* __restrict_ // Column-wise: tile m_tile (128 columns), scale rows [first_k, first_k + kChunkRows) clipped to // padded_k. Each input row segment is 128 contiguous bytes; each output block of 4 scale rows is // 512 contiguous bytes. -template +template __device__ __forceinline__ void swizzle_columnwise_chunk(const uint8_t* __restrict__ input, uint8_t* __restrict__ output, int padded_m, int padded_k, int original_m, @@ -879,7 +876,8 @@ __device__ __forceinline__ void swizzle_columnwise_chunk(const uint8_t* __restri int4* smem) { const int num_rows = min(kChunkRows, padded_k - first_k); // multiple of kScalesPerWord const int num_input_vectors = - (kCompactInput ? max(0, min(num_rows, original_k - first_k)) : num_rows) * kVectorsPerMSpan; + (kUnpaddedOuterDim ? max(0, min(num_rows, original_k - first_k)) : num_rows) * + kVectorsPerMSpan; for (int i = threadIdx.x; i < num_input_vectors; i += kColumnwiseThreadsPerBlock) { const int row = i / kVectorsPerMSpan; const int vector_in_row = i % kVectorsPerMSpan; @@ -920,23 +918,23 @@ __device__ __forceinline__ void swizzle_columnwise_chunk(const uint8_t* __restri } } -template +template __global__ void __launch_bounds__(kRowwiseThreadsPerBlock) swizzle_rowwise_full_width_kernel(const uint8_t* __restrict__ input, uint8_t* __restrict__ output, int padded_k, uint32_t words_per_row_magic, int original_m, int original_k) { extern __shared__ uint32_t rowwise_smem[]; - swizzle_rowwise_tiles(input, output, padded_k, words_per_row_magic, original_m, - original_k, blockIdx.x, 1, rowwise_smem); + swizzle_rowwise_tiles(input, output, padded_k, words_per_row_magic, original_m, + original_k, blockIdx.x, 1, rowwise_smem); } -template +template __global__ void __launch_bounds__(kColumnwiseThreadsPerBlock) swizzle_columnwise_chunk_kernel(const uint8_t* __restrict__ input, uint8_t* __restrict__ output, int padded_m, int padded_k, int original_m, int original_k) { __shared__ int4 columnwise_smem[kColumnwiseSmallChunkRows * kVectorsPerMSpan]; - swizzle_columnwise_chunk( + swizzle_columnwise_chunk( input, output, padded_m, padded_k, original_m, original_k, blockIdx.x, blockIdx.y * kColumnwiseSmallChunkRows, columnwise_smem); } @@ -948,7 +946,7 @@ struct SwizzleTensorDescriptor { const uint8_t* input; uint8_t* output; int padded_m, padded_k, original_m, original_k; - bool compact_input; + bool input_has_unpadded_outer_dim; }; struct MultiTensorSwizzleArgs { @@ -965,7 +963,7 @@ __device__ __forceinline__ int tensor_of_block(const MultiTensorSwizzleArgs& arg return tensor; } -template +template __global__ void __launch_bounds__(kRowwiseThreadsPerBlock) multi_tensor_swizzle_rowwise_full_width_kernel(const MultiTensorSwizzleArgs args) { extern __shared__ uint32_t rowwise_multi_smem[]; @@ -974,13 +972,13 @@ __global__ void __launch_bounds__(kRowwiseThreadsPerBlock) const int first_tile = (blockIdx.x - args.first_block[t]) * row_tiles_per_block; const int num_tiles = min(row_tiles_per_block, args.tensors[t].padded_m / kSwizzleTileDimM - first_tile); - swizzle_rowwise_tiles(args.tensors[t].input, args.tensors[t].output, - args.tensors[t].padded_k, args.words_per_row_magic[t], - args.tensors[t].original_m, args.tensors[t].original_k, - first_tile, num_tiles, rowwise_multi_smem); + swizzle_rowwise_tiles(args.tensors[t].input, args.tensors[t].output, + args.tensors[t].padded_k, args.words_per_row_magic[t], + args.tensors[t].original_m, args.tensors[t].original_k, + first_tile, num_tiles, rowwise_multi_smem); } -template +template __global__ void __launch_bounds__(kColumnwiseThreadsPerBlock) multi_tensor_swizzle_columnwise_chunk_kernel(const MultiTensorSwizzleArgs args) { __shared__ int4 columnwise_multi_smem[kChunkRows * kVectorsPerMSpan]; @@ -989,7 +987,7 @@ __global__ void __launch_bounds__(kColumnwiseThreadsPerBlock) const int num_m_tiles = args.tensors[t].padded_m / kSwizzleTileDimM; const int m_tile = block_in_tensor % num_m_tiles; const int first_k = (block_in_tensor / num_m_tiles) * kChunkRows; - swizzle_columnwise_chunk( + swizzle_columnwise_chunk( args.tensors[t].input, args.tensors[t].output, args.tensors[t].padded_m, args.tensors[t].padded_k, args.tensors[t].original_m, args.tensors[t].original_k, m_tile, first_k, columnwise_multi_smem); @@ -1019,7 +1017,7 @@ inline bool should_use_rowwise_full_width_swizzle(int padded_k, int64_t total_ro // Launches the rowwise full-width kernel, where each block stages the entire padded K dimension, // for up to kMaxTensorsPerLaunch tensors. The row tiles per block are the smallest power of two // that keeps the grid within kRowwiseMaxBlocks, subject to the shared-memory limit. -template +template inline void launch_multi_tensor_rowwise_full_width_swizzle(const SwizzleTensorDescriptor* tensors, int num_tensors, cudaStream_t stream) { MultiTensorSwizzleArgs args{}; @@ -1051,7 +1049,7 @@ inline void launch_multi_tensor_rowwise_full_width_swizzle(const SwizzleTensorDe } const int grid = args.first_block[num_tensors]; if (grid == 0) return; - multi_tensor_swizzle_rowwise_full_width_kernel + multi_tensor_swizzle_rowwise_full_width_kernel <<>>( args); @@ -1061,7 +1059,7 @@ inline void launch_multi_tensor_rowwise_full_width_swizzle(const SwizzleTensorDe // Launches the columnwise chunk kernel, where each block stages part of K for one 128-column M // tile, for up to kMaxTensorsPerLaunch tensors. It uses 128-row K chunks once 32-row chunks would // exceed kColumnwiseMaxBlocksWith32RowChunks blocks. -template +template inline void launch_multi_tensor_columnwise_chunk_swizzle(const SwizzleTensorDescriptor* tensors, int num_tensors, cudaStream_t stream) { MultiTensorSwizzleArgs args{}; @@ -1087,10 +1085,10 @@ inline void launch_multi_tensor_columnwise_chunk_swizzle(const SwizzleTensorDesc const int grid = args.first_block[num_tensors]; if (grid == 0) return; if (chunk_rows == kColumnwiseLargeChunkRows) { - multi_tensor_swizzle_columnwise_chunk_kernel + multi_tensor_swizzle_columnwise_chunk_kernel <<>>(args); } else { - multi_tensor_swizzle_columnwise_chunk_kernel + multi_tensor_swizzle_columnwise_chunk_kernel <<>>(args); } NVTE_CHECK_CUDA(cudaGetLastError()); @@ -1101,7 +1099,7 @@ inline void launch_rowwise_full_width_swizzle(const SwizzleTensorDescriptor& ten cudaStream_t stream) { const int grid = tensor.padded_m / kSwizzleTileDimM; const int smem = kSwizzleTileDimM * ((tensor.padded_k / kScalesPerWord) | 1) * kBytesPerWord; - if (tensor.compact_input) { + if (tensor.input_has_unpadded_outer_dim) { swizzle_rowwise_full_width_kernel<<>>( tensor.input, tensor.output, tensor.padded_k, division_magic(tensor.padded_k / kScalesPerWord), tensor.original_m, tensor.original_k); @@ -1117,7 +1115,7 @@ inline void launch_rowwise_full_width_swizzle(const SwizzleTensorDescriptor& ten inline void launch_columnwise_chunk_swizzle(const SwizzleTensorDescriptor& tensor, cudaStream_t stream) { dim3 grid(tensor.padded_m / kSwizzleTileDimM, DIVUP(tensor.padded_k, kColumnwiseSmallChunkRows)); - if (tensor.compact_input) { + if (tensor.input_has_unpadded_outer_dim) { swizzle_columnwise_chunk_kernel<<>>( tensor.input, tensor.output, tensor.padded_m, tensor.padded_k, tensor.original_m, tensor.original_k); @@ -1137,8 +1135,13 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s NVTE_CHECK(scaling_mode == NVTE_MXFP8_1D_SCALING || scaling_mode == NVTE_NVFP4_1D_SCALING, "Input tensor has invalid scaling mode (", to_string(input->scaling_mode), ")."); - const bool compact_input = column_padded_compact_scales(*input); - CheckInputTensor(*input, "scaling_factor_input", !compact_input); + const bool has_rowwise_scale_inv = input->scale_inv.has_data(); + const bool has_columnwise_scale_inv = input->columnwise_scale_inv.has_data(); + NVTE_CHECK(!has_rowwise_scale_inv || !has_columnwise_scale_inv, + "Input tensor has both row-wise and column-wise scaling factors"); + const bool input_has_unpadded_outer_dim = + has_unpadded_outer_scale_dim(*input, has_rowwise_scale_inv); + CheckInputTensor(*input, "scaling_factor_input", !input_has_unpadded_outer_dim); CheckInputTensor(*output, "scaling_factor_output"); NVTE_CHECK( output->scaling_mode == scaling_mode && output->flat_2d_dims() == input->flat_2d_dims(), @@ -1161,10 +1164,6 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s } // Check if scaling factors are non-trivial - const bool has_rowwise_scale_inv = input->scale_inv.has_data(); - const bool has_columnwise_scale_inv = input->columnwise_scale_inv.has_data(); - NVTE_CHECK(!has_rowwise_scale_inv || !has_columnwise_scale_inv, - "Input tensor has both row-wise and column-wise scaling factors"); if (!has_rowwise_scale_inv && !has_columnwise_scale_inv) { return; } @@ -1300,11 +1299,11 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s k, original_M, original_K, - compact_input}; + input_has_unpadded_outer_dim}; if (should_use_rowwise_full_width_swizzle(descriptor.padded_k, descriptor.padded_m / SF_TILE_DIM_M)) { if (m / SF_TILE_DIM_M > kRowwiseMaxBlocks) { - if (compact_input) { + if (input_has_unpadded_outer_dim) { launch_multi_tensor_rowwise_full_width_swizzle(&descriptor, 1, stream); } else { launch_multi_tensor_rowwise_full_width_swizzle(&descriptor, 1, stream); @@ -1362,10 +1361,11 @@ void swizzle_scaling_factors(const Tensor* input, Tensor* output, cudaStream_t s k, original_M, original_K, - compact_input}; - const int64_t grid_size = static_cast(m / SF_TILE_DIM_M) * ((k + 31) / 32); + input_has_unpadded_outer_dim}; + const int64_t grid_size = + static_cast(m / SF_TILE_DIM_M) * DIVUP(k, kColumnwiseSmallChunkRows); if (grid_size > kColumnwiseMaxBlocksWith32RowChunks) { - if (compact_input) { + if (input_has_unpadded_outer_dim) { launch_multi_tensor_columnwise_chunk_swizzle(&descriptor, 1, stream); } else { launch_multi_tensor_columnwise_chunk_swizzle(&descriptor, 1, stream); @@ -1516,8 +1516,8 @@ void launch_multi_tensor_unswizzle_scaling_factors(MultiSwizzleArgs& kernel_args namespace { // Multi-tensor swizzle using rowwise full-width or columnwise chunk kernels with aligned int4 -// (16-byte) loads and stores. -bool int4_vectorized_multi_tensor_swizzle(const std::vector& input, +// (16-byte) stores and loads, except for scalar 4-byte loads for a partial rowwise input vector. +void int4_vectorized_multi_tensor_swizzle(const std::vector& input, std::vector& output, bool rowwise, cudaStream_t stream) { const size_t num_tensors = input.size(); @@ -1545,34 +1545,37 @@ bool int4_vectorized_multi_tensor_swizzle(const std::vector& input, const int original_m = static_cast(selected_rowwise ? first_dim : last_dim); const int original_k = static_cast( DIVUP(selected_rowwise ? last_dim : first_dim, static_cast(block_size))); - const bool compact_input = in_scales.numel() != static_cast(padded_m) * padded_k; - NVTE_CHECK(!compact_input || column_padded_compact_scales(*input[i], selected_rowwise), - "Input scaling factors must either use the padded GEMM layout or omit only " - "outer-dimension padding."); + const bool input_has_unpadded_outer_dim = + in_scales.numel() != static_cast(padded_m) * padded_k; + NVTE_CHECK( + !input_has_unpadded_outer_dim || has_unpadded_outer_scale_dim(*input[i], selected_rowwise), + "Input scaling factors must either use the padded GEMM layout or omit only " + "outer-dimension padding."); tensors[i] = {static_cast(in_scales.dptr), static_cast(out_scales.dptr), padded_m, padded_k, original_m, original_k, - compact_input}; + input_has_unpadded_outer_dim}; } for (size_t first = 0; first < num_tensors;) { size_t end = first + 1; while (end < num_tensors && end - first < kMaxTensorsPerLaunch && - tensors[end].compact_input == tensors[first].compact_input) { + tensors[end].input_has_unpadded_outer_dim == + tensors[first].input_has_unpadded_outer_dim) { ++end; } const int count = static_cast(end - first); if (rowwise) { - if (tensors[first].compact_input) { + if (tensors[first].input_has_unpadded_outer_dim) { launch_multi_tensor_rowwise_full_width_swizzle(tensors.data() + first, count, stream); } else { launch_multi_tensor_rowwise_full_width_swizzle(tensors.data() + first, count, stream); } } else { - if (tensors[first].compact_input) { + if (tensors[first].input_has_unpadded_outer_dim) { launch_multi_tensor_columnwise_chunk_swizzle(tensors.data() + first, count, stream); } else { launch_multi_tensor_columnwise_chunk_swizzle(tensors.data() + first, count, stream); @@ -1580,7 +1583,6 @@ bool int4_vectorized_multi_tensor_swizzle(const std::vector& input, } first = end; } - return true; } } // namespace @@ -1606,9 +1608,6 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, // We don't allow empty tensors. They should be filtered out before calling this function. NVTE_CHECK(input[i]->numel() != 0, "Tensor input[", i, "] is empty."); - const bool compact_input = column_padded_compact_scales(*input[i]); - CheckInputTensor(*input[i], "scaling_factor_input", check_scale_inv_shapes && !compact_input); - CheckInputTensor(*output[i], "scaling_factor_output", check_scale_inv_shapes); NVTE_CHECK(output[i]->scaling_mode == scaling_mode && output[i]->flat_2d_dims() == input[i]->flat_2d_dims(), "Input and output tensor ", i, " layouts must match."); @@ -1629,6 +1628,11 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, for (size_t i = 0; i < num_tensors; i++) { const bool use_rowwise_buffer = input[i]->scale_inv.has_data(); const bool selected_rowwise = rowwise_swizzle && use_rowwise_buffer; + const bool input_has_unpadded_outer_dim = + has_unpadded_outer_scale_dim(*input[i], selected_rowwise); + CheckInputTensor(*input[i], "scaling_factor_input", + check_scale_inv_shapes && !input_has_unpadded_outer_dim); + CheckInputTensor(*output[i], "scaling_factor_output", check_scale_inv_shapes); const auto& in_scales = selected_rowwise ? input[i]->scale_inv : input[i]->columnwise_scale_inv; const auto& out_scales = selected_rowwise ? output[i]->scale_inv : output[i]->columnwise_scale_inv; @@ -1638,15 +1642,17 @@ void multi_tensor_swizzle_scaling_factors(const std::vector& input, NVTE_CHECK(in_scales.shape.size() == 2 && out_scales.shape.size() == 2, "Expected two-dimensional scaling factors."); max_rowwise_padded_k = std::max(max_rowwise_padded_k, static_cast(out_scales.shape[1])); - total_rowwise_tiles += static_cast(out_scales.shape[0]) / 128; + total_rowwise_tiles += static_cast(out_scales.shape[0]) / kSwizzleTileDimM; } } if (rowwise_swizzle) { if (should_use_rowwise_full_width_swizzle(max_rowwise_padded_k, total_rowwise_tiles)) { - if (int4_vectorized_multi_tensor_swizzle(input, output, true, stream)) return; + int4_vectorized_multi_tensor_swizzle(input, output, true, stream); + return; } } else if (columnwise_swizzle) { - if (int4_vectorized_multi_tensor_swizzle(input, output, false, stream)) return; + int4_vectorized_multi_tensor_swizzle(input, output, false, stream); + return; } // Fallback to the generic rowwise swizzle kernel. constexpr int SF_TILE_DIM_M = 128; From e56032e63067ca5a8d840b937fbf6cf590d7de78 Mon Sep 17 00:00:00 2001 From: Varun Thumbe Date: Tue, 17 Nov 2026 21:37:30 -0800 Subject: [PATCH 5/9] Simplify swizzle padding checks using outer and inner dimensions Signed-off-by: Varun Thumbe --- transformer_engine/common/swizzle/swizzle.cu | 37 +++++++++----------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/transformer_engine/common/swizzle/swizzle.cu b/transformer_engine/common/swizzle/swizzle.cu index e81293f44fe..d149c4f64c9 100644 --- a/transformer_engine/common/swizzle/swizzle.cu +++ b/transformer_engine/common/swizzle/swizzle.cu @@ -738,28 +738,23 @@ bool has_unpadded_outer_scale_dim(const Tensor& input, bool rowwise) { if (shape.size() != 2) return false; const auto [rows, cols] = input.flat_2d_dims(); - if (input.scaling_mode == NVTE_MXFP8_1D_SCALING && rowwise) { - const size_t padded_cols = DIVUP_TO_MULTIPLE(DIVUP(cols, size_t{MXFP8_BLOCK_SIZE}), size_t{4}); - return shape[0] == rows && shape[1] == padded_cols && - rows != DIVUP_TO_MULTIPLE(rows, size_t{128}); - } + size_t outer_dim, inner_dim; + size_t outer_alignment, inner_alignment; if (input.scaling_mode == NVTE_MXFP8_1D_SCALING) { - const size_t scale_rows = DIVUP(rows, size_t{MXFP8_BLOCK_SIZE}); - const size_t padded_cols = DIVUP_TO_MULTIPLE(cols, size_t{128}); - return shape[0] == scale_rows && shape[1] == padded_cols && - scale_rows != DIVUP_TO_MULTIPLE(scale_rows, size_t{4}); - } - if (input.scaling_mode == NVTE_NVFP4_1D_SCALING && rowwise) { - const size_t padded_cols = DIVUP_TO_MULTIPLE(DIVUP(cols, size_t{NVFP4_BLOCK_SIZE}), size_t{4}); - return shape[0] == rows && shape[1] == padded_cols && - rows != DIVUP_TO_MULTIPLE(rows, size_t{128}); - } - if (input.scaling_mode == NVTE_NVFP4_1D_SCALING && !rowwise) { - const size_t padded_cols = DIVUP_TO_MULTIPLE(DIVUP(rows, size_t{NVFP4_BLOCK_SIZE}), size_t{4}); - return shape[0] == cols && shape[1] == padded_cols && - cols != DIVUP_TO_MULTIPLE(cols, size_t{128}); - } - return false; + outer_dim = rowwise ? rows : DIVUP(rows, size_t{MXFP8_BLOCK_SIZE}); + inner_dim = rowwise ? DIVUP(cols, size_t{MXFP8_BLOCK_SIZE}) : cols; + outer_alignment = rowwise ? kSwizzleTileDimM : kScalesPerWord; + inner_alignment = rowwise ? kScalesPerWord : kSwizzleTileDimM; + } else if (input.scaling_mode == NVTE_NVFP4_1D_SCALING) { + outer_dim = rowwise ? rows : cols; + inner_dim = DIVUP(rowwise ? cols : rows, size_t{NVFP4_BLOCK_SIZE}); + outer_alignment = kSwizzleTileDimM; + inner_alignment = kScalesPerWord; + } else { + return false; + } + return shape[0] == outer_dim && shape[1] == DIVUP_TO_MULTIPLE(inner_dim, inner_alignment) && + outer_dim != DIVUP_TO_MULTIPLE(outer_dim, outer_alignment); } // Optimized scale swizzle for narrow scale widths. "Rowwise full width" means that a block stages From 5251842a848b48b02e06994e73cff4f1e3089330 Mon Sep 17 00:00:00 2001 From: Varun Thumbe Date: Tue, 17 Nov 2026 22:46:48 -0800 Subject: [PATCH 6/9] Test swizzling scales without outer-dimension padding Signed-off-by: Varun Thumbe --- tests/cpp/operator/test_swizzle.cu | 125 +++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/tests/cpp/operator/test_swizzle.cu b/tests/cpp/operator/test_swizzle.cu index 4ebb23fdc51..9a40d3a32c3 100644 --- a/tests/cpp/operator/test_swizzle.cu +++ b/tests/cpp/operator/test_swizzle.cu @@ -224,6 +224,131 @@ static void zero_scale_inv_padding(uint8_t *buf, set_scale_inv_padding(buf, padded_rows, padded_cols, orig_rows, orig_cols, 0); } +class SwizzleUnpaddedOuterDimTestSuite + : public ::testing::TestWithParam< + std::tuple, NVTEScalingMode, bool>> { + protected: + void Run(size_t num_tensors, bool unchecked = false) { + using namespace test; + const auto& [shape, scaling_mode, rowwise] = GetParam(); + const auto [rows, cols] = shape; + SCOPED_TRACE(::testing::Message() << "num_tensors=" << num_tensors); + const bool nvfp4 = scaling_mode == NVTE_NVFP4_1D_SCALING; + const bool row_major_scales = rowwise || nvfp4; + const size_t block_size = nvfp4 ? 16 : 32; + const size_t outer_dim = + nvfp4 ? (rowwise ? rows : cols) : (rowwise ? rows : divide_round_up(rows, block_size)); + const size_t inner_dim = nvfp4 ? divide_round_up(rowwise ? cols : rows, block_size) + : (rowwise ? divide_round_up(cols, block_size) : cols); + const size_t padded_outer = round_up_to_nearest_multiple(outer_dim, row_major_scales ? 128 : 4); + const size_t padded_inner = round_up_to_nearest_multiple(inner_dim, row_major_scales ? 4 : 128); + ASSERT_LT(outer_dim, padded_outer); + const std::vector input_scale_shape{outer_dim, padded_inner}; + const std::vector output_scale_shape{padded_outer, padded_inner}; + const std::vector data_shape = + nvfp4 && !rowwise ? std::vector{cols, rows} : std::vector{rows, cols}; + const DType data_dtype = nvfp4 ? DType::kFloat4E2M1 : DType::kFloat8E4M3; + const DType scale_dtype = nvfp4 ? DType::kFloat8E4M3 : DType::kFloat8E8M0; + const size_t input_size = outer_dim * padded_inner; + const size_t output_size = padded_outer * padded_inner; + + std::vector> input_buffers, output_buffers; + std::vector inputs, outputs; + std::vector input_handles, output_handles; + std::vector> references; + inputs.reserve(num_tensors); + outputs.reserve(num_tensors); + for (size_t tensor_index = 0; tensor_index < num_tensors; ++tensor_index) { + std::vector input_scales(input_size, 0xFF); + std::vector padded_scales(output_size, 0); + for (size_t outer = 0; outer < outer_dim; ++outer) { + for (size_t inner = 0; inner < inner_dim; ++inner) { + const uint8_t value = 1 + (37 * outer + 13 * inner + 17 * tensor_index) % 254; + input_scales[outer * padded_inner + inner] = value; + padded_scales[outer * padded_inner + inner] = value; + } + } + references.emplace_back(output_size); + if (row_major_scales) { + compute_ref_swizzle<128, 4, true>(padded_scales.data(), references.back().data(), + padded_outer, padded_inner); + } else { + compute_ref_swizzle<128, 4, false>(padded_scales.data(), references.back().data(), + padded_inner, padded_outer); + } + + input_buffers.push_back(cuda_alloc(input_size)); + output_buffers.push_back(cuda_alloc(output_size)); + NVTE_CHECK_CUDA(cudaMemcpy(input_buffers.back().get(), input_scales.data(), input_size, + cudaMemcpyHostToDevice)); + NVTE_CHECK_CUDA(cudaMemset(output_buffers.back().get(), 0xA5, output_size)); + inputs.emplace_back(scaling_mode); + outputs.emplace_back(scaling_mode); + if (rowwise) { + inputs.back().set_rowwise_data(nullptr, data_dtype, data_shape); + inputs.back().set_rowwise_scale_inv(input_buffers.back().get(), scale_dtype, + input_scale_shape); + outputs.back().set_rowwise_data(nullptr, data_dtype, data_shape); + outputs.back().set_rowwise_scale_inv(output_buffers.back().get(), scale_dtype, + output_scale_shape); + } else { + inputs.back().set_columnwise_data(nullptr, data_dtype, data_shape); + inputs.back().set_columnwise_scale_inv(input_buffers.back().get(), scale_dtype, + input_scale_shape); + outputs.back().set_columnwise_data(nullptr, data_dtype, data_shape); + outputs.back().set_columnwise_scale_inv(output_buffers.back().get(), scale_dtype, + output_scale_shape); + } + outputs.back().set_with_gemm_swizzled_scales(true); + input_handles.push_back(inputs.back().data()); + output_handles.push_back(outputs.back().data()); + } + + if (num_tensors == 1) { + nvte_swizzle_scaling_factors(input_handles.front(), output_handles.front(), 0); + } else if (unchecked) { + nvte_multi_tensor_swizzle_scaling_factors_unchecked(input_handles.data(), + output_handles.data(), num_tensors, 0); + } else { + nvte_multi_tensor_swizzle_scaling_factors(input_handles.data(), output_handles.data(), + num_tensors, 0); + } + NVTE_CHECK_CUDA(cudaDeviceSynchronize()); + for (size_t tensor_index = 0; tensor_index < num_tensors; ++tensor_index) { + std::vector actual(output_size); + NVTE_CHECK_CUDA(cudaMemcpy(actual.data(), output_buffers[tensor_index].get(), output_size, + cudaMemcpyDeviceToHost)); + compareResults("unpadded_outer_dim_" + std::to_string(tensor_index), actual.data(), + references[tensor_index].data(), output_size); + } + } +}; + +TEST_P(SwizzleUnpaddedOuterDimTestSuite, SingleTensor) { Run(1); } + +TEST_P(SwizzleUnpaddedOuterDimTestSuite, MultiTensor) { + Run(4); + Run(65); +} + +TEST_P(SwizzleUnpaddedOuterDimTestSuite, MultiTensorUnchecked) { + Run(4, true); + Run(65, true); +} + +INSTANTIATE_TEST_SUITE_P( + OperatorTest, SwizzleUnpaddedOuterDimTestSuite, + ::testing::Combine( + ::testing::ValuesIn(std::vector>{ + {32, 32}, {160, 96}, {4063, 96}, {4192, 2016}, {160, 8096}, {8096, 160}}), + ::testing::Values(NVTE_MXFP8_1D_SCALING, NVTE_NVFP4_1D_SCALING), ::testing::Bool()), + [](const testing::TestParamInfo& info) { + const auto& [shape, scaling_mode, rowwise] = info.param; + return "M" + std::to_string(shape.first) + "_K" + std::to_string(shape.second) + + (scaling_mode == NVTE_NVFP4_1D_SCALING ? "_NVFP4" : "_MXFP8") + + (rowwise ? "_rowwise" : "_columnwise"); + }); + void performTestSwizzleNVFP4(const size_t M, const size_t K, bool rowwise) { using namespace test; constexpr size_t BLOCK_SIZE = 16; From 26affc6f644f491257d709c833ac3bee760c1136 Mon Sep 17 00:00:00 2001 From: Varun Thumbe Date: Tue, 17 Nov 2026 23:56:39 -0800 Subject: [PATCH 7/9] Batch swizzle descriptors by outer-dimension padding Signed-off-by: Varun Thumbe --- tests/cpp/operator/test_swizzle.cu | 78 +++++++++++++++---- .../mxfp8/test_compact_scale_swizzle.py | 7 +- transformer_engine/common/swizzle/swizzle.cu | 4 + 3 files changed, 73 insertions(+), 16 deletions(-) diff --git a/tests/cpp/operator/test_swizzle.cu b/tests/cpp/operator/test_swizzle.cu index 9a40d3a32c3..5c3d33665d4 100644 --- a/tests/cpp/operator/test_swizzle.cu +++ b/tests/cpp/operator/test_swizzle.cu @@ -228,7 +228,7 @@ class SwizzleUnpaddedOuterDimTestSuite : public ::testing::TestWithParam< std::tuple, NVTEScalingMode, bool>> { protected: - void Run(size_t num_tensors, bool unchecked = false) { + void Run(size_t num_tensors, bool unchecked = false, bool mixed_padding = false) { using namespace test; const auto& [shape, scaling_mode, rowwise] = GetParam(); const auto [rows, cols] = shape; @@ -243,13 +243,11 @@ class SwizzleUnpaddedOuterDimTestSuite const size_t padded_outer = round_up_to_nearest_multiple(outer_dim, row_major_scales ? 128 : 4); const size_t padded_inner = round_up_to_nearest_multiple(inner_dim, row_major_scales ? 4 : 128); ASSERT_LT(outer_dim, padded_outer); - const std::vector input_scale_shape{outer_dim, padded_inner}; const std::vector output_scale_shape{padded_outer, padded_inner}; const std::vector data_shape = nvfp4 && !rowwise ? std::vector{cols, rows} : std::vector{rows, cols}; const DType data_dtype = nvfp4 ? DType::kFloat4E2M1 : DType::kFloat8E4M3; const DType scale_dtype = nvfp4 ? DType::kFloat8E4M3 : DType::kFloat8E8M0; - const size_t input_size = outer_dim * padded_inner; const size_t output_size = padded_outer * padded_inner; std::vector> input_buffers, output_buffers; @@ -259,7 +257,10 @@ class SwizzleUnpaddedOuterDimTestSuite inputs.reserve(num_tensors); outputs.reserve(num_tensors); for (size_t tensor_index = 0; tensor_index < num_tensors; ++tensor_index) { - std::vector input_scales(input_size, 0xFF); + const bool unpadded = !mixed_padding || tensor_index % 2 == 0; + const std::vector input_scale_shape{unpadded ? outer_dim : padded_outer, padded_inner}; + const size_t input_size = input_scale_shape[0] * padded_inner; + std::vector input_scales(input_size, unpadded ? 0xFF : 0); std::vector padded_scales(output_size, 0); for (size_t outer = 0; outer < outer_dim; ++outer) { for (size_t inner = 0; inner < inner_dim; ++inner) { @@ -304,16 +305,43 @@ class SwizzleUnpaddedOuterDimTestSuite output_handles.push_back(outputs.back().data()); } - if (num_tensors == 1) { - nvte_swizzle_scaling_factors(input_handles.front(), output_handles.front(), 0); - } else if (unchecked) { - nvte_multi_tensor_swizzle_scaling_factors_unchecked(input_handles.data(), - output_handles.data(), num_tensors, 0); - } else { - nvte_multi_tensor_swizzle_scaling_factors(input_handles.data(), output_handles.data(), - num_tensors, 0); - } + const auto launch = [&](cudaStream_t stream) { + if (num_tensors == 1) { + nvte_swizzle_scaling_factors(input_handles.front(), output_handles.front(), stream); + } else if (unchecked) { + nvte_multi_tensor_swizzle_scaling_factors_unchecked( + input_handles.data(), output_handles.data(), num_tensors, stream); + } else { + nvte_multi_tensor_swizzle_scaling_factors(input_handles.data(), output_handles.data(), + num_tensors, stream); + } + }; + launch(0); NVTE_CHECK_CUDA(cudaDeviceSynchronize()); + if (mixed_padding) { + cudaStream_t stream; + NVTE_CHECK_CUDA(cudaStreamCreate(&stream)); + NVTE_CHECK_CUDA(cudaStreamBeginCapture(stream, cudaStreamCaptureModeRelaxed)); + launch(stream); + cudaGraph_t graph; + NVTE_CHECK_CUDA(cudaStreamEndCapture(stream, &graph)); + size_t num_nodes = 0; + NVTE_CHECK_CUDA(cudaGraphGetNodes(graph, nullptr, &num_nodes)); + std::vector nodes(num_nodes); + NVTE_CHECK_CUDA(cudaGraphGetNodes(graph, nodes.data(), &num_nodes)); + size_t num_kernels = 0; + for (const auto node : nodes) { + cudaGraphNodeType node_type; + NVTE_CHECK_CUDA(cudaGraphNodeGetType(node, &node_type)); + num_kernels += node_type == cudaGraphNodeTypeKernel; + } + NVTE_CHECK_CUDA(cudaGraphDestroy(graph)); + NVTE_CHECK_CUDA(cudaStreamDestroy(stream)); + const size_t num_padded = num_tensors / 2; + const size_t num_unpadded = num_tensors - num_padded; + EXPECT_EQ(num_kernels, divide_round_up(num_padded, size_t{64}) + + divide_round_up(num_unpadded, size_t{64})); + } for (size_t tensor_index = 0; tensor_index < num_tensors; ++tensor_index) { std::vector actual(output_size); NVTE_CHECK_CUDA(cudaMemcpy(actual.data(), output_buffers[tensor_index].get(), output_size, @@ -343,12 +371,34 @@ INSTANTIATE_TEST_SUITE_P( {32, 32}, {160, 96}, {4063, 96}, {4192, 2016}, {160, 8096}, {8096, 160}}), ::testing::Values(NVTE_MXFP8_1D_SCALING, NVTE_NVFP4_1D_SCALING), ::testing::Bool()), [](const testing::TestParamInfo& info) { - const auto& [shape, scaling_mode, rowwise] = info.param; + const auto& shape = std::get<0>(info.param); + const auto scaling_mode = std::get<1>(info.param); + const auto rowwise = std::get<2>(info.param); return "M" + std::to_string(shape.first) + "_K" + std::to_string(shape.second) + (scaling_mode == NVTE_NVFP4_1D_SCALING ? "_NVFP4" : "_MXFP8") + (rowwise ? "_rowwise" : "_columnwise"); }); +class SwizzleMixedPaddingTestSuite : public SwizzleUnpaddedOuterDimTestSuite {}; + +TEST_P(SwizzleMixedPaddingTestSuite, MultiTensor) { + for (const size_t num_tensors : {64, 65, 129, 130}) { + Run(num_tensors, false, true); + } +} + +TEST_P(SwizzleMixedPaddingTestSuite, MultiTensorUnchecked) { + for (const size_t num_tensors : {64, 65, 129, 130}) { + Run(num_tensors, true, true); + } +} + +INSTANTIATE_TEST_SUITE_P( + OperatorTest, SwizzleMixedPaddingTestSuite, + ::testing::Combine(::testing::Values(std::pair{160, 96}), + ::testing::Values(NVTE_MXFP8_1D_SCALING, NVTE_NVFP4_1D_SCALING), + ::testing::Bool())); + void performTestSwizzleNVFP4(const size_t M, const size_t K, bool rowwise) { using namespace test; constexpr size_t BLOCK_SIZE = 16; diff --git a/tests/pytorch/mxfp8/test_compact_scale_swizzle.py b/tests/pytorch/mxfp8/test_compact_scale_swizzle.py index 8a262894a82..1b85fa2c1e9 100644 --- a/tests/pytorch/mxfp8/test_compact_scale_swizzle.py +++ b/tests/pytorch/mxfp8/test_compact_scale_swizzle.py @@ -72,10 +72,13 @@ def test_compact_scale_swizzle(shape, rowwise): @pytest.mark.skipif(not te.is_mxfp8_available(), reason="MXFP8 requires Blackwell") @pytest.mark.parametrize("rowwise", [False, True]) -def test_multi_tensor_compact_scale_swizzle(rowwise): +@pytest.mark.parametrize("num_tensors", [3, 64, 130]) +def test_multi_tensor_compact_scale_swizzle(rowwise, num_tensors): """Multi-tensor swizzle handles compact and already outer-aligned inputs in one batch.""" + shapes = ((160, 96), (256, 128), (288, 224)) tensors_and_references = [ - _make_tensor_and_reference(shape, rowwise) for shape in ((160, 96), (256, 128), (288, 224)) + _make_tensor_and_reference(shapes[index % len(shapes)], rowwise) + for index in range(num_tensors) ] tensors = [item[0] for item in tensors_and_references] tex.multi_tensor_swizzle_scales_for_gemm_(tensors, rowwise, not rowwise) diff --git a/transformer_engine/common/swizzle/swizzle.cu b/transformer_engine/common/swizzle/swizzle.cu index 315c0f28ab9..a90252b031f 100644 --- a/transformer_engine/common/swizzle/swizzle.cu +++ b/transformer_engine/common/swizzle/swizzle.cu @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -1610,6 +1611,9 @@ void int4_vectorized_multi_tensor_swizzle(const std::vector& input, original_k, input_has_unpadded_outer_dim}; } + std::partition(tensors.begin(), tensors.end(), [](const SwizzleTensorDescriptor& tensor) { + return tensor.input_has_unpadded_outer_dim; + }); for (size_t first = 0; first < num_tensors;) { size_t end = first + 1; while (end < num_tensors && end - first < kMaxTensorsPerLaunch && From 6e49c99590e8cd1ba639c615728058d057066565 Mon Sep 17 00:00:00 2001 From: vthumbe1503 Date: Wed, 18 Nov 2026 09:47:43 -0800 Subject: [PATCH 8/9] Fix import and reference to transformer_engine Signed-off-by: vthumbe1503 --- tests/pytorch/mxfp8/test_compact_scale_swizzle.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/pytorch/mxfp8/test_compact_scale_swizzle.py b/tests/pytorch/mxfp8/test_compact_scale_swizzle.py index 1b85fa2c1e9..cbd113e047f 100644 --- a/tests/pytorch/mxfp8/test_compact_scale_swizzle.py +++ b/tests/pytorch/mxfp8/test_compact_scale_swizzle.py @@ -7,8 +7,8 @@ import torch from mxfp8_utils import swizzle_mxfp8_scale -import transformer_engine_torch as tex import transformer_engine.pytorch as te +import transformer_engine_torch as tex from transformer_engine.pytorch.tensor.mxfp8_tensor import MXFP8Quantizer, MXFP8Tensor @@ -51,8 +51,8 @@ def _make_tensor_and_reference(shape, rowwise): columnwise_data=None if rowwise else torch.empty(shape, dtype=torch.uint8, device="cuda"), rowwise_scale_inv=scales if rowwise else None, columnwise_scale_inv=None if rowwise else scales, - fp8_dtype=tex.DType.kFloat8E4M3, - quantizer=MXFP8Quantizer(tex.DType.kFloat8E4M3, rowwise=rowwise, columnwise=not rowwise), + fp8_dtype=te.DType.kFloat8E4M3, + quantizer=MXFP8Quantizer(te.DType.kFloat8E4M3, rowwise=rowwise, columnwise=not rowwise), with_gemm_swizzled_scales=False, ) return tensor, reference From 7f296fe5c84ce74eed66f4aa9765e07ef50cda42 Mon Sep 17 00:00:00 2001 From: Varun Thumbe Date: Thu, 19 Nov 2026 20:14:40 -0800 Subject: [PATCH 9/9] Address swizzle padding review feedback Signed-off-by: Varun Thumbe --- ...t_scale_swizzle.py => test_swizzle_unpadded_scales.py} | 0 .../common/include/transformer_engine/swizzle.h | 5 +++-- transformer_engine/pytorch/csrc/extensions/swizzle.cpp | 8 ++++---- 3 files changed, 7 insertions(+), 6 deletions(-) rename tests/pytorch/mxfp8/{test_compact_scale_swizzle.py => test_swizzle_unpadded_scales.py} (100%) diff --git a/tests/pytorch/mxfp8/test_compact_scale_swizzle.py b/tests/pytorch/mxfp8/test_swizzle_unpadded_scales.py similarity index 100% rename from tests/pytorch/mxfp8/test_compact_scale_swizzle.py rename to tests/pytorch/mxfp8/test_swizzle_unpadded_scales.py diff --git a/transformer_engine/common/include/transformer_engine/swizzle.h b/transformer_engine/common/include/transformer_engine/swizzle.h index f0d1f6e4fdc..013b679c394 100644 --- a/transformer_engine/common/include/transformer_engine/swizzle.h +++ b/transformer_engine/common/include/transformer_engine/swizzle.h @@ -77,8 +77,9 @@ void nvte_pack_mxfp8_columnwise_scales_for_batched_gemm(const NVTETensor input, * Requirements: * - scale_inv is stored in row-major. * - Output scale_inv size is padded to 128x4 for row-scale and 4x128 for col-scale. - * - Input scale_inv may omit outer-dimension padding under the same conditions as - * nvte_swizzle_scaling_factors. + * - MXFP8 and NVFP4 input scale_inv may omit padding on the outer dimension, while retaining + * contiguous-dimension padding so each scale row is 4-byte aligned. Missing outer-dimension + * padding is filled with zeros during swizzling. * - Input and output scale_inv buffer pointers used by this operation must be 16-byte aligned. * - data is quantized along K-dimension, i.e. 1D-scaling block lies along the K-dimension. */ diff --git a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp index 3b28d33d6da..08b38da8a99 100644 --- a/transformer_engine/pytorch/csrc/extensions/swizzle.cpp +++ b/transformer_engine/pytorch/csrc/extensions/swizzle.cpp @@ -49,11 +49,11 @@ NVTEShape padded_scale_shape_for_gemm(NVTEShape shape, NVTEScalingMode scaling_m if (shape.ndim != 2) return shape; const bool scales_are_row_major = rowwise || scaling_mode == NVTE_NVFP4_1D_SCALING; if (scales_are_row_major) { - shape.data[0] = ceildiv(shape.data[0], size_t{128}) * 128; - shape.data[1] = ceildiv(shape.data[1], size_t{4}) * 4; + shape.data[0] = DIVUP_TO_MULTIPLE(shape.data[0], 128); + shape.data[1] = DIVUP_TO_MULTIPLE(shape.data[1], 4); } else { - shape.data[0] = ceildiv(shape.data[0], size_t{4}) * 4; - shape.data[1] = ceildiv(shape.data[1], size_t{128}) * 128; + shape.data[0] = DIVUP_TO_MULTIPLE(shape.data[0], 4); + shape.data[1] = DIVUP_TO_MULTIPLE(shape.data[1], 128); } return shape; }