Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
175 changes: 175 additions & 0 deletions tests/cpp/operator/test_swizzle.cu
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,181 @@ static void zero_scale_inv_padding(uint8_t *buf,
set_scale_inv_padding(buf, padded_rows, padded_cols, orig_rows, orig_cols, 0);
}

class SwizzleUnpaddedOuterDimTestSuite
: public ::testing::TestWithParam<
std::tuple<std::pair<size_t, size_t>, NVTEScalingMode, bool>> {
protected:
void Run(size_t num_tensors, bool unchecked = false, bool mixed_padding = false) {
using namespace test;
const auto& [shape, scaling_mode, rowwise] = GetParam();
const auto [rows, cols] = shape;
SCOPED_TRACE(::testing::Message() << "num_tensors=" << num_tensors);
const bool nvfp4 = scaling_mode == NVTE_NVFP4_1D_SCALING;
const bool row_major_scales = rowwise || nvfp4;
const size_t block_size = nvfp4 ? 16 : 32;
const size_t outer_dim =
nvfp4 ? (rowwise ? rows : cols) : (rowwise ? rows : divide_round_up(rows, block_size));
const size_t inner_dim = nvfp4 ? divide_round_up(rowwise ? cols : rows, block_size)
: (rowwise ? divide_round_up(cols, block_size) : cols);
const size_t padded_outer = round_up_to_nearest_multiple(outer_dim, row_major_scales ? 128 : 4);
const size_t padded_inner = round_up_to_nearest_multiple(inner_dim, row_major_scales ? 4 : 128);
ASSERT_LT(outer_dim, padded_outer);
const std::vector<size_t> output_scale_shape{padded_outer, padded_inner};
const std::vector<size_t> data_shape =
nvfp4 && !rowwise ? std::vector<size_t>{cols, rows} : std::vector<size_t>{rows, cols};
const DType data_dtype = nvfp4 ? DType::kFloat4E2M1 : DType::kFloat8E4M3;
const DType scale_dtype = nvfp4 ? DType::kFloat8E4M3 : DType::kFloat8E8M0;
const size_t output_size = padded_outer * padded_inner;

std::vector<CudaPtr<uint8_t>> input_buffers, output_buffers;
std::vector<TensorWrapper> inputs, outputs;
std::vector<NVTETensor> input_handles, output_handles;
std::vector<std::vector<uint8_t>> references;
inputs.reserve(num_tensors);
outputs.reserve(num_tensors);
for (size_t tensor_index = 0; tensor_index < num_tensors; ++tensor_index) {
const bool unpadded = !mixed_padding || tensor_index % 2 == 0;
const std::vector<size_t> input_scale_shape{unpadded ? outer_dim : padded_outer, padded_inner};
const size_t input_size = input_scale_shape[0] * padded_inner;
std::vector<uint8_t> input_scales(input_size, unpadded ? 0xFF : 0);
std::vector<uint8_t> padded_scales(output_size, 0);
for (size_t outer = 0; outer < outer_dim; ++outer) {
for (size_t inner = 0; inner < inner_dim; ++inner) {
const uint8_t value = 1 + (37 * outer + 13 * inner + 17 * tensor_index) % 254;
input_scales[outer * padded_inner + inner] = value;
padded_scales[outer * padded_inner + inner] = value;
}
}
references.emplace_back(output_size);
if (row_major_scales) {
compute_ref_swizzle<128, 4, true>(padded_scales.data(), references.back().data(),
padded_outer, padded_inner);
} else {
compute_ref_swizzle<128, 4, false>(padded_scales.data(), references.back().data(),
padded_inner, padded_outer);
}

input_buffers.push_back(cuda_alloc<uint8_t>(input_size));
output_buffers.push_back(cuda_alloc<uint8_t>(output_size));
NVTE_CHECK_CUDA(cudaMemcpy(input_buffers.back().get(), input_scales.data(), input_size,
cudaMemcpyHostToDevice));
NVTE_CHECK_CUDA(cudaMemset(output_buffers.back().get(), 0xA5, output_size));
inputs.emplace_back(scaling_mode);
outputs.emplace_back(scaling_mode);
if (rowwise) {
inputs.back().set_rowwise_data(nullptr, data_dtype, data_shape);
inputs.back().set_rowwise_scale_inv(input_buffers.back().get(), scale_dtype,
input_scale_shape);
outputs.back().set_rowwise_data(nullptr, data_dtype, data_shape);
outputs.back().set_rowwise_scale_inv(output_buffers.back().get(), scale_dtype,
output_scale_shape);
} else {
inputs.back().set_columnwise_data(nullptr, data_dtype, data_shape);
inputs.back().set_columnwise_scale_inv(input_buffers.back().get(), scale_dtype,
input_scale_shape);
outputs.back().set_columnwise_data(nullptr, data_dtype, data_shape);
outputs.back().set_columnwise_scale_inv(output_buffers.back().get(), scale_dtype,
output_scale_shape);
}
outputs.back().set_with_gemm_swizzled_scales(true);
input_handles.push_back(inputs.back().data());
output_handles.push_back(outputs.back().data());
}

const auto launch = [&](cudaStream_t stream) {
if (num_tensors == 1) {
nvte_swizzle_scaling_factors(input_handles.front(), output_handles.front(), stream);
} else if (unchecked) {
nvte_multi_tensor_swizzle_scaling_factors_unchecked(
input_handles.data(), output_handles.data(), num_tensors, stream);
} else {
nvte_multi_tensor_swizzle_scaling_factors(input_handles.data(), output_handles.data(),
num_tensors, stream);
}
};
launch(0);
NVTE_CHECK_CUDA(cudaDeviceSynchronize());
if (mixed_padding) {
cudaStream_t stream;
NVTE_CHECK_CUDA(cudaStreamCreate(&stream));
NVTE_CHECK_CUDA(cudaStreamBeginCapture(stream, cudaStreamCaptureModeRelaxed));
launch(stream);
cudaGraph_t graph;
NVTE_CHECK_CUDA(cudaStreamEndCapture(stream, &graph));
size_t num_nodes = 0;
NVTE_CHECK_CUDA(cudaGraphGetNodes(graph, nullptr, &num_nodes));
std::vector<cudaGraphNode_t> nodes(num_nodes);
NVTE_CHECK_CUDA(cudaGraphGetNodes(graph, nodes.data(), &num_nodes));
size_t num_kernels = 0;
for (const auto node : nodes) {
cudaGraphNodeType node_type;
NVTE_CHECK_CUDA(cudaGraphNodeGetType(node, &node_type));
num_kernels += node_type == cudaGraphNodeTypeKernel;
}
NVTE_CHECK_CUDA(cudaGraphDestroy(graph));
NVTE_CHECK_CUDA(cudaStreamDestroy(stream));
const size_t num_padded = num_tensors / 2;
const size_t num_unpadded = num_tensors - num_padded;
EXPECT_EQ(num_kernels, divide_round_up(num_padded, size_t{64}) +
divide_round_up(num_unpadded, size_t{64}));
}
for (size_t tensor_index = 0; tensor_index < num_tensors; ++tensor_index) {
std::vector<uint8_t> actual(output_size);
NVTE_CHECK_CUDA(cudaMemcpy(actual.data(), output_buffers[tensor_index].get(), output_size,
cudaMemcpyDeviceToHost));
compareResults("unpadded_outer_dim_" + std::to_string(tensor_index), actual.data(),
references[tensor_index].data(), output_size);
}
}
};

TEST_P(SwizzleUnpaddedOuterDimTestSuite, SingleTensor) { Run(1); }

TEST_P(SwizzleUnpaddedOuterDimTestSuite, MultiTensor) {
Run(4);
Run(65);
}

TEST_P(SwizzleUnpaddedOuterDimTestSuite, MultiTensorUnchecked) {
Run(4, true);
Run(65, true);
}

INSTANTIATE_TEST_SUITE_P(
OperatorTest, SwizzleUnpaddedOuterDimTestSuite,
::testing::Combine(
::testing::ValuesIn(std::vector<std::pair<size_t, size_t>>{
{32, 32}, {160, 96}, {4063, 96}, {4192, 2016}, {160, 8096}, {8096, 160}}),
::testing::Values(NVTE_MXFP8_1D_SCALING, NVTE_NVFP4_1D_SCALING), ::testing::Bool()),
[](const testing::TestParamInfo<SwizzleUnpaddedOuterDimTestSuite::ParamType>& info) {
const auto& shape = std::get<0>(info.param);
const auto scaling_mode = std::get<1>(info.param);
const auto rowwise = std::get<2>(info.param);
return "M" + std::to_string(shape.first) + "_K" + std::to_string(shape.second) +
(scaling_mode == NVTE_NVFP4_1D_SCALING ? "_NVFP4" : "_MXFP8") +
(rowwise ? "_rowwise" : "_columnwise");
});

class SwizzleMixedPaddingTestSuite : public SwizzleUnpaddedOuterDimTestSuite {};

TEST_P(SwizzleMixedPaddingTestSuite, MultiTensor) {
for (const size_t num_tensors : {64, 65, 129, 130}) {
Run(num_tensors, false, true);
}
}

TEST_P(SwizzleMixedPaddingTestSuite, MultiTensorUnchecked) {
for (const size_t num_tensors : {64, 65, 129, 130}) {
Run(num_tensors, true, true);
}
}

INSTANTIATE_TEST_SUITE_P(
OperatorTest, SwizzleMixedPaddingTestSuite,
::testing::Combine(::testing::Values(std::pair<size_t, size_t>{160, 96}),
::testing::Values(NVTE_MXFP8_1D_SCALING, NVTE_NVFP4_1D_SCALING),
::testing::Bool()));

void performTestSwizzleNVFP4(const size_t M, const size_t K, bool rowwise) {
using namespace test;
constexpr size_t BLOCK_SIZE = 16;
Expand Down
87 changes: 87 additions & 0 deletions tests/pytorch/mxfp8/test_swizzle_unpadded_scales.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
# Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# See LICENSE for license information.

"""Scale swizzling directly pads compact MXFP8 scale buffers."""

import pytest
import torch
from mxfp8_utils import swizzle_mxfp8_scale

import transformer_engine.pytorch as te
import transformer_engine_torch as tex
from transformer_engine.pytorch.tensor.mxfp8_tensor import MXFP8Quantizer, MXFP8Tensor


def _round_up(value, multiple):
return (value + multiple - 1) // multiple * multiple


def _scale_shape(rows, cols, rowwise, compact):
if rowwise:
return (
rows if compact else _round_up(rows, 128),
_round_up((cols + 31) // 32, 4),
)
return (
(rows + 31) // 32 if compact else _round_up((rows + 31) // 32, 4),
_round_up(cols, 128),
)


def _make_tensor_and_reference(shape, rowwise):
rows, cols = shape
compact_shape = _scale_shape(rows, cols, rowwise, compact=True)
logical_shape = (rows, (cols + 31) // 32) if rowwise else ((rows + 31) // 32, cols)
scales = torch.zeros(compact_shape, dtype=torch.uint8, device="cuda")
scales[: logical_shape[0], : logical_shape[1]] = torch.randint(
1, 256, logical_shape, dtype=torch.uint8, device="cuda"
)

padded_rows, padded_cols = _round_up(rows, 128), _round_up(cols, 128)
padded_shape = _scale_shape(padded_rows, padded_cols, rowwise, compact=False)
padded = torch.zeros(padded_shape, dtype=torch.uint8, device="cuda")
padded[: compact_shape[0], : compact_shape[1]].copy_(scales)
reference = swizzle_mxfp8_scale(padded_rows, padded_cols, padded, columnwise=not rowwise)

tensor = MXFP8Tensor(
shape=shape,
dtype=torch.bfloat16,
device="cuda",
rowwise_data=torch.empty(shape, dtype=torch.uint8, device="cuda") if rowwise else None,
columnwise_data=None if rowwise else torch.empty(shape, dtype=torch.uint8, device="cuda"),
rowwise_scale_inv=scales if rowwise else None,
columnwise_scale_inv=None if rowwise else scales,
fp8_dtype=te.DType.kFloat8E4M3,
quantizer=MXFP8Quantizer(te.DType.kFloat8E4M3, rowwise=rowwise, columnwise=not rowwise),
with_gemm_swizzled_scales=False,
)
return tensor, reference


@pytest.mark.skipif(not te.is_mxfp8_available(), reason="MXFP8 requires Blackwell")
@pytest.mark.parametrize("shape", [(32, 32), (160, 96), (288, 224), (4096, 4128)])
@pytest.mark.parametrize("rowwise", [False, True])
def test_compact_scale_swizzle(shape, rowwise):
"""Single-tensor swizzle pads compact scales and zero-fills the missing outer extent."""
tensor, reference = _make_tensor_and_reference(shape, rowwise)
tex.swizzle_scales_for_gemm_(tensor)
actual = tensor._rowwise_scale_inv if rowwise else tensor._columnwise_scale_inv
torch.testing.assert_close(actual, reference, rtol=0, atol=0)
assert tensor._with_gemm_swizzled_scales


@pytest.mark.skipif(not te.is_mxfp8_available(), reason="MXFP8 requires Blackwell")
@pytest.mark.parametrize("rowwise", [False, True])
@pytest.mark.parametrize("num_tensors", [3, 64, 130])
def test_multi_tensor_compact_scale_swizzle(rowwise, num_tensors):
"""Multi-tensor swizzle handles compact and already outer-aligned inputs in one batch."""
shapes = ((160, 96), (256, 128), (288, 224))
tensors_and_references = [
_make_tensor_and_reference(shapes[index % len(shapes)], rowwise)
for index in range(num_tensors)
]
tensors = [item[0] for item in tensors_and_references]
tex.multi_tensor_swizzle_scales_for_gemm_(tensors, rowwise, not rowwise)
for tensor, reference in tensors_and_references:
actual = tensor._rowwise_scale_inv if rowwise else tensor._columnwise_scale_inv
torch.testing.assert_close(actual, reference, rtol=0, atol=0)
10 changes: 8 additions & 2 deletions transformer_engine/common/include/transformer_engine/swizzle.h
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,10 @@ extern "C" {
*
* Requirements:
* - scale_inv is stored in row-major.
* - scale_inv size is padded to 128x4 for row-scale and 4x128 for col-scale.
* - Output scale_inv size is padded to 128x4 for row-scale and 4x128 for col-scale.
* - MXFP8 and NVFP4 input scale_inv may omit padding on the outer dimension, while retaining
* contiguous-dimension padding so each scale row is 4-byte aligned. Missing outer-dimension
* padding is filled with zeros during swizzling.
* - Input and output scale_inv buffer pointers used by this operation must be 16-byte aligned.
* - data is quantized along K-dimension, i.e. 1D-scaling block lies along the K-dimension.
*/
Expand Down Expand Up @@ -73,7 +76,10 @@ void nvte_pack_mxfp8_columnwise_scales_for_batched_gemm(const NVTETensor input,
*
* Requirements:
* - scale_inv is stored in row-major.
* - scale_inv size is padded to 128x4 for row-scale and 4x128 for col-scale.
* - Output scale_inv size is padded to 128x4 for row-scale and 4x128 for col-scale.
* - MXFP8 and NVFP4 input scale_inv may omit padding on the outer dimension, while retaining
* contiguous-dimension padding so each scale row is 4-byte aligned. Missing outer-dimension
* padding is filled with zeros during swizzling.
* - Input and output scale_inv buffer pointers used by this operation must be 16-byte aligned.
* - data is quantized along K-dimension, i.e. 1D-scaling block lies along the K-dimension.
*/
Expand Down
Loading
Loading