Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 11 additions & 16 deletions tests/cpp/operator/test_requantize_mxfp8_grouped.cu
Original file line number Diff line number Diff line change
Expand Up @@ -520,6 +520,9 @@ TEST_P(GroupedRequantizeMXFP8TestSuite, MatchesDequantizeThenQuantize) {
}

const auto &test_case = std::get<0>(GetParam());
if (test_case.shapes.size() > 64 && test::getDeviceComputeCapability() != 107) {
GTEST_SKIP() << "More than 64 grouped TMA descriptors requires Rubin overrides.";
}
Comment on lines +523 to +525

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Larger groups lack test coverage

The new skip never runs: kGroupShapeCases has at most 40 members, and both varying-width cases have only three. Add cases with at least 65 members for VARYING_LAST_DIM and VARYING_BOTH_DIMS, including empty members near the old limit. Otherwise, this suite cannot catch regressions in the newly allowed path that bypasses the 64-entry table.

Knowledge Base Used: Verification and CI matrix

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

const DType input_dtype = std::get<1>(GetParam());
const bool return_dequantized = std::get<2>(GetParam());
const bool output_swizzled = std::get<3>(GetParam());
Expand Down Expand Up @@ -662,25 +665,17 @@ const std::vector<GroupShapeCase> kGroupShapeCases = {
{"CapacityTail", {{512, 8192}, {256, 8192}, {512, 8192}}, 2048},
{"EmptyGroupsAndTail", {{0, 512}, {512, 512}, {0, 512}, {256, 512}}, 1024},
{"UnsupportedCuTeStrideAndTail", {{0, 640}, {384, 640}, {128, 640}}, 1024},
{"SameBothDims_1024x4096",
{{1024, 4096}, {1024, 4096}, {1024, 4096}, {1024, 4096}}},
{"SameBothDims_2048x8192",
{{2048, 8192}, {2048, 8192}, {2048, 8192}, {2048, 8192}}},
{"SameBothDims_4096x16384",
{{4096, 16384}, {4096, 16384}, {4096, 16384}, {4096, 16384}}},
{"VaryingFirstDim_512to2048x4096",
{{512, 4096}, {512, 4096}, {1024, 4096}, {2048, 4096}}},
{"VaryingFirstDim_1024to4096x8192",
{{1024, 8192}, {1024, 8192}, {2048, 8192}, {4096, 8192}}},
{"SameBothDims_1024x4096", {{1024, 4096}, {1024, 4096}, {1024, 4096}, {1024, 4096}}},
{"SameBothDims_2048x8192", {{2048, 8192}, {2048, 8192}, {2048, 8192}, {2048, 8192}}},
{"SameBothDims_4096x16384", {{4096, 16384}, {4096, 16384}, {4096, 16384}, {4096, 16384}}},
{"VaryingFirstDim_512to2048x4096", {{512, 4096}, {512, 4096}, {1024, 4096}, {2048, 4096}}},
{"VaryingFirstDim_1024to4096x8192", {{1024, 8192}, {1024, 8192}, {2048, 8192}, {4096, 8192}}},
{"VaryingFirstDim_2048to8192x16384",
{{2048, 16384}, {2048, 16384}, {4096, 16384}, {8192, 16384}}},
// An empty member in the middle must not terminate the persistent work loop.
{"VaryingFirstDimWithEmpty",
{{128, 256}, {0, 256}, {384, 256}, {512, 256}}},
{"VaryingLastDim",
{{256, 128}, {256, 384}, {256, 640}}},
{"VaryingBothDims",
{{128, 128}, {256, 384}, {512, 640}}},
{"VaryingFirstDimWithEmpty", {{128, 256}, {0, 256}, {384, 256}, {512, 256}}},
{"VaryingLastDim", {{256, 128}, {256, 384}, {256, 640}}},
{"VaryingBothDims", {{128, 128}, {256, 384}, {512, 640}}},
};

std::string make_test_name(
Expand Down
11 changes: 11 additions & 0 deletions transformer_engine/common/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,14 @@ if(NOT arch_101_index EQUAL -1)
list(APPEND NVTE_SPECIFIC_ARCHS "101a")
endif()

# Check for architecture 107
list(FIND CMAKE_CUDA_ARCHITECTURES "107" arch_107_index)
if(NOT arch_107_index EQUAL -1)
list(REMOVE_ITEM CMAKE_CUDA_ARCHITECTURES "107")
list(APPEND NVTE_GENERIC_ARCHS "107")
list(APPEND NVTE_SPECIFIC_ARCHS "107a")
endif()

# Check for architecture 110 (if we see this we are in toolkit >= 13.0)
list(FIND CMAKE_CUDA_ARCHITECTURES "110" arch_110_index)
if(NOT arch_110_index EQUAL -1)
Expand All @@ -147,6 +155,9 @@ if(NOT arch_120_index EQUAL -1)
endif()
endif()

# Explicit 107 and the 100 family target can both request 107a.
list(REMOVE_DUPLICATES NVTE_SPECIFIC_ARCHS)

# Move remaining standard (pre-Blackwell) architectures into NVTE_STANDARD_ARCHS.
# These are applied to all CUDA sources (both generic and arch-specific).
set(NVTE_STANDARD_ARCHS ${CMAKE_CUDA_ARCHITECTURES})
Expand Down
149 changes: 131 additions & 18 deletions transformer_engine/common/cast/core/grouped_tma.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include <transformer_engine/transformer_engine.h>

#include "../../common.h"
#include "../../util/cuda_runtime.h"
#include "../../util/ptx.cuh"
#include "../../utils.cuh"
#include "grouped_layout.cuh"
Expand All @@ -31,6 +32,100 @@ namespace transformer_engine {
namespace dispatch {
namespace common {

inline bool use_tma_descriptor_override() {
#if CUDA_VERSION >= 13040
return transformer_engine::cuda::sm_arch() == 107;
#else
return false;
#endif
}

inline void create_grouped_tma_map(CUtensorMap &map, const SimpleTensor &tensor,
const bool use_override, const uint64_t rows,
const uint64_t cols, const uint32_t tile_rows,
const uint32_t tile_cols, const size_t type_bits,
const CUtensorMapSwizzle swizzle = CU_TENSOR_MAP_SWIZZLE_NONE) {
const uint64_t map_rows = use_override ? 1 : rows;
const uint64_t map_cols = use_override ? tile_cols : cols;
create_2D_tensor_map(map, tensor, map_rows, map_cols, tile_rows, tile_cols, map_cols, 0,
type_bits, swizzle);
}

struct TmaDescriptorOverride2D {
uint64_t address;
uint16_t size_x;
uint16_t size_y;
uint32_t lower_stride;
uint16_t upper_stride;
};

template <size_t TILE_DIM_Y, size_t TILE_DIM_X, typename T>
__device__ __forceinline__ TmaDescriptorOverride2D make_tma_descriptor_override_2d(
const T *data, const size_t rows, const size_t cols, const size_t x, const size_t y) {
static_assert((TILE_DIM_X > 0) && (TILE_DIM_X < 256) && (TILE_DIM_Y > 0) && (TILE_DIM_Y < 256));
const uint64_t stride_bytes = static_cast<uint64_t>(cols) * sizeof(T);
const uint64_t address = reinterpret_cast<uint64_t>(data + y * cols + x);

if (address % TMA_GMEM_ALIGNMENT != 0) {
NVTE_DEVICE_ERROR("TMA override address must be 16B aligned.");
}
if (stride_bytes % TMA_GMEM_ALIGNMENT != 0) {
NVTE_DEVICE_ERROR("TMA override stride must be 16B aligned.");
}
const uint64_t stride = stride_bytes >> 4;
if ((stride >> 36) != 0) {
NVTE_DEVICE_ERROR("TMA override stride exceeds 36 bits.");
}
const TmaDescriptorOverride2D descriptor{
address, static_cast<uint16_t>(min(TILE_DIM_X, cols - x)),
static_cast<uint16_t>(min(TILE_DIM_Y, rows - y)), static_cast<uint32_t>(stride),
static_cast<uint16_t>(stride >> 32)};
return descriptor;
}

template <size_t TILE_DIM_Y, size_t TILE_DIM_X, typename T>
__device__ __forceinline__ void load_grouped_tma_2d(T *dst, const CUtensorMap &map, const size_t x,
const size_t y, uint64_t *barrier,
const T *data, const size_t rows,
const size_t cols) {
constexpr bool has_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE;
if constexpr (has_override) {
if (data != nullptr) {
const auto info =
make_tma_descriptor_override_2d<TILE_DIM_Y, TILE_DIM_X>(data, rows, cols, x, y);
ptx::cp_async_bulk_tensor_2d_global_to_shared_rubin(
reinterpret_cast<uint64_t *>(dst), reinterpret_cast<const uint64_t *>(&map), info.address,
info.size_x, info.size_y, info.lower_stride, info.upper_stride, barrier);
return;
}
}
ptx::cp_async_bulk_tensor_2d_global_to_shared(
reinterpret_cast<uint64_t *>(dst), reinterpret_cast<const uint64_t *>(&map), x, y, barrier);
}

template <size_t TILE_DIM_Y, size_t TILE_DIM_X, typename T>
__device__ __forceinline__ void store_grouped_tma_2d(T *src, const CUtensorMap &map, const size_t x,
const size_t y, const T *data,
const size_t rows, const size_t cols) {
constexpr bool has_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE;
if constexpr (has_override) {
if (data != nullptr) {
if (x >= cols || y >= rows) {
return;
}

const auto info =
make_tma_descriptor_override_2d<TILE_DIM_Y, TILE_DIM_X>(data, rows, cols, x, y);
ptx::cp_async_bulk_tensor_2d_shared_to_global_rubin(
reinterpret_cast<const uint64_t *>(&map), info.address, info.size_x, info.size_y,
info.lower_stride, info.upper_stride, reinterpret_cast<uint64_t *>(src));
return;
}
}
ptx::cp_async_bulk_tensor_2d_shared_to_global(reinterpret_cast<const uint64_t *>(&map), x, y,
reinterpret_cast<uint64_t *>(src));
}

constexpr int MAX_SUPPORTED_TENSOR_DESCRIPTORS = 64;

struct alignas(128) TensorMapStorage {
Expand Down Expand Up @@ -184,47 +279,65 @@ __device__ __forceinline__ void fence_acquire_tensormap(const CUtensorMap *tenso
}

// Issue TMA global->shared transfer for one stage of input (and optional activation input).
template <typename IType, bool IS_DACT>
template <typename IType, bool IS_DACT, size_t TILE_DIM_Y, size_t TILE_DIM_X>
__device__ __forceinline__ void prefetch_input_stage(
IType *in_sh, IType *act_in_sh, const CUtensorMap &tensor_map_input,
const CUtensorMap &tensor_map_act_input, const size_t global_offset_X,
const size_t global_offset_Y, const size_t buff_offset, const size_t shmem_buff_size,
uint64_t *barrier, const bool leading_thread) {
uint64_t *barrier, const bool leading_thread, const IType *input_data = nullptr,
const IType *act_data = nullptr, const size_t rows = 0, const size_t cols = 0) {
constexpr bool has_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE;
if constexpr (has_override) {
if ((input_data != nullptr) && (global_offset_X >= cols || global_offset_Y >= rows)) {
for (size_t i = threadIdx.x; i < TILE_DIM_Y * TILE_DIM_X; i += blockDim.x) {
in_sh[buff_offset + i] = static_cast<IType>(0.0f);
if constexpr (IS_DACT) {
act_in_sh[buff_offset + i] = static_cast<IType>(0.0f);
}
}
__syncthreads();
if (leading_thread) {
ptx::mbarrier_arrive(barrier);
}

return;
}
}
if (leading_thread) {
ptx::mbarrier_arrive_expect_tx(barrier, shmem_buff_size);
ptx::cp_async_bulk_tensor_2d_global_to_shared(
reinterpret_cast<uint64_t *>(&in_sh[buff_offset]),
reinterpret_cast<const uint64_t *>(&tensor_map_input), global_offset_X, global_offset_Y,
barrier);
load_grouped_tma_2d<TILE_DIM_Y, TILE_DIM_X>(&in_sh[buff_offset], tensor_map_input,
global_offset_X, global_offset_Y, barrier,
input_data, rows, cols);
if constexpr (IS_DACT) {
ptx::cp_async_bulk_tensor_2d_global_to_shared(
reinterpret_cast<uint64_t *>(&act_in_sh[buff_offset]),
reinterpret_cast<const uint64_t *>(&tensor_map_act_input), global_offset_X,
global_offset_Y, barrier);
load_grouped_tma_2d<TILE_DIM_Y, TILE_DIM_X>(&act_in_sh[buff_offset], tensor_map_act_input,
global_offset_X, global_offset_Y, barrier,
act_data, rows, cols);
}
}
}

// Issue TMA shared->global transfer for one stage of outputs.
template <typename OType, bool ROWWISE_SCALING, bool COLWISE_SCALING>
template <typename OType, bool ROWWISE_SCALING, bool COLWISE_SCALING, size_t TILE_DIM_Y = 32,
size_t TILE_DIM_X = 128>
__device__ __forceinline__ void store_output_stage(
OType *out_rowwise_data_sh, OType *out_colwise_data_sh,
const CUtensorMap &tensor_map_output_rowwise, const CUtensorMap &tensor_map_output_colwise,
const size_t global_offset_X, const size_t global_offset_Y, const size_t buff_offset,
const bool leading_thread) {
const bool leading_thread, const OType *rowwise_data = nullptr,
const OType *colwise_data = nullptr, const size_t rows = 0, const size_t cols = 0) {
if (!leading_thread) {
return;
}

if constexpr (ROWWISE_SCALING) {
ptx::cp_async_bulk_tensor_2d_shared_to_global(
reinterpret_cast<const uint64_t *>(&tensor_map_output_rowwise), global_offset_X,
global_offset_Y, reinterpret_cast<uint64_t *>(&out_rowwise_data_sh[buff_offset]));
store_grouped_tma_2d<TILE_DIM_Y, TILE_DIM_X>(&out_rowwise_data_sh[buff_offset],
tensor_map_output_rowwise, global_offset_X,
global_offset_Y, rowwise_data, rows, cols);
}
if constexpr (COLWISE_SCALING) {
ptx::cp_async_bulk_tensor_2d_shared_to_global(
reinterpret_cast<const uint64_t *>(&tensor_map_output_colwise), global_offset_X,
global_offset_Y, reinterpret_cast<uint64_t *>(&out_colwise_data_sh[buff_offset]));
store_grouped_tma_2d<TILE_DIM_Y, TILE_DIM_X>(&out_colwise_data_sh[buff_offset],
tensor_map_output_colwise, global_offset_X,
global_offset_Y, colwise_data, rows, cols);
}
if constexpr (ROWWISE_SCALING || COLWISE_SCALING) {
ptx::cp_async_bulk_commit_group();
Expand Down
Loading
Loading