From e4b3877b052466fee2f90f6b0bc0714d8151284d Mon Sep 17 00:00:00 2001 From: Oleg Goncharov Date: Tue, 9 Jun 2026 18:39:14 +0000 Subject: [PATCH 1/8] Implemented TMA override path for Rubin arch Signed-off-by: Oleg Goncharov --- .../operator/test_requantize_mxfp8_grouped.cu | 103 ++++++++++-- transformer_engine/common/CMakeLists.txt | 11 ++ .../common/cast/core/grouped_tma.cuh | 141 +++++++++++++--- .../cast/mxfp8/group_dequantize_mxfp8.cuh | 119 +++++++------- .../cast/mxfp8/group_quantize_mxfp8.cuh | 150 ++++++++++-------- .../common/cast/mxfp8/requantize_mxfp8.cu | 70 ++++---- transformer_engine/common/util/ptx.cuh | 65 ++++++++ 7 files changed, 478 insertions(+), 181 deletions(-) diff --git a/tests/cpp/operator/test_requantize_mxfp8_grouped.cu b/tests/cpp/operator/test_requantize_mxfp8_grouped.cu index c039d2fb2ff..f854dc878a0 100644 --- a/tests/cpp/operator/test_requantize_mxfp8_grouped.cu +++ b/tests/cpp/operator/test_requantize_mxfp8_grouped.cu @@ -520,6 +520,9 @@ TEST_P(GroupedRequantizeMXFP8TestSuite, MatchesDequantizeThenQuantize) { } const auto &test_case = std::get<0>(GetParam()); + if (test_case.shapes.size() > 64 && test::getDeviceComputeCapability() != 107) { + GTEST_SKIP() << "More than 64 grouped TMA descriptors requires Rubin overrides."; + } const DType input_dtype = std::get<1>(GetParam()); const bool return_dequantized = std::get<2>(GetParam()); const bool output_swizzled = std::get<3>(GetParam()); @@ -649,7 +652,83 @@ GroupShapeCase many_groups_case() { return test_case; } +GroupShapeCase make_large_group_case(const std::string &name, bool varying_rows, + bool varying_cols) { + GroupShapeCase result{name, {}}; + for (size_t i = 0; i < 65; ++i) { + result.shapes.emplace_back(varying_rows ? 128 * (1 + i % 3) : 128, + varying_cols ? 128 * (1 + i % 5) : 128); + } + return result; +} + +// In addition to numerical comparisons below, inspect the graph to catch a +// reintroduced descriptor-update launch. Warm up every path before capture. +TEST(GroupedTmaOverrideTest, DescriptorUpdateLaunches) { + const int cc = test::getDeviceComputeCapability(); + if (cc < test::blackwellComputeCapability) GTEST_SKIP(); + const std::vector cases = { + {"VaryingFirst", {{128, 256}, {0, 256}, {384, 256}}}, + {"VaryingLast", {{256, 128}, {256, 384}, {256, 640}}}, + {"VaryingBoth", {{128, 128}, {0, 384}, {256, 640}}}, + }; + for (const auto &test_case : cases) { + SCOPED_TRACE(test_case.name); + const auto info = make_shape_info(test_case); + auto source = + make_grouped_tensor(info, DType::kBFloat16, NVTE_DELAYED_TENSOR_SCALING, true, false); + auto quantized = + make_grouped_tensor(info, DType::kFloat8E4M3, NVTE_MXFP8_1D_SCALING, true, false); + auto decoded = + make_grouped_tensor(info, DType::kBFloat16, NVTE_DELAYED_TENSOR_SCALING, true, false); + auto requantized = + make_grouped_tensor(info, DType::kFloat8E4M3, NVTE_MXFP8_1D_SCALING, false, true); + fill_source(source, info); + cudaStream_t stream; + NVTE_CHECK_CUDA(cudaStreamCreate(&stream)); + for (int op = 0; op < 3; ++op) { + SCOPED_TRACE(op); + auto launch = [&] { + if (op == 0) { + nvte_group_quantize(source.data(), quantized.data(), nullptr, stream); + } else if (op == 1) { + nvte_group_dequantize(quantized.data(), decoded.data(), stream); + } else { + nvte_group_requantize(quantized.data(), requantized.data(), nullptr, nullptr, stream); + } + }; + launch(); + NVTE_CHECK_CUDA(cudaStreamSynchronize(stream)); + cudaGraph_t graph; + NVTE_CHECK_CUDA(cudaStreamBeginCapture(stream, cudaStreamCaptureModeThreadLocal)); + launch(); + NVTE_CHECK_CUDA(cudaStreamEndCapture(stream, &graph)); + size_t node_count = 0; + NVTE_CHECK_CUDA(cudaGraphGetNodes(graph, nullptr, &node_count)); + std::vector nodes(node_count); + NVTE_CHECK_CUDA(cudaGraphGetNodes(graph, nodes.data(), &node_count)); + size_t kernel_count = 0; + for (auto node : nodes) { + cudaGraphNodeType type; + NVTE_CHECK_CUDA(cudaGraphNodeGetType(node, &type)); + kernel_count += type == cudaGraphNodeTypeKernel; + } + EXPECT_EQ(kernel_count, (cc == 107 || info.same_last_dim) ? 1 : 2); + cudaGraphExec_t executable; + NVTE_CHECK_CUDA(cudaGraphInstantiate(&executable, graph, nullptr, nullptr, 0)); + NVTE_CHECK_CUDA(cudaGraphLaunch(executable, stream)); + NVTE_CHECK_CUDA(cudaStreamSynchronize(stream)); + NVTE_CHECK_CUDA(cudaGraphExecDestroy(executable)); + NVTE_CHECK_CUDA(cudaGraphDestroy(graph)); + } + NVTE_CHECK_CUDA(cudaStreamDestroy(stream)); + } +} + const std::vector kGroupShapeCases = { + make_large_group_case("Rubin65VaryingFirst", true, false), + make_large_group_case("Rubin65VaryingLast", false, true), + make_large_group_case("Rubin65VaryingBoth", true, true), {"Single", {{1024, 512}}}, {"UniformSmall", {{512, 512}, {512, 512}, {512, 512}, {512, 512}}}, {"VaryingFirstSmall", {{256, 256}, {1024, 256}, {128, 256}, {640, 256}}}, @@ -662,25 +741,17 @@ const std::vector kGroupShapeCases = { {"CapacityTail", {{512, 8192}, {256, 8192}, {512, 8192}}, 2048}, {"EmptyGroupsAndTail", {{0, 512}, {512, 512}, {0, 512}, {256, 512}}, 1024}, {"UnsupportedCuTeStrideAndTail", {{0, 640}, {384, 640}, {128, 640}}, 1024}, - {"SameBothDims_1024x4096", - {{1024, 4096}, {1024, 4096}, {1024, 4096}, {1024, 4096}}}, - {"SameBothDims_2048x8192", - {{2048, 8192}, {2048, 8192}, {2048, 8192}, {2048, 8192}}}, - {"SameBothDims_4096x16384", - {{4096, 16384}, {4096, 16384}, {4096, 16384}, {4096, 16384}}}, - {"VaryingFirstDim_512to2048x4096", - {{512, 4096}, {512, 4096}, {1024, 4096}, {2048, 4096}}}, - {"VaryingFirstDim_1024to4096x8192", - {{1024, 8192}, {1024, 8192}, {2048, 8192}, {4096, 8192}}}, + {"SameBothDims_1024x4096", {{1024, 4096}, {1024, 4096}, {1024, 4096}, {1024, 4096}}}, + {"SameBothDims_2048x8192", {{2048, 8192}, {2048, 8192}, {2048, 8192}, {2048, 8192}}}, + {"SameBothDims_4096x16384", {{4096, 16384}, {4096, 16384}, {4096, 16384}, {4096, 16384}}}, + {"VaryingFirstDim_512to2048x4096", {{512, 4096}, {512, 4096}, {1024, 4096}, {2048, 4096}}}, + {"VaryingFirstDim_1024to4096x8192", {{1024, 8192}, {1024, 8192}, {2048, 8192}, {4096, 8192}}}, {"VaryingFirstDim_2048to8192x16384", {{2048, 16384}, {2048, 16384}, {4096, 16384}, {8192, 16384}}}, // An empty member in the middle must not terminate the persistent work loop. - {"VaryingFirstDimWithEmpty", - {{128, 256}, {0, 256}, {384, 256}, {512, 256}}}, - {"VaryingLastDim", - {{256, 128}, {256, 384}, {256, 640}}}, - {"VaryingBothDims", - {{128, 128}, {256, 384}, {512, 640}}}, + {"VaryingFirstDimWithEmpty", {{128, 256}, {0, 256}, {384, 256}, {512, 256}}}, + {"VaryingLastDim", {{256, 128}, {256, 384}, {256, 640}}}, + {"VaryingBothDims", {{128, 128}, {256, 384}, {512, 640}}}, }; std::string make_test_name( diff --git a/transformer_engine/common/CMakeLists.txt b/transformer_engine/common/CMakeLists.txt index f90f74449c5..68a34661203 100644 --- a/transformer_engine/common/CMakeLists.txt +++ b/transformer_engine/common/CMakeLists.txt @@ -127,6 +127,14 @@ if(NOT arch_101_index EQUAL -1) list(APPEND NVTE_SPECIFIC_ARCHS "101a") endif() +# Check for architecture 107 +list(FIND CMAKE_CUDA_ARCHITECTURES "107" arch_107_index) +if(NOT arch_107_index EQUAL -1) + list(REMOVE_ITEM CMAKE_CUDA_ARCHITECTURES "107") + list(APPEND NVTE_GENERIC_ARCHS "107") + list(APPEND NVTE_SPECIFIC_ARCHS "107a") +endif() + # Check for architecture 110 (if we see this we are in toolkit >= 13.0) list(FIND CMAKE_CUDA_ARCHITECTURES "110" arch_110_index) if(NOT arch_110_index EQUAL -1) @@ -147,6 +155,9 @@ if(NOT arch_120_index EQUAL -1) endif() endif() +# Explicit 107 and the 100 family target can both request 107a. +list(REMOVE_DUPLICATES NVTE_SPECIFIC_ARCHS) + # Move remaining standard (pre-Blackwell) architectures into NVTE_STANDARD_ARCHS. # These are applied to all CUDA sources (both generic and arch-specific). set(NVTE_STANDARD_ARCHS ${CMAKE_CUDA_ARCHITECTURES}) diff --git a/transformer_engine/common/cast/core/grouped_tma.cuh b/transformer_engine/common/cast/core/grouped_tma.cuh index 1e7f67e5301..79af6c8dea5 100644 --- a/transformer_engine/common/cast/core/grouped_tma.cuh +++ b/transformer_engine/common/cast/core/grouped_tma.cuh @@ -23,6 +23,7 @@ #include #include "../../common.h" +#include "../../util/cuda_runtime.h" #include "../../util/ptx.cuh" #include "../../utils.cuh" #include "grouped_layout.cuh" @@ -31,6 +32,98 @@ namespace transformer_engine { namespace dispatch { namespace common { +// Keep host dispatch in sync with ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE. SM100/103 and +// other architectures retain the descriptor publication/acquire protocol. +inline bool use_tma_descriptor_override() { +#if CUDA_VERSION >= 13040 + return transformer_engine::cuda::sm_arch() == 107; +#else + return false; +#endif +} + +// Overrides replace all geometry, so encode only a small template layout. In +// particular, do not inherit the large-allocation class from the logical group: +// a stage address near its end need not have the 128 KiB backing window required +// by that class on Rubin (CUDA 13.5+). +inline void create_grouped_tma_map(CUtensorMap &map, const SimpleTensor &tensor, + const bool use_override, const uint64_t rows, + const uint64_t cols, const uint32_t tile_rows, + const uint32_t tile_cols, const size_t type_bits, + const CUtensorMapSwizzle swizzle = CU_TENSOR_MAP_SWIZZLE_NONE) { + const uint64_t map_rows = use_override ? 1 : rows; + const uint64_t map_cols = use_override ? tile_cols : cols; + create_2D_tensor_map(map, tensor, map_rows, map_cols, tile_rows, tile_cols, map_cols, 0, + type_bits, swizzle); +} + +struct TmaDescriptorOverride2D { + uint64_t address; + uint16_t size_x; + uint16_t size_y; + uint32_t lower_stride; + uint16_t upper_stride; +}; + +// Attribute overrides require zero coordinates. Fold the tile origin into the +// address and bound the dimensions by the stage, preserving OOB zero-fill/stores. +// The dimensions occupy eight bits even though the PTX operands are b16. +template +__device__ __forceinline__ TmaDescriptorOverride2D make_tma_descriptor_override_2d( + const T *data, const size_t rows, const size_t cols, const size_t x, const size_t y) { + static_assert(TILE_X > 0 && TILE_X < 256 && TILE_Y > 0 && TILE_Y < 256); + const uint64_t stride_bytes = static_cast(cols) * sizeof(T); + const uint64_t address = reinterpret_cast(data + y * cols + x); + if (stride_bytes % TMA_GMEM_ALIGNMENT != 0 || address % TMA_GMEM_ALIGNMENT != 0) { + NVTE_DEVICE_ERROR("TMA override address and stride must be 16B aligned."); + } + const uint64_t stride = stride_bytes >> 4; + if ((stride >> 36) != 0) { + NVTE_DEVICE_ERROR("TMA override stride exceeds 36 bits."); + } + return {address, static_cast(min(TILE_X, cols - x)), + static_cast(min(TILE_Y, rows - y)), static_cast(stride), + static_cast(stride >> 32)}; +} + +template +__device__ __forceinline__ void load_grouped_tma_2d(T *dst, const CUtensorMap &map, const size_t x, + const size_t y, uint64_t *barrier, + const T *data, const size_t rows, + const size_t cols) { + constexpr bool has_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; + if constexpr (has_override) { + if (data != nullptr) { + const auto info = make_tma_descriptor_override_2d(data, rows, cols, x, y); + ptx::cp_async_bulk_tensor_2d_global_to_shared_rubin( + reinterpret_cast(dst), reinterpret_cast(&map), info.address, + info.size_x, info.size_y, info.lower_stride, info.upper_stride, barrier); + return; + } + } + ptx::cp_async_bulk_tensor_2d_global_to_shared( + reinterpret_cast(dst), reinterpret_cast(&map), x, y, barrier); +} + +template +__device__ __forceinline__ void store_grouped_tma_2d(T *src, const CUtensorMap &map, const size_t x, + const size_t y, const T *data, + const size_t rows, const size_t cols) { + constexpr bool has_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; + if constexpr (has_override) { + if (data != nullptr) { + if (x >= cols || y >= rows) return; + const auto info = make_tma_descriptor_override_2d(data, rows, cols, x, y); + ptx::cp_async_bulk_tensor_2d_shared_to_global_rubin( + reinterpret_cast(&map), info.address, info.size_x, info.size_y, + info.lower_stride, info.upper_stride, reinterpret_cast(src)); + return; + } + } + ptx::cp_async_bulk_tensor_2d_shared_to_global(reinterpret_cast(&map), x, y, + reinterpret_cast(src)); +} + constexpr int MAX_SUPPORTED_TENSOR_DESCRIPTORS = 64; struct alignas(128) TensorMapStorage { @@ -184,47 +277,59 @@ __device__ __forceinline__ void fence_acquire_tensormap(const CUtensorMap *tenso } // Issue TMA global->shared transfer for one stage of input (and optional activation input). -template +template __device__ __forceinline__ void prefetch_input_stage( IType *in_sh, IType *act_in_sh, const CUtensorMap &tensor_map_input, const CUtensorMap &tensor_map_act_input, const size_t global_offset_X, const size_t global_offset_Y, const size_t buff_offset, const size_t shmem_buff_size, - uint64_t *barrier, const bool leading_thread) { + uint64_t *barrier, const bool leading_thread, const IType *input_data = nullptr, + const IType *act_data = nullptr, const size_t rows = 0, const size_t cols = 0) { + constexpr bool has_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; + if constexpr (has_override) { + if (input_data != nullptr && (global_offset_X >= cols || global_offset_Y >= rows)) { + for (size_t i = threadIdx.x; i < TILE_Y * TILE_X; i += blockDim.x) { + in_sh[buff_offset + i] = static_cast(0.0f); + if constexpr (IS_DACT) act_in_sh[buff_offset + i] = static_cast(0.0f); + } + __syncthreads(); + if (leading_thread) ptx::mbarrier_arrive(barrier); + return; + } + } if (leading_thread) { ptx::mbarrier_arrive_expect_tx(barrier, shmem_buff_size); - ptx::cp_async_bulk_tensor_2d_global_to_shared( - reinterpret_cast(&in_sh[buff_offset]), - reinterpret_cast(&tensor_map_input), global_offset_X, global_offset_Y, - barrier); + load_grouped_tma_2d(&in_sh[buff_offset], tensor_map_input, global_offset_X, + global_offset_Y, barrier, input_data, rows, cols); if constexpr (IS_DACT) { - ptx::cp_async_bulk_tensor_2d_global_to_shared( - reinterpret_cast(&act_in_sh[buff_offset]), - reinterpret_cast(&tensor_map_act_input), global_offset_X, - global_offset_Y, barrier); + load_grouped_tma_2d(&act_in_sh[buff_offset], tensor_map_act_input, + global_offset_X, global_offset_Y, barrier, act_data, rows, + cols); } } } // Issue TMA shared->global transfer for one stage of outputs. -template +template __device__ __forceinline__ void store_output_stage( OType *out_rowwise_data_sh, OType *out_colwise_data_sh, const CUtensorMap &tensor_map_output_rowwise, const CUtensorMap &tensor_map_output_colwise, const size_t global_offset_X, const size_t global_offset_Y, const size_t buff_offset, - const bool leading_thread) { + const bool leading_thread, const OType *rowwise_data = nullptr, + const OType *colwise_data = nullptr, const size_t rows = 0, const size_t cols = 0) { if (!leading_thread) { return; } if constexpr (ROWWISE_SCALING) { - ptx::cp_async_bulk_tensor_2d_shared_to_global( - reinterpret_cast(&tensor_map_output_rowwise), global_offset_X, - global_offset_Y, reinterpret_cast(&out_rowwise_data_sh[buff_offset])); + store_grouped_tma_2d(&out_rowwise_data_sh[buff_offset], + tensor_map_output_rowwise, global_offset_X, + global_offset_Y, rowwise_data, rows, cols); } if constexpr (COLWISE_SCALING) { - ptx::cp_async_bulk_tensor_2d_shared_to_global( - reinterpret_cast(&tensor_map_output_colwise), global_offset_X, - global_offset_Y, reinterpret_cast(&out_colwise_data_sh[buff_offset])); + store_grouped_tma_2d(&out_colwise_data_sh[buff_offset], + tensor_map_output_colwise, global_offset_X, + global_offset_Y, colwise_data, rows, cols); } if constexpr (ROWWISE_SCALING || COLWISE_SCALING) { ptx::cp_async_bulk_commit_group(); diff --git a/transformer_engine/common/cast/mxfp8/group_dequantize_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/group_dequantize_mxfp8.cuh index fcf8819e5f8..242aef9f21e 100644 --- a/transformer_engine/common/cast/mxfp8/group_dequantize_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/group_dequantize_mxfp8.cuh @@ -122,6 +122,7 @@ template __global__ void __launch_bounds__(128) group_dequantize_mxfp8_kernel(const __grid_constant__ CUtensorMap tensor_map_input_static, const __grid_constant__ CUtensorMap tensor_map_output_static, + const IType *const input_data, OType *const output_data, const ShapeRepresentation shape_rep, const size_t num_tensors, const size_t first_logical_dim, const size_t last_logical_dim, const int64_t *const __restrict__ offsets_ptr, @@ -154,6 +155,8 @@ __global__ void __launch_bounds__(128) constexpr size_t THREADS_PER_SCALE_X_ROWWISE = DIVUP(SCALE_DIM_X, ELEMS_PER_THREAD); + constexpr bool has_tma_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; + // Group-awareness: determine which tensor this block belongs to const bool is_single_tensor = (shape_rep == ShapeRepresentation::SAME_BOTH_DIMS || shape_rep == ShapeRepresentation::VARYING_FIRST_DIM); @@ -210,16 +213,23 @@ __global__ void __launch_bounds__(128) const size_t tensor_base = is_single_tensor ? 0 : static_cast(offsets_ptr[tensor_id]); // Select TMA descriptors (static for single tensor, per-tensor for multi-tensor) - const CUtensorMap &tensor_map_input = - is_single_tensor ? tensor_map_input_static : g_tensor_maps_input[tensor_id]; - const CUtensorMap &tensor_map_output = - is_single_tensor ? tensor_map_output_static : g_tensor_maps_output[tensor_id]; - - if (!is_single_tensor) { + const CUtensorMap &tensor_map_input = (is_single_tensor || has_tma_override) + ? tensor_map_input_static + : g_tensor_maps_input[tensor_id]; + const CUtensorMap &tensor_map_output = (is_single_tensor || has_tma_override) + ? tensor_map_output_static + : g_tensor_maps_output[tensor_id]; + + if (!is_single_tensor && !has_tma_override) { fence_acquire_tensormap(&tensor_map_input); fence_acquire_tensormap(&tensor_map_output); } + const IType *const override_input = + (!is_single_tensor && has_tma_override) ? input_data + tensor_base : nullptr; + OType *const override_output = + (!is_single_tensor && has_tma_override) ? output_data + tensor_base : nullptr; + const int chunk_offset_Y = block_id_Y * CHUNK_DIM_Y; const int chunk_offset_X = block_id_X * CHUNK_DIM_X; @@ -273,40 +283,40 @@ __global__ void __launch_bounds__(128) initialize_barriers(mbar, is_master_thread); + // Every thread arrives, while only the leader issues the transfer. For a + // fully OOB stage, publish zeroes without registering asynchronous tx bytes. + auto prefetch_stage = [&](const int buff, const int x, const int y, const int iter) { + if (is_master_thread) { + if (override_input != nullptr && (x >= cols || y >= rows)) { + IType *const dst = reinterpret_cast(&in_sh[buff]); + for (size_t i = 0; i < SHMEM_DIM_Y * SHMEM_DIM_X; ++i) { + dst[i] = static_cast(0.0f); + } + ptx::mbarrier_arrive(&mbar[iter]); + } else { + common::load_grouped_tma_2d( + reinterpret_cast(&in_sh[buff]), tensor_map_input, x, y, &mbar[iter], + override_input, rows, cols); + ptx::mbarrier_arrive_expect_tx(&mbar[iter], transaction_size); + } + } else { + ptx::mbarrier_arrive(&mbar[iter]); + } + }; + int parity = 0; constexpr int iteration_zero = 0; constexpr int buffer_zero = 0; - if (is_master_thread) { - const int chunk_stage_offset_Y = chunk_offset_Y; - const int chunk_stage_offset_X = chunk_offset_X; - ptx::cp_async_bulk_tensor_2d_global_to_shared( - reinterpret_cast(&in_sh[buffer_zero]), - reinterpret_cast(&tensor_map_input), chunk_stage_offset_X, - chunk_stage_offset_Y, &mbar[iteration_zero]); - - ptx::mbarrier_arrive_expect_tx(&mbar[iteration_zero], transaction_size); - } else { - ptx::mbarrier_arrive(&mbar[iteration_zero]); - } + prefetch_stage(buffer_zero, chunk_offset_X, chunk_offset_Y, iteration_zero); #pragma unroll for (int iter = 0; iter < ITERATIONS; ++iter) { const int buff = iter % BUFFERS_NUM; const int next_iter = iter + 1; if (next_iter < ITERATIONS) { - if (is_master_thread) { - const int next_buff = next_iter % BUFFERS_NUM; - const int chunk_it_offset_y = chunk_offset_Y + next_iter * BUFFER_DIM_Y; - const int chunk_it_offset_x = chunk_offset_X; - ptx::cp_async_bulk_tensor_2d_global_to_shared( - reinterpret_cast(&in_sh[next_buff]), - reinterpret_cast(&tensor_map_input), chunk_it_offset_x, - chunk_it_offset_y, &mbar[next_iter]); - - ptx::mbarrier_arrive_expect_tx(&mbar[next_iter], transaction_size); - } else { - ptx::mbarrier_arrive(&mbar[next_iter]); - } + const int next_buff = next_iter % BUFFERS_NUM; + prefetch_stage(next_buff, chunk_offset_X, chunk_offset_Y + next_iter * BUFFER_DIM_Y, + next_iter); } // Wait for the data to have arrived @@ -354,9 +364,9 @@ __global__ void __launch_bounds__(128) if (is_master_thread) { const int chunk_it_offset_y = chunk_offset_Y + iter * BUFFER_DIM_Y; const int chunk_it_offset_x = chunk_offset_X; - ptx::cp_async_bulk_tensor_2d_shared_to_global( - reinterpret_cast(&tensor_map_output), chunk_it_offset_x, - chunk_it_offset_y, reinterpret_cast(&out_sh[buff])); + common::store_grouped_tma_2d( + reinterpret_cast(&out_sh[buff]), tensor_map_output, chunk_it_offset_x, + chunk_it_offset_y, override_output, rows, cols); ptx::cp_async_bulk_commit_group(); ptx::cp_async_bulk_wait_group_read(); @@ -430,9 +440,10 @@ inline void group_dequantize(const GroupedTensor *input, GroupedTensor *output, const size_t blocks_X = DIVUP(last_logical_dim, CHUNK_DIM_X); blocks = blocks_Y * blocks_X; } else { - NVTE_CHECK(num_tensors <= MAX_SUPPORTED_TENSOR_DESCRIPTORS, - "Number of tensors in a group is larger than " - "the MAX number of supported descriptors (64)."); + NVTE_CHECK( + common::use_tma_descriptor_override() || num_tensors <= MAX_SUPPORTED_TENSOR_DESCRIPTORS, + "Number of tensors in a group is larger than " + "the MAX number of supported descriptors (64)."); NVTE_CHECK(last_logical_dim % CHUNK_DIM_X == 0, "Last dimension of a grouped tensor should be divisible by 128."); if (shape_rep == ShapeRepresentation::VARYING_LAST_DIM) { @@ -463,18 +474,18 @@ inline void group_dequantize(const GroupedTensor *input, GroupedTensor *output, alignas(64) CUtensorMap tensor_map_input{}; alignas(64) CUtensorMap tensor_map_output{}; - create_2D_tensor_map(tensor_map_input, input_data, first_logical_dim, last_logical_dim, - SHMEM_DIM_Y, SHMEM_DIM_X, last_logical_dim, 0, - typeToNumBits(input->dtype())); - create_2D_tensor_map(tensor_map_output, output->data, first_logical_dim, last_logical_dim, - SHMEM_DIM_Y, SHMEM_DIM_X, last_logical_dim, 0, - typeToNumBits(output->dtype())); - - // Update tensor descriptors before launching the kernel - if (!is_single_tensor) { - const IType *const input_dptr = reinterpret_cast(input_data.dptr); - OType *const output_dptr = reinterpret_cast(output->data.dptr); - + const bool override_map = !is_single_tensor && common::use_tma_descriptor_override(); + common::create_grouped_tma_map(tensor_map_input, input_data, override_map, + first_logical_dim, last_logical_dim, SHMEM_DIM_Y, + SHMEM_DIM_X, typeToNumBits(input->dtype())); + common::create_grouped_tma_map(tensor_map_output, output->data, override_map, + first_logical_dim, last_logical_dim, SHMEM_DIM_Y, + SHMEM_DIM_X, typeToNumBits(output->dtype())); + + const IType *const input_dptr = reinterpret_cast(input_data.dptr); + OType *const output_dptr = reinterpret_cast(output->data.dptr); + // Rubin consumes instruction-local overrides from the static descriptors. + if (!is_single_tensor && !common::use_tma_descriptor_override()) { update_tma_descriptors<<>>( tensor_map_input, tensor_map_output, input_dptr, output_dptr, shape_rep, num_tensors, first_logical_dim, last_logical_dim, offsets_ptr, first_dims_ptr, @@ -483,12 +494,14 @@ inline void group_dequantize(const GroupedTensor *input, GroupedTensor *output, if (use_rowwise_scaling) { group_dequantize_mxfp8_kernel<<>>( - tensor_map_input, tensor_map_output, shape_rep, num_tensors, first_logical_dim, - last_logical_dim, offsets_ptr, first_dims_ptr, last_dims_ptr, scales_ptr); + tensor_map_input, tensor_map_output, input_dptr, output_dptr, shape_rep, + num_tensors, first_logical_dim, last_logical_dim, offsets_ptr, first_dims_ptr, + last_dims_ptr, scales_ptr); } else { group_dequantize_mxfp8_kernel<<>>( - tensor_map_input, tensor_map_output, shape_rep, num_tensors, first_logical_dim, - last_logical_dim, offsets_ptr, first_dims_ptr, last_dims_ptr, scales_ptr); + tensor_map_input, tensor_map_output, input_dptr, output_dptr, shape_rep, + num_tensors, first_logical_dim, last_logical_dim, offsets_ptr, first_dims_ptr, + last_dims_ptr, scales_ptr); }); // NOLINT(*) ); // NOLINT(*) NVTE_CHECK_CUDA(cudaGetLastError()); diff --git a/transformer_engine/common/cast/mxfp8/group_quantize_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/group_quantize_mxfp8.cuh index 08605a27867..f0e398fecc4 100644 --- a/transformer_engine/common/cast/mxfp8/group_quantize_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/group_quantize_mxfp8.cuh @@ -619,9 +619,12 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ const __grid_constant__ CUtensorMap tensor_map_input_static, const __grid_constant__ CUtensorMap tensor_map_act_input_static, const __grid_constant__ CUtensorMap tensor_map_output_rowwise_static, - const __grid_constant__ CUtensorMap tensor_map_output_colwise_static, const size_t num_tensors, - const size_t first_logical_dim, const size_t last_logical_dim, const size_t same_both_rows, - const int64_t *const __restrict__ offsets_ptr, const int64_t *const __restrict__ first_dims_ptr, + const __grid_constant__ CUtensorMap tensor_map_output_colwise_static, + const IType *const input_data_ptr, const IType *const act_input_data_ptr, + OType *const output_rowwise_data_ptr, OType *const output_colwise_data_ptr, + const size_t num_tensors, const size_t first_logical_dim, const size_t last_logical_dim, + const size_t same_both_rows, const int64_t *const __restrict__ offsets_ptr, + const int64_t *const __restrict__ first_dims_ptr, const int64_t *const __restrict__ last_dims_ptr, e8m0_t *const __restrict__ scales_rowwise_ptr, e8m0_t *const __restrict__ scales_colwise_ptr, const float *__restrict__ noop, float *const __restrict__ dbias_workspace, float *const __restrict__ amax_ptr, @@ -664,6 +667,7 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ constexpr bool use_direct_varying_first_mapper = (shape_rep == VARYING_FIRST_DIM); constexpr bool use_direct_mapper = use_direct_same_both_mapper || use_direct_varying_first_mapper; constexpr bool is_single_tensor = (shape_rep == SAME_BOTH_DIMS || shape_rep == VARYING_FIRST_DIM); + constexpr bool use_tma_override = !is_single_tensor && ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; const bool leading_thread = (threadIdx.x == 0); // Keep mapper metadata separate from dynamic shared memory. The latter is the destination of @@ -671,15 +675,13 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ __shared__ DirectVaryingFirstMapperStorage direct_mapper_storage; if constexpr (use_direct_varying_first_mapper) { - static_assert(CastTraits::THREADS_PER_CHUNK >= MAX_SUPPORTED_TENSOR_DESCRIPTORS, - "The first CTA must have enough threads to validate every tensor."); - // The direct mapper relies on every tensor boundary being TILE_DIM_Y-aligned. The legacy - // mapper validated this while decoding each job, but the direct path no longer calls it. - // Validate all per-tensor row counts once, using the first CTA. num_tensors is bounded by - // MAX_SUPPORTED_TENSOR_DESCRIPTORS (64), which is smaller than the CTA size. - if (blockIdx.x == 0 && threadIdx.x < num_tensors) { - get_tensor_rows_num(threadIdx.x, first_logical_dim, - first_dims_ptr, num_tensors); + // Validate every tensor boundary once in the first CTA. Rubin's override + // path is independent of the legacy 64-entry descriptor table. + if (blockIdx.x == 0) { + for (size_t id = threadIdx.x; id < num_tensors; id += blockDim.x) { + get_tensor_rows_num(id, first_logical_dim, + first_dims_ptr, num_tensors); + } } } @@ -785,9 +787,17 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ if (fixed_tensor_id >= num_tensors) { return; } - fixed_rows = g_tensor_maps.rows[fixed_tensor_id]; - fixed_cols = g_tensor_maps.cols[fixed_tensor_id]; - fixed_tensor_base = g_tensor_maps.offsets[fixed_tensor_id]; + if constexpr (use_tma_override) { + fixed_rows = get_tensor_rows_num(fixed_tensor_id, first_logical_dim, + first_dims_ptr, num_tensors); + fixed_cols = + get_tensor_cols_num(fixed_tensor_id, last_logical_dim, last_dims_ptr); + fixed_tensor_base = static_cast(offsets_ptr[fixed_tensor_id]); + } else { + fixed_rows = g_tensor_maps.rows[fixed_tensor_id]; + fixed_cols = g_tensor_maps.cols[fixed_tensor_id]; + fixed_tensor_base = g_tensor_maps.offsets[fixed_tensor_id]; + } if (fixed_rows == 0 || fixed_cols == 0) { return; } @@ -906,18 +916,21 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ const size_t dbias_offset_Y = block_offset_Y / TILE_DIM_Y; - const CUtensorMap &tensor_map_input = - is_single_tensor ? tensor_map_input_static : g_tensor_maps.input[tensor_id]; - const CUtensorMap &tensor_map_act_input = - is_single_tensor ? tensor_map_act_input_static : g_tensor_maps.act_input[tensor_id]; - const CUtensorMap &tensor_map_output_rowwise = is_single_tensor + const CUtensorMap &tensor_map_input = (is_single_tensor || use_tma_override) + ? tensor_map_input_static + : g_tensor_maps.input[tensor_id]; + const CUtensorMap &tensor_map_act_input = (is_single_tensor || use_tma_override) + ? tensor_map_act_input_static + : g_tensor_maps.act_input[tensor_id]; + const CUtensorMap &tensor_map_output_rowwise = (is_single_tensor || use_tma_override) ? tensor_map_output_rowwise_static : g_tensor_maps.output_rowwise[tensor_id]; - const CUtensorMap &tensor_map_output_colwise = is_single_tensor + const CUtensorMap &tensor_map_output_colwise = (is_single_tensor || use_tma_override) ? tensor_map_output_colwise_static : g_tensor_maps.output_colwise[tensor_id]; - if (leading_thread && (!is_single_tensor) && (last_acquired_tensor_id != tensor_id)) { + if (leading_thread && !is_single_tensor && !use_tma_override && + (last_acquired_tensor_id != tensor_id)) { fence_acquire_tensormap(&tensor_map_input); if constexpr (COMPUTE_ACTIVATIONS) { fence_acquire_tensormap(&tensor_map_act_input); @@ -933,6 +946,14 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ __syncthreads(); int buff_in = 0; + const IType *const override_input = use_tma_override ? input_data_ptr + tensor_base : nullptr; + const IType *const override_act = + (use_tma_override && IS_DACT) ? act_input_data_ptr + tensor_base : nullptr; + OType *const override_rowwise = + (use_tma_override && ROWWISE_SCALING) ? output_rowwise_data_ptr + tensor_base : nullptr; + OType *const override_colwise = + (use_tma_override && COLWISE_SCALING) ? output_colwise_data_ptr + tensor_base : nullptr; + size_t chunk_rows = CHUNK_DIM_Y; if constexpr (CHUNK_DIM_Y > TILE_DIM_Y) { chunk_rows = min(static_cast(CHUNK_DIM_Y), rows - tensor_offset_Y); @@ -954,9 +975,10 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ const size_t global_offset_X = block_offset_X + stage_offset_X; const size_t buff_offset = buff * BUFF_DIM; uint64_t *barrier = &IN_buff_readable_mbar[buff]; - prefetch_input_stage(sIn_ptr, sActIn_ptr, tensor_map_input, - tensor_map_act_input, global_offset_X, global_offset_Y, - buff_offset, shmem_buff_size, barrier, leading_thread); + prefetch_input_stage( + sIn_ptr, sActIn_ptr, tensor_map_input, tensor_map_act_input, global_offset_X, + global_offset_Y, buff_offset, shmem_buff_size, barrier, leading_thread, override_input, + override_act, rows, cols); } float partial_dbias_colwise[STAGES_X] = {0.0f}; @@ -992,9 +1014,10 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ const size_t next_prefetch_buff_offset = next_prefetch_buff * BUFF_DIM; uint64_t *barrier = &IN_buff_readable_mbar[next_prefetch_buff]; - prefetch_input_stage( + prefetch_input_stage( sIn_ptr, sActIn_ptr, tensor_map_input, tensor_map_act_input, global_offset_X, - global_offset_Y, next_prefetch_buff_offset, shmem_buff_size, barrier, leading_thread); + global_offset_Y, next_prefetch_buff_offset, shmem_buff_size, barrier, leading_thread, + override_input, override_act, rows, cols); } ptx::mbarrier_wait_parity_acquire_cta_shared_cta(&IN_buff_readable_mbar[buff_in], @@ -1039,9 +1062,10 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ const size_t global_offset_Y = block_offset_Y + stage_offset_Y; const size_t global_offset_X = block_offset_X + stage_offset_X; const size_t buff_offset = buff * BUFF_DIM; - store_output_stage( + store_output_stage( sOutRowwise_ptr, sOutColwise_ptr, tensor_map_output_rowwise, tensor_map_output_colwise, - global_offset_X, global_offset_Y, buff_offset, leading_thread); + global_offset_X, global_offset_Y, buff_offset, leading_thread, override_rowwise, + override_colwise, rows, cols); buff_in = (buff_in + 1) % BUFFS_NUM; } @@ -1177,7 +1201,8 @@ void group_quantize(const GroupedTensor *input, const GroupedTensor *activations const size_t elts_total = first_logical_dim * last_logical_dim; const size_t num_tensors = input->num_tensors; - NVTE_CHECK(num_tensors > 0 && num_tensors <= MAX_SUPPORTED_TENSOR_DESCRIPTORS, + NVTE_CHECK(num_tensors > 0 && (common::use_tma_descriptor_override() || + num_tensors <= MAX_SUPPORTED_TENSOR_DESCRIPTORS), "Number of tensors in a group must be between 1 and ", MAX_SUPPORTED_TENSOR_DESCRIPTORS, "."); @@ -1257,29 +1282,25 @@ void group_quantize(const GroupedTensor *input, const GroupedTensor *activations constexpr size_t input_type_bit_size = TypeInfo::size; constexpr size_t output_type_bit_size = TypeInfo::size; - create_2D_tensor_map(tensor_map_input, input->data, first_logical_dim, - last_logical_dim, BUFF_DIM_Y, BUFF_DIM_X, - last_logical_dim, 0, input_type_bit_size); - - if constexpr (IS_DACT) { - create_2D_tensor_map(tensor_map_act_input, activations->data, + const bool override_map = + !is_single_tensor && common::use_tma_descriptor_override(); + create_grouped_tma_map(tensor_map_input, input->data, override_map, first_logical_dim, last_logical_dim, BUFF_DIM_Y, - BUFF_DIM_X, last_logical_dim, 0, - input_type_bit_size); + BUFF_DIM_X, input_type_bit_size); + if constexpr (IS_DACT) { + create_grouped_tma_map(tensor_map_act_input, activations->data, + override_map, first_logical_dim, last_logical_dim, + BUFF_DIM_Y, BUFF_DIM_X, input_type_bit_size); } - if (use_rowwise_scaling) { - create_2D_tensor_map(tensor_map_output_rowwise, output->data, - first_logical_dim, last_logical_dim, BUFF_DIM_Y, - BUFF_DIM_X, last_logical_dim, 0, - output_type_bit_size); + create_grouped_tma_map(tensor_map_output_rowwise, output->data, + override_map, first_logical_dim, last_logical_dim, + BUFF_DIM_Y, BUFF_DIM_X, output_type_bit_size); } - if (use_colwise_scaling) { - create_2D_tensor_map(tensor_map_output_colwise, output->columnwise_data, - first_logical_dim, last_logical_dim, BUFF_DIM_Y, - BUFF_DIM_X, last_logical_dim, 0, - output_type_bit_size); + create_grouped_tma_map(tensor_map_output_colwise, output->columnwise_data, + override_map, first_logical_dim, last_logical_dim, + BUFF_DIM_Y, BUFF_DIM_X, output_type_bit_size); } constexpr size_t buff_elems = BUFF_DIM_Y * BUFF_DIM_X; @@ -1305,19 +1326,19 @@ void group_quantize(const GroupedTensor *input, const GroupedTensor *activations const size_t dshmem_size = in_mem + out_mem + TMA_SHMEM_ALIGNMENT; // Update tensor descriptors before launching the kernel - if (!is_single_tensor) { - const IType *const input_dptr = - reinterpret_cast(input->data.dptr); - const IType *const act_input_dptr = - IS_DACT ? reinterpret_cast(activations->data.dptr) - : nullptr; - OType *const output_rowwise_dptr = - use_rowwise_scaling ? reinterpret_cast(output->data.dptr) - : nullptr; - OType *const output_colwise_dptr = - use_colwise_scaling - ? reinterpret_cast(output->columnwise_data.dptr) - : nullptr; + const IType *const input_dptr = + reinterpret_cast(input->data.dptr); + const IType *const act_input_dptr = + IS_DACT ? reinterpret_cast(activations->data.dptr) + : nullptr; + OType *const output_rowwise_dptr = + use_rowwise_scaling ? reinterpret_cast(output->data.dptr) + : nullptr; + OType *const output_colwise_dptr = + use_colwise_scaling + ? reinterpret_cast(output->columnwise_data.dptr) + : nullptr; + if (!is_single_tensor && !common::use_tma_descriptor_override()) { update_tma_descriptors <<>>( tensor_map_input, tensor_map_act_input, tensor_map_output_rowwise, @@ -1340,9 +1361,10 @@ void group_quantize(const GroupedTensor *input, const GroupedTensor *activations kernel<<>>( tensor_map_input, tensor_map_act_input, tensor_map_output_rowwise, - tensor_map_output_colwise, num_tensors, first_logical_dim, - last_logical_dim, launch_config.same_both_rows, offsets_ptr, - first_dims_ptr, last_dims_ptr, scales_rowwise_ptr, + tensor_map_output_colwise, input_dptr, act_input_dptr, + output_rowwise_dptr, output_colwise_dptr, num_tensors, + first_logical_dim, last_logical_dim, launch_config.same_both_rows, + offsets_ptr, first_dims_ptr, last_dims_ptr, scales_rowwise_ptr, scales_colwise_ptr, noop_ptr, workspace_ptr, amax_ptr, launch_config.work_blocks_X, launch_config.work_blocks_Y); }); diff --git a/transformer_engine/common/cast/mxfp8/requantize_mxfp8.cu b/transformer_engine/common/cast/mxfp8/requantize_mxfp8.cu index 7f406f8c8aa..58f643b3358 100644 --- a/transformer_engine/common/cast/mxfp8/requantize_mxfp8.cu +++ b/transformer_engine/common/cast/mxfp8/requantize_mxfp8.cu @@ -130,7 +130,7 @@ LaunchConfig get_launch_config(const size_t first_logical_dim, const size_t last "VARYING_FIRST_DIM requires a non-empty logical tensor."); config.grid = dim3(config.work_blocks_x, work_blocks_y); } else { - NVTE_CHECK(num_tensors <= MAX_SUPPORTED_TENSOR_DESCRIPTORS, + NVTE_CHECK(use_tma_descriptor_override() || num_tensors <= MAX_SUPPORTED_TENSOR_DESCRIPTORS, "Number of tensors exceeds the grouped TMA descriptor limit (", MAX_SUPPORTED_TENSOR_DESCRIPTORS, ")."); const size_t estimated_work_blocks = DIVUP(total_elements, chunk_dim_y * chunk_dim_x); @@ -467,7 +467,7 @@ __device__ __forceinline__ void process_chunk( const size_t block_offset_y, const size_t block_offset_x, const size_t tma_offset_y, IType *const input_shared, void *const dequantized_shared, OType *const output_shared, uint64_t *const input_barriers, int *const input_barrier_parity, const bool leading_thread, - bf16 *const dequantized_out) { + bf16 *const dequantized_out, const IType *const override_input, OType *const override_output) { constexpr size_t tile_dim_x = Traits::TILE_DIM_X; constexpr size_t chunk_dim_y = Traits::CHUNK_DIM_Y; constexpr size_t chunk_dim_x = Traits::CHUNK_DIM_X; @@ -498,9 +498,10 @@ __device__ __forceinline__ void process_chunk( const size_t global_y = tma_offset_y + stage_y * buff_dim_y; const size_t global_x = block_offset_x + stage_x * tile_dim_x; const size_t buffer_offset = stage * buff_dim; - prefetch_input_stage(input_shared, nullptr, tensor_map_input, tensor_map_input, - global_x, global_y, buffer_offset, input_buffer_bytes, - &input_barriers[stage], leading_thread); + prefetch_input_stage( + input_shared, nullptr, tensor_map_input, tensor_map_input, global_x, global_y, + buffer_offset, input_buffer_bytes, &input_barriers[stage], leading_thread, override_input, + nullptr, rows, cols); } int input_buffer = 0; @@ -519,9 +520,10 @@ __device__ __forceinline__ void process_chunk( const size_t global_y = tma_offset_y + next_stage_y * buff_dim_y; const size_t global_x = block_offset_x + next_stage_x * tile_dim_x; const size_t buffer_offset = next_buffer * buff_dim; - prefetch_input_stage(input_shared, nullptr, tensor_map_input, tensor_map_input, - global_x, global_y, buffer_offset, input_buffer_bytes, - &input_barriers[next_buffer], leading_thread); + prefetch_input_stage( + input_shared, nullptr, tensor_map_input, tensor_map_input, global_x, global_y, + buffer_offset, input_buffer_bytes, &input_barriers[next_buffer], leading_thread, + override_input, nullptr, rows, cols); } ptx::mbarrier_wait_parity_acquire_cta_shared_cta(&input_barriers[input_buffer], @@ -545,9 +547,9 @@ __device__ __forceinline__ void process_chunk( const size_t global_y = tma_offset_y + stage_offset_y; const size_t global_x = block_offset_x + stage_offset_x; const size_t buffer_offset = input_buffer * buff_dim; - store_output_stage(nullptr, output_shared, tensor_map_output, - tensor_map_output, global_x, global_y, buffer_offset, - leading_thread); + store_output_stage( + nullptr, output_shared, tensor_map_output, tensor_map_output, global_x, global_y, + buffer_offset, leading_thread, nullptr, override_output, rows, cols); input_buffer = (input_buffer + 1) % buffs_num; } @@ -563,8 +565,9 @@ template __global__ void __launch_bounds__(Traits::THREADS_PER_CHUNK) group_requantize_mxfp8_kernel( const __grid_constant__ CUtensorMap tensor_map_input_static, - const __grid_constant__ CUtensorMap tensor_map_output_static, const size_t num_tensors, - const size_t first_logical_dim, const size_t last_logical_dim, const size_t same_both_rows, + const __grid_constant__ CUtensorMap tensor_map_output_static, const IType *const input_data, + OType *const output_data, const size_t num_tensors, const size_t first_logical_dim, + const size_t last_logical_dim, const size_t same_both_rows, const int64_t *const __restrict__ offsets_ptr, const int64_t *const __restrict__ first_dims_ptr, const int64_t *const __restrict__ last_dims_ptr, const e8m0_t *const __restrict__ input_scales, e8m0_t *const __restrict__ output_rowwise_scales, e8m0_t *const __restrict__ output_scales, @@ -575,6 +578,7 @@ __global__ void __launch_bounds__(Traits::THREADS_PER_CHUNK) group_requantize_mx constexpr bool direct_varying_first = shape_rep == ShapeRepresentation::VARYING_FIRST_DIM; constexpr bool direct_mapper = direct_same || direct_varying_first; constexpr bool single_tma_tensor = direct_mapper; + constexpr bool use_tma_override = !single_tma_tensor && ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; constexpr size_t buffs_num = Traits::BUFFS_NUM; constexpr size_t buff_dim = Traits::BUFF_DIM; constexpr size_t input_bytes = @@ -652,11 +656,13 @@ __global__ void __launch_bounds__(Traits::THREADS_PER_CHUNK) group_requantize_mx } } - const CUtensorMap &tensor_map_input = - single_tma_tensor ? tensor_map_input_static : g_tensor_maps.input[tensor_id]; - const CUtensorMap &tensor_map_output = - single_tma_tensor ? tensor_map_output_static : g_tensor_maps.output_colwise[tensor_id]; - if constexpr (!single_tma_tensor) { + const CUtensorMap &tensor_map_input = (single_tma_tensor || use_tma_override) + ? tensor_map_input_static + : g_tensor_maps.input[tensor_id]; + const CUtensorMap &tensor_map_output = (single_tma_tensor || use_tma_override) + ? tensor_map_output_static + : g_tensor_maps.output_colwise[tensor_id]; + if constexpr (!single_tma_tensor && !use_tma_override) { if (leading_thread) { fence_acquire_tensormap(&tensor_map_input); fence_acquire_tensormap(&tensor_map_output); @@ -689,12 +695,14 @@ __global__ void __launch_bounds__(Traits::THREADS_PER_CHUNK) group_requantize_mx initialize_barriers(input_barriers, leading_thread); int input_barrier_parity[buffs_num] = {0}; + const IType *const override_input = use_tma_override ? input_data + tensor_base : nullptr; + OType *const override_output = use_tma_override ? output_data + tensor_base : nullptr; if constexpr (direct_mapper) { process_chunk( tensor_map_input, tensor_map_output, input_scales, output_scales, output_rowwise_scales, input_scale_base, output_scale_base, rows, cols, block_offset_y, block_offset_x, tma_offset_y, input_shared, dequantized_shared, output_shared, input_barriers, - input_barrier_parity, leading_thread, dequantized_group); + input_barrier_parity, leading_thread, dequantized_group, override_input, override_output); } else { const size_t blocks_x = DIVUP(cols, static_cast(Traits::CHUNK_DIM_X)); const size_t blocks_y = DIVUP(rows, static_cast(Traits::CHUNK_DIM_Y)); @@ -707,7 +715,7 @@ __global__ void __launch_bounds__(Traits::THREADS_PER_CHUNK) group_requantize_mx input_scale_base, output_scale_base, rows, cols, block_y * Traits::CHUNK_DIM_Y, block_x * Traits::CHUNK_DIM_X, block_y * Traits::CHUNK_DIM_Y, input_shared, dequantized_shared, output_shared, input_barriers, input_barrier_parity, leading_thread, - dequantized_group); + dequantized_group, override_input, override_output); } } @@ -749,17 +757,18 @@ void launch_group_requantize(const GroupedTensor &input, GroupedTensor *output, alignas(64) CUtensorMap tensor_map_input{}; alignas(64) CUtensorMap tensor_map_output{}; - create_2D_tensor_map(tensor_map_input, input.data, first_logical_dim, last_logical_dim, - Traits::BUFF_DIM_Y, Traits::BUFF_DIM_X, last_logical_dim, 0, - TypeInfo::size, tma_swizzle); - create_2D_tensor_map(tensor_map_output, output->columnwise_data, first_logical_dim, - last_logical_dim, Traits::BUFF_DIM_Y, Traits::BUFF_DIM_X, last_logical_dim, - 0, TypeInfo::size, tma_swizzle); - constexpr bool single_tma_tensor = Traits::SHAPE_REPRESENTATION == ShapeRepresentation::SAME_BOTH_DIMS || Traits::SHAPE_REPRESENTATION == ShapeRepresentation::VARYING_FIRST_DIM; - if constexpr (!single_tma_tensor) { + const bool override_map = !single_tma_tensor && use_tma_descriptor_override(); + create_grouped_tma_map(tensor_map_input, input.data, override_map, first_logical_dim, + last_logical_dim, Traits::BUFF_DIM_Y, Traits::BUFF_DIM_X, + TypeInfo::size, tma_swizzle); + create_grouped_tma_map(tensor_map_output, output->columnwise_data, override_map, + first_logical_dim, last_logical_dim, Traits::BUFF_DIM_Y, + Traits::BUFF_DIM_X, TypeInfo::size, tma_swizzle); + + if (!single_tma_tensor && !use_tma_descriptor_override()) { alignas(64) CUtensorMap empty_tensor_map{}; update_tma_descriptors<<>>( tensor_map_input, empty_tensor_map, empty_tensor_map, tensor_map_output, @@ -774,8 +783,9 @@ void launch_group_requantize(const GroupedTensor &input, GroupedTensor *output, NVTE_CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, dynamic_shared_bytes)); kernel<<>>( - tensor_map_input, tensor_map_output, num_tensors, first_logical_dim, last_logical_dim, - launch_config.same_both_rows, offsets_ptr, first_dims_ptr, last_dims_ptr, + tensor_map_input, tensor_map_output, reinterpret_cast(input.data.dptr), + reinterpret_cast(output->columnwise_data.dptr), num_tensors, first_logical_dim, + last_logical_dim, launch_config.same_both_rows, offsets_ptr, first_dims_ptr, last_dims_ptr, reinterpret_cast(input.scale_inv.dptr), output->scale_inv.dptr == input.scale_inv.dptr ? nullptr diff --git a/transformer_engine/common/util/ptx.cuh b/transformer_engine/common/util/ptx.cuh index e37e147b77c..54e47378512 100644 --- a/transformer_engine/common/util/ptx.cuh +++ b/transformer_engine/common/util/ptx.cuh @@ -143,6 +143,15 @@ constexpr bool is_supported_arch() { #define ARCH_HAS_STOCHASTIC_ROUNDING \ NVTE_CUDA_ARCH_MATCHES(ptx::ArchSpecific<100>, ptx::ArchSpecific<103>, ptx::ArchSpecific<107>) +// Attribute override is a Rubin family feature, available starting with CUDA 13.4. +// Do not enable it for earlier members of the SM10x family. +#define ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE \ + [&] { \ + __NVTE_CURRENT_ARCH__ \ + __NVTE_ARCH_FAMILY_SPECIFIC__ \ + return current_arch == 1070 && FamilySpecific == current_arch; \ + }(); + // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init __device__ __forceinline__ void mbarrier_init(uint64_t *mbar, const uint32_t count) { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -1938,6 +1947,62 @@ __device__ __forceinline__ void prefetch_l2_evict_last(const void *addr) { NVTE_DEVICE_ERROR("prefetch_l2_evict_last is only supported on SM 8.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 800) } +// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-overriding-tensor-property-value +// global -> shared::cluster, with tensor base address, dimensions, and stride overridden. +__device__ __forceinline__ void cp_async_bulk_tensor_2d_global_to_shared_rubin( + uint64_t *dst_shmem, const uint64_t *tensor_map_ptr, const uint64_t gaddr_to_override, + const uint16_t tensor_size_x, const uint16_t tensor_size_y, const uint32_t tensor_lower_stride, + const uint16_t tensor_upper_stride, uint64_t *mbar) { +#if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ == 1070) + constexpr bool has_tma_descriptor_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; + if constexpr (has_tma_descriptor_override) { + uint32_t dst_shmem_ptr = __cvta_generic_to_shared(dst_shmem); + uint32_t mbar_ptr = __cvta_generic_to_shared(mbar); + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global.tile" + ".mbarrier::complete_tx::bytes" + ".override::global_address.override::global_dim_stride " + "[%0], [%1, %2, {%3, %4}, {%5}, %6, {0, 0}], [%7];" ::"r"(dst_shmem_ptr), + "l"(tensor_map_ptr), "l"(gaddr_to_override), "h"(tensor_size_x), "h"(tensor_size_y), + "r"(tensor_lower_stride), "h"(tensor_upper_stride), "r"(mbar_ptr) + : "memory"); + } else { + NVTE_DEVICE_ERROR( + "cp_async_bulk_tensor_2d_global_to_shared_rubin is only supported on SM 10.7f+."); + } +#else + NVTE_DEVICE_ERROR( + "cp_async_bulk_tensor_2d_global_to_shared_rubin is only supported on SM 10.7f+."); +#endif // __CUDA_ARCH__ == 1070 +} + +// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-overriding-tensor-property-value +// shared::cta -> global, with tensor base address, dimensions, and stride overridden. +__device__ __forceinline__ void cp_async_bulk_tensor_2d_shared_to_global_rubin( + const uint64_t *tensor_map_ptr, const uint64_t gaddr_to_override, const uint16_t tensor_size_x, + const uint16_t tensor_size_y, const uint32_t tensor_lower_stride, + const uint16_t tensor_upper_stride, uint64_t *src_shmem) { +#if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ == 1070) + constexpr bool has_tma_descriptor_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; + if constexpr (has_tma_descriptor_override) { + uint32_t src_shmem_ptr = __cvta_generic_to_shared(src_shmem); + asm volatile( + "cp.async.bulk.tensor.2d.global.shared::cta.bulk_group.tile" + ".override::global_address.override::global_dim_stride " + "[%0, %1, {%2, %3}, {%4}, %5, {0, 0}], [%6];" ::"l"(tensor_map_ptr), + "l"(gaddr_to_override), "h"(tensor_size_x), "h"(tensor_size_y), "r"(tensor_lower_stride), + "h"(tensor_upper_stride), "r"(src_shmem_ptr) + : "memory"); + } else { + NVTE_DEVICE_ERROR( + "cp_async_bulk_tensor_2d_shared_to_global_rubin is only supported on SM 10.7f+."); + } +#else + NVTE_DEVICE_ERROR( + "cp_async_bulk_tensor_2d_shared_to_global_rubin is only supported on SM 10.7f+."); +#endif // __CUDA_ARCH__ == 1070 +} + } // namespace ptx namespace { From a09df0b51eb72379e98934c61f9de537bea10abf Mon Sep 17 00:00:00 2001 From: Oleg Goncharov Date: Fri, 20 Nov 2026 13:37:27 +0000 Subject: [PATCH 2/8] Small refactoring Signed-off-by: Oleg Goncharov --- transformer_engine/common/util/ptx.cuh | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/transformer_engine/common/util/ptx.cuh b/transformer_engine/common/util/ptx.cuh index 54e47378512..cc761eab3d4 100644 --- a/transformer_engine/common/util/ptx.cuh +++ b/transformer_engine/common/util/ptx.cuh @@ -1947,6 +1947,7 @@ __device__ __forceinline__ void prefetch_l2_evict_last(const void *addr) { NVTE_DEVICE_ERROR("prefetch_l2_evict_last is only supported on SM 8.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 800) } + // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-overriding-tensor-property-value // global -> shared::cluster, with tensor base address, dimensions, and stride overridden. __device__ __forceinline__ void cp_async_bulk_tensor_2d_global_to_shared_rubin( @@ -1966,14 +1967,11 @@ __device__ __forceinline__ void cp_async_bulk_tensor_2d_global_to_shared_rubin( "l"(tensor_map_ptr), "l"(gaddr_to_override), "h"(tensor_size_x), "h"(tensor_size_y), "r"(tensor_lower_stride), "h"(tensor_upper_stride), "r"(mbar_ptr) : "memory"); - } else { - NVTE_DEVICE_ERROR( - "cp_async_bulk_tensor_2d_global_to_shared_rubin is only supported on SM 10.7f+."); + return; } -#else +#endif // __CUDA_ARCH__ == 1070 NVTE_DEVICE_ERROR( "cp_async_bulk_tensor_2d_global_to_shared_rubin is only supported on SM 10.7f+."); -#endif // __CUDA_ARCH__ == 1070 } // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-overriding-tensor-property-value @@ -1993,14 +1991,11 @@ __device__ __forceinline__ void cp_async_bulk_tensor_2d_shared_to_global_rubin( "l"(gaddr_to_override), "h"(tensor_size_x), "h"(tensor_size_y), "r"(tensor_lower_stride), "h"(tensor_upper_stride), "r"(src_shmem_ptr) : "memory"); - } else { - NVTE_DEVICE_ERROR( - "cp_async_bulk_tensor_2d_shared_to_global_rubin is only supported on SM 10.7f+."); + return; } -#else +#endif // __CUDA_ARCH__ == 1070 NVTE_DEVICE_ERROR( "cp_async_bulk_tensor_2d_shared_to_global_rubin is only supported on SM 10.7f+."); -#endif // __CUDA_ARCH__ == 1070 } } // namespace ptx From 111d6cde3adfc695a179c4420df62b331350e40c Mon Sep 17 00:00:00 2001 From: Oleg Goncharov Date: Fri, 20 Nov 2026 14:22:21 +0000 Subject: [PATCH 3/8] Refactoring Signed-off-by: Oleg Goncharov --- .../operator/test_requantize_mxfp8_grouped.cu | 76 ------------------- .../common/cast/core/grouped_tma.cuh | 75 +++++++++++------- 2 files changed, 46 insertions(+), 105 deletions(-) diff --git a/tests/cpp/operator/test_requantize_mxfp8_grouped.cu b/tests/cpp/operator/test_requantize_mxfp8_grouped.cu index f854dc878a0..793be629666 100644 --- a/tests/cpp/operator/test_requantize_mxfp8_grouped.cu +++ b/tests/cpp/operator/test_requantize_mxfp8_grouped.cu @@ -652,83 +652,7 @@ GroupShapeCase many_groups_case() { return test_case; } -GroupShapeCase make_large_group_case(const std::string &name, bool varying_rows, - bool varying_cols) { - GroupShapeCase result{name, {}}; - for (size_t i = 0; i < 65; ++i) { - result.shapes.emplace_back(varying_rows ? 128 * (1 + i % 3) : 128, - varying_cols ? 128 * (1 + i % 5) : 128); - } - return result; -} - -// In addition to numerical comparisons below, inspect the graph to catch a -// reintroduced descriptor-update launch. Warm up every path before capture. -TEST(GroupedTmaOverrideTest, DescriptorUpdateLaunches) { - const int cc = test::getDeviceComputeCapability(); - if (cc < test::blackwellComputeCapability) GTEST_SKIP(); - const std::vector cases = { - {"VaryingFirst", {{128, 256}, {0, 256}, {384, 256}}}, - {"VaryingLast", {{256, 128}, {256, 384}, {256, 640}}}, - {"VaryingBoth", {{128, 128}, {0, 384}, {256, 640}}}, - }; - for (const auto &test_case : cases) { - SCOPED_TRACE(test_case.name); - const auto info = make_shape_info(test_case); - auto source = - make_grouped_tensor(info, DType::kBFloat16, NVTE_DELAYED_TENSOR_SCALING, true, false); - auto quantized = - make_grouped_tensor(info, DType::kFloat8E4M3, NVTE_MXFP8_1D_SCALING, true, false); - auto decoded = - make_grouped_tensor(info, DType::kBFloat16, NVTE_DELAYED_TENSOR_SCALING, true, false); - auto requantized = - make_grouped_tensor(info, DType::kFloat8E4M3, NVTE_MXFP8_1D_SCALING, false, true); - fill_source(source, info); - cudaStream_t stream; - NVTE_CHECK_CUDA(cudaStreamCreate(&stream)); - for (int op = 0; op < 3; ++op) { - SCOPED_TRACE(op); - auto launch = [&] { - if (op == 0) { - nvte_group_quantize(source.data(), quantized.data(), nullptr, stream); - } else if (op == 1) { - nvte_group_dequantize(quantized.data(), decoded.data(), stream); - } else { - nvte_group_requantize(quantized.data(), requantized.data(), nullptr, nullptr, stream); - } - }; - launch(); - NVTE_CHECK_CUDA(cudaStreamSynchronize(stream)); - cudaGraph_t graph; - NVTE_CHECK_CUDA(cudaStreamBeginCapture(stream, cudaStreamCaptureModeThreadLocal)); - launch(); - NVTE_CHECK_CUDA(cudaStreamEndCapture(stream, &graph)); - size_t node_count = 0; - NVTE_CHECK_CUDA(cudaGraphGetNodes(graph, nullptr, &node_count)); - std::vector nodes(node_count); - NVTE_CHECK_CUDA(cudaGraphGetNodes(graph, nodes.data(), &node_count)); - size_t kernel_count = 0; - for (auto node : nodes) { - cudaGraphNodeType type; - NVTE_CHECK_CUDA(cudaGraphNodeGetType(node, &type)); - kernel_count += type == cudaGraphNodeTypeKernel; - } - EXPECT_EQ(kernel_count, (cc == 107 || info.same_last_dim) ? 1 : 2); - cudaGraphExec_t executable; - NVTE_CHECK_CUDA(cudaGraphInstantiate(&executable, graph, nullptr, nullptr, 0)); - NVTE_CHECK_CUDA(cudaGraphLaunch(executable, stream)); - NVTE_CHECK_CUDA(cudaStreamSynchronize(stream)); - NVTE_CHECK_CUDA(cudaGraphExecDestroy(executable)); - NVTE_CHECK_CUDA(cudaGraphDestroy(graph)); - } - NVTE_CHECK_CUDA(cudaStreamDestroy(stream)); - } -} - const std::vector kGroupShapeCases = { - make_large_group_case("Rubin65VaryingFirst", true, false), - make_large_group_case("Rubin65VaryingLast", false, true), - make_large_group_case("Rubin65VaryingBoth", true, true), {"Single", {{1024, 512}}}, {"UniformSmall", {{512, 512}, {512, 512}, {512, 512}, {512, 512}}}, {"VaryingFirstSmall", {{256, 256}, {1024, 256}, {128, 256}, {640, 256}}}, diff --git a/transformer_engine/common/cast/core/grouped_tma.cuh b/transformer_engine/common/cast/core/grouped_tma.cuh index 79af6c8dea5..3dde7c7a448 100644 --- a/transformer_engine/common/cast/core/grouped_tma.cuh +++ b/transformer_engine/common/cast/core/grouped_tma.cuh @@ -32,8 +32,6 @@ namespace transformer_engine { namespace dispatch { namespace common { -// Keep host dispatch in sync with ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE. SM100/103 and -// other architectures retain the descriptor publication/acquire protocol. inline bool use_tma_descriptor_override() { #if CUDA_VERSION >= 13040 return transformer_engine::cuda::sm_arch() == 107; @@ -42,15 +40,13 @@ inline bool use_tma_descriptor_override() { #endif } -// Overrides replace all geometry, so encode only a small template layout. In -// particular, do not inherit the large-allocation class from the logical group: -// a stage address near its end need not have the 128 KiB backing window required -// by that class on Rubin (CUDA 13.5+). -inline void create_grouped_tma_map(CUtensorMap &map, const SimpleTensor &tensor, - const bool use_override, const uint64_t rows, - const uint64_t cols, const uint32_t tile_rows, - const uint32_t tile_cols, const size_t type_bits, - const CUtensorMapSwizzle swizzle = CU_TENSOR_MAP_SWIZZLE_NONE) { +inline void +create_grouped_tma_map(CUtensorMap &map, const SimpleTensor &tensor, + const bool use_override, const uint64_t rows, + const uint64_t cols, const uint32_t tile_rows, + const uint32_t tile_cols, const size_t type_bits, + const CUtensorMapSwizzle swizzle = CU_TENSOR_MAP_SWIZZLE_NONE) +{ const uint64_t map_rows = use_override ? 1 : rows; const uint64_t map_cols = use_override ? tile_cols : cols; create_2D_tensor_map(map, tensor, map_rows, map_cols, tile_rows, tile_cols, map_cols, 0, @@ -65,13 +61,15 @@ struct TmaDescriptorOverride2D { uint16_t upper_stride; }; -// Attribute overrides require zero coordinates. Fold the tile origin into the -// address and bound the dimensions by the stage, preserving OOB zero-fill/stores. -// The dimensions occupy eight bits even though the PTX operands are b16. template -__device__ __forceinline__ TmaDescriptorOverride2D make_tma_descriptor_override_2d( - const T *data, const size_t rows, const size_t cols, const size_t x, const size_t y) { - static_assert(TILE_X > 0 && TILE_X < 256 && TILE_Y > 0 && TILE_Y < 256); +__device__ __forceinline__ TmaDescriptorOverride2D +make_tma_descriptor_override_2d(const T *data, + const size_t rows, + const size_t cols, + const size_t x, + const size_t y) +{ + static_assert((TILE_X > 0) && (TILE_X < 256) && (TILE_Y > 0) && (TILE_Y < 256)); const uint64_t stride_bytes = static_cast(cols) * sizeof(T); const uint64_t address = reinterpret_cast(data + y * cols + x); if (stride_bytes % TMA_GMEM_ALIGNMENT != 0 || address % TMA_GMEM_ALIGNMENT != 0) { @@ -87,10 +85,15 @@ __device__ __forceinline__ TmaDescriptorOverride2D make_tma_descriptor_override_ } template -__device__ __forceinline__ void load_grouped_tma_2d(T *dst, const CUtensorMap &map, const size_t x, - const size_t y, uint64_t *barrier, - const T *data, const size_t rows, - const size_t cols) { +__device__ __forceinline__ void +load_grouped_tma_2d(T *dst, const CUtensorMap &map, + const size_t x, + const size_t y, + uint64_t *barrier, + const T *data, + const size_t rows, + const size_t cols) +{ constexpr bool has_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; if constexpr (has_override) { if (data != nullptr) { @@ -106,13 +109,22 @@ __device__ __forceinline__ void load_grouped_tma_2d(T *dst, const CUtensorMap &m } template -__device__ __forceinline__ void store_grouped_tma_2d(T *src, const CUtensorMap &map, const size_t x, - const size_t y, const T *data, - const size_t rows, const size_t cols) { +__device__ __forceinline__ void +store_grouped_tma_2d(T *src, + const CUtensorMap &map, + const size_t x, + const size_t y, + const T *data, + const size_t rows, + const size_t cols) +{ constexpr bool has_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; if constexpr (has_override) { if (data != nullptr) { - if (x >= cols || y >= rows) return; + if (x >= cols || y >= rows) { + return; + } + const auto info = make_tma_descriptor_override_2d(data, rows, cols, x, y); ptx::cp_async_bulk_tensor_2d_shared_to_global_rubin( reinterpret_cast(&map), info.address, info.size_x, info.size_y, @@ -277,7 +289,7 @@ __device__ __forceinline__ void fence_acquire_tensormap(const CUtensorMap *tenso } // Issue TMA global->shared transfer for one stage of input (and optional activation input). -template +template __device__ __forceinline__ void prefetch_input_stage( IType *in_sh, IType *act_in_sh, const CUtensorMap &tensor_map_input, const CUtensorMap &tensor_map_act_input, const size_t global_offset_X, @@ -286,13 +298,18 @@ __device__ __forceinline__ void prefetch_input_stage( const IType *act_data = nullptr, const size_t rows = 0, const size_t cols = 0) { constexpr bool has_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; if constexpr (has_override) { - if (input_data != nullptr && (global_offset_X >= cols || global_offset_Y >= rows)) { + if ((input_data != nullptr) && (global_offset_X >= cols || global_offset_Y >= rows)) { for (size_t i = threadIdx.x; i < TILE_Y * TILE_X; i += blockDim.x) { in_sh[buff_offset + i] = static_cast(0.0f); - if constexpr (IS_DACT) act_in_sh[buff_offset + i] = static_cast(0.0f); + if constexpr (IS_DACT) { + act_in_sh[buff_offset + i] = static_cast(0.0f); + } } __syncthreads(); - if (leading_thread) ptx::mbarrier_arrive(barrier); + if (leading_thread) { + ptx::mbarrier_arrive(barrier); + } + return; } } From 519ba9a64e3721742fde4b7a8f391f4eebb54ee8 Mon Sep 17 00:00:00 2001 From: Oleg Goncharov Date: Fri, 20 Nov 2026 14:48:13 +0000 Subject: [PATCH 4/8] Small refactoring Signed-off-by: Oleg Goncharov --- transformer_engine/common/util/ptx.cuh | 2 -- 1 file changed, 2 deletions(-) diff --git a/transformer_engine/common/util/ptx.cuh b/transformer_engine/common/util/ptx.cuh index cc761eab3d4..999c102aeec 100644 --- a/transformer_engine/common/util/ptx.cuh +++ b/transformer_engine/common/util/ptx.cuh @@ -143,8 +143,6 @@ constexpr bool is_supported_arch() { #define ARCH_HAS_STOCHASTIC_ROUNDING \ NVTE_CUDA_ARCH_MATCHES(ptx::ArchSpecific<100>, ptx::ArchSpecific<103>, ptx::ArchSpecific<107>) -// Attribute override is a Rubin family feature, available starting with CUDA 13.4. -// Do not enable it for earlier members of the SM10x family. #define ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE \ [&] { \ __NVTE_CURRENT_ARCH__ \ From 2fad2364a22d13b97d3c5e823de97a1729ede924 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 20 Nov 2026 14:50:39 +0000 Subject: [PATCH 5/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../common/cast/core/grouped_tma.cuh | 46 ++++++------------- 1 file changed, 14 insertions(+), 32 deletions(-) diff --git a/transformer_engine/common/cast/core/grouped_tma.cuh b/transformer_engine/common/cast/core/grouped_tma.cuh index 3dde7c7a448..a14d24368fb 100644 --- a/transformer_engine/common/cast/core/grouped_tma.cuh +++ b/transformer_engine/common/cast/core/grouped_tma.cuh @@ -40,13 +40,11 @@ inline bool use_tma_descriptor_override() { #endif } -inline void -create_grouped_tma_map(CUtensorMap &map, const SimpleTensor &tensor, - const bool use_override, const uint64_t rows, - const uint64_t cols, const uint32_t tile_rows, - const uint32_t tile_cols, const size_t type_bits, - const CUtensorMapSwizzle swizzle = CU_TENSOR_MAP_SWIZZLE_NONE) -{ +inline void create_grouped_tma_map(CUtensorMap &map, const SimpleTensor &tensor, + const bool use_override, const uint64_t rows, + const uint64_t cols, const uint32_t tile_rows, + const uint32_t tile_cols, const size_t type_bits, + const CUtensorMapSwizzle swizzle = CU_TENSOR_MAP_SWIZZLE_NONE) { const uint64_t map_rows = use_override ? 1 : rows; const uint64_t map_cols = use_override ? tile_cols : cols; create_2D_tensor_map(map, tensor, map_rows, map_cols, tile_rows, tile_cols, map_cols, 0, @@ -62,13 +60,8 @@ struct TmaDescriptorOverride2D { }; template -__device__ __forceinline__ TmaDescriptorOverride2D -make_tma_descriptor_override_2d(const T *data, - const size_t rows, - const size_t cols, - const size_t x, - const size_t y) -{ +__device__ __forceinline__ TmaDescriptorOverride2D make_tma_descriptor_override_2d( + const T *data, const size_t rows, const size_t cols, const size_t x, const size_t y) { static_assert((TILE_X > 0) && (TILE_X < 256) && (TILE_Y > 0) && (TILE_Y < 256)); const uint64_t stride_bytes = static_cast(cols) * sizeof(T); const uint64_t address = reinterpret_cast(data + y * cols + x); @@ -85,15 +78,10 @@ make_tma_descriptor_override_2d(const T *data, } template -__device__ __forceinline__ void -load_grouped_tma_2d(T *dst, const CUtensorMap &map, - const size_t x, - const size_t y, - uint64_t *barrier, - const T *data, - const size_t rows, - const size_t cols) -{ +__device__ __forceinline__ void load_grouped_tma_2d(T *dst, const CUtensorMap &map, const size_t x, + const size_t y, uint64_t *barrier, + const T *data, const size_t rows, + const size_t cols) { constexpr bool has_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; if constexpr (has_override) { if (data != nullptr) { @@ -109,15 +97,9 @@ load_grouped_tma_2d(T *dst, const CUtensorMap &map, } template -__device__ __forceinline__ void -store_grouped_tma_2d(T *src, - const CUtensorMap &map, - const size_t x, - const size_t y, - const T *data, - const size_t rows, - const size_t cols) -{ +__device__ __forceinline__ void store_grouped_tma_2d(T *src, const CUtensorMap &map, const size_t x, + const size_t y, const T *data, + const size_t rows, const size_t cols) { constexpr bool has_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; if constexpr (has_override) { if (data != nullptr) { From e1ea40f5669776e3925eea3791aa3bd01edf7d12 Mon Sep 17 00:00:00 2001 From: Oleg Goncharov Date: Fri, 20 Nov 2026 15:25:58 +0000 Subject: [PATCH 6/8] Small refactoring Signed-off-by: Oleg Goncharov --- .../common/cast/core/grouped_tma.cuh | 49 ++++++++++--------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/transformer_engine/common/cast/core/grouped_tma.cuh b/transformer_engine/common/cast/core/grouped_tma.cuh index a14d24368fb..700bc6e7b3e 100644 --- a/transformer_engine/common/cast/core/grouped_tma.cuh +++ b/transformer_engine/common/cast/core/grouped_tma.cuh @@ -59,10 +59,10 @@ struct TmaDescriptorOverride2D { uint16_t upper_stride; }; -template +template __device__ __forceinline__ TmaDescriptorOverride2D make_tma_descriptor_override_2d( const T *data, const size_t rows, const size_t cols, const size_t x, const size_t y) { - static_assert((TILE_X > 0) && (TILE_X < 256) && (TILE_Y > 0) && (TILE_Y < 256)); + static_assert((TILE_DIM_X > 0) && (TILE_DIM_X < 256) && (TILE_DIM_Y > 0) && (TILE_DIM_Y < 256)); const uint64_t stride_bytes = static_cast(cols) * sizeof(T); const uint64_t address = reinterpret_cast(data + y * cols + x); if (stride_bytes % TMA_GMEM_ALIGNMENT != 0 || address % TMA_GMEM_ALIGNMENT != 0) { @@ -72,12 +72,12 @@ __device__ __forceinline__ TmaDescriptorOverride2D make_tma_descriptor_override_ if ((stride >> 36) != 0) { NVTE_DEVICE_ERROR("TMA override stride exceeds 36 bits."); } - return {address, static_cast(min(TILE_X, cols - x)), - static_cast(min(TILE_Y, rows - y)), static_cast(stride), + return {address, static_cast(min(TILE_DIM_X, cols - x)), + static_cast(min(TILE_DIM_Y, rows - y)), static_cast(stride), static_cast(stride >> 32)}; } -template +template __device__ __forceinline__ void load_grouped_tma_2d(T *dst, const CUtensorMap &map, const size_t x, const size_t y, uint64_t *barrier, const T *data, const size_t rows, @@ -85,7 +85,8 @@ __device__ __forceinline__ void load_grouped_tma_2d(T *dst, const CUtensorMap &m constexpr bool has_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; if constexpr (has_override) { if (data != nullptr) { - const auto info = make_tma_descriptor_override_2d(data, rows, cols, x, y); + const auto info = + make_tma_descriptor_override_2d(data, rows, cols, x, y); ptx::cp_async_bulk_tensor_2d_global_to_shared_rubin( reinterpret_cast(dst), reinterpret_cast(&map), info.address, info.size_x, info.size_y, info.lower_stride, info.upper_stride, barrier); @@ -96,7 +97,7 @@ __device__ __forceinline__ void load_grouped_tma_2d(T *dst, const CUtensorMap &m reinterpret_cast(dst), reinterpret_cast(&map), x, y, barrier); } -template +template __device__ __forceinline__ void store_grouped_tma_2d(T *src, const CUtensorMap &map, const size_t x, const size_t y, const T *data, const size_t rows, const size_t cols) { @@ -107,7 +108,8 @@ __device__ __forceinline__ void store_grouped_tma_2d(T *src, const CUtensorMap & return; } - const auto info = make_tma_descriptor_override_2d(data, rows, cols, x, y); + const auto info = + make_tma_descriptor_override_2d(data, rows, cols, x, y); ptx::cp_async_bulk_tensor_2d_shared_to_global_rubin( reinterpret_cast(&map), info.address, info.size_x, info.size_y, info.lower_stride, info.upper_stride, reinterpret_cast(src)); @@ -271,7 +273,7 @@ __device__ __forceinline__ void fence_acquire_tensormap(const CUtensorMap *tenso } // Issue TMA global->shared transfer for one stage of input (and optional activation input). -template +template __device__ __forceinline__ void prefetch_input_stage( IType *in_sh, IType *act_in_sh, const CUtensorMap &tensor_map_input, const CUtensorMap &tensor_map_act_input, const size_t global_offset_X, @@ -281,7 +283,7 @@ __device__ __forceinline__ void prefetch_input_stage( constexpr bool has_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; if constexpr (has_override) { if ((input_data != nullptr) && (global_offset_X >= cols || global_offset_Y >= rows)) { - for (size_t i = threadIdx.x; i < TILE_Y * TILE_X; i += blockDim.x) { + for (size_t i = threadIdx.x; i < TILE_DIM_Y * TILE_DIM_X; i += blockDim.x) { in_sh[buff_offset + i] = static_cast(0.0f); if constexpr (IS_DACT) { act_in_sh[buff_offset + i] = static_cast(0.0f); @@ -297,19 +299,20 @@ __device__ __forceinline__ void prefetch_input_stage( } if (leading_thread) { ptx::mbarrier_arrive_expect_tx(barrier, shmem_buff_size); - load_grouped_tma_2d(&in_sh[buff_offset], tensor_map_input, global_offset_X, - global_offset_Y, barrier, input_data, rows, cols); + load_grouped_tma_2d(&in_sh[buff_offset], tensor_map_input, + global_offset_X, global_offset_Y, barrier, + input_data, rows, cols); if constexpr (IS_DACT) { - load_grouped_tma_2d(&act_in_sh[buff_offset], tensor_map_act_input, - global_offset_X, global_offset_Y, barrier, act_data, rows, - cols); + load_grouped_tma_2d(&act_in_sh[buff_offset], tensor_map_act_input, + global_offset_X, global_offset_Y, barrier, + act_data, rows, cols); } } } // Issue TMA shared->global transfer for one stage of outputs. -template +template __device__ __forceinline__ void store_output_stage( OType *out_rowwise_data_sh, OType *out_colwise_data_sh, const CUtensorMap &tensor_map_output_rowwise, const CUtensorMap &tensor_map_output_colwise, @@ -321,14 +324,14 @@ __device__ __forceinline__ void store_output_stage( } if constexpr (ROWWISE_SCALING) { - store_grouped_tma_2d(&out_rowwise_data_sh[buff_offset], - tensor_map_output_rowwise, global_offset_X, - global_offset_Y, rowwise_data, rows, cols); + store_grouped_tma_2d(&out_rowwise_data_sh[buff_offset], + tensor_map_output_rowwise, global_offset_X, + global_offset_Y, rowwise_data, rows, cols); } if constexpr (COLWISE_SCALING) { - store_grouped_tma_2d(&out_colwise_data_sh[buff_offset], - tensor_map_output_colwise, global_offset_X, - global_offset_Y, colwise_data, rows, cols); + store_grouped_tma_2d(&out_colwise_data_sh[buff_offset], + tensor_map_output_colwise, global_offset_X, + global_offset_Y, colwise_data, rows, cols); } if constexpr (ROWWISE_SCALING || COLWISE_SCALING) { ptx::cp_async_bulk_commit_group(); From e1d21b72c73448d235c32612a1e068d0f4011ff6 Mon Sep 17 00:00:00 2001 From: Oleg Goncharov Date: Fri, 20 Nov 2026 15:48:22 +0000 Subject: [PATCH 7/8] Small refactoring Signed-off-by: Oleg Goncharov --- .../common/cast/core/grouped_tma.cuh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/transformer_engine/common/cast/core/grouped_tma.cuh b/transformer_engine/common/cast/core/grouped_tma.cuh index 700bc6e7b3e..14c6d417662 100644 --- a/transformer_engine/common/cast/core/grouped_tma.cuh +++ b/transformer_engine/common/cast/core/grouped_tma.cuh @@ -65,16 +65,23 @@ __device__ __forceinline__ TmaDescriptorOverride2D make_tma_descriptor_override_ static_assert((TILE_DIM_X > 0) && (TILE_DIM_X < 256) && (TILE_DIM_Y > 0) && (TILE_DIM_Y < 256)); const uint64_t stride_bytes = static_cast(cols) * sizeof(T); const uint64_t address = reinterpret_cast(data + y * cols + x); - if (stride_bytes % TMA_GMEM_ALIGNMENT != 0 || address % TMA_GMEM_ALIGNMENT != 0) { - NVTE_DEVICE_ERROR("TMA override address and stride must be 16B aligned."); + + if (address % TMA_GMEM_ALIGNMENT != 0) { + NVTE_DEVICE_ERROR("TMA override address must be 16B aligned."); + } + if (stride_bytes % TMA_GMEM_ALIGNMENT != 0) { + NVTE_DEVICE_ERROR("TMA override stride must be 16B aligned."); } const uint64_t stride = stride_bytes >> 4; if ((stride >> 36) != 0) { NVTE_DEVICE_ERROR("TMA override stride exceeds 36 bits."); } - return {address, static_cast(min(TILE_DIM_X, cols - x)), - static_cast(min(TILE_DIM_Y, rows - y)), static_cast(stride), - static_cast(stride >> 32)}; + const TmaDescriptorOverride2D descriptor{address, + static_cast(min(TILE_DIM_X, cols - x)), + static_cast(min(TILE_DIM_Y, rows - y)), + static_cast(stride), + static_cast(stride >> 32)}; + return descriptor; } template From 43d08b5969e55439edce365e49952fb345b83c6f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 20 Nov 2026 15:51:56 +0000 Subject: [PATCH 8/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- transformer_engine/common/cast/core/grouped_tma.cuh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/transformer_engine/common/cast/core/grouped_tma.cuh b/transformer_engine/common/cast/core/grouped_tma.cuh index 14c6d417662..a9751a90072 100644 --- a/transformer_engine/common/cast/core/grouped_tma.cuh +++ b/transformer_engine/common/cast/core/grouped_tma.cuh @@ -76,11 +76,10 @@ __device__ __forceinline__ TmaDescriptorOverride2D make_tma_descriptor_override_ if ((stride >> 36) != 0) { NVTE_DEVICE_ERROR("TMA override stride exceeds 36 bits."); } - const TmaDescriptorOverride2D descriptor{address, - static_cast(min(TILE_DIM_X, cols - x)), - static_cast(min(TILE_DIM_Y, rows - y)), - static_cast(stride), - static_cast(stride >> 32)}; + const TmaDescriptorOverride2D descriptor{ + address, static_cast(min(TILE_DIM_X, cols - x)), + static_cast(min(TILE_DIM_Y, rows - y)), static_cast(stride), + static_cast(stride >> 32)}; return descriptor; }