diff --git a/tests/cpp/operator/test_requantize_mxfp8_grouped.cu b/tests/cpp/operator/test_requantize_mxfp8_grouped.cu index c039d2fb2ff..793be629666 100644 --- a/tests/cpp/operator/test_requantize_mxfp8_grouped.cu +++ b/tests/cpp/operator/test_requantize_mxfp8_grouped.cu @@ -520,6 +520,9 @@ TEST_P(GroupedRequantizeMXFP8TestSuite, MatchesDequantizeThenQuantize) { } const auto &test_case = std::get<0>(GetParam()); + if (test_case.shapes.size() > 64 && test::getDeviceComputeCapability() != 107) { + GTEST_SKIP() << "More than 64 grouped TMA descriptors requires Rubin overrides."; + } const DType input_dtype = std::get<1>(GetParam()); const bool return_dequantized = std::get<2>(GetParam()); const bool output_swizzled = std::get<3>(GetParam()); @@ -662,25 +665,17 @@ const std::vector kGroupShapeCases = { {"CapacityTail", {{512, 8192}, {256, 8192}, {512, 8192}}, 2048}, {"EmptyGroupsAndTail", {{0, 512}, {512, 512}, {0, 512}, {256, 512}}, 1024}, {"UnsupportedCuTeStrideAndTail", {{0, 640}, {384, 640}, {128, 640}}, 1024}, - {"SameBothDims_1024x4096", - {{1024, 4096}, {1024, 4096}, {1024, 4096}, {1024, 4096}}}, - {"SameBothDims_2048x8192", - {{2048, 8192}, {2048, 8192}, {2048, 8192}, {2048, 8192}}}, - {"SameBothDims_4096x16384", - {{4096, 16384}, {4096, 16384}, {4096, 16384}, {4096, 16384}}}, - {"VaryingFirstDim_512to2048x4096", - {{512, 4096}, {512, 4096}, {1024, 4096}, {2048, 4096}}}, - {"VaryingFirstDim_1024to4096x8192", - {{1024, 8192}, {1024, 8192}, {2048, 8192}, {4096, 8192}}}, + {"SameBothDims_1024x4096", {{1024, 4096}, {1024, 4096}, {1024, 4096}, {1024, 4096}}}, + {"SameBothDims_2048x8192", {{2048, 8192}, {2048, 8192}, {2048, 8192}, {2048, 8192}}}, + {"SameBothDims_4096x16384", {{4096, 16384}, {4096, 16384}, {4096, 16384}, {4096, 16384}}}, + {"VaryingFirstDim_512to2048x4096", {{512, 4096}, {512, 4096}, {1024, 4096}, {2048, 4096}}}, + {"VaryingFirstDim_1024to4096x8192", {{1024, 8192}, {1024, 8192}, {2048, 8192}, {4096, 8192}}}, {"VaryingFirstDim_2048to8192x16384", {{2048, 16384}, {2048, 16384}, {4096, 16384}, {8192, 16384}}}, // An empty member in the middle must not terminate the persistent work loop. - {"VaryingFirstDimWithEmpty", - {{128, 256}, {0, 256}, {384, 256}, {512, 256}}}, - {"VaryingLastDim", - {{256, 128}, {256, 384}, {256, 640}}}, - {"VaryingBothDims", - {{128, 128}, {256, 384}, {512, 640}}}, + {"VaryingFirstDimWithEmpty", {{128, 256}, {0, 256}, {384, 256}, {512, 256}}}, + {"VaryingLastDim", {{256, 128}, {256, 384}, {256, 640}}}, + {"VaryingBothDims", {{128, 128}, {256, 384}, {512, 640}}}, }; std::string make_test_name( diff --git a/transformer_engine/common/CMakeLists.txt b/transformer_engine/common/CMakeLists.txt index f90f74449c5..68a34661203 100644 --- a/transformer_engine/common/CMakeLists.txt +++ b/transformer_engine/common/CMakeLists.txt @@ -127,6 +127,14 @@ if(NOT arch_101_index EQUAL -1) list(APPEND NVTE_SPECIFIC_ARCHS "101a") endif() +# Check for architecture 107 +list(FIND CMAKE_CUDA_ARCHITECTURES "107" arch_107_index) +if(NOT arch_107_index EQUAL -1) + list(REMOVE_ITEM CMAKE_CUDA_ARCHITECTURES "107") + list(APPEND NVTE_GENERIC_ARCHS "107") + list(APPEND NVTE_SPECIFIC_ARCHS "107a") +endif() + # Check for architecture 110 (if we see this we are in toolkit >= 13.0) list(FIND CMAKE_CUDA_ARCHITECTURES "110" arch_110_index) if(NOT arch_110_index EQUAL -1) @@ -147,6 +155,9 @@ if(NOT arch_120_index EQUAL -1) endif() endif() +# Explicit 107 and the 100 family target can both request 107a. +list(REMOVE_DUPLICATES NVTE_SPECIFIC_ARCHS) + # Move remaining standard (pre-Blackwell) architectures into NVTE_STANDARD_ARCHS. # These are applied to all CUDA sources (both generic and arch-specific). set(NVTE_STANDARD_ARCHS ${CMAKE_CUDA_ARCHITECTURES}) diff --git a/transformer_engine/common/cast/core/grouped_tma.cuh b/transformer_engine/common/cast/core/grouped_tma.cuh index 1e7f67e5301..a9751a90072 100644 --- a/transformer_engine/common/cast/core/grouped_tma.cuh +++ b/transformer_engine/common/cast/core/grouped_tma.cuh @@ -23,6 +23,7 @@ #include #include "../../common.h" +#include "../../util/cuda_runtime.h" #include "../../util/ptx.cuh" #include "../../utils.cuh" #include "grouped_layout.cuh" @@ -31,6 +32,100 @@ namespace transformer_engine { namespace dispatch { namespace common { +inline bool use_tma_descriptor_override() { +#if CUDA_VERSION >= 13040 + return transformer_engine::cuda::sm_arch() == 107; +#else + return false; +#endif +} + +inline void create_grouped_tma_map(CUtensorMap &map, const SimpleTensor &tensor, + const bool use_override, const uint64_t rows, + const uint64_t cols, const uint32_t tile_rows, + const uint32_t tile_cols, const size_t type_bits, + const CUtensorMapSwizzle swizzle = CU_TENSOR_MAP_SWIZZLE_NONE) { + const uint64_t map_rows = use_override ? 1 : rows; + const uint64_t map_cols = use_override ? tile_cols : cols; + create_2D_tensor_map(map, tensor, map_rows, map_cols, tile_rows, tile_cols, map_cols, 0, + type_bits, swizzle); +} + +struct TmaDescriptorOverride2D { + uint64_t address; + uint16_t size_x; + uint16_t size_y; + uint32_t lower_stride; + uint16_t upper_stride; +}; + +template +__device__ __forceinline__ TmaDescriptorOverride2D make_tma_descriptor_override_2d( + const T *data, const size_t rows, const size_t cols, const size_t x, const size_t y) { + static_assert((TILE_DIM_X > 0) && (TILE_DIM_X < 256) && (TILE_DIM_Y > 0) && (TILE_DIM_Y < 256)); + const uint64_t stride_bytes = static_cast(cols) * sizeof(T); + const uint64_t address = reinterpret_cast(data + y * cols + x); + + if (address % TMA_GMEM_ALIGNMENT != 0) { + NVTE_DEVICE_ERROR("TMA override address must be 16B aligned."); + } + if (stride_bytes % TMA_GMEM_ALIGNMENT != 0) { + NVTE_DEVICE_ERROR("TMA override stride must be 16B aligned."); + } + const uint64_t stride = stride_bytes >> 4; + if ((stride >> 36) != 0) { + NVTE_DEVICE_ERROR("TMA override stride exceeds 36 bits."); + } + const TmaDescriptorOverride2D descriptor{ + address, static_cast(min(TILE_DIM_X, cols - x)), + static_cast(min(TILE_DIM_Y, rows - y)), static_cast(stride), + static_cast(stride >> 32)}; + return descriptor; +} + +template +__device__ __forceinline__ void load_grouped_tma_2d(T *dst, const CUtensorMap &map, const size_t x, + const size_t y, uint64_t *barrier, + const T *data, const size_t rows, + const size_t cols) { + constexpr bool has_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; + if constexpr (has_override) { + if (data != nullptr) { + const auto info = + make_tma_descriptor_override_2d(data, rows, cols, x, y); + ptx::cp_async_bulk_tensor_2d_global_to_shared_rubin( + reinterpret_cast(dst), reinterpret_cast(&map), info.address, + info.size_x, info.size_y, info.lower_stride, info.upper_stride, barrier); + return; + } + } + ptx::cp_async_bulk_tensor_2d_global_to_shared( + reinterpret_cast(dst), reinterpret_cast(&map), x, y, barrier); +} + +template +__device__ __forceinline__ void store_grouped_tma_2d(T *src, const CUtensorMap &map, const size_t x, + const size_t y, const T *data, + const size_t rows, const size_t cols) { + constexpr bool has_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; + if constexpr (has_override) { + if (data != nullptr) { + if (x >= cols || y >= rows) { + return; + } + + const auto info = + make_tma_descriptor_override_2d(data, rows, cols, x, y); + ptx::cp_async_bulk_tensor_2d_shared_to_global_rubin( + reinterpret_cast(&map), info.address, info.size_x, info.size_y, + info.lower_stride, info.upper_stride, reinterpret_cast(src)); + return; + } + } + ptx::cp_async_bulk_tensor_2d_shared_to_global(reinterpret_cast(&map), x, y, + reinterpret_cast(src)); +} + constexpr int MAX_SUPPORTED_TENSOR_DESCRIPTORS = 64; struct alignas(128) TensorMapStorage { @@ -184,47 +279,65 @@ __device__ __forceinline__ void fence_acquire_tensormap(const CUtensorMap *tenso } // Issue TMA global->shared transfer for one stage of input (and optional activation input). -template +template __device__ __forceinline__ void prefetch_input_stage( IType *in_sh, IType *act_in_sh, const CUtensorMap &tensor_map_input, const CUtensorMap &tensor_map_act_input, const size_t global_offset_X, const size_t global_offset_Y, const size_t buff_offset, const size_t shmem_buff_size, - uint64_t *barrier, const bool leading_thread) { + uint64_t *barrier, const bool leading_thread, const IType *input_data = nullptr, + const IType *act_data = nullptr, const size_t rows = 0, const size_t cols = 0) { + constexpr bool has_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; + if constexpr (has_override) { + if ((input_data != nullptr) && (global_offset_X >= cols || global_offset_Y >= rows)) { + for (size_t i = threadIdx.x; i < TILE_DIM_Y * TILE_DIM_X; i += blockDim.x) { + in_sh[buff_offset + i] = static_cast(0.0f); + if constexpr (IS_DACT) { + act_in_sh[buff_offset + i] = static_cast(0.0f); + } + } + __syncthreads(); + if (leading_thread) { + ptx::mbarrier_arrive(barrier); + } + + return; + } + } if (leading_thread) { ptx::mbarrier_arrive_expect_tx(barrier, shmem_buff_size); - ptx::cp_async_bulk_tensor_2d_global_to_shared( - reinterpret_cast(&in_sh[buff_offset]), - reinterpret_cast(&tensor_map_input), global_offset_X, global_offset_Y, - barrier); + load_grouped_tma_2d(&in_sh[buff_offset], tensor_map_input, + global_offset_X, global_offset_Y, barrier, + input_data, rows, cols); if constexpr (IS_DACT) { - ptx::cp_async_bulk_tensor_2d_global_to_shared( - reinterpret_cast(&act_in_sh[buff_offset]), - reinterpret_cast(&tensor_map_act_input), global_offset_X, - global_offset_Y, barrier); + load_grouped_tma_2d(&act_in_sh[buff_offset], tensor_map_act_input, + global_offset_X, global_offset_Y, barrier, + act_data, rows, cols); } } } // Issue TMA shared->global transfer for one stage of outputs. -template +template __device__ __forceinline__ void store_output_stage( OType *out_rowwise_data_sh, OType *out_colwise_data_sh, const CUtensorMap &tensor_map_output_rowwise, const CUtensorMap &tensor_map_output_colwise, const size_t global_offset_X, const size_t global_offset_Y, const size_t buff_offset, - const bool leading_thread) { + const bool leading_thread, const OType *rowwise_data = nullptr, + const OType *colwise_data = nullptr, const size_t rows = 0, const size_t cols = 0) { if (!leading_thread) { return; } if constexpr (ROWWISE_SCALING) { - ptx::cp_async_bulk_tensor_2d_shared_to_global( - reinterpret_cast(&tensor_map_output_rowwise), global_offset_X, - global_offset_Y, reinterpret_cast(&out_rowwise_data_sh[buff_offset])); + store_grouped_tma_2d(&out_rowwise_data_sh[buff_offset], + tensor_map_output_rowwise, global_offset_X, + global_offset_Y, rowwise_data, rows, cols); } if constexpr (COLWISE_SCALING) { - ptx::cp_async_bulk_tensor_2d_shared_to_global( - reinterpret_cast(&tensor_map_output_colwise), global_offset_X, - global_offset_Y, reinterpret_cast(&out_colwise_data_sh[buff_offset])); + store_grouped_tma_2d(&out_colwise_data_sh[buff_offset], + tensor_map_output_colwise, global_offset_X, + global_offset_Y, colwise_data, rows, cols); } if constexpr (ROWWISE_SCALING || COLWISE_SCALING) { ptx::cp_async_bulk_commit_group(); diff --git a/transformer_engine/common/cast/mxfp8/group_dequantize_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/group_dequantize_mxfp8.cuh index fcf8819e5f8..242aef9f21e 100644 --- a/transformer_engine/common/cast/mxfp8/group_dequantize_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/group_dequantize_mxfp8.cuh @@ -122,6 +122,7 @@ template __global__ void __launch_bounds__(128) group_dequantize_mxfp8_kernel(const __grid_constant__ CUtensorMap tensor_map_input_static, const __grid_constant__ CUtensorMap tensor_map_output_static, + const IType *const input_data, OType *const output_data, const ShapeRepresentation shape_rep, const size_t num_tensors, const size_t first_logical_dim, const size_t last_logical_dim, const int64_t *const __restrict__ offsets_ptr, @@ -154,6 +155,8 @@ __global__ void __launch_bounds__(128) constexpr size_t THREADS_PER_SCALE_X_ROWWISE = DIVUP(SCALE_DIM_X, ELEMS_PER_THREAD); + constexpr bool has_tma_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; + // Group-awareness: determine which tensor this block belongs to const bool is_single_tensor = (shape_rep == ShapeRepresentation::SAME_BOTH_DIMS || shape_rep == ShapeRepresentation::VARYING_FIRST_DIM); @@ -210,16 +213,23 @@ __global__ void __launch_bounds__(128) const size_t tensor_base = is_single_tensor ? 0 : static_cast(offsets_ptr[tensor_id]); // Select TMA descriptors (static for single tensor, per-tensor for multi-tensor) - const CUtensorMap &tensor_map_input = - is_single_tensor ? tensor_map_input_static : g_tensor_maps_input[tensor_id]; - const CUtensorMap &tensor_map_output = - is_single_tensor ? tensor_map_output_static : g_tensor_maps_output[tensor_id]; - - if (!is_single_tensor) { + const CUtensorMap &tensor_map_input = (is_single_tensor || has_tma_override) + ? tensor_map_input_static + : g_tensor_maps_input[tensor_id]; + const CUtensorMap &tensor_map_output = (is_single_tensor || has_tma_override) + ? tensor_map_output_static + : g_tensor_maps_output[tensor_id]; + + if (!is_single_tensor && !has_tma_override) { fence_acquire_tensormap(&tensor_map_input); fence_acquire_tensormap(&tensor_map_output); } + const IType *const override_input = + (!is_single_tensor && has_tma_override) ? input_data + tensor_base : nullptr; + OType *const override_output = + (!is_single_tensor && has_tma_override) ? output_data + tensor_base : nullptr; + const int chunk_offset_Y = block_id_Y * CHUNK_DIM_Y; const int chunk_offset_X = block_id_X * CHUNK_DIM_X; @@ -273,40 +283,40 @@ __global__ void __launch_bounds__(128) initialize_barriers(mbar, is_master_thread); + // Every thread arrives, while only the leader issues the transfer. For a + // fully OOB stage, publish zeroes without registering asynchronous tx bytes. + auto prefetch_stage = [&](const int buff, const int x, const int y, const int iter) { + if (is_master_thread) { + if (override_input != nullptr && (x >= cols || y >= rows)) { + IType *const dst = reinterpret_cast(&in_sh[buff]); + for (size_t i = 0; i < SHMEM_DIM_Y * SHMEM_DIM_X; ++i) { + dst[i] = static_cast(0.0f); + } + ptx::mbarrier_arrive(&mbar[iter]); + } else { + common::load_grouped_tma_2d( + reinterpret_cast(&in_sh[buff]), tensor_map_input, x, y, &mbar[iter], + override_input, rows, cols); + ptx::mbarrier_arrive_expect_tx(&mbar[iter], transaction_size); + } + } else { + ptx::mbarrier_arrive(&mbar[iter]); + } + }; + int parity = 0; constexpr int iteration_zero = 0; constexpr int buffer_zero = 0; - if (is_master_thread) { - const int chunk_stage_offset_Y = chunk_offset_Y; - const int chunk_stage_offset_X = chunk_offset_X; - ptx::cp_async_bulk_tensor_2d_global_to_shared( - reinterpret_cast(&in_sh[buffer_zero]), - reinterpret_cast(&tensor_map_input), chunk_stage_offset_X, - chunk_stage_offset_Y, &mbar[iteration_zero]); - - ptx::mbarrier_arrive_expect_tx(&mbar[iteration_zero], transaction_size); - } else { - ptx::mbarrier_arrive(&mbar[iteration_zero]); - } + prefetch_stage(buffer_zero, chunk_offset_X, chunk_offset_Y, iteration_zero); #pragma unroll for (int iter = 0; iter < ITERATIONS; ++iter) { const int buff = iter % BUFFERS_NUM; const int next_iter = iter + 1; if (next_iter < ITERATIONS) { - if (is_master_thread) { - const int next_buff = next_iter % BUFFERS_NUM; - const int chunk_it_offset_y = chunk_offset_Y + next_iter * BUFFER_DIM_Y; - const int chunk_it_offset_x = chunk_offset_X; - ptx::cp_async_bulk_tensor_2d_global_to_shared( - reinterpret_cast(&in_sh[next_buff]), - reinterpret_cast(&tensor_map_input), chunk_it_offset_x, - chunk_it_offset_y, &mbar[next_iter]); - - ptx::mbarrier_arrive_expect_tx(&mbar[next_iter], transaction_size); - } else { - ptx::mbarrier_arrive(&mbar[next_iter]); - } + const int next_buff = next_iter % BUFFERS_NUM; + prefetch_stage(next_buff, chunk_offset_X, chunk_offset_Y + next_iter * BUFFER_DIM_Y, + next_iter); } // Wait for the data to have arrived @@ -354,9 +364,9 @@ __global__ void __launch_bounds__(128) if (is_master_thread) { const int chunk_it_offset_y = chunk_offset_Y + iter * BUFFER_DIM_Y; const int chunk_it_offset_x = chunk_offset_X; - ptx::cp_async_bulk_tensor_2d_shared_to_global( - reinterpret_cast(&tensor_map_output), chunk_it_offset_x, - chunk_it_offset_y, reinterpret_cast(&out_sh[buff])); + common::store_grouped_tma_2d( + reinterpret_cast(&out_sh[buff]), tensor_map_output, chunk_it_offset_x, + chunk_it_offset_y, override_output, rows, cols); ptx::cp_async_bulk_commit_group(); ptx::cp_async_bulk_wait_group_read(); @@ -430,9 +440,10 @@ inline void group_dequantize(const GroupedTensor *input, GroupedTensor *output, const size_t blocks_X = DIVUP(last_logical_dim, CHUNK_DIM_X); blocks = blocks_Y * blocks_X; } else { - NVTE_CHECK(num_tensors <= MAX_SUPPORTED_TENSOR_DESCRIPTORS, - "Number of tensors in a group is larger than " - "the MAX number of supported descriptors (64)."); + NVTE_CHECK( + common::use_tma_descriptor_override() || num_tensors <= MAX_SUPPORTED_TENSOR_DESCRIPTORS, + "Number of tensors in a group is larger than " + "the MAX number of supported descriptors (64)."); NVTE_CHECK(last_logical_dim % CHUNK_DIM_X == 0, "Last dimension of a grouped tensor should be divisible by 128."); if (shape_rep == ShapeRepresentation::VARYING_LAST_DIM) { @@ -463,18 +474,18 @@ inline void group_dequantize(const GroupedTensor *input, GroupedTensor *output, alignas(64) CUtensorMap tensor_map_input{}; alignas(64) CUtensorMap tensor_map_output{}; - create_2D_tensor_map(tensor_map_input, input_data, first_logical_dim, last_logical_dim, - SHMEM_DIM_Y, SHMEM_DIM_X, last_logical_dim, 0, - typeToNumBits(input->dtype())); - create_2D_tensor_map(tensor_map_output, output->data, first_logical_dim, last_logical_dim, - SHMEM_DIM_Y, SHMEM_DIM_X, last_logical_dim, 0, - typeToNumBits(output->dtype())); - - // Update tensor descriptors before launching the kernel - if (!is_single_tensor) { - const IType *const input_dptr = reinterpret_cast(input_data.dptr); - OType *const output_dptr = reinterpret_cast(output->data.dptr); - + const bool override_map = !is_single_tensor && common::use_tma_descriptor_override(); + common::create_grouped_tma_map(tensor_map_input, input_data, override_map, + first_logical_dim, last_logical_dim, SHMEM_DIM_Y, + SHMEM_DIM_X, typeToNumBits(input->dtype())); + common::create_grouped_tma_map(tensor_map_output, output->data, override_map, + first_logical_dim, last_logical_dim, SHMEM_DIM_Y, + SHMEM_DIM_X, typeToNumBits(output->dtype())); + + const IType *const input_dptr = reinterpret_cast(input_data.dptr); + OType *const output_dptr = reinterpret_cast(output->data.dptr); + // Rubin consumes instruction-local overrides from the static descriptors. + if (!is_single_tensor && !common::use_tma_descriptor_override()) { update_tma_descriptors<<>>( tensor_map_input, tensor_map_output, input_dptr, output_dptr, shape_rep, num_tensors, first_logical_dim, last_logical_dim, offsets_ptr, first_dims_ptr, @@ -483,12 +494,14 @@ inline void group_dequantize(const GroupedTensor *input, GroupedTensor *output, if (use_rowwise_scaling) { group_dequantize_mxfp8_kernel<<>>( - tensor_map_input, tensor_map_output, shape_rep, num_tensors, first_logical_dim, - last_logical_dim, offsets_ptr, first_dims_ptr, last_dims_ptr, scales_ptr); + tensor_map_input, tensor_map_output, input_dptr, output_dptr, shape_rep, + num_tensors, first_logical_dim, last_logical_dim, offsets_ptr, first_dims_ptr, + last_dims_ptr, scales_ptr); } else { group_dequantize_mxfp8_kernel<<>>( - tensor_map_input, tensor_map_output, shape_rep, num_tensors, first_logical_dim, - last_logical_dim, offsets_ptr, first_dims_ptr, last_dims_ptr, scales_ptr); + tensor_map_input, tensor_map_output, input_dptr, output_dptr, shape_rep, + num_tensors, first_logical_dim, last_logical_dim, offsets_ptr, first_dims_ptr, + last_dims_ptr, scales_ptr); }); // NOLINT(*) ); // NOLINT(*) NVTE_CHECK_CUDA(cudaGetLastError()); diff --git a/transformer_engine/common/cast/mxfp8/group_quantize_mxfp8.cuh b/transformer_engine/common/cast/mxfp8/group_quantize_mxfp8.cuh index 08605a27867..f0e398fecc4 100644 --- a/transformer_engine/common/cast/mxfp8/group_quantize_mxfp8.cuh +++ b/transformer_engine/common/cast/mxfp8/group_quantize_mxfp8.cuh @@ -619,9 +619,12 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ const __grid_constant__ CUtensorMap tensor_map_input_static, const __grid_constant__ CUtensorMap tensor_map_act_input_static, const __grid_constant__ CUtensorMap tensor_map_output_rowwise_static, - const __grid_constant__ CUtensorMap tensor_map_output_colwise_static, const size_t num_tensors, - const size_t first_logical_dim, const size_t last_logical_dim, const size_t same_both_rows, - const int64_t *const __restrict__ offsets_ptr, const int64_t *const __restrict__ first_dims_ptr, + const __grid_constant__ CUtensorMap tensor_map_output_colwise_static, + const IType *const input_data_ptr, const IType *const act_input_data_ptr, + OType *const output_rowwise_data_ptr, OType *const output_colwise_data_ptr, + const size_t num_tensors, const size_t first_logical_dim, const size_t last_logical_dim, + const size_t same_both_rows, const int64_t *const __restrict__ offsets_ptr, + const int64_t *const __restrict__ first_dims_ptr, const int64_t *const __restrict__ last_dims_ptr, e8m0_t *const __restrict__ scales_rowwise_ptr, e8m0_t *const __restrict__ scales_colwise_ptr, const float *__restrict__ noop, float *const __restrict__ dbias_workspace, float *const __restrict__ amax_ptr, @@ -664,6 +667,7 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ constexpr bool use_direct_varying_first_mapper = (shape_rep == VARYING_FIRST_DIM); constexpr bool use_direct_mapper = use_direct_same_both_mapper || use_direct_varying_first_mapper; constexpr bool is_single_tensor = (shape_rep == SAME_BOTH_DIMS || shape_rep == VARYING_FIRST_DIM); + constexpr bool use_tma_override = !is_single_tensor && ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; const bool leading_thread = (threadIdx.x == 0); // Keep mapper metadata separate from dynamic shared memory. The latter is the destination of @@ -671,15 +675,13 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ __shared__ DirectVaryingFirstMapperStorage direct_mapper_storage; if constexpr (use_direct_varying_first_mapper) { - static_assert(CastTraits::THREADS_PER_CHUNK >= MAX_SUPPORTED_TENSOR_DESCRIPTORS, - "The first CTA must have enough threads to validate every tensor."); - // The direct mapper relies on every tensor boundary being TILE_DIM_Y-aligned. The legacy - // mapper validated this while decoding each job, but the direct path no longer calls it. - // Validate all per-tensor row counts once, using the first CTA. num_tensors is bounded by - // MAX_SUPPORTED_TENSOR_DESCRIPTORS (64), which is smaller than the CTA size. - if (blockIdx.x == 0 && threadIdx.x < num_tensors) { - get_tensor_rows_num(threadIdx.x, first_logical_dim, - first_dims_ptr, num_tensors); + // Validate every tensor boundary once in the first CTA. Rubin's override + // path is independent of the legacy 64-entry descriptor table. + if (blockIdx.x == 0) { + for (size_t id = threadIdx.x; id < num_tensors; id += blockDim.x) { + get_tensor_rows_num(id, first_logical_dim, + first_dims_ptr, num_tensors); + } } } @@ -785,9 +787,17 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ if (fixed_tensor_id >= num_tensors) { return; } - fixed_rows = g_tensor_maps.rows[fixed_tensor_id]; - fixed_cols = g_tensor_maps.cols[fixed_tensor_id]; - fixed_tensor_base = g_tensor_maps.offsets[fixed_tensor_id]; + if constexpr (use_tma_override) { + fixed_rows = get_tensor_rows_num(fixed_tensor_id, first_logical_dim, + first_dims_ptr, num_tensors); + fixed_cols = + get_tensor_cols_num(fixed_tensor_id, last_logical_dim, last_dims_ptr); + fixed_tensor_base = static_cast(offsets_ptr[fixed_tensor_id]); + } else { + fixed_rows = g_tensor_maps.rows[fixed_tensor_id]; + fixed_cols = g_tensor_maps.cols[fixed_tensor_id]; + fixed_tensor_base = g_tensor_maps.offsets[fixed_tensor_id]; + } if (fixed_rows == 0 || fixed_cols == 0) { return; } @@ -906,18 +916,21 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ const size_t dbias_offset_Y = block_offset_Y / TILE_DIM_Y; - const CUtensorMap &tensor_map_input = - is_single_tensor ? tensor_map_input_static : g_tensor_maps.input[tensor_id]; - const CUtensorMap &tensor_map_act_input = - is_single_tensor ? tensor_map_act_input_static : g_tensor_maps.act_input[tensor_id]; - const CUtensorMap &tensor_map_output_rowwise = is_single_tensor + const CUtensorMap &tensor_map_input = (is_single_tensor || use_tma_override) + ? tensor_map_input_static + : g_tensor_maps.input[tensor_id]; + const CUtensorMap &tensor_map_act_input = (is_single_tensor || use_tma_override) + ? tensor_map_act_input_static + : g_tensor_maps.act_input[tensor_id]; + const CUtensorMap &tensor_map_output_rowwise = (is_single_tensor || use_tma_override) ? tensor_map_output_rowwise_static : g_tensor_maps.output_rowwise[tensor_id]; - const CUtensorMap &tensor_map_output_colwise = is_single_tensor + const CUtensorMap &tensor_map_output_colwise = (is_single_tensor || use_tma_override) ? tensor_map_output_colwise_static : g_tensor_maps.output_colwise[tensor_id]; - if (leading_thread && (!is_single_tensor) && (last_acquired_tensor_id != tensor_id)) { + if (leading_thread && !is_single_tensor && !use_tma_override && + (last_acquired_tensor_id != tensor_id)) { fence_acquire_tensormap(&tensor_map_input); if constexpr (COMPUTE_ACTIVATIONS) { fence_acquire_tensormap(&tensor_map_act_input); @@ -933,6 +946,14 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ __syncthreads(); int buff_in = 0; + const IType *const override_input = use_tma_override ? input_data_ptr + tensor_base : nullptr; + const IType *const override_act = + (use_tma_override && IS_DACT) ? act_input_data_ptr + tensor_base : nullptr; + OType *const override_rowwise = + (use_tma_override && ROWWISE_SCALING) ? output_rowwise_data_ptr + tensor_base : nullptr; + OType *const override_colwise = + (use_tma_override && COLWISE_SCALING) ? output_colwise_data_ptr + tensor_base : nullptr; + size_t chunk_rows = CHUNK_DIM_Y; if constexpr (CHUNK_DIM_Y > TILE_DIM_Y) { chunk_rows = min(static_cast(CHUNK_DIM_Y), rows - tensor_offset_Y); @@ -954,9 +975,10 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ const size_t global_offset_X = block_offset_X + stage_offset_X; const size_t buff_offset = buff * BUFF_DIM; uint64_t *barrier = &IN_buff_readable_mbar[buff]; - prefetch_input_stage(sIn_ptr, sActIn_ptr, tensor_map_input, - tensor_map_act_input, global_offset_X, global_offset_Y, - buff_offset, shmem_buff_size, barrier, leading_thread); + prefetch_input_stage( + sIn_ptr, sActIn_ptr, tensor_map_input, tensor_map_act_input, global_offset_X, + global_offset_Y, buff_offset, shmem_buff_size, barrier, leading_thread, override_input, + override_act, rows, cols); } float partial_dbias_colwise[STAGES_X] = {0.0f}; @@ -992,9 +1014,10 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ const size_t next_prefetch_buff_offset = next_prefetch_buff * BUFF_DIM; uint64_t *barrier = &IN_buff_readable_mbar[next_prefetch_buff]; - prefetch_input_stage( + prefetch_input_stage( sIn_ptr, sActIn_ptr, tensor_map_input, tensor_map_act_input, global_offset_X, - global_offset_Y, next_prefetch_buff_offset, shmem_buff_size, barrier, leading_thread); + global_offset_Y, next_prefetch_buff_offset, shmem_buff_size, barrier, leading_thread, + override_input, override_act, rows, cols); } ptx::mbarrier_wait_parity_acquire_cta_shared_cta(&IN_buff_readable_mbar[buff_in], @@ -1039,9 +1062,10 @@ __global__ void __launch_bounds__(CastTraits::THREADS_PER_CHUNK) group_quantize_ const size_t global_offset_Y = block_offset_Y + stage_offset_Y; const size_t global_offset_X = block_offset_X + stage_offset_X; const size_t buff_offset = buff * BUFF_DIM; - store_output_stage( + store_output_stage( sOutRowwise_ptr, sOutColwise_ptr, tensor_map_output_rowwise, tensor_map_output_colwise, - global_offset_X, global_offset_Y, buff_offset, leading_thread); + global_offset_X, global_offset_Y, buff_offset, leading_thread, override_rowwise, + override_colwise, rows, cols); buff_in = (buff_in + 1) % BUFFS_NUM; } @@ -1177,7 +1201,8 @@ void group_quantize(const GroupedTensor *input, const GroupedTensor *activations const size_t elts_total = first_logical_dim * last_logical_dim; const size_t num_tensors = input->num_tensors; - NVTE_CHECK(num_tensors > 0 && num_tensors <= MAX_SUPPORTED_TENSOR_DESCRIPTORS, + NVTE_CHECK(num_tensors > 0 && (common::use_tma_descriptor_override() || + num_tensors <= MAX_SUPPORTED_TENSOR_DESCRIPTORS), "Number of tensors in a group must be between 1 and ", MAX_SUPPORTED_TENSOR_DESCRIPTORS, "."); @@ -1257,29 +1282,25 @@ void group_quantize(const GroupedTensor *input, const GroupedTensor *activations constexpr size_t input_type_bit_size = TypeInfo::size; constexpr size_t output_type_bit_size = TypeInfo::size; - create_2D_tensor_map(tensor_map_input, input->data, first_logical_dim, - last_logical_dim, BUFF_DIM_Y, BUFF_DIM_X, - last_logical_dim, 0, input_type_bit_size); - - if constexpr (IS_DACT) { - create_2D_tensor_map(tensor_map_act_input, activations->data, + const bool override_map = + !is_single_tensor && common::use_tma_descriptor_override(); + create_grouped_tma_map(tensor_map_input, input->data, override_map, first_logical_dim, last_logical_dim, BUFF_DIM_Y, - BUFF_DIM_X, last_logical_dim, 0, - input_type_bit_size); + BUFF_DIM_X, input_type_bit_size); + if constexpr (IS_DACT) { + create_grouped_tma_map(tensor_map_act_input, activations->data, + override_map, first_logical_dim, last_logical_dim, + BUFF_DIM_Y, BUFF_DIM_X, input_type_bit_size); } - if (use_rowwise_scaling) { - create_2D_tensor_map(tensor_map_output_rowwise, output->data, - first_logical_dim, last_logical_dim, BUFF_DIM_Y, - BUFF_DIM_X, last_logical_dim, 0, - output_type_bit_size); + create_grouped_tma_map(tensor_map_output_rowwise, output->data, + override_map, first_logical_dim, last_logical_dim, + BUFF_DIM_Y, BUFF_DIM_X, output_type_bit_size); } - if (use_colwise_scaling) { - create_2D_tensor_map(tensor_map_output_colwise, output->columnwise_data, - first_logical_dim, last_logical_dim, BUFF_DIM_Y, - BUFF_DIM_X, last_logical_dim, 0, - output_type_bit_size); + create_grouped_tma_map(tensor_map_output_colwise, output->columnwise_data, + override_map, first_logical_dim, last_logical_dim, + BUFF_DIM_Y, BUFF_DIM_X, output_type_bit_size); } constexpr size_t buff_elems = BUFF_DIM_Y * BUFF_DIM_X; @@ -1305,19 +1326,19 @@ void group_quantize(const GroupedTensor *input, const GroupedTensor *activations const size_t dshmem_size = in_mem + out_mem + TMA_SHMEM_ALIGNMENT; // Update tensor descriptors before launching the kernel - if (!is_single_tensor) { - const IType *const input_dptr = - reinterpret_cast(input->data.dptr); - const IType *const act_input_dptr = - IS_DACT ? reinterpret_cast(activations->data.dptr) - : nullptr; - OType *const output_rowwise_dptr = - use_rowwise_scaling ? reinterpret_cast(output->data.dptr) - : nullptr; - OType *const output_colwise_dptr = - use_colwise_scaling - ? reinterpret_cast(output->columnwise_data.dptr) - : nullptr; + const IType *const input_dptr = + reinterpret_cast(input->data.dptr); + const IType *const act_input_dptr = + IS_DACT ? reinterpret_cast(activations->data.dptr) + : nullptr; + OType *const output_rowwise_dptr = + use_rowwise_scaling ? reinterpret_cast(output->data.dptr) + : nullptr; + OType *const output_colwise_dptr = + use_colwise_scaling + ? reinterpret_cast(output->columnwise_data.dptr) + : nullptr; + if (!is_single_tensor && !common::use_tma_descriptor_override()) { update_tma_descriptors <<>>( tensor_map_input, tensor_map_act_input, tensor_map_output_rowwise, @@ -1340,9 +1361,10 @@ void group_quantize(const GroupedTensor *input, const GroupedTensor *activations kernel<<>>( tensor_map_input, tensor_map_act_input, tensor_map_output_rowwise, - tensor_map_output_colwise, num_tensors, first_logical_dim, - last_logical_dim, launch_config.same_both_rows, offsets_ptr, - first_dims_ptr, last_dims_ptr, scales_rowwise_ptr, + tensor_map_output_colwise, input_dptr, act_input_dptr, + output_rowwise_dptr, output_colwise_dptr, num_tensors, + first_logical_dim, last_logical_dim, launch_config.same_both_rows, + offsets_ptr, first_dims_ptr, last_dims_ptr, scales_rowwise_ptr, scales_colwise_ptr, noop_ptr, workspace_ptr, amax_ptr, launch_config.work_blocks_X, launch_config.work_blocks_Y); }); diff --git a/transformer_engine/common/cast/mxfp8/requantize_mxfp8.cu b/transformer_engine/common/cast/mxfp8/requantize_mxfp8.cu index 7f406f8c8aa..58f643b3358 100644 --- a/transformer_engine/common/cast/mxfp8/requantize_mxfp8.cu +++ b/transformer_engine/common/cast/mxfp8/requantize_mxfp8.cu @@ -130,7 +130,7 @@ LaunchConfig get_launch_config(const size_t first_logical_dim, const size_t last "VARYING_FIRST_DIM requires a non-empty logical tensor."); config.grid = dim3(config.work_blocks_x, work_blocks_y); } else { - NVTE_CHECK(num_tensors <= MAX_SUPPORTED_TENSOR_DESCRIPTORS, + NVTE_CHECK(use_tma_descriptor_override() || num_tensors <= MAX_SUPPORTED_TENSOR_DESCRIPTORS, "Number of tensors exceeds the grouped TMA descriptor limit (", MAX_SUPPORTED_TENSOR_DESCRIPTORS, ")."); const size_t estimated_work_blocks = DIVUP(total_elements, chunk_dim_y * chunk_dim_x); @@ -467,7 +467,7 @@ __device__ __forceinline__ void process_chunk( const size_t block_offset_y, const size_t block_offset_x, const size_t tma_offset_y, IType *const input_shared, void *const dequantized_shared, OType *const output_shared, uint64_t *const input_barriers, int *const input_barrier_parity, const bool leading_thread, - bf16 *const dequantized_out) { + bf16 *const dequantized_out, const IType *const override_input, OType *const override_output) { constexpr size_t tile_dim_x = Traits::TILE_DIM_X; constexpr size_t chunk_dim_y = Traits::CHUNK_DIM_Y; constexpr size_t chunk_dim_x = Traits::CHUNK_DIM_X; @@ -498,9 +498,10 @@ __device__ __forceinline__ void process_chunk( const size_t global_y = tma_offset_y + stage_y * buff_dim_y; const size_t global_x = block_offset_x + stage_x * tile_dim_x; const size_t buffer_offset = stage * buff_dim; - prefetch_input_stage(input_shared, nullptr, tensor_map_input, tensor_map_input, - global_x, global_y, buffer_offset, input_buffer_bytes, - &input_barriers[stage], leading_thread); + prefetch_input_stage( + input_shared, nullptr, tensor_map_input, tensor_map_input, global_x, global_y, + buffer_offset, input_buffer_bytes, &input_barriers[stage], leading_thread, override_input, + nullptr, rows, cols); } int input_buffer = 0; @@ -519,9 +520,10 @@ __device__ __forceinline__ void process_chunk( const size_t global_y = tma_offset_y + next_stage_y * buff_dim_y; const size_t global_x = block_offset_x + next_stage_x * tile_dim_x; const size_t buffer_offset = next_buffer * buff_dim; - prefetch_input_stage(input_shared, nullptr, tensor_map_input, tensor_map_input, - global_x, global_y, buffer_offset, input_buffer_bytes, - &input_barriers[next_buffer], leading_thread); + prefetch_input_stage( + input_shared, nullptr, tensor_map_input, tensor_map_input, global_x, global_y, + buffer_offset, input_buffer_bytes, &input_barriers[next_buffer], leading_thread, + override_input, nullptr, rows, cols); } ptx::mbarrier_wait_parity_acquire_cta_shared_cta(&input_barriers[input_buffer], @@ -545,9 +547,9 @@ __device__ __forceinline__ void process_chunk( const size_t global_y = tma_offset_y + stage_offset_y; const size_t global_x = block_offset_x + stage_offset_x; const size_t buffer_offset = input_buffer * buff_dim; - store_output_stage(nullptr, output_shared, tensor_map_output, - tensor_map_output, global_x, global_y, buffer_offset, - leading_thread); + store_output_stage( + nullptr, output_shared, tensor_map_output, tensor_map_output, global_x, global_y, + buffer_offset, leading_thread, nullptr, override_output, rows, cols); input_buffer = (input_buffer + 1) % buffs_num; } @@ -563,8 +565,9 @@ template __global__ void __launch_bounds__(Traits::THREADS_PER_CHUNK) group_requantize_mxfp8_kernel( const __grid_constant__ CUtensorMap tensor_map_input_static, - const __grid_constant__ CUtensorMap tensor_map_output_static, const size_t num_tensors, - const size_t first_logical_dim, const size_t last_logical_dim, const size_t same_both_rows, + const __grid_constant__ CUtensorMap tensor_map_output_static, const IType *const input_data, + OType *const output_data, const size_t num_tensors, const size_t first_logical_dim, + const size_t last_logical_dim, const size_t same_both_rows, const int64_t *const __restrict__ offsets_ptr, const int64_t *const __restrict__ first_dims_ptr, const int64_t *const __restrict__ last_dims_ptr, const e8m0_t *const __restrict__ input_scales, e8m0_t *const __restrict__ output_rowwise_scales, e8m0_t *const __restrict__ output_scales, @@ -575,6 +578,7 @@ __global__ void __launch_bounds__(Traits::THREADS_PER_CHUNK) group_requantize_mx constexpr bool direct_varying_first = shape_rep == ShapeRepresentation::VARYING_FIRST_DIM; constexpr bool direct_mapper = direct_same || direct_varying_first; constexpr bool single_tma_tensor = direct_mapper; + constexpr bool use_tma_override = !single_tma_tensor && ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; constexpr size_t buffs_num = Traits::BUFFS_NUM; constexpr size_t buff_dim = Traits::BUFF_DIM; constexpr size_t input_bytes = @@ -652,11 +656,13 @@ __global__ void __launch_bounds__(Traits::THREADS_PER_CHUNK) group_requantize_mx } } - const CUtensorMap &tensor_map_input = - single_tma_tensor ? tensor_map_input_static : g_tensor_maps.input[tensor_id]; - const CUtensorMap &tensor_map_output = - single_tma_tensor ? tensor_map_output_static : g_tensor_maps.output_colwise[tensor_id]; - if constexpr (!single_tma_tensor) { + const CUtensorMap &tensor_map_input = (single_tma_tensor || use_tma_override) + ? tensor_map_input_static + : g_tensor_maps.input[tensor_id]; + const CUtensorMap &tensor_map_output = (single_tma_tensor || use_tma_override) + ? tensor_map_output_static + : g_tensor_maps.output_colwise[tensor_id]; + if constexpr (!single_tma_tensor && !use_tma_override) { if (leading_thread) { fence_acquire_tensormap(&tensor_map_input); fence_acquire_tensormap(&tensor_map_output); @@ -689,12 +695,14 @@ __global__ void __launch_bounds__(Traits::THREADS_PER_CHUNK) group_requantize_mx initialize_barriers(input_barriers, leading_thread); int input_barrier_parity[buffs_num] = {0}; + const IType *const override_input = use_tma_override ? input_data + tensor_base : nullptr; + OType *const override_output = use_tma_override ? output_data + tensor_base : nullptr; if constexpr (direct_mapper) { process_chunk( tensor_map_input, tensor_map_output, input_scales, output_scales, output_rowwise_scales, input_scale_base, output_scale_base, rows, cols, block_offset_y, block_offset_x, tma_offset_y, input_shared, dequantized_shared, output_shared, input_barriers, - input_barrier_parity, leading_thread, dequantized_group); + input_barrier_parity, leading_thread, dequantized_group, override_input, override_output); } else { const size_t blocks_x = DIVUP(cols, static_cast(Traits::CHUNK_DIM_X)); const size_t blocks_y = DIVUP(rows, static_cast(Traits::CHUNK_DIM_Y)); @@ -707,7 +715,7 @@ __global__ void __launch_bounds__(Traits::THREADS_PER_CHUNK) group_requantize_mx input_scale_base, output_scale_base, rows, cols, block_y * Traits::CHUNK_DIM_Y, block_x * Traits::CHUNK_DIM_X, block_y * Traits::CHUNK_DIM_Y, input_shared, dequantized_shared, output_shared, input_barriers, input_barrier_parity, leading_thread, - dequantized_group); + dequantized_group, override_input, override_output); } } @@ -749,17 +757,18 @@ void launch_group_requantize(const GroupedTensor &input, GroupedTensor *output, alignas(64) CUtensorMap tensor_map_input{}; alignas(64) CUtensorMap tensor_map_output{}; - create_2D_tensor_map(tensor_map_input, input.data, first_logical_dim, last_logical_dim, - Traits::BUFF_DIM_Y, Traits::BUFF_DIM_X, last_logical_dim, 0, - TypeInfo::size, tma_swizzle); - create_2D_tensor_map(tensor_map_output, output->columnwise_data, first_logical_dim, - last_logical_dim, Traits::BUFF_DIM_Y, Traits::BUFF_DIM_X, last_logical_dim, - 0, TypeInfo::size, tma_swizzle); - constexpr bool single_tma_tensor = Traits::SHAPE_REPRESENTATION == ShapeRepresentation::SAME_BOTH_DIMS || Traits::SHAPE_REPRESENTATION == ShapeRepresentation::VARYING_FIRST_DIM; - if constexpr (!single_tma_tensor) { + const bool override_map = !single_tma_tensor && use_tma_descriptor_override(); + create_grouped_tma_map(tensor_map_input, input.data, override_map, first_logical_dim, + last_logical_dim, Traits::BUFF_DIM_Y, Traits::BUFF_DIM_X, + TypeInfo::size, tma_swizzle); + create_grouped_tma_map(tensor_map_output, output->columnwise_data, override_map, + first_logical_dim, last_logical_dim, Traits::BUFF_DIM_Y, + Traits::BUFF_DIM_X, TypeInfo::size, tma_swizzle); + + if (!single_tma_tensor && !use_tma_descriptor_override()) { alignas(64) CUtensorMap empty_tensor_map{}; update_tma_descriptors<<>>( tensor_map_input, empty_tensor_map, empty_tensor_map, tensor_map_output, @@ -774,8 +783,9 @@ void launch_group_requantize(const GroupedTensor &input, GroupedTensor *output, NVTE_CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, dynamic_shared_bytes)); kernel<<>>( - tensor_map_input, tensor_map_output, num_tensors, first_logical_dim, last_logical_dim, - launch_config.same_both_rows, offsets_ptr, first_dims_ptr, last_dims_ptr, + tensor_map_input, tensor_map_output, reinterpret_cast(input.data.dptr), + reinterpret_cast(output->columnwise_data.dptr), num_tensors, first_logical_dim, + last_logical_dim, launch_config.same_both_rows, offsets_ptr, first_dims_ptr, last_dims_ptr, reinterpret_cast(input.scale_inv.dptr), output->scale_inv.dptr == input.scale_inv.dptr ? nullptr diff --git a/transformer_engine/common/util/ptx.cuh b/transformer_engine/common/util/ptx.cuh index e37e147b77c..999c102aeec 100644 --- a/transformer_engine/common/util/ptx.cuh +++ b/transformer_engine/common/util/ptx.cuh @@ -143,6 +143,13 @@ constexpr bool is_supported_arch() { #define ARCH_HAS_STOCHASTIC_ROUNDING \ NVTE_CUDA_ARCH_MATCHES(ptx::ArchSpecific<100>, ptx::ArchSpecific<103>, ptx::ArchSpecific<107>) +#define ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE \ + [&] { \ + __NVTE_CURRENT_ARCH__ \ + __NVTE_ARCH_FAMILY_SPECIFIC__ \ + return current_arch == 1070 && FamilySpecific == current_arch; \ + }(); + // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init __device__ __forceinline__ void mbarrier_init(uint64_t *mbar, const uint32_t count) { #if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -1938,6 +1945,57 @@ __device__ __forceinline__ void prefetch_l2_evict_last(const void *addr) { NVTE_DEVICE_ERROR("prefetch_l2_evict_last is only supported on SM 8.0+."); #endif // (defined __CUDA_ARCH__) && (__CUDA_ARCH__ >= 800) } + +// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-overriding-tensor-property-value +// global -> shared::cluster, with tensor base address, dimensions, and stride overridden. +__device__ __forceinline__ void cp_async_bulk_tensor_2d_global_to_shared_rubin( + uint64_t *dst_shmem, const uint64_t *tensor_map_ptr, const uint64_t gaddr_to_override, + const uint16_t tensor_size_x, const uint16_t tensor_size_y, const uint32_t tensor_lower_stride, + const uint16_t tensor_upper_stride, uint64_t *mbar) { +#if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ == 1070) + constexpr bool has_tma_descriptor_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; + if constexpr (has_tma_descriptor_override) { + uint32_t dst_shmem_ptr = __cvta_generic_to_shared(dst_shmem); + uint32_t mbar_ptr = __cvta_generic_to_shared(mbar); + asm volatile( + "cp.async.bulk.tensor.2d.shared::cluster.global.tile" + ".mbarrier::complete_tx::bytes" + ".override::global_address.override::global_dim_stride " + "[%0], [%1, %2, {%3, %4}, {%5}, %6, {0, 0}], [%7];" ::"r"(dst_shmem_ptr), + "l"(tensor_map_ptr), "l"(gaddr_to_override), "h"(tensor_size_x), "h"(tensor_size_y), + "r"(tensor_lower_stride), "h"(tensor_upper_stride), "r"(mbar_ptr) + : "memory"); + return; + } +#endif // __CUDA_ARCH__ == 1070 + NVTE_DEVICE_ERROR( + "cp_async_bulk_tensor_2d_global_to_shared_rubin is only supported on SM 10.7f+."); +} + +// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-overriding-tensor-property-value +// shared::cta -> global, with tensor base address, dimensions, and stride overridden. +__device__ __forceinline__ void cp_async_bulk_tensor_2d_shared_to_global_rubin( + const uint64_t *tensor_map_ptr, const uint64_t gaddr_to_override, const uint16_t tensor_size_x, + const uint16_t tensor_size_y, const uint32_t tensor_lower_stride, + const uint16_t tensor_upper_stride, uint64_t *src_shmem) { +#if (defined __CUDA_ARCH__) && (__CUDA_ARCH__ == 1070) + constexpr bool has_tma_descriptor_override = ARCH_HAS_TMA_DESCRIPTOR_OVERRIDE; + if constexpr (has_tma_descriptor_override) { + uint32_t src_shmem_ptr = __cvta_generic_to_shared(src_shmem); + asm volatile( + "cp.async.bulk.tensor.2d.global.shared::cta.bulk_group.tile" + ".override::global_address.override::global_dim_stride " + "[%0, %1, {%2, %3}, {%4}, %5, {0, 0}], [%6];" ::"l"(tensor_map_ptr), + "l"(gaddr_to_override), "h"(tensor_size_x), "h"(tensor_size_y), "r"(tensor_lower_stride), + "h"(tensor_upper_stride), "r"(src_shmem_ptr) + : "memory"); + return; + } +#endif // __CUDA_ARCH__ == 1070 + NVTE_DEVICE_ERROR( + "cp_async_bulk_tensor_2d_shared_to_global_rubin is only supported on SM 10.7f+."); +} + } // namespace ptx namespace {