From 9e7bf8ab3339c3a27d07b172cfc27ddb2fb3faa0 Mon Sep 17 00:00:00 2001 From: Akira Naruse Date: Thu, 19 Nov 2026 20:04:17 -0800 Subject: [PATCH 1/2] Optimize grouped FP8 blockwise quantization on Hopper Add a grouped BF16-to-FP8 kernel for compatible matrices with a common K and variable M, producing both rowwise and columnwise GEMM-ready outputs. Specialize output modes and use packed BF16x2 amax operations. Select the optimized path automatically through existing quantization APIs, preserving fallback behavior for unsupported configurations. Document the dispatch control and add coverage for boundary shapes, empty tensors, fallbacks, and CUDA Graph replay. --- docs/envvars.rst | 12 + tests/pytorch/test_float8blockwisetensor.py | 200 ++++++++++++++ transformer_engine/common/cast/cast.cu | 9 + .../common/include/transformer_engine/cast.h | 4 + .../common/transpose/cast_transpose.h | 8 + .../quantize_transpose_vector_blockwise.cu | 261 +++++++++++++++--- .../pytorch/csrc/extensions/cast.cpp | 50 ++++ 7 files changed, 508 insertions(+), 36 deletions(-) diff --git a/docs/envvars.rst b/docs/envvars.rst index a0a64b3f637..50eb8f43a9e 100644 --- a/docs/envvars.rst +++ b/docs/envvars.rst @@ -272,6 +272,18 @@ Context Parallelism FP8 Configuration ^^^^^^^^^^^^^^^^^ +.. envvar:: NVTE_FP8_BLOCK_QUANTIZATION_GROUPED + + :Type: ``int`` (0 or 1) + :Default: ``1`` + :Description: Enable grouped 1D blockwise FP8 quantization on Hopper for compatible + lists of at least two BF16 matrices. Inputs must share K, but M may differ, and + all quantizers must request both rowwise and columnwise outputs + with matching FP8 dtype, epsilon, and power-of-two scaling options. + The existing split and multi-tensor APIs select this path + automatically. Other configurations retain their existing path. + Set to ``0`` to disable the optimization for comparison. + .. envvar:: NVTE_UNFUSED_FP8_UPDATE :Type: ``int`` (0 or 1) diff --git a/tests/pytorch/test_float8blockwisetensor.py b/tests/pytorch/test_float8blockwisetensor.py index ab00786e4fe..c185738b05a 100644 --- a/tests/pytorch/test_float8blockwisetensor.py +++ b/tests/pytorch/test_float8blockwisetensor.py @@ -641,3 +641,203 @@ def test_clone_detach( # Make sure we are not trivially passing tests with pytest.raises(AssertionError): torch.testing.assert_close(x_clone.dequantize(), -x_hp, **_tols[fp8_dtype]) + + +def _snapshot_blockwise_outputs(outputs, shapes, block_dim): + snapshots = [] + for out, (m, k) in zip(outputs, shapes): + for direction in ("rowwise", "columnwise"): + data = getattr(out, f"_{direction}_data") + scale = getattr(out, f"_{direction}_scale_inv") + if data is None: + snapshots.append(None) + continue + # Consume on the caller stream immediately, before synchronizing. + snapshots.append(data.clone()) + if block_dim == 2: + rows, cols = (m + 127) // 128, (k + 127) // 128 + if direction == "columnwise": + rows, cols = cols, rows + elif direction == "rowwise": + rows, cols = (k + 127) // 128, m + else: + rows, cols = (m + 127) // 128, k + # Scale storage includes unspecified GEMM-alignment padding. + snapshots.append(scale[:rows, :cols].clone()) + return snapshots + + +def _assert_blockwise_snapshots_equal(actual, expected): + assert len(actual) == len(expected) + for a, b in zip(actual, expected): + if a is None or b is None: + assert a is None and b is None + else: + torch.testing.assert_close(a, b, rtol=0, atol=0) + + +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +@pytest.mark.parametrize( + "splits", [[128, 0, 256, 384] * 4, [0, 1, 127, 129, 255], [129] * 33, [0, 0], [129]] +) +@pytest.mark.parametrize("k", [256, 257]) +@pytest.mark.parametrize("bulk", [True, False]) +@pytest.mark.parametrize("fp8_dtype", [DType.kFloat8E4M3, DType.kFloat8E5M2]) +def test_split_quantize_grouped(monkeypatch, splits, k, bulk, fp8_dtype): + """Variable M, empty/partial tiles, chunk boundaries, and caller ordering.""" + if get_device_compute_capability()[0] != 9: + pytest.skip("Grouped BF16 blockwise path targets Hopper") + shapes = [(m, k) for m in splits] + quantizers = [ + Float8BlockQuantizer( + fp8_dtype=fp8_dtype, + rowwise=True, + columnwise=True, + block_scaling_dim=1, + force_pow_2_scales=(k == 256), + amax_epsilon=0.01, + ) + for _ in splits + ] + caller = torch.cuda.Stream() + with torch.cuda.stream(caller): + for _ in range(2): + x = torch.randn(sum(splits), k, dtype=torch.bfloat16, device="cuda") + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "0") + reference = tex.split_quantize(x, splits, quantizers, disable_bulk_allocation=not bulk) + expected = _snapshot_blockwise_outputs(reference, shapes, 1) + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "1") + outputs = tex.split_quantize(x, splits, quantizers, disable_bulk_allocation=not bulk) + actual = _snapshot_blockwise_outputs(outputs, shapes, 1) + assert torch.cuda.current_stream() == caller + caller.synchronize() + _assert_blockwise_snapshots_equal(actual, expected) + + +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +@pytest.mark.parametrize( + "usage,block_dim,dtype,mixed_options", + [ + ((True, False), 1, torch.bfloat16, False), + ((False, True), 1, torch.bfloat16, False), + ((True, True), 2, torch.bfloat16, False), + ((True, True), 1, torch.float16, False), + ((True, True), 1, torch.bfloat16, True), + ], +) +def test_split_quantize_grouped_fallback(monkeypatch, usage, block_dim, dtype, mixed_options): + """Keep single-direction, 2D scaling, other dtypes and per-expert options.""" + splits, k = [128, 256, 0], 256 + quantizers = [ + Float8BlockQuantizer( + fp8_dtype=DType.kFloat8E4M3, + rowwise=usage[0], + columnwise=usage[1], + block_scaling_dim=block_dim, + amax_epsilon=(0.1 * i if mixed_options else 0.0), + ) + for i in range(len(splits)) + ] + x = torch.randn(sum(splits), k, dtype=dtype, device="cuda") + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "0") + reference = tex.split_quantize(x, splits, quantizers) + expected = _snapshot_blockwise_outputs(reference, [(m, k) for m in splits], block_dim) + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "1") + outputs = tex.split_quantize(x, splits, quantizers) + actual = _snapshot_blockwise_outputs(outputs, [(m, k) for m in splits], block_dim) + torch.cuda.synchronize() + _assert_blockwise_snapshots_equal(actual, expected) + + +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +@pytest.mark.parametrize("common_k", [True, False]) +def test_multi_tensor_quantize_grouped(monkeypatch, common_k): + """Support separate allocations; mixed K must retain the existing path.""" + shapes = [(129, 256), (256, 256 if common_k else 384)] + inputs = [torch.randn(shape, dtype=torch.bfloat16, device="cuda") for shape in shapes] + quantizers = [ + Float8BlockQuantizer( + fp8_dtype=DType.kFloat8E4M3, + rowwise=True, + columnwise=True, + block_scaling_dim=1, + ) + for _ in inputs + ] + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "0") + reference = tex.multi_tensor_quantize(inputs, quantizers) + expected = _snapshot_blockwise_outputs(reference, shapes, 1) + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "1") + outputs = tex.multi_tensor_quantize(inputs, quantizers) + actual = _snapshot_blockwise_outputs(outputs, shapes, 1) + torch.cuda.synchronize() + _assert_blockwise_snapshots_equal(actual, expected) + + +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +def test_split_quantize_grouped_dispatch(monkeypatch): + """Ensure a qualifying call actually executes one grouped CUDA kernel.""" + if get_device_compute_capability()[0] != 9: + pytest.skip("Grouped BF16 blockwise path targets Hopper") + splits, k = [128, 0, 256, 384], 256 + quantizers = [ + Float8BlockQuantizer( + fp8_dtype=DType.kFloat8E4M3, + rowwise=True, + columnwise=True, + block_scaling_dim=1, + ) + for _ in splits + ] + x = torch.randn(sum(splits), k, dtype=torch.bfloat16, device="cuda") + monkeypatch.delenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", raising=False) + tex.split_quantize(x, splits, quantizers) + torch.cuda.synchronize() + with torch.profiler.profile( + activities=[torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA] + ) as profile: + tex.split_quantize(x, splits, quantizers) + torch.cuda.synchronize() + kernels = [ + event + for event in profile.events() + if "grouped_block_scaled_1d_cast_transpose_kernel" in event.name + ] + assert len(kernels) == 1, "Expected one grouped kernel; rebuild the TE core library as well" + + +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +def test_split_quantize_grouped_cuda_graph(monkeypatch): + """By-value metadata remains valid through capture and changed-input replays.""" + if get_device_compute_capability()[0] != 9: + pytest.skip("Grouped BF16 blockwise path targets Hopper") + splits, k = [0, 129, 256, 511], 256 + quantizers = [ + Float8BlockQuantizer( + fp8_dtype=DType.kFloat8E4M3, + rowwise=True, + columnwise=True, + block_scaling_dim=1, + ) + for _ in splits + ] + x = torch.ones(sum(splits), k, dtype=torch.bfloat16, device="cuda") + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "1") + warmup = torch.cuda.Stream() + warmup.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(warmup): + for _ in range(3): + tex.split_quantize(x, splits, quantizers) + torch.cuda.current_stream().wait_stream(warmup) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outputs = tex.split_quantize(x, splits, quantizers) + for value in (0.0, 0.5, 2.0, -3.0): + x.fill_(value) + graph.replay() + actual = _snapshot_blockwise_outputs(outputs, [(m, k) for m in splits], 1) + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "0") + reference = tex.split_quantize(x, splits, quantizers) + expected = _snapshot_blockwise_outputs(reference, [(m, k) for m in splits], 1) + torch.cuda.synchronize() + _assert_blockwise_snapshots_equal(actual, expected) diff --git a/transformer_engine/common/cast/cast.cu b/transformer_engine/common/cast/cast.cu index 1e3c04573be..f579cf83437 100644 --- a/transformer_engine/common/cast/cast.cu +++ b/transformer_engine/common/cast/cast.cu @@ -60,6 +60,15 @@ void nvte_multi_tensor_quantize(const NVTETensor *inputs, NVTETensor *outputs, NVTE_API_CALL(nvte_multi_tensor_quantize); using namespace transformer_engine; + QuantizationConfig config; + if (quant_configs != nullptr) { + config = *reinterpret_cast(quant_configs); + } + if (detail::try_grouped_quantize_transpose_vector_blockwise(inputs, outputs, num_tensors, config, + stream)) { + return; + } + constexpr bool IS_ACT = false; const size_t num_streams = nvte_get_num_compute_streams(); diff --git a/transformer_engine/common/include/transformer_engine/cast.h b/transformer_engine/common/include/transformer_engine/cast.h index 1af93d42919..3d0a0553e64 100644 --- a/transformer_engine/common/include/transformer_engine/cast.h +++ b/transformer_engine/common/include/transformer_engine/cast.h @@ -455,6 +455,10 @@ void nvte_group_requantize(const NVTEGroupedTensor input, NVTEGroupedTensor outp cudaStream_t stream); /*! \brief Casts multiple input tensors to quantized output tensors. + * + * On Hopper, compatible BF16 matrices with common K and both GEMM-ready 1D + * block-scaled FP8 outputs use grouped quantization. M may vary per tensor. + * Other configurations retain their existing quantization path. * * \param[in] inputs List of input tensors to be cast. * \param[in,out] outputs List of output quantized tensors. diff --git a/transformer_engine/common/transpose/cast_transpose.h b/transformer_engine/common/transpose/cast_transpose.h index c462b301475..309564490a3 100644 --- a/transformer_engine/common/transpose/cast_transpose.h +++ b/transformer_engine/common/transpose/cast_transpose.h @@ -54,6 +54,14 @@ enum class FP8BlockwiseColumnwiseOption { COLUMNWISE_COMPACT }; +// Try grouped BF16-to-FP8 quantization with both GEMM-ready orientations. +// All inputs must have the same K; M may vary. Returns false before launching +// work if any tensor/configuration needs the existing per-tensor path. +bool try_grouped_quantize_transpose_vector_blockwise(const NVTETensor *inputs, NVTETensor *outputs, + size_t num_tensors, + const QuantizationConfig &config, + cudaStream_t stream); + void quantize_transpose_vector_blockwise(const SimpleTensor &input, SimpleTensor &scale_inv, SimpleTensor &scale_inv_t, SimpleTensor &output, SimpleTensor &output_t, const float epsilon, diff --git a/transformer_engine/common/transpose/quantize_transpose_vector_blockwise.cu b/transformer_engine/common/transpose/quantize_transpose_vector_blockwise.cu index df869b43317..6874e042e6b 100644 --- a/transformer_engine/common/transpose/quantize_transpose_vector_blockwise.cu +++ b/transformer_engine/common/transpose/quantize_transpose_vector_blockwise.cu @@ -11,13 +11,17 @@ #include #include +#include #include +#include #include +#include #include "common/common.h" #include "common/recipe/recipe_common.cuh" #include "common/transpose/cast_transpose.h" #include "common/util/cuda_runtime.h" +#include "common/util/system.h" #include "common/utils.cuh" namespace transformer_engine { @@ -166,24 +170,39 @@ static_assert(kNumThreadsLoad <= kThreadsPerWarp, "kNumThreadsLoad must be <= kT static_assert(kNumThreadsStore <= kThreadsPerWarp, "kNumThreadsStore must be <= kThreadsPerWarp"); constexpr int kNumWarps = kThreadsPerBlock / kThreadsPerWarp; -template -__global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpose_kernel( +// Packed BF16 local amax, used only by the grouped BF16 specialization. +// The padded shared-memory layout and eight-lane shuffle reductions are kept. +template +__device__ __forceinline__ __nv_bfloat162 +local_bf16_pair_amax(const Vec (&values)[Count]) { + __nv_bfloat162 amax = __floats2bfloat162_rn(0.f, 0.f); +#pragma unroll + for (int i = 0; i < Count; ++i) { + const auto pair = __halves2bfloat162(values[i].data.elt[0], values[i].data.elt[1]); + amax = __hmax2(amax, __habs2(pair)); + } + return amax; +} + +template +__device__ __forceinline__ void block_scaled_1d_cast_transpose_tile( const IType* const input, OType* const output_c, OType* const output_t, CType* const tile_scales_inv_c, CType* const tile_scales_inv_t, const size_t row_length, const size_t num_rows, const size_t scale_stride_x, const size_t scale_stride_y, const size_t scale_t_stride_x, const size_t scale_t_stride_y, const float epsilon, FP8BlockwiseRowwiseOption rowwise_option, FP8BlockwiseColumnwiseOption columnwise_option, - const bool pow_2_scaling, const float* noop_ptr) { + const bool pow_2_scaling, const float* noop_ptr, size_t tile_x, size_t tile_y) { // skip execution if noop if (noop_ptr != nullptr && noop_ptr[0] == 1.0f) { return; } - bool return_rowwise = rowwise_option != FP8BlockwiseRowwiseOption::NONE; + const bool return_rowwise = kBothGemmReady || rowwise_option != FP8BlockwiseRowwiseOption::NONE; bool return_columnwise_gemm_ready = - columnwise_option == FP8BlockwiseColumnwiseOption::COLUMNWISE_GEMM_READY; + kBothGemmReady || columnwise_option == FP8BlockwiseColumnwiseOption::COLUMNWISE_GEMM_READY; bool return_columnwise_compact = - columnwise_option == FP8BlockwiseColumnwiseOption::COLUMNWISE_COMPACT; + !kBothGemmReady && columnwise_option == FP8BlockwiseColumnwiseOption::COLUMNWISE_COMPACT; using SMemVec = Vec; using OVec = Vec; @@ -200,11 +219,10 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo constexpr int r_stride = kThreadsPerBlock / kNumThreadsLoad; // stride in rows of shared memory constexpr int num_iterations = kTileDim / r_stride; const int c_s = - (threadIdx.x % kNumThreadsLoad) * (kNVecIn / kNVecSMem); // Column in shared memory - int r_s = threadIdx.x / kNumThreadsLoad; // Row in shared memory - const size_t c_g = - static_cast(blockIdx.x) * kTileDim + c_s * kNVecSMem; // Column in global memory - size_t r_g = static_cast(blockIdx.y) * kTileDim + r_s; // Row in global memory + (threadIdx.x % kNumThreadsLoad) * (kNVecIn / kNVecSMem); // Column in shared memory + int r_s = threadIdx.x / kNumThreadsLoad; // Row in shared memory + const size_t c_g = tile_x * kTileDim + c_s * kNVecSMem; // Column in global memory + size_t r_g = tile_y * kTileDim + r_s; // Row in global memory const size_t stride_g = static_cast(r_stride) * row_length; // Stride in global memory const size_t num_ele = c_g < row_length ? min(static_cast(kNVecIn), row_length - c_g) : 0; // For not aligned case @@ -246,11 +264,10 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo kThreadsPerBlock / kNumThreadsStore; // stride in rows of shared memory constexpr int num_iterations = kTileDim / r_stride; const int c_s = - (threadIdx.x % kNumThreadsStore) * (kNVecOut / kNVecSMem); // Column in shared memory - int r_s = threadIdx.x / kNumThreadsStore; // Row in shared memory - const size_t c_g = - static_cast(blockIdx.x) * kTileDim + c_s * kNVecSMem; // Column in global memory - size_t r_g = static_cast(blockIdx.y) * kTileDim + r_s; // Row in global memory + (threadIdx.x % kNumThreadsStore) * (kNVecOut / kNVecSMem); // Column in shared memory + int r_s = threadIdx.x / kNumThreadsStore; // Row in shared memory + const size_t c_g = tile_x * kTileDim + c_s * kNVecSMem; // Column in global memory + size_t r_g = tile_y * kTileDim + r_s; // Row in global memory const size_t stride_g = static_cast(r_stride) * row_length; // Stride in global memory const size_t num_ele = c_g < row_length ? min(static_cast(kNVecOut), row_length - c_g) : 0; // For not aligned case @@ -273,12 +290,17 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo } // Step 2.2: Compute local amax CType amax = 0; + if constexpr (kBothGemmReady) { + const auto pair_amax = local_bf16_pair_amax(smem_vec); + amax = fmaxf(__low2float(pair_amax), __high2float(pair_amax)); + } else { #pragma unroll - for (int i = 0; i < kNVecOut / kNVecSMem; ++i) { + for (int i = 0; i < kNVecOut / kNVecSMem; ++i) { #pragma unroll - for (int j = 0; j < kNVecSMem; ++j) { - __builtin_assume(amax >= 0); - amax = fmaxf(amax, fabsf(smem_vec[i].data.elt[j])); + for (int j = 0; j < kNVecSMem; ++j) { + __builtin_assume(amax >= 0); + amax = fmaxf(amax, fabsf(smem_vec[i].data.elt[j])); + } } } // Step 2.3: Reduce amax @@ -300,8 +322,8 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo } if (write_scale_inv) { CType scale_inv = 1.0 / scale; - size_t row_idx = static_cast(blockIdx.y) * kTileDim + r_s; - size_t col_idx = static_cast(blockIdx.x); + size_t row_idx = tile_y * kTileDim + r_s; + size_t col_idx = tile_x; tile_scales_inv_c[row_idx * scale_stride_y + col_idx * scale_stride_x] = scale_inv; } // Step 2.6: Quantize @@ -338,9 +360,8 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo constexpr int num_iterations = kTileDim / (c_stride * kNVecSMem); const int r_s = (threadIdx.x % kNumThreadsStore) * kNVecOut; // Row in shared memory int c_s = threadIdx.x / kNumThreadsStore; // Column in shared memory - size_t r_g = - static_cast(blockIdx.x) * kTileDim + c_s * kNVecSMem; // Row in global memory - const size_t c_g = static_cast(blockIdx.y) * kTileDim + r_s; // Column in global memory + size_t r_g = tile_x * kTileDim + c_s * kNVecSMem; // Row in global memory + const size_t c_g = tile_y * kTileDim + r_s; // Column in global memory const size_t stride_g = static_cast(c_stride) * kNVecSMem * num_rows; // Stride in global memory const size_t num_ele = c_g < num_rows ? min(static_cast(kNVecOut), num_rows - c_g) @@ -362,13 +383,19 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo int c = c_s; smem_vec[i] = smem[r * kSMemCol + c]; } + __nv_bfloat162 pair_amax; + if constexpr (kBothGemmReady) pair_amax = local_bf16_pair_amax(smem_vec); #pragma unroll for (int smem_idx = 0; smem_idx < kNVecSMem; ++smem_idx) { // Step 3.2: Compute local amax CType amax = 0; + if constexpr (kBothGemmReady) { + amax = smem_idx == 0 ? __low2float(pair_amax) : __high2float(pair_amax); + } else { #pragma unroll - for (int i = 0; i < kNVecOut; ++i) { - amax = fmaxf(amax, fabsf(smem_vec[i].data.elt[smem_idx])); + for (int i = 0; i < kNVecOut; ++i) { + amax = fmaxf(amax, fabsf(smem_vec[i].data.elt[smem_idx])); + } } // Step 3.3: Reduce amax #pragma unroll @@ -389,8 +416,8 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo } if (write_scale_inv) { CType scale_inv = 1.0 / scale; - size_t row_idx = static_cast(blockIdx.x) * kTileDim + c_s * kNVecSMem + smem_idx; - size_t col_idx = static_cast(blockIdx.y); + size_t row_idx = tile_x * kTileDim + c_s * kNVecSMem + smem_idx; + size_t col_idx = tile_y; tile_scales_inv_t[row_idx * scale_t_stride_y + col_idx * scale_t_stride_x] = scale_inv; } // Step 3.6: Quantize @@ -434,11 +461,10 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo "num_iterations should be 1 for columnwise non-transpose case"); const int thr_idx_in_warp = threadIdx.x % kThreadsPerWarp; const int warp_idx = threadIdx.x / kThreadsPerWarp; - const int r_s = thr_idx_in_warp * kThreadTileRow; // Row in shared memory - int c_s = warp_idx * num_smem_reads; // Column in shared memory - size_t r_g = static_cast(blockIdx.y) * kTileDim + r_s; // Row in global memory - const size_t c_g = - static_cast(blockIdx.x) * kTileDim + c_s * kNVecSMem; // Column in global memory + const int r_s = thr_idx_in_warp * kThreadTileRow; // Row in shared memory + int c_s = warp_idx * num_smem_reads; // Column in shared memory + size_t r_g = tile_y * kTileDim + r_s; // Row in global memory + const size_t c_g = tile_x * kTileDim + c_s * kNVecSMem; // Column in global memory const size_t num_ele = c_g < row_length ? min(static_cast(kThreadTileCol), row_length - c_g) : 0; // For not aligned case @@ -486,8 +512,8 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo } if (write_scale_inv) { CType scale_inv = 1.0 / scale; - size_t row_idx = static_cast(blockIdx.y); - size_t col_idx = static_cast(blockIdx.x) * kTileDim + c_s * kNVecSMem + reg_idx; + size_t row_idx = tile_y; + size_t col_idx = tile_x * kTileDim + c_s * kNVecSMem + reg_idx; tile_scales_inv_t[row_idx * scale_t_stride_y + col_idx * scale_t_stride_x] = scale_inv; } } @@ -516,6 +542,74 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo } } +template +__global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpose_kernel( + const IType* const input, OType* const output_c, OType* const output_t, + CType* const tile_scales_inv_c, CType* const tile_scales_inv_t, const size_t row_length, + const size_t num_rows, const size_t scale_stride_x, const size_t scale_stride_y, + const size_t scale_t_stride_x, const size_t scale_t_stride_y, const float epsilon, + FP8BlockwiseRowwiseOption rowwise_option, FP8BlockwiseColumnwiseOption columnwise_option, + const bool pow_2_scaling, const float* noop_ptr) { + block_scaled_1d_cast_transpose_tile( + input, output_c, output_t, tile_scales_inv_c, tile_scales_inv_t, row_length, num_rows, + scale_stride_x, scale_stride_y, scale_t_stride_x, scale_t_stride_y, epsilon, rowwise_option, + columnwise_option, pow_2_scaling, noop_ptr, blockIdx.x, blockIdx.y); +} + +// By-value descriptors avoid device allocation/copies and host-buffer lifetime +// hazards during CUDA graph capture. Split larger lists into bounded launches. +constexpr int kMaxGroupedBlockwiseTensors = 32; +constexpr size_t kMaxGroupedGridSize = std::numeric_limits::max(); +struct GroupedBlockwiseTensor { + const bf16* input; + void* row; + void* col; + float* row_scale; + float* col_scale; + size_t m; + size_t row_scale_stride; + size_t col_scale_stride; + size_t tile_end; +}; +struct GroupedBlockwiseParams { + GroupedBlockwiseTensor tensors[kMaxGroupedBlockwiseTensors]; + size_t k; + int count; + float epsilon; + bool pow2_scale; +}; +static_assert(sizeof(GroupedBlockwiseParams) < 4096, + "Grouped descriptors must fit the portable kernel argument budget"); + +template +__global__ void __launch_bounds__(kThreadsPerBlock) grouped_block_scaled_1d_cast_transpose_kernel( + const __grid_constant__ GroupedBlockwiseParams params) { +#if __CUDA_ARCH__ >= 900 && __CUDA_ARCH__ < 1000 + const size_t tile = blockIdx.x; + int lo = 0, hi = params.count; + while (lo < hi) { + const int mid = lo + (hi - lo) / 2; + if (tile < params.tensors[mid].tile_end) { + hi = mid; + } else { + lo = mid + 1; + } + } + const auto tensor = params.tensors[lo]; + const size_t begin = lo == 0 ? 0 : params.tensors[lo - 1].tile_end; + const size_t local_tile = tile - begin; + const size_t tiles_k = DIVUP(params.k, size_t{kTileDim}); + block_scaled_1d_cast_transpose_tile( + tensor.input, static_cast(tensor.row), static_cast(tensor.col), + tensor.row_scale, tensor.col_scale, params.k, tensor.m, tensor.row_scale_stride, 1, + tensor.col_scale_stride, 1, params.epsilon, FP8BlockwiseRowwiseOption::ROWWISE_GEMM_READY, + FP8BlockwiseColumnwiseOption::COLUMNWISE_GEMM_READY, params.pow2_scale, nullptr, + local_tile % tiles_k, local_tile / tiles_k); +#else + asm volatile("trap;"); +#endif +} + } // namespace } // namespace transformer_engine @@ -638,4 +732,99 @@ void quantize_transpose_vector_blockwise(const SimpleTensor& input, SimpleTensor NVTE_CHECK_CUDA(cudaGetLastError()); } +// Return false before launching anything if any operand needs another path. +bool try_grouped_quantize_transpose_vector_blockwise(const NVTETensor* inputs, NVTETensor* outputs, + size_t num_tensors, + const QuantizationConfig& config, + cudaStream_t stream) { + if (num_tensors < 2 || transformer_engine::cuda::sm_arch() != 90 || + getenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", 1) == 0 || config.noop_tensor != nullptr || + config.stochastic_rounding) { + return false; + } + std::vector tensors; + tensors.reserve(num_tensors); + size_t common_k = 0; + DType output_dtype = DType::kFloat8E4M3; + for (size_t i = 0; i < num_tensors; ++i) { + const auto* input = convertNVTETensorCheck(inputs[i]); + const auto* output = convertNVTETensorCheck(outputs[i]); + if (input->data.dtype != DType::kBFloat16 || input->data.shape.size() != 2 || + output->scaling_mode != NVTE_BLOCK_SCALING_1D || output->with_gemm_swizzled_scales) { + return false; + } + const size_t m = input->data.shape[0], k = input->data.shape[1]; + if (k == 0 || (i != 0 && k != common_k)) { + return false; + } + common_k = k; + // Empty tensors may have null data/scales and contribute no tiles. + if (m == 0) { + continue; + } + if (input->data.dptr == nullptr || output->data.dptr == nullptr || + output->columnwise_data.dptr == nullptr || output->scale_inv.dptr == nullptr || + output->columnwise_scale_inv.dptr == nullptr || output->data.shape != input->data.shape || + output->columnwise_data.shape != std::vector{k, m} || + output->data.dtype != output->columnwise_data.dtype || + (output->data.dtype != DType::kFloat8E4M3 && output->data.dtype != DType::kFloat8E5M2) || + output->scale_inv.dtype != DType::kFloat32 || + output->columnwise_scale_inv.dtype != DType::kFloat32 || + output->scale_inv.shape.size() != 2 || output->columnwise_scale_inv.shape.size() != 2) { + return false; + } + if (!tensors.empty() && output->data.dtype != output_dtype) { + return false; + } + output_dtype = output->data.dtype; + const auto tiles_m = DIVUP(m, size_t{kTileDim}); + const auto tiles_k = DIVUP(k, size_t{kTileDim}); + if (tiles_m > kMaxGroupedGridSize / tiles_k || output->scale_inv.shape[0] < tiles_k || + output->scale_inv.shape[1] < m || output->columnwise_scale_inv.shape[0] < tiles_m || + output->columnwise_scale_inv.shape[1] < k) { + return false; + } + tensors.push_back({static_cast(input->data.dptr), output->data.dptr, + output->columnwise_data.dptr, static_cast(output->scale_inv.dptr), + static_cast(output->columnwise_scale_inv.dptr), m, + output->scale_inv.shape[1], output->columnwise_scale_inv.shape[1], + tiles_m * tiles_k}); + } + + // Every pointer is passed by value: works for bulk and separate allocations, + // caller-stream producer/consumer ordering, and repeated CUDA graph replay. + size_t begin = 0; + while (begin < tensors.size()) { + GroupedBlockwiseParams params{}; + params.k = common_k; + params.epsilon = config.amax_epsilon; + params.pow2_scale = config.force_pow_2_scales; + size_t tiles = 0; + bool aligned = common_k % kTileDim == 0; + while (begin < tensors.size() && params.count < kMaxGroupedBlockwiseTensors) { + const auto tensor_tiles = tensors[begin].tile_end; + if (tensor_tiles > kMaxGroupedGridSize - tiles) { + break; + } + auto tensor = tensors[begin++]; + tiles += tensor_tiles; + tensor.tile_end = tiles; + params.tensors[params.count++] = tensor; + aligned &= tensor.m % kTileDim == 0 && reinterpret_cast(tensor.input) % 16 == 0 && + reinterpret_cast(tensor.row) % 16 == 0 && + reinterpret_cast(tensor.col) % 16 == 0; + } + // The validation pass ensures a nonempty chunk always fits the grid limit. + const dim3 grid(static_cast(tiles)); + TRANSFORMER_ENGINE_TYPE_SWITCH_FP8ONLY( + output_dtype, OutputType, + TRANSFORMER_ENGINE_SWITCH_CONDITION( + aligned, kAligned, + grouped_block_scaled_1d_cast_transpose_kernel + <<>>(params););); + NVTE_CHECK_CUDA(cudaGetLastError()); + } + return true; +} + } // namespace transformer_engine::detail diff --git a/transformer_engine/pytorch/csrc/extensions/cast.cpp b/transformer_engine/pytorch/csrc/extensions/cast.cpp index 13dd26f6259..b3a1f85baaa 100644 --- a/transformer_engine/pytorch/csrc/extensions/cast.cpp +++ b/transformer_engine/pytorch/csrc/extensions/cast.cpp @@ -1196,6 +1196,51 @@ py::object group_requantize_inplace(py::handle grouped_x, py::handle quantizer, namespace { +bool try_grouped_blockwise_quantize(const std::vector &inputs, + const std::vector &quantizer_py_list, + std::vector> &quantizers, + std::vector &outputs) { + if (inputs.size() < 2 || + transformer_engine::getenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", 1) == 0 || + transformer_engine::cuda::sm_arch() != 90 || + !std::all_of(quantizer_py_list.begin(), quantizer_py_list.end(), [](const py::handle &q) { + return detail::IsFloat8BlockwiseQuantizers(q.ptr()); + })) { + return false; + } + const auto *reference = static_cast(quantizers[0].get()); + if (reference->dtype != DType::kFloat8E4M3 && reference->dtype != DType::kFloat8E5M2) { + return false; + } + size_t k = 0; + std::vector input_tensors, output_tensors; + input_tensors.reserve(inputs.size()); + output_tensors.reserve(outputs.size()); + for (size_t i = 0; i < inputs.size(); ++i) { + const auto *quantizer = static_cast(quantizers[i].get()); + const auto shape = inputs[i].shape(); + if (inputs[i].dtype() != DType::kBFloat16 || shape.ndim != 2 || shape.data[1] == 0 || + (i != 0 && shape.data[1] != k) || !quantizer->rowwise_usage || + !quantizer->columnwise_usage || quantizer->get_scaling_mode() != NVTE_BLOCK_SCALING_1D || + quantizer->dtype != reference->dtype || + quantizer->force_pow_2_scales != reference->force_pow_2_scales || + quantizer->amax_epsilon != reference->amax_epsilon) { + return false; + } + k = shape.data[1]; + input_tensors.push_back(inputs[i].data()); + output_tensors.push_back(outputs[i].data()); + } + QuantizationConfigWrapper config; + config.set_force_pow_2_scales(reference->force_pow_2_scales); + config.set_amax_epsilon(reference->amax_epsilon); + NVTE_SCOPED_GIL_RELEASE({ + nvte_multi_tensor_quantize(input_tensors.data(), output_tensors.data(), config, + input_tensors.size(), at::cuda::getCurrentCUDAStream()); + }); + return true; +} + void multi_tensor_quantize_impl(const std::vector &input_list, std::vector &quantizer_py_list, std::vector> &quantizer_cpp_list, @@ -1209,6 +1254,11 @@ void multi_tensor_quantize_impl(const std::vector &input_list, NVTE_CHECK(output_list.size() == num_tensors, "Expected ", num_tensors, " output tensors, but got ", output_list.size()); + if (try_grouped_blockwise_quantize(input_list, quantizer_py_list, quantizer_cpp_list, + output_list)) { + return; + } + // Choose implementation // Note: Currently only have fused kernel for FP8 delayed scaling bool with_fused_kernel = true; From abf2e551ebded4088bb6cb778174ee0557cba372 Mon Sep 17 00:00:00 2001 From: Akira Naruse Date: Thu, 19 Nov 2026 20:54:53 -0800 Subject: [PATCH 2/2] Fix grouped FP8 quantization PTX compatibility and extend coverage Compile the grouped kernel body for lower PTX targets while retaining runtime dispatch on Hopper. Clarify that CUDA Graph replay uses captured shapes and buffer addresses. Add special-value equivalence tests for packed BF16 amax and vary matrix width independently of power-of-two scaling. --- docs/envvars.rst | 5 ++ tests/pytorch/test_float8blockwisetensor.py | 65 +++++++++++++++++-- .../common/include/transformer_engine/cast.h | 3 + .../quantize_transpose_vector_blockwise.cu | 14 ++-- 4 files changed, 76 insertions(+), 11 deletions(-) diff --git a/docs/envvars.rst b/docs/envvars.rst index 50eb8f43a9e..9910b5bad7c 100644 --- a/docs/envvars.rst +++ b/docs/envvars.rst @@ -284,6 +284,11 @@ FP8 Configuration automatically. Other configurations retain their existing path. Set to ``0`` to disable the optimization for comparison. + M may vary across tensors and between API calls. CUDA Graph replay + uses the shapes and buffer addresses recorded during capture; + input values may change in place between replays. Changing M + between replays requires updating or recapturing the graph. + .. envvar:: NVTE_UNFUSED_FP8_UPDATE :Type: ``int`` (0 or 1) diff --git a/tests/pytorch/test_float8blockwisetensor.py b/tests/pytorch/test_float8blockwisetensor.py index c185738b05a..34ed82bc45f 100644 --- a/tests/pytorch/test_float8blockwisetensor.py +++ b/tests/pytorch/test_float8blockwisetensor.py @@ -653,7 +653,7 @@ def _snapshot_blockwise_outputs(outputs, shapes, block_dim): snapshots.append(None) continue # Consume on the caller stream immediately, before synchronizing. - snapshots.append(data.clone()) + snapshots.append(data.view(torch.uint8).clone()) if block_dim == 2: rows, cols = (m + 127) // 128, (k + 127) // 128 if direction == "columnwise": @@ -683,7 +683,8 @@ def _assert_blockwise_snapshots_equal(actual, expected): @pytest.mark.parametrize("k", [256, 257]) @pytest.mark.parametrize("bulk", [True, False]) @pytest.mark.parametrize("fp8_dtype", [DType.kFloat8E4M3, DType.kFloat8E5M2]) -def test_split_quantize_grouped(monkeypatch, splits, k, bulk, fp8_dtype): +@pytest.mark.parametrize("pow2_scale", [False, True]) +def test_split_quantize_grouped(monkeypatch, splits, k, bulk, fp8_dtype, pow2_scale): """Variable M, empty/partial tiles, chunk boundaries, and caller ordering.""" if get_device_compute_capability()[0] != 9: pytest.skip("Grouped BF16 blockwise path targets Hopper") @@ -694,7 +695,7 @@ def test_split_quantize_grouped(monkeypatch, splits, k, bulk, fp8_dtype): rowwise=True, columnwise=True, block_scaling_dim=1, - force_pow_2_scales=(k == 256), + force_pow_2_scales=pow2_scale, amax_epsilon=0.01, ) for _ in splits @@ -714,6 +715,62 @@ def test_split_quantize_grouped(monkeypatch, splits, k, bulk, fp8_dtype): _assert_blockwise_snapshots_equal(actual, expected) +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +@pytest.mark.parametrize("shape", [(128, 384), (129, 385)]) +@pytest.mark.parametrize("fp8_dtype", [DType.kFloat8E4M3, DType.kFloat8E5M2]) +@pytest.mark.parametrize("pow2_scale", [False, True]) +@pytest.mark.parametrize("case", ["nan", "inf", "zero", "tiny", "scaling_boundary"]) +def test_split_quantize_grouped_special_values(monkeypatch, shape, fp8_dtype, pow2_scale, case): + """Match scalar amax for special values in both aligned and partial tiles.""" + if get_device_compute_capability()[0] != 9: + pytest.skip("Grouped BF16 blockwise path targets Hopper") + m, k = shape + if case == "nan": + values = [float("nan")] + elif case == "inf": + values = [float("inf"), -float("inf"), 1.0, -1.0] + elif case == "zero": + values = [0.0, -0.0] + elif case == "tiny": + # Smallest/largest BF16 subnormals and smallest normal, with both signs. + values = [sign * n * 2.0**-133 for n in (1, 127, 128) for sign in (1.0, -1.0)] + else: + fp8_max = 448.0 if fp8_dtype == DType.kFloat8E4M3 else 57344.0 + boundary = fp8_max / 128.0 + ulp = math.ldexp(1.0, math.frexp(boundary)[1] - 8) + values = [sign * (boundary + delta * ulp) for sign in (1.0, -1.0) for delta in (-1, 0, 1)] + x = torch.empty(2 * m, k, dtype=torch.bfloat16, device="cuda") + tile = 0 + for expert in x.split(m): + for row in range(0, m, 128): + for col in range(0, k, 128): + # Uniform tiles keep each candidate visible as a block amax. + expert[row : row + 128, col : col + 128].fill_(values[tile % len(values)]) + tile += 1 + if case == "nan": + # Mix finite values with NaNs, retaining all-NaN rows and columns. + x[1::2, 1::2] = 1.0 + quantizers = [ + Float8BlockQuantizer( + fp8_dtype=fp8_dtype, + rowwise=True, + columnwise=True, + block_scaling_dim=1, + force_pow_2_scales=pow2_scale, + amax_epsilon=0.0, + ) + for _ in range(2) + ] + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "0") + reference = tex.split_quantize(x, [m, m], quantizers) + expected = _snapshot_blockwise_outputs(reference, [shape, shape], 1) + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "1") + outputs = tex.split_quantize(x, [m, m], quantizers) + actual = _snapshot_blockwise_outputs(outputs, [shape, shape], 1) + torch.cuda.synchronize() + _assert_blockwise_snapshots_equal(actual, expected) + + @pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) @pytest.mark.parametrize( "usage,block_dim,dtype,mixed_options", @@ -808,7 +865,7 @@ def test_split_quantize_grouped_dispatch(monkeypatch): @pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) def test_split_quantize_grouped_cuda_graph(monkeypatch): - """By-value metadata remains valid through capture and changed-input replays.""" + """Replay with fixed shapes and buffer addresses, changing only input values.""" if get_device_compute_capability()[0] != 9: pytest.skip("Grouped BF16 blockwise path targets Hopper") splits, k = [0, 129, 256, 511], 256 diff --git a/transformer_engine/common/include/transformer_engine/cast.h b/transformer_engine/common/include/transformer_engine/cast.h index 3d0a0553e64..044d9590f30 100644 --- a/transformer_engine/common/include/transformer_engine/cast.h +++ b/transformer_engine/common/include/transformer_engine/cast.h @@ -459,6 +459,9 @@ void nvte_group_requantize(const NVTEGroupedTensor input, NVTEGroupedTensor outp * On Hopper, compatible BF16 matrices with common K and both GEMM-ready 1D * block-scaled FP8 outputs use grouped quantization. M may vary per tensor. * Other configurations retain their existing quantization path. + * With CUDA Graph capture, shapes and buffer addresses are fixed for replay; + * input values may change in place. Different M values between API calls do not + * imply support for changing M between replays of the same captured graph. * * \param[in] inputs List of input tensors to be cast. * \param[in,out] outputs List of output quantized tensors. diff --git a/transformer_engine/common/transpose/quantize_transpose_vector_blockwise.cu b/transformer_engine/common/transpose/quantize_transpose_vector_blockwise.cu index 6874e042e6b..ac60204e352 100644 --- a/transformer_engine/common/transpose/quantize_transpose_vector_blockwise.cu +++ b/transformer_engine/common/transpose/quantize_transpose_vector_blockwise.cu @@ -557,7 +557,9 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo } // By-value descriptors avoid device allocation/copies and host-buffer lifetime -// hazards during CUDA graph capture. Split larger lists into bounded launches. +// hazards during CUDA graph capture. Captured descriptors fix shapes and buffer +// addresses for replay; input values may change in place. Split larger lists into +// bounded launches. constexpr int kMaxGroupedBlockwiseTensors = 32; constexpr size_t kMaxGroupedGridSize = std::numeric_limits::max(); struct GroupedBlockwiseTensor { @@ -581,10 +583,11 @@ struct GroupedBlockwiseParams { static_assert(sizeof(GroupedBlockwiseParams) < 4096, "Grouped descriptors must fit the portable kernel argument budget"); +// Compile the full body for lower PTX targets that may be JIT-compiled on Hopper. +// The host dispatcher restricts execution to SM90. template __global__ void __launch_bounds__(kThreadsPerBlock) grouped_block_scaled_1d_cast_transpose_kernel( const __grid_constant__ GroupedBlockwiseParams params) { -#if __CUDA_ARCH__ >= 900 && __CUDA_ARCH__ < 1000 const size_t tile = blockIdx.x; int lo = 0, hi = params.count; while (lo < hi) { @@ -605,9 +608,6 @@ __global__ void __launch_bounds__(kThreadsPerBlock) grouped_block_scaled_1d_cast tensor.col_scale_stride, 1, params.epsilon, FP8BlockwiseRowwiseOption::ROWWISE_GEMM_READY, FP8BlockwiseColumnwiseOption::COLUMNWISE_GEMM_READY, params.pow2_scale, nullptr, local_tile % tiles_k, local_tile / tiles_k); -#else - asm volatile("trap;"); -#endif } } // namespace @@ -791,8 +791,8 @@ bool try_grouped_quantize_transpose_vector_blockwise(const NVTETensor* inputs, N tiles_m * tiles_k}); } - // Every pointer is passed by value: works for bulk and separate allocations, - // caller-stream producer/consumer ordering, and repeated CUDA graph replay. + // By-value descriptors support bulk and separate allocations on the caller + // stream. Graph replays reuse captured shapes and pointers with new input values. size_t begin = 0; while (begin < tensors.size()) { GroupedBlockwiseParams params{};