diff --git a/docs/envvars.rst b/docs/envvars.rst index a0a64b3f637..9910b5bad7c 100644 --- a/docs/envvars.rst +++ b/docs/envvars.rst @@ -272,6 +272,23 @@ Context Parallelism FP8 Configuration ^^^^^^^^^^^^^^^^^ +.. envvar:: NVTE_FP8_BLOCK_QUANTIZATION_GROUPED + + :Type: ``int`` (0 or 1) + :Default: ``1`` + :Description: Enable grouped 1D blockwise FP8 quantization on Hopper for compatible + lists of at least two BF16 matrices. Inputs must share K, but M may differ, and + all quantizers must request both rowwise and columnwise outputs + with matching FP8 dtype, epsilon, and power-of-two scaling options. + The existing split and multi-tensor APIs select this path + automatically. Other configurations retain their existing path. + Set to ``0`` to disable the optimization for comparison. + + M may vary across tensors and between API calls. CUDA Graph replay + uses the shapes and buffer addresses recorded during capture; + input values may change in place between replays. Changing M + between replays requires updating or recapturing the graph. + .. envvar:: NVTE_UNFUSED_FP8_UPDATE :Type: ``int`` (0 or 1) diff --git a/tests/pytorch/test_float8blockwisetensor.py b/tests/pytorch/test_float8blockwisetensor.py index ab00786e4fe..34ed82bc45f 100644 --- a/tests/pytorch/test_float8blockwisetensor.py +++ b/tests/pytorch/test_float8blockwisetensor.py @@ -641,3 +641,260 @@ def test_clone_detach( # Make sure we are not trivially passing tests with pytest.raises(AssertionError): torch.testing.assert_close(x_clone.dequantize(), -x_hp, **_tols[fp8_dtype]) + + +def _snapshot_blockwise_outputs(outputs, shapes, block_dim): + snapshots = [] + for out, (m, k) in zip(outputs, shapes): + for direction in ("rowwise", "columnwise"): + data = getattr(out, f"_{direction}_data") + scale = getattr(out, f"_{direction}_scale_inv") + if data is None: + snapshots.append(None) + continue + # Consume on the caller stream immediately, before synchronizing. + snapshots.append(data.view(torch.uint8).clone()) + if block_dim == 2: + rows, cols = (m + 127) // 128, (k + 127) // 128 + if direction == "columnwise": + rows, cols = cols, rows + elif direction == "rowwise": + rows, cols = (k + 127) // 128, m + else: + rows, cols = (m + 127) // 128, k + # Scale storage includes unspecified GEMM-alignment padding. + snapshots.append(scale[:rows, :cols].clone()) + return snapshots + + +def _assert_blockwise_snapshots_equal(actual, expected): + assert len(actual) == len(expected) + for a, b in zip(actual, expected): + if a is None or b is None: + assert a is None and b is None + else: + torch.testing.assert_close(a, b, rtol=0, atol=0) + + +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +@pytest.mark.parametrize( + "splits", [[128, 0, 256, 384] * 4, [0, 1, 127, 129, 255], [129] * 33, [0, 0], [129]] +) +@pytest.mark.parametrize("k", [256, 257]) +@pytest.mark.parametrize("bulk", [True, False]) +@pytest.mark.parametrize("fp8_dtype", [DType.kFloat8E4M3, DType.kFloat8E5M2]) +@pytest.mark.parametrize("pow2_scale", [False, True]) +def test_split_quantize_grouped(monkeypatch, splits, k, bulk, fp8_dtype, pow2_scale): + """Variable M, empty/partial tiles, chunk boundaries, and caller ordering.""" + if get_device_compute_capability()[0] != 9: + pytest.skip("Grouped BF16 blockwise path targets Hopper") + shapes = [(m, k) for m in splits] + quantizers = [ + Float8BlockQuantizer( + fp8_dtype=fp8_dtype, + rowwise=True, + columnwise=True, + block_scaling_dim=1, + force_pow_2_scales=pow2_scale, + amax_epsilon=0.01, + ) + for _ in splits + ] + caller = torch.cuda.Stream() + with torch.cuda.stream(caller): + for _ in range(2): + x = torch.randn(sum(splits), k, dtype=torch.bfloat16, device="cuda") + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "0") + reference = tex.split_quantize(x, splits, quantizers, disable_bulk_allocation=not bulk) + expected = _snapshot_blockwise_outputs(reference, shapes, 1) + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "1") + outputs = tex.split_quantize(x, splits, quantizers, disable_bulk_allocation=not bulk) + actual = _snapshot_blockwise_outputs(outputs, shapes, 1) + assert torch.cuda.current_stream() == caller + caller.synchronize() + _assert_blockwise_snapshots_equal(actual, expected) + + +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +@pytest.mark.parametrize("shape", [(128, 384), (129, 385)]) +@pytest.mark.parametrize("fp8_dtype", [DType.kFloat8E4M3, DType.kFloat8E5M2]) +@pytest.mark.parametrize("pow2_scale", [False, True]) +@pytest.mark.parametrize("case", ["nan", "inf", "zero", "tiny", "scaling_boundary"]) +def test_split_quantize_grouped_special_values(monkeypatch, shape, fp8_dtype, pow2_scale, case): + """Match scalar amax for special values in both aligned and partial tiles.""" + if get_device_compute_capability()[0] != 9: + pytest.skip("Grouped BF16 blockwise path targets Hopper") + m, k = shape + if case == "nan": + values = [float("nan")] + elif case == "inf": + values = [float("inf"), -float("inf"), 1.0, -1.0] + elif case == "zero": + values = [0.0, -0.0] + elif case == "tiny": + # Smallest/largest BF16 subnormals and smallest normal, with both signs. + values = [sign * n * 2.0**-133 for n in (1, 127, 128) for sign in (1.0, -1.0)] + else: + fp8_max = 448.0 if fp8_dtype == DType.kFloat8E4M3 else 57344.0 + boundary = fp8_max / 128.0 + ulp = math.ldexp(1.0, math.frexp(boundary)[1] - 8) + values = [sign * (boundary + delta * ulp) for sign in (1.0, -1.0) for delta in (-1, 0, 1)] + x = torch.empty(2 * m, k, dtype=torch.bfloat16, device="cuda") + tile = 0 + for expert in x.split(m): + for row in range(0, m, 128): + for col in range(0, k, 128): + # Uniform tiles keep each candidate visible as a block amax. + expert[row : row + 128, col : col + 128].fill_(values[tile % len(values)]) + tile += 1 + if case == "nan": + # Mix finite values with NaNs, retaining all-NaN rows and columns. + x[1::2, 1::2] = 1.0 + quantizers = [ + Float8BlockQuantizer( + fp8_dtype=fp8_dtype, + rowwise=True, + columnwise=True, + block_scaling_dim=1, + force_pow_2_scales=pow2_scale, + amax_epsilon=0.0, + ) + for _ in range(2) + ] + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "0") + reference = tex.split_quantize(x, [m, m], quantizers) + expected = _snapshot_blockwise_outputs(reference, [shape, shape], 1) + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "1") + outputs = tex.split_quantize(x, [m, m], quantizers) + actual = _snapshot_blockwise_outputs(outputs, [shape, shape], 1) + torch.cuda.synchronize() + _assert_blockwise_snapshots_equal(actual, expected) + + +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +@pytest.mark.parametrize( + "usage,block_dim,dtype,mixed_options", + [ + ((True, False), 1, torch.bfloat16, False), + ((False, True), 1, torch.bfloat16, False), + ((True, True), 2, torch.bfloat16, False), + ((True, True), 1, torch.float16, False), + ((True, True), 1, torch.bfloat16, True), + ], +) +def test_split_quantize_grouped_fallback(monkeypatch, usage, block_dim, dtype, mixed_options): + """Keep single-direction, 2D scaling, other dtypes and per-expert options.""" + splits, k = [128, 256, 0], 256 + quantizers = [ + Float8BlockQuantizer( + fp8_dtype=DType.kFloat8E4M3, + rowwise=usage[0], + columnwise=usage[1], + block_scaling_dim=block_dim, + amax_epsilon=(0.1 * i if mixed_options else 0.0), + ) + for i in range(len(splits)) + ] + x = torch.randn(sum(splits), k, dtype=dtype, device="cuda") + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "0") + reference = tex.split_quantize(x, splits, quantizers) + expected = _snapshot_blockwise_outputs(reference, [(m, k) for m in splits], block_dim) + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "1") + outputs = tex.split_quantize(x, splits, quantizers) + actual = _snapshot_blockwise_outputs(outputs, [(m, k) for m in splits], block_dim) + torch.cuda.synchronize() + _assert_blockwise_snapshots_equal(actual, expected) + + +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +@pytest.mark.parametrize("common_k", [True, False]) +def test_multi_tensor_quantize_grouped(monkeypatch, common_k): + """Support separate allocations; mixed K must retain the existing path.""" + shapes = [(129, 256), (256, 256 if common_k else 384)] + inputs = [torch.randn(shape, dtype=torch.bfloat16, device="cuda") for shape in shapes] + quantizers = [ + Float8BlockQuantizer( + fp8_dtype=DType.kFloat8E4M3, + rowwise=True, + columnwise=True, + block_scaling_dim=1, + ) + for _ in inputs + ] + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "0") + reference = tex.multi_tensor_quantize(inputs, quantizers) + expected = _snapshot_blockwise_outputs(reference, shapes, 1) + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "1") + outputs = tex.multi_tensor_quantize(inputs, quantizers) + actual = _snapshot_blockwise_outputs(outputs, shapes, 1) + torch.cuda.synchronize() + _assert_blockwise_snapshots_equal(actual, expected) + + +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +def test_split_quantize_grouped_dispatch(monkeypatch): + """Ensure a qualifying call actually executes one grouped CUDA kernel.""" + if get_device_compute_capability()[0] != 9: + pytest.skip("Grouped BF16 blockwise path targets Hopper") + splits, k = [128, 0, 256, 384], 256 + quantizers = [ + Float8BlockQuantizer( + fp8_dtype=DType.kFloat8E4M3, + rowwise=True, + columnwise=True, + block_scaling_dim=1, + ) + for _ in splits + ] + x = torch.randn(sum(splits), k, dtype=torch.bfloat16, device="cuda") + monkeypatch.delenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", raising=False) + tex.split_quantize(x, splits, quantizers) + torch.cuda.synchronize() + with torch.profiler.profile( + activities=[torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA] + ) as profile: + tex.split_quantize(x, splits, quantizers) + torch.cuda.synchronize() + kernels = [ + event + for event in profile.events() + if "grouped_block_scaled_1d_cast_transpose_kernel" in event.name + ] + assert len(kernels) == 1, "Expected one grouped kernel; rebuild the TE core library as well" + + +@pytest.mark.skipif(not recipe_available, reason=reason_for_no_recipe) +def test_split_quantize_grouped_cuda_graph(monkeypatch): + """Replay with fixed shapes and buffer addresses, changing only input values.""" + if get_device_compute_capability()[0] != 9: + pytest.skip("Grouped BF16 blockwise path targets Hopper") + splits, k = [0, 129, 256, 511], 256 + quantizers = [ + Float8BlockQuantizer( + fp8_dtype=DType.kFloat8E4M3, + rowwise=True, + columnwise=True, + block_scaling_dim=1, + ) + for _ in splits + ] + x = torch.ones(sum(splits), k, dtype=torch.bfloat16, device="cuda") + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "1") + warmup = torch.cuda.Stream() + warmup.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(warmup): + for _ in range(3): + tex.split_quantize(x, splits, quantizers) + torch.cuda.current_stream().wait_stream(warmup) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outputs = tex.split_quantize(x, splits, quantizers) + for value in (0.0, 0.5, 2.0, -3.0): + x.fill_(value) + graph.replay() + actual = _snapshot_blockwise_outputs(outputs, [(m, k) for m in splits], 1) + monkeypatch.setenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", "0") + reference = tex.split_quantize(x, splits, quantizers) + expected = _snapshot_blockwise_outputs(reference, [(m, k) for m in splits], 1) + torch.cuda.synchronize() + _assert_blockwise_snapshots_equal(actual, expected) diff --git a/transformer_engine/common/cast/cast.cu b/transformer_engine/common/cast/cast.cu index 1e3c04573be..f579cf83437 100644 --- a/transformer_engine/common/cast/cast.cu +++ b/transformer_engine/common/cast/cast.cu @@ -60,6 +60,15 @@ void nvte_multi_tensor_quantize(const NVTETensor *inputs, NVTETensor *outputs, NVTE_API_CALL(nvte_multi_tensor_quantize); using namespace transformer_engine; + QuantizationConfig config; + if (quant_configs != nullptr) { + config = *reinterpret_cast(quant_configs); + } + if (detail::try_grouped_quantize_transpose_vector_blockwise(inputs, outputs, num_tensors, config, + stream)) { + return; + } + constexpr bool IS_ACT = false; const size_t num_streams = nvte_get_num_compute_streams(); diff --git a/transformer_engine/common/include/transformer_engine/cast.h b/transformer_engine/common/include/transformer_engine/cast.h index 1af93d42919..044d9590f30 100644 --- a/transformer_engine/common/include/transformer_engine/cast.h +++ b/transformer_engine/common/include/transformer_engine/cast.h @@ -455,6 +455,13 @@ void nvte_group_requantize(const NVTEGroupedTensor input, NVTEGroupedTensor outp cudaStream_t stream); /*! \brief Casts multiple input tensors to quantized output tensors. + * + * On Hopper, compatible BF16 matrices with common K and both GEMM-ready 1D + * block-scaled FP8 outputs use grouped quantization. M may vary per tensor. + * Other configurations retain their existing quantization path. + * With CUDA Graph capture, shapes and buffer addresses are fixed for replay; + * input values may change in place. Different M values between API calls do not + * imply support for changing M between replays of the same captured graph. * * \param[in] inputs List of input tensors to be cast. * \param[in,out] outputs List of output quantized tensors. diff --git a/transformer_engine/common/transpose/cast_transpose.h b/transformer_engine/common/transpose/cast_transpose.h index c462b301475..309564490a3 100644 --- a/transformer_engine/common/transpose/cast_transpose.h +++ b/transformer_engine/common/transpose/cast_transpose.h @@ -54,6 +54,14 @@ enum class FP8BlockwiseColumnwiseOption { COLUMNWISE_COMPACT }; +// Try grouped BF16-to-FP8 quantization with both GEMM-ready orientations. +// All inputs must have the same K; M may vary. Returns false before launching +// work if any tensor/configuration needs the existing per-tensor path. +bool try_grouped_quantize_transpose_vector_blockwise(const NVTETensor *inputs, NVTETensor *outputs, + size_t num_tensors, + const QuantizationConfig &config, + cudaStream_t stream); + void quantize_transpose_vector_blockwise(const SimpleTensor &input, SimpleTensor &scale_inv, SimpleTensor &scale_inv_t, SimpleTensor &output, SimpleTensor &output_t, const float epsilon, diff --git a/transformer_engine/common/transpose/quantize_transpose_vector_blockwise.cu b/transformer_engine/common/transpose/quantize_transpose_vector_blockwise.cu index df869b43317..ac60204e352 100644 --- a/transformer_engine/common/transpose/quantize_transpose_vector_blockwise.cu +++ b/transformer_engine/common/transpose/quantize_transpose_vector_blockwise.cu @@ -11,13 +11,17 @@ #include #include +#include #include +#include #include +#include #include "common/common.h" #include "common/recipe/recipe_common.cuh" #include "common/transpose/cast_transpose.h" #include "common/util/cuda_runtime.h" +#include "common/util/system.h" #include "common/utils.cuh" namespace transformer_engine { @@ -166,24 +170,39 @@ static_assert(kNumThreadsLoad <= kThreadsPerWarp, "kNumThreadsLoad must be <= kT static_assert(kNumThreadsStore <= kThreadsPerWarp, "kNumThreadsStore must be <= kThreadsPerWarp"); constexpr int kNumWarps = kThreadsPerBlock / kThreadsPerWarp; -template -__global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpose_kernel( +// Packed BF16 local amax, used only by the grouped BF16 specialization. +// The padded shared-memory layout and eight-lane shuffle reductions are kept. +template +__device__ __forceinline__ __nv_bfloat162 +local_bf16_pair_amax(const Vec (&values)[Count]) { + __nv_bfloat162 amax = __floats2bfloat162_rn(0.f, 0.f); +#pragma unroll + for (int i = 0; i < Count; ++i) { + const auto pair = __halves2bfloat162(values[i].data.elt[0], values[i].data.elt[1]); + amax = __hmax2(amax, __habs2(pair)); + } + return amax; +} + +template +__device__ __forceinline__ void block_scaled_1d_cast_transpose_tile( const IType* const input, OType* const output_c, OType* const output_t, CType* const tile_scales_inv_c, CType* const tile_scales_inv_t, const size_t row_length, const size_t num_rows, const size_t scale_stride_x, const size_t scale_stride_y, const size_t scale_t_stride_x, const size_t scale_t_stride_y, const float epsilon, FP8BlockwiseRowwiseOption rowwise_option, FP8BlockwiseColumnwiseOption columnwise_option, - const bool pow_2_scaling, const float* noop_ptr) { + const bool pow_2_scaling, const float* noop_ptr, size_t tile_x, size_t tile_y) { // skip execution if noop if (noop_ptr != nullptr && noop_ptr[0] == 1.0f) { return; } - bool return_rowwise = rowwise_option != FP8BlockwiseRowwiseOption::NONE; + const bool return_rowwise = kBothGemmReady || rowwise_option != FP8BlockwiseRowwiseOption::NONE; bool return_columnwise_gemm_ready = - columnwise_option == FP8BlockwiseColumnwiseOption::COLUMNWISE_GEMM_READY; + kBothGemmReady || columnwise_option == FP8BlockwiseColumnwiseOption::COLUMNWISE_GEMM_READY; bool return_columnwise_compact = - columnwise_option == FP8BlockwiseColumnwiseOption::COLUMNWISE_COMPACT; + !kBothGemmReady && columnwise_option == FP8BlockwiseColumnwiseOption::COLUMNWISE_COMPACT; using SMemVec = Vec; using OVec = Vec; @@ -200,11 +219,10 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo constexpr int r_stride = kThreadsPerBlock / kNumThreadsLoad; // stride in rows of shared memory constexpr int num_iterations = kTileDim / r_stride; const int c_s = - (threadIdx.x % kNumThreadsLoad) * (kNVecIn / kNVecSMem); // Column in shared memory - int r_s = threadIdx.x / kNumThreadsLoad; // Row in shared memory - const size_t c_g = - static_cast(blockIdx.x) * kTileDim + c_s * kNVecSMem; // Column in global memory - size_t r_g = static_cast(blockIdx.y) * kTileDim + r_s; // Row in global memory + (threadIdx.x % kNumThreadsLoad) * (kNVecIn / kNVecSMem); // Column in shared memory + int r_s = threadIdx.x / kNumThreadsLoad; // Row in shared memory + const size_t c_g = tile_x * kTileDim + c_s * kNVecSMem; // Column in global memory + size_t r_g = tile_y * kTileDim + r_s; // Row in global memory const size_t stride_g = static_cast(r_stride) * row_length; // Stride in global memory const size_t num_ele = c_g < row_length ? min(static_cast(kNVecIn), row_length - c_g) : 0; // For not aligned case @@ -246,11 +264,10 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo kThreadsPerBlock / kNumThreadsStore; // stride in rows of shared memory constexpr int num_iterations = kTileDim / r_stride; const int c_s = - (threadIdx.x % kNumThreadsStore) * (kNVecOut / kNVecSMem); // Column in shared memory - int r_s = threadIdx.x / kNumThreadsStore; // Row in shared memory - const size_t c_g = - static_cast(blockIdx.x) * kTileDim + c_s * kNVecSMem; // Column in global memory - size_t r_g = static_cast(blockIdx.y) * kTileDim + r_s; // Row in global memory + (threadIdx.x % kNumThreadsStore) * (kNVecOut / kNVecSMem); // Column in shared memory + int r_s = threadIdx.x / kNumThreadsStore; // Row in shared memory + const size_t c_g = tile_x * kTileDim + c_s * kNVecSMem; // Column in global memory + size_t r_g = tile_y * kTileDim + r_s; // Row in global memory const size_t stride_g = static_cast(r_stride) * row_length; // Stride in global memory const size_t num_ele = c_g < row_length ? min(static_cast(kNVecOut), row_length - c_g) : 0; // For not aligned case @@ -273,12 +290,17 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo } // Step 2.2: Compute local amax CType amax = 0; + if constexpr (kBothGemmReady) { + const auto pair_amax = local_bf16_pair_amax(smem_vec); + amax = fmaxf(__low2float(pair_amax), __high2float(pair_amax)); + } else { #pragma unroll - for (int i = 0; i < kNVecOut / kNVecSMem; ++i) { + for (int i = 0; i < kNVecOut / kNVecSMem; ++i) { #pragma unroll - for (int j = 0; j < kNVecSMem; ++j) { - __builtin_assume(amax >= 0); - amax = fmaxf(amax, fabsf(smem_vec[i].data.elt[j])); + for (int j = 0; j < kNVecSMem; ++j) { + __builtin_assume(amax >= 0); + amax = fmaxf(amax, fabsf(smem_vec[i].data.elt[j])); + } } } // Step 2.3: Reduce amax @@ -300,8 +322,8 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo } if (write_scale_inv) { CType scale_inv = 1.0 / scale; - size_t row_idx = static_cast(blockIdx.y) * kTileDim + r_s; - size_t col_idx = static_cast(blockIdx.x); + size_t row_idx = tile_y * kTileDim + r_s; + size_t col_idx = tile_x; tile_scales_inv_c[row_idx * scale_stride_y + col_idx * scale_stride_x] = scale_inv; } // Step 2.6: Quantize @@ -338,9 +360,8 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo constexpr int num_iterations = kTileDim / (c_stride * kNVecSMem); const int r_s = (threadIdx.x % kNumThreadsStore) * kNVecOut; // Row in shared memory int c_s = threadIdx.x / kNumThreadsStore; // Column in shared memory - size_t r_g = - static_cast(blockIdx.x) * kTileDim + c_s * kNVecSMem; // Row in global memory - const size_t c_g = static_cast(blockIdx.y) * kTileDim + r_s; // Column in global memory + size_t r_g = tile_x * kTileDim + c_s * kNVecSMem; // Row in global memory + const size_t c_g = tile_y * kTileDim + r_s; // Column in global memory const size_t stride_g = static_cast(c_stride) * kNVecSMem * num_rows; // Stride in global memory const size_t num_ele = c_g < num_rows ? min(static_cast(kNVecOut), num_rows - c_g) @@ -362,13 +383,19 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo int c = c_s; smem_vec[i] = smem[r * kSMemCol + c]; } + __nv_bfloat162 pair_amax; + if constexpr (kBothGemmReady) pair_amax = local_bf16_pair_amax(smem_vec); #pragma unroll for (int smem_idx = 0; smem_idx < kNVecSMem; ++smem_idx) { // Step 3.2: Compute local amax CType amax = 0; + if constexpr (kBothGemmReady) { + amax = smem_idx == 0 ? __low2float(pair_amax) : __high2float(pair_amax); + } else { #pragma unroll - for (int i = 0; i < kNVecOut; ++i) { - amax = fmaxf(amax, fabsf(smem_vec[i].data.elt[smem_idx])); + for (int i = 0; i < kNVecOut; ++i) { + amax = fmaxf(amax, fabsf(smem_vec[i].data.elt[smem_idx])); + } } // Step 3.3: Reduce amax #pragma unroll @@ -389,8 +416,8 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo } if (write_scale_inv) { CType scale_inv = 1.0 / scale; - size_t row_idx = static_cast(blockIdx.x) * kTileDim + c_s * kNVecSMem + smem_idx; - size_t col_idx = static_cast(blockIdx.y); + size_t row_idx = tile_x * kTileDim + c_s * kNVecSMem + smem_idx; + size_t col_idx = tile_y; tile_scales_inv_t[row_idx * scale_t_stride_y + col_idx * scale_t_stride_x] = scale_inv; } // Step 3.6: Quantize @@ -434,11 +461,10 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo "num_iterations should be 1 for columnwise non-transpose case"); const int thr_idx_in_warp = threadIdx.x % kThreadsPerWarp; const int warp_idx = threadIdx.x / kThreadsPerWarp; - const int r_s = thr_idx_in_warp * kThreadTileRow; // Row in shared memory - int c_s = warp_idx * num_smem_reads; // Column in shared memory - size_t r_g = static_cast(blockIdx.y) * kTileDim + r_s; // Row in global memory - const size_t c_g = - static_cast(blockIdx.x) * kTileDim + c_s * kNVecSMem; // Column in global memory + const int r_s = thr_idx_in_warp * kThreadTileRow; // Row in shared memory + int c_s = warp_idx * num_smem_reads; // Column in shared memory + size_t r_g = tile_y * kTileDim + r_s; // Row in global memory + const size_t c_g = tile_x * kTileDim + c_s * kNVecSMem; // Column in global memory const size_t num_ele = c_g < row_length ? min(static_cast(kThreadTileCol), row_length - c_g) : 0; // For not aligned case @@ -486,8 +512,8 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo } if (write_scale_inv) { CType scale_inv = 1.0 / scale; - size_t row_idx = static_cast(blockIdx.y); - size_t col_idx = static_cast(blockIdx.x) * kTileDim + c_s * kNVecSMem + reg_idx; + size_t row_idx = tile_y; + size_t col_idx = tile_x * kTileDim + c_s * kNVecSMem + reg_idx; tile_scales_inv_t[row_idx * scale_t_stride_y + col_idx * scale_t_stride_x] = scale_inv; } } @@ -516,6 +542,74 @@ __global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpo } } +template +__global__ void __launch_bounds__(kThreadsPerBlock) block_scaled_1d_cast_transpose_kernel( + const IType* const input, OType* const output_c, OType* const output_t, + CType* const tile_scales_inv_c, CType* const tile_scales_inv_t, const size_t row_length, + const size_t num_rows, const size_t scale_stride_x, const size_t scale_stride_y, + const size_t scale_t_stride_x, const size_t scale_t_stride_y, const float epsilon, + FP8BlockwiseRowwiseOption rowwise_option, FP8BlockwiseColumnwiseOption columnwise_option, + const bool pow_2_scaling, const float* noop_ptr) { + block_scaled_1d_cast_transpose_tile( + input, output_c, output_t, tile_scales_inv_c, tile_scales_inv_t, row_length, num_rows, + scale_stride_x, scale_stride_y, scale_t_stride_x, scale_t_stride_y, epsilon, rowwise_option, + columnwise_option, pow_2_scaling, noop_ptr, blockIdx.x, blockIdx.y); +} + +// By-value descriptors avoid device allocation/copies and host-buffer lifetime +// hazards during CUDA graph capture. Captured descriptors fix shapes and buffer +// addresses for replay; input values may change in place. Split larger lists into +// bounded launches. +constexpr int kMaxGroupedBlockwiseTensors = 32; +constexpr size_t kMaxGroupedGridSize = std::numeric_limits::max(); +struct GroupedBlockwiseTensor { + const bf16* input; + void* row; + void* col; + float* row_scale; + float* col_scale; + size_t m; + size_t row_scale_stride; + size_t col_scale_stride; + size_t tile_end; +}; +struct GroupedBlockwiseParams { + GroupedBlockwiseTensor tensors[kMaxGroupedBlockwiseTensors]; + size_t k; + int count; + float epsilon; + bool pow2_scale; +}; +static_assert(sizeof(GroupedBlockwiseParams) < 4096, + "Grouped descriptors must fit the portable kernel argument budget"); + +// Compile the full body for lower PTX targets that may be JIT-compiled on Hopper. +// The host dispatcher restricts execution to SM90. +template +__global__ void __launch_bounds__(kThreadsPerBlock) grouped_block_scaled_1d_cast_transpose_kernel( + const __grid_constant__ GroupedBlockwiseParams params) { + const size_t tile = blockIdx.x; + int lo = 0, hi = params.count; + while (lo < hi) { + const int mid = lo + (hi - lo) / 2; + if (tile < params.tensors[mid].tile_end) { + hi = mid; + } else { + lo = mid + 1; + } + } + const auto tensor = params.tensors[lo]; + const size_t begin = lo == 0 ? 0 : params.tensors[lo - 1].tile_end; + const size_t local_tile = tile - begin; + const size_t tiles_k = DIVUP(params.k, size_t{kTileDim}); + block_scaled_1d_cast_transpose_tile( + tensor.input, static_cast(tensor.row), static_cast(tensor.col), + tensor.row_scale, tensor.col_scale, params.k, tensor.m, tensor.row_scale_stride, 1, + tensor.col_scale_stride, 1, params.epsilon, FP8BlockwiseRowwiseOption::ROWWISE_GEMM_READY, + FP8BlockwiseColumnwiseOption::COLUMNWISE_GEMM_READY, params.pow2_scale, nullptr, + local_tile % tiles_k, local_tile / tiles_k); +} + } // namespace } // namespace transformer_engine @@ -638,4 +732,99 @@ void quantize_transpose_vector_blockwise(const SimpleTensor& input, SimpleTensor NVTE_CHECK_CUDA(cudaGetLastError()); } +// Return false before launching anything if any operand needs another path. +bool try_grouped_quantize_transpose_vector_blockwise(const NVTETensor* inputs, NVTETensor* outputs, + size_t num_tensors, + const QuantizationConfig& config, + cudaStream_t stream) { + if (num_tensors < 2 || transformer_engine::cuda::sm_arch() != 90 || + getenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", 1) == 0 || config.noop_tensor != nullptr || + config.stochastic_rounding) { + return false; + } + std::vector tensors; + tensors.reserve(num_tensors); + size_t common_k = 0; + DType output_dtype = DType::kFloat8E4M3; + for (size_t i = 0; i < num_tensors; ++i) { + const auto* input = convertNVTETensorCheck(inputs[i]); + const auto* output = convertNVTETensorCheck(outputs[i]); + if (input->data.dtype != DType::kBFloat16 || input->data.shape.size() != 2 || + output->scaling_mode != NVTE_BLOCK_SCALING_1D || output->with_gemm_swizzled_scales) { + return false; + } + const size_t m = input->data.shape[0], k = input->data.shape[1]; + if (k == 0 || (i != 0 && k != common_k)) { + return false; + } + common_k = k; + // Empty tensors may have null data/scales and contribute no tiles. + if (m == 0) { + continue; + } + if (input->data.dptr == nullptr || output->data.dptr == nullptr || + output->columnwise_data.dptr == nullptr || output->scale_inv.dptr == nullptr || + output->columnwise_scale_inv.dptr == nullptr || output->data.shape != input->data.shape || + output->columnwise_data.shape != std::vector{k, m} || + output->data.dtype != output->columnwise_data.dtype || + (output->data.dtype != DType::kFloat8E4M3 && output->data.dtype != DType::kFloat8E5M2) || + output->scale_inv.dtype != DType::kFloat32 || + output->columnwise_scale_inv.dtype != DType::kFloat32 || + output->scale_inv.shape.size() != 2 || output->columnwise_scale_inv.shape.size() != 2) { + return false; + } + if (!tensors.empty() && output->data.dtype != output_dtype) { + return false; + } + output_dtype = output->data.dtype; + const auto tiles_m = DIVUP(m, size_t{kTileDim}); + const auto tiles_k = DIVUP(k, size_t{kTileDim}); + if (tiles_m > kMaxGroupedGridSize / tiles_k || output->scale_inv.shape[0] < tiles_k || + output->scale_inv.shape[1] < m || output->columnwise_scale_inv.shape[0] < tiles_m || + output->columnwise_scale_inv.shape[1] < k) { + return false; + } + tensors.push_back({static_cast(input->data.dptr), output->data.dptr, + output->columnwise_data.dptr, static_cast(output->scale_inv.dptr), + static_cast(output->columnwise_scale_inv.dptr), m, + output->scale_inv.shape[1], output->columnwise_scale_inv.shape[1], + tiles_m * tiles_k}); + } + + // By-value descriptors support bulk and separate allocations on the caller + // stream. Graph replays reuse captured shapes and pointers with new input values. + size_t begin = 0; + while (begin < tensors.size()) { + GroupedBlockwiseParams params{}; + params.k = common_k; + params.epsilon = config.amax_epsilon; + params.pow2_scale = config.force_pow_2_scales; + size_t tiles = 0; + bool aligned = common_k % kTileDim == 0; + while (begin < tensors.size() && params.count < kMaxGroupedBlockwiseTensors) { + const auto tensor_tiles = tensors[begin].tile_end; + if (tensor_tiles > kMaxGroupedGridSize - tiles) { + break; + } + auto tensor = tensors[begin++]; + tiles += tensor_tiles; + tensor.tile_end = tiles; + params.tensors[params.count++] = tensor; + aligned &= tensor.m % kTileDim == 0 && reinterpret_cast(tensor.input) % 16 == 0 && + reinterpret_cast(tensor.row) % 16 == 0 && + reinterpret_cast(tensor.col) % 16 == 0; + } + // The validation pass ensures a nonempty chunk always fits the grid limit. + const dim3 grid(static_cast(tiles)); + TRANSFORMER_ENGINE_TYPE_SWITCH_FP8ONLY( + output_dtype, OutputType, + TRANSFORMER_ENGINE_SWITCH_CONDITION( + aligned, kAligned, + grouped_block_scaled_1d_cast_transpose_kernel + <<>>(params););); + NVTE_CHECK_CUDA(cudaGetLastError()); + } + return true; +} + } // namespace transformer_engine::detail diff --git a/transformer_engine/pytorch/csrc/extensions/cast.cpp b/transformer_engine/pytorch/csrc/extensions/cast.cpp index 13dd26f6259..b3a1f85baaa 100644 --- a/transformer_engine/pytorch/csrc/extensions/cast.cpp +++ b/transformer_engine/pytorch/csrc/extensions/cast.cpp @@ -1196,6 +1196,51 @@ py::object group_requantize_inplace(py::handle grouped_x, py::handle quantizer, namespace { +bool try_grouped_blockwise_quantize(const std::vector &inputs, + const std::vector &quantizer_py_list, + std::vector> &quantizers, + std::vector &outputs) { + if (inputs.size() < 2 || + transformer_engine::getenv("NVTE_FP8_BLOCK_QUANTIZATION_GROUPED", 1) == 0 || + transformer_engine::cuda::sm_arch() != 90 || + !std::all_of(quantizer_py_list.begin(), quantizer_py_list.end(), [](const py::handle &q) { + return detail::IsFloat8BlockwiseQuantizers(q.ptr()); + })) { + return false; + } + const auto *reference = static_cast(quantizers[0].get()); + if (reference->dtype != DType::kFloat8E4M3 && reference->dtype != DType::kFloat8E5M2) { + return false; + } + size_t k = 0; + std::vector input_tensors, output_tensors; + input_tensors.reserve(inputs.size()); + output_tensors.reserve(outputs.size()); + for (size_t i = 0; i < inputs.size(); ++i) { + const auto *quantizer = static_cast(quantizers[i].get()); + const auto shape = inputs[i].shape(); + if (inputs[i].dtype() != DType::kBFloat16 || shape.ndim != 2 || shape.data[1] == 0 || + (i != 0 && shape.data[1] != k) || !quantizer->rowwise_usage || + !quantizer->columnwise_usage || quantizer->get_scaling_mode() != NVTE_BLOCK_SCALING_1D || + quantizer->dtype != reference->dtype || + quantizer->force_pow_2_scales != reference->force_pow_2_scales || + quantizer->amax_epsilon != reference->amax_epsilon) { + return false; + } + k = shape.data[1]; + input_tensors.push_back(inputs[i].data()); + output_tensors.push_back(outputs[i].data()); + } + QuantizationConfigWrapper config; + config.set_force_pow_2_scales(reference->force_pow_2_scales); + config.set_amax_epsilon(reference->amax_epsilon); + NVTE_SCOPED_GIL_RELEASE({ + nvte_multi_tensor_quantize(input_tensors.data(), output_tensors.data(), config, + input_tensors.size(), at::cuda::getCurrentCUDAStream()); + }); + return true; +} + void multi_tensor_quantize_impl(const std::vector &input_list, std::vector &quantizer_py_list, std::vector> &quantizer_cpp_list, @@ -1209,6 +1254,11 @@ void multi_tensor_quantize_impl(const std::vector &input_list, NVTE_CHECK(output_list.size() == num_tensors, "Expected ", num_tensors, " output tensors, but got ", output_list.size()); + if (try_grouped_blockwise_quantize(input_list, quantizer_py_list, quantizer_cpp_list, + output_list)) { + return; + } + // Choose implementation // Note: Currently only have fused kernel for FP8 delayed scaling bool with_fused_kernel = true;